diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml new file mode 100644 index 0000000000000..6acc35392a53d --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -0,0 +1,137 @@ +setup: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [tsdbds-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [k8s*] + data_stream: {} + template: + settings: + index: + number_of_replicas: 0 + number_of_shards: 2 + mode: time_series + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + ip: + type: ip + network: + properties: + tx: + type: long + rx: + type: long + + - do: + bulk: + refresh: true + index: k8s + body: + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}' + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}' + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}' + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}' + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}' + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}' + - '{"create": {}}' + - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}' + +--- +created the data stream: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + indices.get_data_stream: + name: '*' + - length: { data_streams: 1 } + - match: { data_streams.0.name: 'k8s' } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-k8s-\d{4}\.\d{2}\.\d{2}-000001/' } + - match: { data_streams.0.status: 'GREEN' } + - match: { data_streams.0.template: 'my-template1' } + - match: { data_streams.0.hidden: false } + - match: { data_streams.0.system: false } + +--- +fetch the tsid: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + search: + index: k8s + body: + runtime_mappings: # TODO replace this with tsid once it is generated + tsid: + type: keyword + script: emit('k8s.pod.uid:' + doc['k8s.pod.uid'].value + ',metricset:' + doc['metricset'].value) + fields: + - field: tsid + query: + query_string: + query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' + + - match: {hits.total.value: 1} + - match: {hits.hits.0.fields.tsid: ['k8s.pod.uid:947e4ced-1786-4e53-9e0c-5c447e959507,metricset:pod']} + +--- +"aggregate the tsid": + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 + + - do: + search: + index: k8s + body: + size: 0 + runtime_mappings: # TODO replace this with tsid once it is generated + tsid: + type: keyword + script: emit('k8s.pod.uid:' + doc['k8s.pod.uid'].value + ',metricset:' + doc['metricset'].value) + aggs: + tsids: + terms: + field: tsid + order: + _key: asc + + - match: {hits.total.value: 8} + - match: {aggregations.tsids.buckets.0.key: 'k8s.pod.uid:947e4ced-1786-4e53-9e0c-5c447e959507,metricset:pod'} + - match: {aggregations.tsids.buckets.0.doc_count: 4} + - match: {aggregations.tsids.buckets.1.key: 'k8s.pod.uid:df3145b3-0563-4d3b-a0f7-897eb2876ea9,metricset:pod'} + - match: {aggregations.tsids.buckets.1.doc_count: 4}