TSDB: Tests for data_stream

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -0,0 +1,137 @@
+setup:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+      features: allowed_warnings
+
+  - do:
+      allowed_warnings:
+        - "index template [tsdbds-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template1
+        body:
+          index_patterns: [k8s*]
+          data_stream: {}
+          template:
+            settings:
+              index:
+                number_of_replicas: 0
+                number_of_shards: 2
+                mode: time_series
+            mappings:
+              properties:
+                "@timestamp":
+                  type: date
+                metricset:
+                  type: keyword
+                  time_series_dimension: true
+                k8s:
+                  properties:
+                    pod:
+                      properties:
+                        uid:
+                          type: keyword
+                          time_series_dimension: true
+                        name:
+                          type: keyword
+                        ip:
+                          type: ip
+                        network:
+                          properties:
+                            tx:
+                              type: long
+                            rx:
+                              type: long
+
+  - do:
+      bulk:
+        refresh: true
+        index: k8s
+        body:
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2005177954, "rx": 801479970}}}}'
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2006223737, "rx": 802337279}}}}'
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.2", "network": {"tx": 2012916202, "rx": 803685721}}}}'
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434521831, "rx": 530575198}}}}'
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434577921, "rx": 530600088}}}}'
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434587694, "rx": 530604797}}}}'
+          - '{"create": {}}'
+          - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 1434595272, "rx": 530605511}}}}'
+
+---
+created the data stream:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+
+  - do:
+      indices.get_data_stream:
+        name: '*'
+  - length: { data_streams: 1 }
+  - match: { data_streams.0.name: 'k8s' }
+  - match: { data_streams.0.timestamp_field.name: '@timestamp' }
+  - match: { data_streams.0.generation: 1 }
+  - length: { data_streams.0.indices: 1 }
+  - match: { data_streams.0.indices.0.index_name: '/\.ds-k8s-\d{4}\.\d{2}\.\d{2}-000001/' }
+  - match: { data_streams.0.status: 'GREEN' }
+  - match: { data_streams.0.template: 'my-template1' }
+  - match: { data_streams.0.hidden: false }
+  - match: { data_streams.0.system: false }
+
+---
+fetch the tsid:
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+
+  - do:
+      search:
+        index: k8s
+        body:
+          runtime_mappings: # TODO replace this with tsid once it is generated
+            tsid:
+              type: keyword
+              script: emit('k8s.pod.uid:' + doc['k8s.pod.uid'].value + ',metricset:' + doc['metricset'].value)
+          fields:
+            - field: tsid
+          query:
+            query_string:
+              query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat'
+
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0.fields.tsid: ['k8s.pod.uid:947e4ced-1786-4e53-9e0c-5c447e959507,metricset:pod']}
+
+---
+"aggregate the tsid":
+  - skip:
+      version: " - 7.99.99"
+      reason: introduced in 8.0.0
+
+  - do:
+      search:
+        index: k8s
+        body:
+          size: 0
+          runtime_mappings: # TODO replace this with tsid once it is generated
+            tsid:
+              type: keyword
+              script: emit('k8s.pod.uid:' + doc['k8s.pod.uid'].value + ',metricset:' + doc['metricset'].value)
+          aggs:
+            tsids:
+              terms:
+                field: tsid
+                order:
+                  _key: asc
+
+  - match: {hits.total.value: 8}
+  - match: {aggregations.tsids.buckets.0.key: 'k8s.pod.uid:947e4ced-1786-4e53-9e0c-5c447e959507,metricset:pod'}
+  - match: {aggregations.tsids.buckets.0.doc_count: 4}
+  - match: {aggregations.tsids.buckets.1.key: 'k8s.pod.uid:df3145b3-0563-4d3b-a0f7-897eb2876ea9,metricset:pod'}
+  - match: {aggregations.tsids.buckets.1.doc_count: 4}