Skip to content

Commit 63bb072

Browse files
darnautovdarnautov
authored
[ML] Transforms: Filter aggregation support (#67591)
* [ML] WIP filter support * [ML] value selector * [ML] only supported filter aggs as options * [ML] WIP apply config * [ML] fix form persistence * [ML] refactor * [ML] support clone * [ML] validation, get es config * [ML] support "exists", fixes for the term form, validation * [ML] fix ts issues * [ML] don't perform request on adding incomplete agg * [ML] basic range number support * [ML] filter bool agg support * [ML] functional tests * [ML] getAggConfigFromEsAgg tests * [ML] fix unit tests * [ML] agg name update on config change, add unit tests * [ML] update snapshot * [ML] range selector enhancements * [ML] improve types * [ML] update step for range selector to support float numbers * [ML] range validation * [ML] term selector improvements * [ML] fix switch between advanced editor * [ML] prefix test ids * [ML] support helper text for aggs item
1 parent 1346b15 commit 63bb072

28 files changed

Lines changed: 1401 additions & 141 deletions

‎x-pack/plugins/ml/common/util/validators.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ export function requiredValidator() {
6565
};
6666
}
6767

68+
export type ValidationResult = object | null;
69+
6870
export function memoryInputValidator(allowedUnits = ALLOWED_DATA_UNITS) {
6971
return (value: any) => {
7072
if (typeof value !== 'string' || value === '') {

‎x-pack/plugins/transform/public/app/common/pivot_aggs.test.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import { getAggConfigFromEsAgg } from './pivot_aggs';
8+
import {
9+
FilterAggForm,
10+
FilterTermForm,
11+
} from '../sections/create_transform/components/step_define/common/filter_agg/components';
12+
13+
describe('getAggConfigFromEsAgg', () => {
14+
test('should throw an error for unsupported agg', () => {
15+
expect(() => getAggConfigFromEsAgg({ terms: {} }, 'test')).toThrowError();
16+
});
17+
18+
test('should return a common config if the agg does not have a custom config defined', () => {
19+
expect(getAggConfigFromEsAgg({ avg: { field: 'region' } }, 'test_1')).toEqual({
20+
agg: 'avg',
21+
aggName: 'test_1',
22+
dropDownName: 'test_1',
23+
field: 'region',
24+
});
25+
});
26+
27+
test('should return a custom config for recognized aggregation type', () => {
28+
expect(
29+
getAggConfigFromEsAgg({ filter: { term: { region: 'sa-west-1' } } }, 'test_2')
30+
).toMatchObject({
31+
agg: 'filter',
32+
aggName: 'test_2',
33+
dropDownName: 'test_2',
34+
field: 'region',
35+
AggFormComponent: FilterAggForm,
36+
aggConfig: {
37+
filterAgg: 'term',
38+
aggTypeConfig: {
39+
FilterAggFormComponent: FilterTermForm,
40+
filterAggConfig: {
41+
value: 'sa-west-1',
42+
},
43+
},
44+
},
45+
});
46+
});
47+
});

‎x-pack/plugins/transform/public/app/common/pivot_aggs.ts‎

Lines changed: 130 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,51 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7+
import { FC } from 'react';
78
import { Dictionary } from '../../../common/types/common';
89
import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/common';
910

1011
import { AggName } from './aggregations';
1112
import { EsFieldName } from './fields';
13+
import { getAggFormConfig } from '../sections/create_transform/components/step_define/common/get_agg_form_config';
14+
import { PivotAggsConfigFilter } from '../sections/create_transform/components/step_define/common/filter_agg/types';
1215

13-
export enum PIVOT_SUPPORTED_AGGS {
14-
AVG = 'avg',
15-
CARDINALITY = 'cardinality',
16-
MAX = 'max',
17-
MIN = 'min',
18-
PERCENTILES = 'percentiles',
19-
SUM = 'sum',
20-
VALUE_COUNT = 'value_count',
16+
export type PivotSupportedAggs = typeof PIVOT_SUPPORTED_AGGS[keyof typeof PIVOT_SUPPORTED_AGGS];
17+
18+
export function isPivotSupportedAggs(arg: any): arg is PivotSupportedAggs {
19+
return Object.values(PIVOT_SUPPORTED_AGGS).includes(arg);
2120
}
2221

22+
export const PIVOT_SUPPORTED_AGGS = {
23+
AVG: 'avg',
24+
CARDINALITY: 'cardinality',
25+
MAX: 'max',
26+
MIN: 'min',
27+
PERCENTILES: 'percentiles',
28+
SUM: 'sum',
29+
VALUE_COUNT: 'value_count',
30+
FILTER: 'filter',
31+
} as const;
32+
2333
export const PERCENTILES_AGG_DEFAULT_PERCENTS = [1, 5, 25, 50, 75, 95, 99];
2434

2535
export const pivotAggsFieldSupport = {
26-
[KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
27-
[KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
36+
[KBN_FIELD_TYPES.ATTACHMENT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
37+
[KBN_FIELD_TYPES.BOOLEAN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
2838
[KBN_FIELD_TYPES.DATE]: [
2939
PIVOT_SUPPORTED_AGGS.MAX,
3040
PIVOT_SUPPORTED_AGGS.MIN,
3141
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
42+
PIVOT_SUPPORTED_AGGS.FILTER,
43+
],
44+
[KBN_FIELD_TYPES.GEO_POINT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
45+
[KBN_FIELD_TYPES.GEO_SHAPE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
46+
[KBN_FIELD_TYPES.IP]: [
47+
PIVOT_SUPPORTED_AGGS.CARDINALITY,
48+
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
49+
PIVOT_SUPPORTED_AGGS.FILTER,
3250
],
33-
[KBN_FIELD_TYPES.GEO_POINT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
34-
[KBN_FIELD_TYPES.GEO_SHAPE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
35-
[KBN_FIELD_TYPES.IP]: [PIVOT_SUPPORTED_AGGS.CARDINALITY, PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
36-
[KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
51+
[KBN_FIELD_TYPES.MURMUR3]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
3752
[KBN_FIELD_TYPES.NUMBER]: [
3853
PIVOT_SUPPORTED_AGGS.AVG,
3954
PIVOT_SUPPORTED_AGGS.CARDINALITY,
@@ -42,49 +57,122 @@ export const pivotAggsFieldSupport = {
4257
PIVOT_SUPPORTED_AGGS.PERCENTILES,
4358
PIVOT_SUPPORTED_AGGS.SUM,
4459
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
60+
PIVOT_SUPPORTED_AGGS.FILTER,
61+
],
62+
[KBN_FIELD_TYPES.STRING]: [
63+
PIVOT_SUPPORTED_AGGS.CARDINALITY,
64+
PIVOT_SUPPORTED_AGGS.VALUE_COUNT,
65+
PIVOT_SUPPORTED_AGGS.FILTER,
4566
],
46-
[KBN_FIELD_TYPES.STRING]: [PIVOT_SUPPORTED_AGGS.CARDINALITY, PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
47-
[KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
48-
[KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
49-
[KBN_FIELD_TYPES.CONFLICT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT],
67+
[KBN_FIELD_TYPES._SOURCE]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
68+
[KBN_FIELD_TYPES.UNKNOWN]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
69+
[KBN_FIELD_TYPES.CONFLICT]: [PIVOT_SUPPORTED_AGGS.VALUE_COUNT, PIVOT_SUPPORTED_AGGS.FILTER],
5070
};
5171

5272
export type PivotAgg = {
53-
[key in PIVOT_SUPPORTED_AGGS]?: {
73+
[key in PivotSupportedAggs]?: {
5474
field: EsFieldName;
5575
};
5676
};
5777

58-
export type PivotAggDict = { [key in AggName]: PivotAgg };
78+
export type PivotAggDict = {
79+
[key in AggName]: PivotAgg;
80+
};
5981

6082
// The internal representation of an aggregation definition.
6183
export interface PivotAggsConfigBase {
62-
agg: PIVOT_SUPPORTED_AGGS;
84+
agg: PivotSupportedAggs;
6385
aggName: AggName;
6486
dropDownName: string;
6587
}
6688

67-
interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
89+
/**
90+
* Resolves agg UI config from provided ES agg definition
91+
*/
92+
export function getAggConfigFromEsAgg(esAggDefinition: Record<string, any>, aggName: string) {
93+
const aggKeys = Object.keys(esAggDefinition);
94+
95+
// Find the main aggregation key
96+
const agg = aggKeys.find((aggKey) => aggKey !== 'aggs');
97+
98+
if (!isPivotSupportedAggs(agg)) {
99+
throw new Error(`Aggregation "${agg}" is not supported`);
100+
}
101+
102+
const commonConfig: PivotAggsConfigBase = {
103+
...esAggDefinition[agg],
104+
agg,
105+
aggName,
106+
dropDownName: aggName,
107+
};
108+
109+
const config = getAggFormConfig(agg, commonConfig);
110+
111+
if (isPivotAggsWithExtendedForm(config)) {
112+
config.setUiConfigFromEs(esAggDefinition[agg]);
113+
}
114+
115+
if (aggKeys.includes('aggs')) {
116+
// TODO process sub-aggregation
117+
}
118+
119+
return config;
120+
}
121+
122+
export interface PivotAggsConfigWithUiBase extends PivotAggsConfigBase {
68123
field: EsFieldName;
69124
}
70125

126+
export interface PivotAggsConfigWithExtra<T> extends PivotAggsConfigWithUiBase {
127+
/** Form component */
128+
AggFormComponent: FC<{
129+
aggConfig: Partial<T>;
130+
onChange: (arg: Partial<T>) => void;
131+
selectedField: string;
132+
}>;
133+
/** Aggregation specific configuration */
134+
aggConfig: Partial<T>;
135+
/** Set UI configuration from ES aggregation definition */
136+
setUiConfigFromEs: (arg: { [key: string]: any }) => void;
137+
/** Converts UI agg config form to ES agg request object */
138+
getEsAggConfig: () => { [key: string]: any } | null;
139+
/** Indicates if the configuration is valid */
140+
isValid: () => boolean;
141+
/** Provides aggregation name generated based on the configuration */
142+
getAggName?: () => string | undefined;
143+
/** Helper text for the aggregation reflecting some configuration info */
144+
helperText?: () => string | undefined;
145+
}
146+
71147
interface PivotAggsConfigPercentiles extends PivotAggsConfigWithUiBase {
72-
agg: PIVOT_SUPPORTED_AGGS.PERCENTILES;
148+
agg: typeof PIVOT_SUPPORTED_AGGS.PERCENTILES;
73149
percents: number[];
74150
}
75151

76-
export type PivotAggsConfigWithUiSupport = PivotAggsConfigWithUiBase | PivotAggsConfigPercentiles;
152+
export type PivotAggsConfigWithUiSupport =
153+
| PivotAggsConfigWithUiBase
154+
| PivotAggsConfigPercentiles
155+
| PivotAggsConfigWithExtendedForm;
77156

78157
export function isPivotAggsConfigWithUiSupport(arg: any): arg is PivotAggsConfigWithUiSupport {
79158
return (
80159
arg.hasOwnProperty('agg') &&
81160
arg.hasOwnProperty('aggName') &&
82161
arg.hasOwnProperty('dropDownName') &&
83162
arg.hasOwnProperty('field') &&
84-
Object.values(PIVOT_SUPPORTED_AGGS).includes(arg.agg)
163+
isPivotSupportedAggs(arg.agg)
85164
);
86165
}
87166

167+
/**
168+
* Union type for agg configs with extended forms
169+
*/
170+
type PivotAggsConfigWithExtendedForm = PivotAggsConfigFilter;
171+
172+
export function isPivotAggsWithExtendedForm(arg: any): arg is PivotAggsConfigWithExtendedForm {
173+
return arg.hasOwnProperty('AggFormComponent');
174+
}
175+
88176
export function isPivotAggsConfigPercentiles(arg: any): arg is PivotAggsConfigPercentiles {
89177
return (
90178
arg.hasOwnProperty('agg') &&
@@ -99,14 +187,28 @@ export type PivotAggsConfig = PivotAggsConfigBase | PivotAggsConfigWithUiSupport
99187
export type PivotAggsConfigWithUiSupportDict = Dictionary<PivotAggsConfigWithUiSupport>;
100188
export type PivotAggsConfigDict = Dictionary<PivotAggsConfig>;
101189

102-
export function getEsAggFromAggConfig(groupByConfig: PivotAggsConfigBase): PivotAgg {
103-
const esAgg = { ...groupByConfig };
190+
/**
191+
* Extracts Elasticsearch-ready aggregation configuration
192+
* from the UI config
193+
*/
194+
export function getEsAggFromAggConfig(
195+
pivotAggsConfig: PivotAggsConfigBase | PivotAggsConfigWithExtendedForm
196+
): PivotAgg | null {
197+
let esAgg: { [key: string]: any } | null = { ...pivotAggsConfig };
104198

105199
delete esAgg.agg;
106200
delete esAgg.aggName;
107201
delete esAgg.dropDownName;
108202

203+
if (isPivotAggsWithExtendedForm(pivotAggsConfig)) {
204+
esAgg = pivotAggsConfig.getEsAggConfig();
205+
206+
if (esAgg === null) {
207+
return null;
208+
}
209+
}
210+
109211
return {
110-
[groupByConfig.agg]: esAgg,
212+
[pivotAggsConfig.agg]: esAgg,
111213
};
112214
}

‎x-pack/plugins/transform/public/app/common/request.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,11 @@ export function getPreviewRequestBody(
115115
});
116116

117117
aggs.forEach((agg) => {
118-
request.pivot.aggregations[agg.aggName] = getEsAggFromAggConfig(agg);
118+
const result = getEsAggFromAggConfig(agg);
119+
if (result === null) {
120+
return;
121+
}
122+
request.pivot.aggregations[agg.aggName] = result;
119123
});
120124

121125
return request;

0 commit comments

Comments
 (0)