Skip to content

Commit e25a879

Browse files
SpencerTorresalyssabull
SpencerTorres
and
alyssabull
authored
Add "Labels" column for log queries (#941)
Co-authored-by: Alyssa (Bull) Joyner <[email protected]>
1 parent 3fb2082 commit e25a879

File tree

13 files changed

+156
-18
lines changed

13 files changed

+156
-18
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
### Features
66

7+
- Added "Labels" column selector to the log query builder
78
- Datasource OTel configuration will now set default table names for logs and traces.
89

910
### Fixes

‎cspell.config.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
"mage_output_file.go"
1111
],
1212
"words": [
13+
"dataframe",
1314
"lowcardinality",
1415
"fixedstring",
1516
"singlequote",

‎gen-db-dashboards.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ function generatePanels(clickHouseDashboards) {
4242
const positionY = panelHeight * Math.floor(i / 2);
4343
const panel = generatePanel(id, name, query, panelWidth, panelHeight, positionX, positionY);
4444

45-
if (lastRowName != dashboardTitle) {
45+
if (lastRowName !== dashboardTitle) {
4646
const rowPanel = generateRowPanel(dashboardTitle, positionY);
4747
panels.push(rowPanel);
4848
lastRowName = dashboardTitle;

‎pkg/plugin/driver.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,8 @@ func (h *Clickhouse) MutateQuery(ctx context.Context, req backend.DataQuery) (co
272272
func (h *Clickhouse) MutateResponse(ctx context.Context, res data.Frames) (data.Frames, error) {
273273
for _, frame := range res {
274274
if frame.Meta.PreferredVisualization != data.VisTypeTrace &&
275-
frame.Meta.PreferredVisualization != data.VisTypeTable {
275+
frame.Meta.PreferredVisualization != data.VisTypeTable &&
276+
frame.Meta.PreferredVisualization != data.VisTypeLogs {
276277
var fields []*data.Field
277278
for _, field := range frame.Fields {
278279
values := make([]*string, field.Len())

‎pkg/plugin/driver_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1145,12 +1145,12 @@ func TestConvertNullableIPv6(t *testing.T) {
11451145
// assert.Equal(t, frames[0].Fields[0].Type(), data.FieldTypeNullableJSON)
11461146
// })
11471147

1148-
// t.Run("doesn't mutate tables", func(t *testing.T) {
1148+
// t.Run("doesn't mutate logs", func(t *testing.T) {
11491149
// rows, err := conn.Query("SELECT * FROM simple_table LIMIT 1")
11501150
// require.NoError(t, err)
11511151
// frame, err := sqlutil.FrameFromRows(rows, 1, converters.ClickhouseConverters...)
11521152
// require.NoError(t, err)
1153-
// frame.Meta = &data.FrameMeta{PreferredVisualization: data.VisType(data.VisTypeTable)}
1153+
// frame.Meta = &data.FrameMeta{PreferredVisualization: data.VisType(data.VisTypeLogs)}
11541154
// frames, err := clickhouse.MutateResponse(context.Background(), []*data.Frame{frame})
11551155
// require.NoError(t, err)
11561156
// require.NotNil(t, frames)
@@ -1162,7 +1162,7 @@ func TestConvertNullableIPv6(t *testing.T) {
11621162
// require.NoError(t, err)
11631163
// frame, err := sqlutil.FrameFromRows(rows, 1, converters.ClickhouseConverters...)
11641164
// require.NoError(t, err)
1165-
// frame.Meta = &data.FrameMeta{PreferredVisualization: data.VisType(data.VisTypeLogs)}
1165+
// frame.Meta = &data.FrameMeta{PreferredVisualization: data.VisType(data.VisTypeGraph)}
11661166
// frames, err := clickhouse.MutateResponse(context.Background(), []*data.Frame{frame})
11671167
// require.NoError(t, err)
11681168
// require.NotNil(t, frames)

‎src/components/queryBuilder/views/LogsQueryBuilder.tsx

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ interface LogsQueryBuilderState {
3232
timeColumn?: SelectedColumn;
3333
logLevelColumn?: SelectedColumn;
3434
messageColumn?: SelectedColumn;
35+
labelsColumn?: SelectedColumn;
3536
// liveView: boolean;
3637
orderBy: OrderBy[];
3738
limit: number;
@@ -50,11 +51,13 @@ export const LogsQueryBuilder = (props: LogsQueryBuilderProps) => {
5051
timeColumn: getColumnByHint(builderOptions, ColumnHint.Time),
5152
logLevelColumn: getColumnByHint(builderOptions, ColumnHint.LogLevel),
5253
messageColumn: getColumnByHint(builderOptions, ColumnHint.LogMessage),
54+
labelsColumn: getColumnByHint(builderOptions, ColumnHint.LogLabels),
5355
selectedColumns: builderOptions.columns?.filter(c => (
5456
// Only select columns that don't have their own box
5557
c.hint !== ColumnHint.Time &&
5658
c.hint !== ColumnHint.LogLevel &&
57-
c.hint !== ColumnHint.LogMessage
59+
c.hint !== ColumnHint.LogMessage &&
60+
c.hint !== ColumnHint.LogLabels
5861
)) || [],
5962
// liveView: builderOptions.meta?.liveView || false,
6063
filters: builderOptions.filters || [],
@@ -75,6 +78,9 @@ export const LogsQueryBuilder = (props: LogsQueryBuilderProps) => {
7578
if (next.messageColumn) {
7679
nextColumns.push(next.messageColumn);
7780
}
81+
if (next.labelsColumn) {
82+
nextColumns.push(next.labelsColumn);
83+
}
7884

7985
builderOptionsDispatch(setOptions({
8086
columns: nextColumns,
@@ -155,6 +161,17 @@ export const LogsQueryBuilder = (props: LogsQueryBuilderProps) => {
155161
label={labels.logMessageColumn.label}
156162
tooltip={labels.logMessageColumn.tooltip}
157163
/>
164+
<ColumnSelect
165+
disabled={builderState.otelEnabled}
166+
allColumns={allColumns}
167+
selectedColumn={builderState.labelsColumn}
168+
invalid={!builderState.labelsColumn}
169+
onColumnChange={onOptionChange('labelsColumn')}
170+
columnHint={ColumnHint.LogLabels}
171+
label={labels.logLabelsColumn.label}
172+
tooltip={labels.logLabelsColumn.tooltip}
173+
inline
174+
/>
158175
{/* <Switch
159176
value={builderState.liveView}
160177
onChange={onOptionChange('liveView')}

‎src/data/CHDatasource.ts

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ import {
4747
import { generateSql, getColumnByHint, logAliasToColumnHints } from './sqlGenerator';
4848
import otel from 'otel';
4949
import { ReactNode } from 'react';
50-
import { transformQueryResponseWithTraceAndLogLinks } from './utils';
50+
import { dataFrameHasLogLabelWithName, transformQueryResponseWithTraceAndLogLinks } from './utils';
5151
import { pluginVersion } from 'utils/version';
5252

5353
export class Datasource
@@ -282,13 +282,15 @@ export class Datasource
282282
}
283283

284284
const columnName = action.options.key;
285+
const actionFrame: DataFrame | undefined = (action as any).frame;
285286
const actionValue = action.options.value;
286287

287288
// Find selected column by alias/name
288289
const lookupByAlias = query.builderOptions.columns?.find(c => c.alias === columnName); // Check all aliases first,
289290
const lookupByName = query.builderOptions.columns?.find(c => c.name === columnName); // then try matching column name
290291
const lookupByLogsAlias = logAliasToColumnHints.has(columnName) ? getColumnByHint(query.builderOptions, logAliasToColumnHints.get(columnName)!) : undefined;
291-
const column = lookupByAlias || lookupByName || lookupByLogsAlias;
292+
const lookupByLogLabels = dataFrameHasLogLabelWithName(actionFrame, columnName) && getColumnByHint(query.builderOptions, ColumnHint.LogLabels);
293+
const column = lookupByAlias || lookupByName || lookupByLogsAlias || lookupByLogLabels;
292294

293295
let nextFilters: Filter[] = (query.builderOptions.filters?.slice() || []);
294296
if (action.type === 'ADD_FILTER') {
@@ -299,13 +301,20 @@ export class Datasource
299301
f.type === 'string' &&
300302
((column && column.hint && f.hint) ? f.hint === column.hint : f.key === columnName) &&
301303
(f.operator === FilterOperator.IsAnything || f.operator === FilterOperator.Equals || f.operator === FilterOperator.NotEquals)
304+
) &&
305+
!(
306+
f.type.toLowerCase().startsWith('map') &&
307+
(column && lookupByLogLabels && f.mapKey === columnName) &&
308+
(f.operator === FilterOperator.IsAnything || f.operator === FilterOperator.Equals || f.operator === FilterOperator.NotEquals)
302309
)
303310
);
311+
304312
nextFilters.push({
305313
condition: 'AND',
306314
key: (column && column.hint) ? '' : columnName,
307315
hint: (column && column.hint) ? column.hint : undefined,
308-
type: 'string',
316+
mapKey: lookupByLogLabels ? columnName : undefined,
317+
type: lookupByLogLabels ? 'Map(String, String)' : 'string',
309318
filterType: 'custom',
310319
operator: FilterOperator.Equals,
311320
value: actionValue,
@@ -325,14 +334,21 @@ export class Datasource
325334
f.type === 'string' &&
326335
((column && column.hint && f.hint) ? f.hint === column.hint : f.key === columnName) &&
327336
(f.operator === FilterOperator.IsAnything || f.operator === FilterOperator.Equals)
337+
) ||
338+
(
339+
f.type.toLowerCase().startsWith('map') &&
340+
(column && lookupByLogLabels && f.mapKey === columnName) &&
341+
(f.operator === FilterOperator.IsAnything || f.operator === FilterOperator.Equals)
328342
)
329343
)
330344
);
345+
331346
nextFilters.push({
332347
condition: 'AND',
333348
key: (column && column.hint) ? '' : columnName,
334349
hint: (column && column.hint) ? column.hint : undefined,
335-
type: 'string',
350+
mapKey: lookupByLogLabels ? columnName : undefined,
351+
type: lookupByLogLabels ? 'Map(String, String)' : 'string',
336352
filterType: 'custom',
337353
operator: FilterOperator.NotEquals,
338354
value: actionValue,

‎src/data/sqlGenerator.ts

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -222,27 +222,33 @@ const generateLogsQuery = (_options: QueryBuilderOptions): string => {
222222
const logTime = getColumnByHint(options, ColumnHint.Time);
223223
if (logTime !== undefined) {
224224
// Must be first column in list.
225-
logTime.alias = 'timestamp';
225+
logTime.alias = logColumnHintsToAlias.get(ColumnHint.Time);
226226
selectParts.push(getColumnIdentifier(logTime));
227227
}
228228

229229
const logMessage = getColumnByHint(options, ColumnHint.LogMessage);
230230
if (logMessage !== undefined) {
231231
// Must be second column in list.
232-
logMessage.alias = 'body';
232+
logMessage.alias = logColumnHintsToAlias.get(ColumnHint.LogMessage);
233233
selectParts.push(getColumnIdentifier(logMessage));
234234
}
235235

236236
const logLevel = getColumnByHint(options, ColumnHint.LogLevel);
237237
if (logLevel !== undefined) {
238238
// TODO: "severity" should be a number, but "level" can be a string? Perhaps we can check the column type here?
239-
logLevel.alias = 'level';
239+
logLevel.alias = logColumnHintsToAlias.get(ColumnHint.LogLevel);
240240
selectParts.push(getColumnIdentifier(logLevel));
241241
}
242242

243+
const logLabels = getColumnByHint(options, ColumnHint.LogLabels);
244+
if (logLabels !== undefined) {
245+
logLabels.alias = logColumnHintsToAlias.get(ColumnHint.LogLabels);
246+
selectParts.push(getColumnIdentifier(logLabels));
247+
}
248+
243249
const traceId = getColumnByHint(options, ColumnHint.TraceId);
244250
if (traceId !== undefined) {
245-
traceId.alias = 'traceID';
251+
traceId.alias = logColumnHintsToAlias.get(ColumnHint.TraceId);
246252
selectParts.push(getColumnIdentifier(traceId));
247253
}
248254

@@ -777,12 +783,15 @@ const isMultiFilter = (type: string, operator: FilterOperator): boolean => isStr
777783
* map from the SQL generator's aliases back to the original column hints
778784
* so that filters can be added properly.
779785
*/
780-
export const logAliasToColumnHints: Map<string, ColumnHint> = new Map([
786+
const logAliasToColumnHintsEntries: ReadonlyArray<[string, ColumnHint]> = [
781787
['timestamp', ColumnHint.Time],
782788
['body', ColumnHint.LogMessage],
783789
['level', ColumnHint.LogLevel],
790+
['labels', ColumnHint.LogLabels],
784791
['traceID', ColumnHint.TraceId],
785-
]);
792+
];
793+
export const logAliasToColumnHints: Map<string, ColumnHint> = new Map(logAliasToColumnHintsEntries);
794+
export const logColumnHintsToAlias: Map<ColumnHint, string> = new Map(logAliasToColumnHintsEntries.map(e => [e[1], e[0]]));
786795

787796

788797
export const _testExports = {

‎src/data/utils.test.ts

Lines changed: 62 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
import { QueryBuilderOptions, QueryType } from "types/queryBuilder";
2-
import { columnLabelToPlaceholder, isBuilderOptionsRunnable, transformQueryResponseWithTraceAndLogLinks } from "./utils";
1+
import { ColumnHint, QueryBuilderOptions, QueryType } from "types/queryBuilder";
2+
import { columnLabelToPlaceholder, dataFrameHasLogLabelWithName, isBuilderOptionsRunnable, transformQueryResponseWithTraceAndLogLinks } from "./utils";
33
import { newMockDatasource } from "__mocks__/datasource";
44
import { CoreApp, DataFrame, DataQueryRequest, DataQueryResponse, Field, FieldType } from "@grafana/data";
55
import { CHBuilderQuery, CHQuery, EditorType } from "types/sql";
6+
import { logColumnHintsToAlias } from "./sqlGenerator";
67

78
describe('isBuilderOptionsRunnable', () => {
89
it('should return false for empty builder options', () => {
@@ -137,3 +138,62 @@ describe('transformQueryResponseWithTraceAndLogLinks', () => {
137138
expect(getDefaultLogsColumns).not.toHaveBeenCalled();
138139
});
139140
});
141+
142+
143+
describe('dataFrameHasLogLabelWithName', () => {
144+
const logLabelsFieldName = logColumnHintsToAlias.get(ColumnHint.LogLabels);
145+
146+
it('should return false for undefined dataframe', () => {
147+
expect(dataFrameHasLogLabelWithName(undefined, 'testLabel')).toBe(false);
148+
});
149+
150+
it('should return false for dataframe with no fields', () => {
151+
const frame: DataFrame = { fields: [] } as any as DataFrame;
152+
expect(dataFrameHasLogLabelWithName(frame, 'testLabel')).toBe(false);
153+
});
154+
155+
it('should return false when log labels field is not present', () => {
156+
const frame: DataFrame = {
157+
fields: [{ name: 'otherField', values: { get: jest.fn(), length: 1 } }],
158+
} as any as DataFrame;
159+
expect(dataFrameHasLogLabelWithName(frame, 'testLabel')).toBe(false);
160+
});
161+
162+
it('should return false when log labels field has no values', () => {
163+
const frame: DataFrame = {
164+
fields: [{ name: logLabelsFieldName, values: { get: jest.fn(), length: 0 } }],
165+
} as any as DataFrame;
166+
expect(dataFrameHasLogLabelWithName(frame, 'testLabel')).toBe(false);
167+
});
168+
169+
it('should return false when log labels field value is null', () => {
170+
const frame: DataFrame = {
171+
fields: [{ name: logLabelsFieldName, values: { get: () => null, length: 1 } }],
172+
} as any as DataFrame;
173+
expect(dataFrameHasLogLabelWithName(frame, 'testLabel')).toBe(false);
174+
});
175+
176+
it('should return true when log label with given name exists', () => {
177+
const frame: DataFrame = {
178+
fields: [
179+
{
180+
name: logLabelsFieldName,
181+
values: { get: () => ({ testLabel: 'value', otherLabel: 'otherValue' }), length: 1 },
182+
},
183+
],
184+
} as any as DataFrame;
185+
expect(dataFrameHasLogLabelWithName(frame, 'testLabel')).toBe(true);
186+
});
187+
188+
it('should return false when log label with given name does not exist', () => {
189+
const frame: DataFrame = {
190+
fields: [
191+
{
192+
name: logLabelsFieldName,
193+
values: { get: () => ({ otherLabel: 'value' }), length: 1 },
194+
},
195+
],
196+
} as any as DataFrame;
197+
expect(dataFrameHasLogLabelWithName(frame, 'testLabel')).toBe(false);
198+
});
199+
});

‎src/data/utils.ts

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { ColumnHint, FilterOperator, OrderByDirection, QueryBuilderOptions, Quer
33
import { CHBuilderQuery, CHQuery, EditorType } from "types/sql";
44
import { Datasource } from "./CHDatasource";
55
import { pluginVersion } from "utils/version";
6+
import { logColumnHintsToAlias } from "./sqlGenerator";
67

78
/**
89
* Returns true if the builder options contain enough information to start showing a query
@@ -253,3 +254,28 @@ export const transformQueryResponseWithTraceAndLogLinks = (datasource: Datasourc
253254

254255
return res;
255256
};
257+
258+
259+
/**
260+
* Returns true if the dataframe contains a log label that matches the provided name.
261+
*
262+
* This function exists for the logs panel, when clicking "filter for value" on a single log row.
263+
* A dataframe will be provided for that single row, and we need to check the labels object to see if it
264+
* contains a field with that name. If it does then we can create a filter using the labels column hint.
265+
*/
266+
export const dataFrameHasLogLabelWithName = (frame: DataFrame | undefined, name: string): boolean => {
267+
if (!frame || !frame.fields || frame.fields.length === 0) {
268+
return false;
269+
}
270+
271+
const logLabelsFieldName = logColumnHintsToAlias.get(ColumnHint.LogLabels);
272+
const field = frame.fields.find(f => f.name === logLabelsFieldName);
273+
if (!field || !field.values || field.values.length < 1 || !field.values.get(0)) {
274+
return false;
275+
}
276+
277+
const labels = (field.values.get(0) || {}) as object;
278+
const labelKeys = Object.keys(labels);
279+
280+
return labelKeys.includes(name);
281+
}

‎src/labels.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,10 @@ export default {
306306
label: 'Message',
307307
tooltip: 'Column that contains the log message'
308308
},
309+
logLabelsColumn: {
310+
label: 'Labels',
311+
tooltip: 'A column with a key/value structure for log labels'
312+
},
309313
liveView: {
310314
label: 'Live View',
311315
tooltip: 'Enable to update logs in real time'
@@ -408,6 +412,7 @@ export default {
408412

409413
[ColumnHint.LogLevel]: 'Level',
410414
[ColumnHint.LogMessage]: 'Message',
415+
[ColumnHint.LogLabels]: 'Labels',
411416

412417
[ColumnHint.TraceId]: 'Trace ID',
413418
[ColumnHint.TraceSpanId]: 'Span ID',

‎src/otel.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const otel129: OtelVersion = {
2626
[ColumnHint.Time, 'Timestamp'],
2727
[ColumnHint.LogMessage, 'Body'],
2828
[ColumnHint.LogLevel, 'SeverityText'],
29+
[ColumnHint.LogLabels, 'LogAttributes'],
2930
[ColumnHint.TraceId, 'TraceId'],
3031
]),
3132
logLevels: [

‎src/types/queryBuilder.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ export enum ColumnHint {
122122

123123
LogLevel = 'log_level',
124124
LogMessage = 'log_message',
125+
LogLabels = 'log_labels',
125126

126127
TraceId = 'trace_id',
127128
TraceSpanId = 'trace_span_id',

0 commit comments

Comments
 (0)