[ES|QL] Fixes the columns selection in Lens when there are ES|QL variables (#221083)

## Summary

It fixes the problem described in the PR. Specifically when you have a
variable in your esql query and you try to select extra columns. Because
we were not passing the esql Variables, the query failed and the columns
array was empty.

This PR is passing the dashboard variables in the component, and now the
query runs successfully

<img width="616" alt="image"
src="https://github.com/user-attachments/assets/cd009af9-9465-48cd-bb3b-918c3a872cff"
/>
 

### Checklist
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

Author: stratoula

x-pack/platform/plugins/shared/lens/public/datasources/form_based/esql_layer/components/dimension_editor.test.tsx

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import React from 'react';
+import { render, screen, waitFor } from '@testing-library/react';
+import { TextBasedDimensionEditor, TextBasedDimensionEditorProps } from './dimension_editor';
+
+// Mock fetchFieldsFromESQL
+jest.mock('@kbn/esql-editor', () => ({
+  fetchFieldsFromESQL: jest.fn(),
+}));
+
+const { fetchFieldsFromESQL } = jest.requireMock('@kbn/esql-editor');
+
+describe('TextBasedDimensionEditor', () => {
+  const defaultProps = {
+    isFullscreen: false,
+    columnId: 'dim1',
+    layerId: 'layer1',
+    state: {
+      layers: {
+        layer1: {
+          query: { esql: 'FROM my_data' },
+          columns: [],
+          indexPatternRefs: [],
+        },
+      },
+      indexPatternRefs: [],
+    },
+    setState: jest.fn(),
+    indexPatterns: {},
+    dateRange: { fromDate: '2023-01-01', toDate: '2023-01-31' },
+    expressions: {},
+    esqlVariables: [
+      {
+        key: 'agent_keyword',
+        value: 'Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1',
+        type: 'values',
+      },
+    ],
+    isMetricDimension: false,
+    filterOperations: jest.fn(() => true),
+    core: { docLinks: {} },
+    groupId: 'rows',
+  } as unknown as TextBasedDimensionEditorProps;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+
+    fetchFieldsFromESQL.mockResolvedValue({
+      columns: [
+        { id: 'field1', name: 'Field One', meta: { type: 'string' } },
+        { id: 'field2', name: 'Field Two', meta: { type: 'number' } },
+      ],
+    });
+  });
+
+  it('renders correctly and fetches columns on mount', async () => {
+    render(<TextBasedDimensionEditor {...defaultProps} />);
+
+    // Check if fetchFieldsFromESQL was called
+    await waitFor(() => {
+      expect(fetchFieldsFromESQL).toHaveBeenCalledTimes(1);
+      expect(fetchFieldsFromESQL).toHaveBeenCalledWith(
+        { esql: 'FROM my_data | limit 0' },
+        {},
+        { from: defaultProps.dateRange.fromDate, to: defaultProps.dateRange.toDate },
+        undefined,
+        undefined, // No index patterns
+        defaultProps.esqlVariables
+      );
+    });
+
+    expect(screen.getByTestId('text-based-dimension-field')).toBeInTheDocument();
+  });
+});

x-pack/platform/plugins/shared/lens/public/datasources/form_based/esql_layer/components/dimension_editor.tsx

@@ -39,6 +39,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
     indexPatterns,
     dateRange,
     expressions,
+    esqlVariables,
   } = props;
 
   useEffect(() => {
@@ -52,7 +53,8 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
           undefined,
           Object.values(indexPatterns).length
             ? Object.values(indexPatterns)[0].timeFieldName
-            : undefined
+            : undefined,
+          esqlVariables
         );
         if (table) {
           const hasNumberTypeColumns = table.columns?.some(isNumeric);
@@ -84,6 +86,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
     indexPatterns,
     props,
     props.expressions,
+    esqlVariables,
     query,
   ]);
 
@@ -122,6 +125,7 @@ export function TextBasedDimensionEditor(props: TextBasedDimensionEditorProps) {
         className="lnsIndexPatternDimensionEditor--padded"
       >
         <FieldSelect
+          data-test-subj="text-based-dimension-field"
           existingFields={allColumns ?? []}
           selectedField={selectedField}
           onChoose={(choice) => {

x-pack/platform/plugins/shared/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx

@@ -16,11 +16,13 @@ import {
   EuiIconTip,
   useEuiTheme,
 } from '@elastic/eui';
+import { BehaviorSubject } from 'rxjs';
 import { i18n } from '@kbn/i18n';
 import { css } from '@emotion/react';
 import { euiThemeVars } from '@kbn/ui-theme';
 import { DragDropIdentifier, ReorderProvider, DropType } from '@kbn/dom-drag-drop';
 import { DimensionButton } from '@kbn/visualization-ui-components';
+import { useStateFromPublishingSubject } from '@kbn/presentation-publishing';
 import { isOfAggregateQueryType } from '@kbn/es-query';
 import { LayerActions } from './layer_actions';
 import { isOperation, LayerAction, VisualizationDimensionGroupConfig } from '../../../types';
@@ -40,6 +42,7 @@ import { getSharedActions } from './layer_actions/layer_actions';
 import { FlyoutContainer } from '../../../shared_components/flyout_container';
 import { FakeDimensionButton } from './buttons/fake_dimension_button';
 import { getLongMessage } from '../../../user_messages_utils';
+import { isApiESQLVariablesCompatible } from '../../../react_embeddable/types';
 import { ESQLEditor } from './esql_editor';
 
 export function LayerPanel(props: LayerPanelProps) {
@@ -78,6 +81,13 @@ export function LayerPanel(props: LayerPanelProps) {
     ...editorProps
   } = props;
 
+  const { parentApi } = editorProps;
+  const esqlVariables = useStateFromPublishingSubject(
+    isApiESQLVariablesCompatible(parentApi)
+      ? parentApi?.esqlVariables$
+      : new BehaviorSubject(undefined)
+  );
+
   const isInlineEditing = Boolean(props?.setIsInlineFlyoutVisible);
 
   const isSaveable = useLensSelector((state) => state.lens.isSaveable);
@@ -812,6 +822,7 @@ export function LayerPanel(props: LayerPanelProps) {
                 layerType: activeVisualization.getLayerType(layerId, visualizationState),
                 indexPatterns: dataViews.indexPatterns,
                 activeData: layerVisualizationConfigProps.activeData,
+                esqlVariables,
                 dataSectionExtra: !isFullscreen &&
                   openDimension.isComplete &&
                   activeVisualization.DimensionEditorDataExtraComponent && (

x-pack/platform/plugins/shared/lens/public/types.ts

@@ -9,6 +9,7 @@ import type { IconType } from '@elastic/eui/src/components/icon/icon';
 import type { CoreStart, SavedObjectReference, ResolvedSimpleSavedObject } from '@kbn/core/public';
 import type { ColorMapping, PaletteOutput } from '@kbn/coloring';
 import type { TopNavMenuData } from '@kbn/navigation-plugin/public';
+import type { ESQLControlVariable } from '@kbn/esql-types';
 import type { MutableRefObject, ReactElement } from 'react';
 import type { Query, AggregateQuery, Filter, TimeRange } from '@kbn/es-query';
 import type {
@@ -673,6 +674,7 @@ export type DatasourceDimensionEditorProps<T = unknown> = DatasourceDimensionPro
     | 'docLinks'
   >;
   dateRange: DateRange;
+  esqlVariables?: ESQLControlVariable[] | undefined;
   dimensionGroups: VisualizationDimensionGroupConfig[];
   toggleFullscreen: () => void;
   isFullscreen: boolean;