[ML] Adding file upload tools to package

.github/CODEOWNERS

@@ -820,6 +820,7 @@ x-pack/platform/packages/shared/ai-infra/inference-common @elastic/appex-ai-infr
 x-pack/platform/packages/shared/ai-infra/inference-langchain @elastic/appex-ai-infra
 x-pack/platform/packages/shared/ai-infra/product-doc-common @elastic/appex-ai-infra
 x-pack/platform/packages/shared/alerting-rule-utils @elastic/obs-ux-management-team
+x-pack/platform/packages/shared/file-upload @elastic/ml-ui
 x-pack/platform/packages/shared/file-upload-common @elastic/ml-ui
 x-pack/platform/packages/shared/index-lifecycle-management/index_lifecycle_management_common_shared @elastic/kibana-management
 x-pack/platform/packages/shared/index-management/index_management_shared_types @elastic/kibana-management

package.json

@@ -548,6 +548,7 @@
     "@kbn/field-types": "link:src/platform/packages/shared/kbn-field-types",
     "@kbn/field-utils": "link:src/platform/packages/shared/kbn-field-utils",
     "@kbn/fields-metadata-plugin": "link:x-pack/platform/plugins/shared/fields_metadata",
+    "@kbn/file-upload": "link:x-pack/platform/packages/shared/file-upload",
     "@kbn/file-upload-common": "link:x-pack/platform/packages/shared/file-upload-common",
     "@kbn/file-upload-plugin": "link:x-pack/platform/plugins/private/file_upload",
     "@kbn/files-example-plugin": "link:examples/files_example",

tsconfig.base.json

@@ -970,6 +970,8 @@
       "@kbn/field-utils/*": ["src/platform/packages/shared/kbn-field-utils/*"],
       "@kbn/fields-metadata-plugin": ["x-pack/platform/plugins/shared/fields_metadata"],
       "@kbn/fields-metadata-plugin/*": ["x-pack/platform/plugins/shared/fields_metadata/*"],
+      "@kbn/file-upload": ["x-pack/platform/packages/shared/file-upload"],
+      "@kbn/file-upload/*": ["x-pack/platform/packages/shared/file-upload/*"],
       "@kbn/file-upload-common": ["x-pack/platform/packages/shared/file-upload-common"],
       "@kbn/file-upload-common/*": ["x-pack/platform/packages/shared/file-upload-common/*"],
       "@kbn/file-upload-plugin": ["x-pack/platform/plugins/private/file_upload"],

x-pack/platform/packages/shared/file-upload-common/index.ts

@@ -6,4 +6,13 @@
  */
 
 export type { FileUploadResults, OpenFileUploadLiteContext } from './src/types';
-export { OPEN_FILE_UPLOAD_LITE_ACTION, OPEN_FILE_UPLOAD_LITE_TRIGGER } from './src/constants';
+export {
+  OPEN_FILE_UPLOAD_LITE_ACTION,
+  OPEN_FILE_UPLOAD_LITE_TRIGGER,
+  FILE_FORMATS,
+  FILE_SIZE_DISPLAY_FORMAT,
+  MB,
+  NO_TIME_FORMAT,
+  TIKA_PREVIEW_CHARS,
+  INDEX_META_DATA_CREATED_BY,
+} from './src/constants';

x-pack/platform/packages/shared/file-upload-common/src/constants.ts

@@ -8,3 +8,23 @@
 export const OPEN_FILE_UPLOAD_LITE_ACTION = 'openFileUploadLiteTrigger';
 
 export const OPEN_FILE_UPLOAD_LITE_TRIGGER = 'OPEN_FILE_UPLOAD_LITE_TRIGGER';
+
+export const FILE_FORMATS = {
+  DELIMITED: 'delimited',
+  NDJSON: 'ndjson',
+  SEMI_STRUCTURED_TEXT: 'semi_structured_text',
+  TIKA: 'tika',
+};
+
+export const UI_SETTING_MAX_FILE_SIZE = 'fileUpload:maxFileSize';
+export const MAX_FILE_SIZE = '100MB';
+export const MAX_FILE_SIZE_BYTES = 524288000; // 500MB
+export const ABSOLUTE_MAX_FILE_SIZE_BYTES = 1073741274; // 1GB
+export const FILE_SIZE_DISPLAY_FORMAT = '0,0.[0] b';
+export const MAX_TIKA_FILE_SIZE_BYTES = 62914560; // 60MB
+
+export const NO_TIME_FORMAT = 'null';
+export const MB = Math.pow(2, 20);
+
+export const TIKA_PREVIEW_CHARS = 100000;
+export const INDEX_META_DATA_CREATED_BY = 'file-data-visualizer';

x-pack/platform/packages/shared/file-upload-common/src/types.ts

@@ -13,13 +13,15 @@ export interface FileUploadResults {
   dataView?: { id: string; title: string };
   inferenceId?: string;
   files: Array<{ fileName: string; docCount: number; fileFormat: string; documentType: string }>;
+  timeFieldName?: string;
 }
 
 export interface OpenFileUploadLiteContext {
   onUploadComplete?: (results: FileUploadResults | null) => void;
   indexSettings?: IndicesIndexSettings;
   autoAddInference?: string;
   autoCreateDataView?: boolean;
+  existingIndex?: string;
   initialIndexName?: string;
   flyoutContent?: FlyoutContent;
 }

x-pack/platform/packages/shared/file-upload-common/tsconfig.json

@@ -13,5 +13,6 @@
   "exclude": [
     "target/**/*"
   ],
-  "kbn_references": []
+  "kbn_references": [
+  ]
 }

x-pack/platform/packages/shared/file-upload/README.md

@@ -0,0 +1,3 @@
+# @kbn/file-upload
+
+Functions from performing file upload

x-pack/platform/packages/shared/file-upload/file_upload_manager/auto_deploy.ts

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type {
+  InferenceInferenceEndpointInfo,
+  InferenceInferenceResponse,
+} from '@elastic/elasticsearch/lib/api/types';
+import type { HttpSetup } from '@kbn/core/public';
+
+const POLL_INTERVAL = 5; // seconds
+
+export class AutoDeploy {
+  private inferError: Error | null = null;
+  private inferFinished: boolean = false;
+  constructor(private readonly http: HttpSetup, private readonly inferenceId: string) {}
+
+  public async deploy() {
+    this.inferError = null;
+    if (await this.isDeployed()) {
+      return;
+    }
+
+    this.infer()
+      .then(() => {
+        this.inferFinished = true;
+      })
+      .catch((e) => {
+        // ignore timeout errors
+        // The deployment may take a long time
+        // we'll know when it's ready from polling the inference endpoints
+        // looking for num_allocations
+        const status = e.response?.status;
+        if (status === 408 || status === 504 || status === 502 || status === 500) {
+          return;
+        }
+        this.inferError = e;
+      });
+    await this.pollIsDeployed();
+  }
+
+  private async infer() {
+    return this.http.fetch<InferenceInferenceResponse>(
+      `/internal/data_visualizer/inference/${this.inferenceId}`,
+      {
+        method: 'POST',
+        version: '1',
+        body: JSON.stringify({ input: '' }),
+      }
+    );
+  }
+
+  private async isDeployed() {
+    const inferenceEndpoints = await this.http.fetch<InferenceInferenceEndpointInfo[]>(
+      '/internal/data_visualizer/inference_endpoints',
+      {
+        method: 'GET',
+        version: '1',
+      }
+    );
+
+    // if the call to inference has already finished, we know the model is deployed
+    if (this.inferFinished) {
+      return true;
+    }
+
+    // otherwise, we need to check the allocation count
+    return inferenceEndpoints.some((endpoint) => {
+      return (
+        endpoint.inference_id === this.inferenceId && endpoint.service_settings.num_allocations > 0
+      );
+    });
+  }
+
+  private async pollIsDeployed() {
+    while (true) {
+      if (this.inferError !== null) {
+        throw this.inferError;
+      }
+      const isDeployed = await this.isDeployed();
+      if (isDeployed) {
+        // break out of the loop once we have a successful deployment
+        return;
+      }
+      await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL * 1000));
+    }
+  }
+}