Merge branch 'main' into lucene_snapshot

Author: ChrisHegarty

.buildkite/scripts/third-party-test-credentials.sh

@@ -7,6 +7,7 @@ set -euo pipefail
 # The second/lowercase export is what the tests expect/require
 
 if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
@@ -19,6 +20,7 @@ if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
 fi
 
 if [[ "${USE_3RD_PARTY_AZURE_SAS_CREDENTIALS:-}" == "true" ]]; then
+  # These credentials expire periodically and must be manually renewed - the process is in the onboarding/process docs.
   json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_sas_test_creds)
 
   AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)

README.asciidoc

@@ -97,6 +97,15 @@ From the `elastic-start-local` folder, check the connection to Elasticsearch usi
 source .env
 curl $ES_LOCAL_URL -H "Authorization: ApiKey ${ES_LOCAL_API_KEY}"
 ----
+
+To use the password for the `elastic` user, set and export the `ES_LOCAL_PASSWORD` environment variable. For example:
+
+[source,sh]
+----
+source .env
+export ES_LOCAL_PASSWORD
+----
+
 // NOTCONSOLE
 
 === Send requests to Elasticsearch
@@ -112,22 +121,23 @@ Here's an example curl command to create a new Elasticsearch index, using basic
 
 [source,sh]
 ----
-curl -u elastic:$ELASTIC_PASSWORD \
+curl -u elastic:$ES_LOCAL_PASSWORD \
   -X PUT \
   http://localhost:9200/my-new-index \
   -H 'Content-Type: application/json'
 ----
+
 // NOTCONSOLE
 
 ==== Using a language client
 
-To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable.
+To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password stored in the `ES_LOCAL_PASSWORD` environment variable.
 
 You'll use the following connection details:
 
 * **Elasticsearch endpoint**: `http://localhost:9200`
 * **Username**: `elastic`
-* **Password**: `$ELASTIC_PASSWORD` (Value you set in the environment variable)
+* **Password**: `$ES_LOCAL_PASSWORD` (Value you set in the environment variable)
 
 For example, to connect with the Python `elasticsearch` client:
 
@@ -137,7 +147,7 @@ import os
 from elasticsearch import Elasticsearch
 
 username = 'elastic'
-password = os.getenv('ELASTIC_PASSWORD') # Value you set in the environment variable
+password = os.getenv('ES_LOCAL_PASSWORD') # Value you set in the environment variable
 
 client = Elasticsearch(
     "http://localhost:9200",

TESTING.asciidoc

@@ -46,14 +46,8 @@ run it using Gradle:
 ==== Launching and debugging from an IDE
 
 If you want to run and debug Elasticsearch from your IDE, the `./gradlew run` task
-supports a remote debugging option. Run the following from your terminal:
-
----------------------------------------------------------------------------
-./gradlew run --debug-jvm
----------------------------------------------------------------------------
-
-Next start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the IDE to connect to the process and allow debug functionality.
-
+supports a remote debugging option. Start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the
+IDE to connect to the process and allow debug functionality.
 
 As such the IDE needs to be instructed to listen for connections on the debug port.
 Since we might run multiple JVMs as part of configuring and starting the cluster it's
@@ -64,6 +58,12 @@ NOTE: If you have imported the project into IntelliJ according to the instructio
 link:/CONTRIBUTING.md#importing-the-project-into-intellij-idea[CONTRIBUTING.md] then a debug run configuration
 named "Debug Elasticsearch" will be created for you and configured appropriately.
 
+Next run the following from your terminal:
+
+---------------------------------------------------------------------------
+./gradlew run --debug-jvm
+---------------------------------------------------------------------------
+
 ===== Debugging the CLI launcher
 
 The gradle task does not start the Elasticsearch server process directly; like in the Elasticsearch distribution,

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java

@@ -21,10 +21,13 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.AggregationOperator;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -282,11 +285,18 @@ private static Block dataBlock(int groups, String dataType) {
         int blockLength = blockLength(groups);
         return switch (dataType) {
             case BYTES_REF -> {
-                try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(blockLength)) {
+                try (
+                    BytesRefVector.Builder dict = blockFactory.newBytesRefVectorBuilder(blockLength);
+                    IntVector.Builder ords = blockFactory.newIntVectorBuilder(blockLength)
+                ) {
+                    final int dictLength = Math.min(blockLength, KEYWORDS.length);
+                    for (int i = 0; i < dictLength; i++) {
+                        dict.appendBytesRef(KEYWORDS[i]);
+                    }
                     for (int i = 0; i < blockLength; i++) {
-                        builder.appendBytesRef(KEYWORDS[i % KEYWORDS.length]);
+                        ords.appendInt(i % dictLength);
                     }
-                    yield builder.build();
+                    yield new OrdinalBytesRefVector(ords.build(), dict.build()).asBlock();
                 }
             }
             case INT -> {

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -30,8 +30,10 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
+import org.elasticsearch.index.mapper.SourceFieldMetrics;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.plugins.PluginsLoader;
@@ -90,7 +92,7 @@ public class ScriptScoreBenchmark {
     private final SearchLookup lookup = new SearchLookup(
         fieldTypes::get,
         (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService),
-        SourceProvider.fromStoredFields()
+        SourceProvider.fromLookup(MappingLookup.EMPTY, null, SourceFieldMetrics.NOOP)
     );
 
     @Param({ "expression", "metal", "painless_cast", "painless_def" })

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -17,7 +17,7 @@
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.simdvec.internal.vectorization.ES91OSQVectorsScorer;
+import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;

build-conventions/settings.gradle

@@ -8,7 +8,7 @@
  */
 
 plugins {
-  id "com.gradle.develocity" version "3.19.2"
+  id "com.gradle.develocity" version "4.0.1"
 }
 
 rootProject.name = 'build-conventions'

build-tools-internal/settings.gradle

@@ -9,7 +9,7 @@ pluginManagement {
 }
 
 plugins {
-  id "com.gradle.develocity" version "3.19.2"
+  id "com.gradle.develocity" version "4.0.1"
 }
 
 dependencyResolutionManagement {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -14,30 +14,27 @@
  */
 public enum DockerBase {
     // "latest" here is intentional, since the image name specifies "9"
-    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "Dockerfile.default"),
+    DEFAULT("redhat/ubi9-minimal:latest", "", "microdnf", "dockerfiles/default/Dockerfile"),
 
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum", "Dockerfile"),
 
     // Chainguard based wolfi image with latest jdk
-    // This is usually updated via renovatebot
-    // spotless:off
     WOLFI(
-        "docker.elastic.co/wolfi/chainguard-base:latest@sha256:29150cd940cc7f69407d978d5a19c86f4d9e67cf44e4d6ded787a497e8f27c9a",
+        null,
         "-wolfi",
         "apk",
-        "Dockerfile"
+        "dockerfiles/wolfi/Dockerfile"
     ),
-    // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.
     CLOUD_ESS(null, "-cloud-ess", "apk", "Dockerfile.ess"),
 
     CLOUD_ESS_FIPS(
-        "docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7@sha256:ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7",
+        null,
         "-cloud-ess-fips",
         "apk",
-        "Dockerfile.ess-fips"
+        "dockerfiles/cloud_ess_fips/Dockerfile"
     );
 
     private final String image;

build-tools/build.gradle

@@ -51,6 +51,10 @@ gradlePlugin {
           id = 'elasticsearch.stable-esplugin'
           implementationClass = 'org.elasticsearch.gradle.plugin.StablePluginBuildPlugin'
         }
+        testBuildInfo {
+          id = 'elasticsearch.test-build-info'
+          implementationClass = 'org.elasticsearch.gradle.test.TestBuildInfoPlugin'
+        }
         javaRestTest {
             id = 'elasticsearch.java-rest-test'
             implementationClass = 'org.elasticsearch.gradle.test.JavaRestTestPlugin'

build-tools/settings.gradle

@@ -10,7 +10,7 @@ pluginManagement {
     includeBuild "../build-conventions"
 }
 plugins {
-  id "com.gradle.develocity" version "3.19.2"
+  id "com.gradle.develocity" version "4.0.1"
 }
 include 'reaper'
 

build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy

@@ -0,0 +1,63 @@
+package org.elasticsearch.gradle.test
+
+import com.fasterxml.jackson.databind.ObjectMapper
+
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.TaskOutcome
+
+class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
+    def "works"() {
+        given:
+        file("src/main/java/com/example/Example.java") << """
+            package com.example;
+
+            public class Example {
+            }
+        """
+
+        file("src/main/java/module-info.java") << """
+            module com.example {
+                exports com.example;
+            }
+        """
+
+        buildFile << """
+        import org.elasticsearch.gradle.plugin.GenerateTestBuildInfoTask;
+
+        plugins {
+            id 'java'
+            id 'elasticsearch.test-build-info'
+        }
+
+        repositories {
+            mavenCentral()
+        }
+
+        tasks.withType(GenerateTestBuildInfoTask.class) {
+            componentName = 'example-component'
+            outputFile = new File('build/generated-build-info/plugin-test-build-info.json')
+        }
+        """
+
+        when:
+        def result = gradleRunner('generateTestBuildInfo').build()
+        def task = result.task(":generateTestBuildInfo")
+
+
+        then:
+        task.outcome == TaskOutcome.SUCCESS
+
+        def output = file("build/generated-build-info/plugin-test-build-info.json")
+        output.exists() == true
+
+        def location = Map.of(
+            "module", "com.example",
+            "representative_class", "com/example/Example.class"
+        )
+        def expectedOutput = Map.of(
+            "component", "example-component",
+            "locations", List.of(location)
+        )
+        new ObjectMapper().readValue(output, Map.class) == expectedOutput
+    }
+}

build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java

@@ -121,7 +121,10 @@ private TaskProvider<Zip> createBundleTasks(final Project project, PluginPropert
             task.getIsLicensed().set(providerFactory.provider(extension::isLicensed));
 
             var mainSourceSet = project.getExtensions().getByType(SourceSetContainer.class).getByName(SourceSet.MAIN_SOURCE_SET_NAME);
-            FileCollection moduleInfoFile = mainSourceSet.getOutput().getAsFileTree().matching(p -> p.include("module-info.class"));
+            FileCollection moduleInfoFile = mainSourceSet.getOutput()
+                .getClassesDirs()
+                .getAsFileTree()
+                .matching(p -> p.include("module-info.class"));
             task.getModuleInfoFile().setFrom(moduleInfoFile);
 
         });