SERVER-103931 Add task to build index for SourceGraph via scip-clang (#36830)

GitOrigin-RevId: 8cd77a3ec23d5dfa93db9e45dbf3cd2d4f114d5f

Author: zackwintermdb

BUILD.bazel

@@ -441,3 +441,21 @@ mongo_install(
         "//src/mongo/replay:mongor",
     ],
 )
+
+sh_binary(
+    name = "scip-clang",
+    srcs = ["//bazel:run_under_working_dir.sh"],
+    data = ["@scip-clang//file"],
+    env = {
+        "BINARY_PATH": "$(location @scip-clang//file)",
+    },
+)
+
+sh_binary(
+    name = "scip-src",
+    srcs = ["//bazel:run_under_working_dir.sh"],
+    data = ["@scip-src//file"],
+    env = {
+        "BINARY_PATH": "$(location @scip-src//file)",
+    },
+)

MODULE.bazel

@@ -11,6 +11,8 @@ use_repo(bazel_features_deps, "bazel_features_globals", "bazel_features_version"
 
 http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
 http_archive(
     name = "bazel_clang_tidy",
     integrity = "sha256-A+TGGfuHdS7vWT20eyAEiA3u4YXFtuzsJDcpTEmDMS0=",
@@ -22,6 +24,30 @@ http_archive(
     ] * 5,
 )
 
+# SourceGraph indexer
+http_file(
+    name = "scip-clang",
+    executable = True,
+    sha256 = "9fc7d1c309f8adee1b8663efa802efa7363ef22ee06f4711117535e406690689",
+    urls = [
+        # Implements retry by relisting each url multiple times to be used as a failover.
+        # TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
+        "https://github.com/sourcegraph/scip-clang/releases/download/v0.3.2/scip-clang-x86_64-linux",
+    ] * 5,
+)
+
+# SourceGraph CLI
+http_file(
+    name = "scip-src",
+    executable = True,
+    sha256 = "eb3d627c4dad56b9f5d06634b90384104db065d410c4b338fa5e6558be4f3ad9",
+    urls = [
+        # Implements retry by relisting each url multiple times to be used as a failover.
+        # TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
+        "https://github.com/sourcegraph/src-cli/releases/download/6.4.0/src_linux_amd64",
+    ] * 5,
+)
+
 bazel_dep(name = "platforms", version = "0.0.9")
 single_version_override(
     module_name = "platforms",

bazel/BUILD.bazel

@@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"])
 # Expose script for external usage through bazel.
 exports_files([
     "install_rules.py",
+    "run_under_working_dir.sh",
 ])
 
 sh_binary(

bazel/run_under_working_dir.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+RUNFILES_WORKING_DIRECTORY="$(pwd)"
+
+if [ -z $BUILD_WORKING_DIRECTORY ]; then
+  echo "ERROR: BUILD_WORKING_DIRECTORY was not set, was this run from bazel?"
+  exit 1
+fi
+
+cd $BUILD_WORKING_DIRECTORY
+
+${RUNFILES_WORKING_DIRECTORY}/${BINARY_PATH} "${@:1}"

etc/evergreen_yml_components/tasks/compile_tasks.yml

@@ -179,6 +179,30 @@ tasks:
             --keep_going
             --build_tag_filters=${bazel_filters_for_cache_hydration}
 
+  - name: build_source_graph_index
+    tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
+    depends_on:
+      - name: version_expansions_gen
+        variant: generate-tasks-for-version
+    commands:
+      - func: "bazel compile"
+        vars:
+          targets: compiledb
+          bazel_args: --intree_compdb
+      - func: "bazel compile"
+        vars:
+          targets: //src/...
+          bazel_args: --output_groups=compilation_outputs --build_enterprise=True --config=dbg --remote_download_outputs=all
+      - func: "bazel run"
+        vars:
+          target: //:scip-clang -- --compdb-path ./compile_commands.json
+      - func: "bazel run"
+        vars:
+          target: //:scip-src -- code-intel upload -file=index.scip
+          args: -github-token=${SOURCE_GRAPH_GITHUB_CLONER_TOKEN}
+          env: SRC_ENDPOINT=https://mongodb.sourcegraphcloud.com SRC_ACCESS_TOKEN=${SOURCE_GRAPH_API_ACCESS_TOKEN}
+          redact_args: true
+
   - name: hydrate_bazel_profile_opt
     tags:
       [
@@ -974,6 +998,12 @@ task_groups:
       - hydrate_bazel_profile_dbg_aubsan
       - hydrate_bazel_profile_dbg_tsan
 
+  - <<: *compile_task_group_template
+    name: build_source_graph_index_TG
+    max_hosts: -1
+    tasks:
+      - build_source_graph_index
+
   - <<: *compile_bazel_task_group_template
     name: compile_bazel_windows_TG
     tasks:

etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml

@@ -45,3 +45,17 @@ buildvariants:
     tasks:
       - name: hydrate_bazel_profile_TG
       - name: .clang_tidy
+
+  - name: &upload_source_graph_index upload_source_graph_index
+    display_name: "~ Upload Source Graph Index"
+    tags: ["source_graph_indexing"]
+    cron: "30 * * * *" # Every hour, 30 min offset
+    # SourceGraph's indexer requires a lot of CPU to execute in a reasonable amount of time
+    run_on:
+      - rhel93-xxlarge
+    stepback: false
+    expansions:
+      compile_variant: *upload_source_graph_index
+      evergreen_remote_exec: on
+    tasks:
+      - name: build_source_graph_index_TG

evergreen/bazel_run.sh

@@ -4,6 +4,8 @@
 # Required environment variables:
 # * ${target} - Build target
 # * ${args} - Extra command line args to pass to "bazel run"
+# * ${env} - Env variable string to set (ex. ENV_VAR_ABC=123)
+# * ${redact_args} - If set, redact the args in the report
 
 # Needed for evergreen scripts that use evergreen expansions and utility methods.
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
@@ -17,7 +19,7 @@ set -o verbose
 activate_venv
 
 # Use `eval` to force evaluation of the environment variables in the echo statement:
-eval echo "Execution environment: Args: ${args} Target: ${target}"
+eval echo "Execution environment: Args: ${args} Target: ${target} Env: ${env} redact_args: ${redact_args}"
 
 source ./evergreen/bazel_utility_functions.sh
 source ./evergreen/bazel_RBE_supported.sh
@@ -46,17 +48,23 @@ if [[ -f /etc/os-release ]]; then
   fi
 fi
 
+if [[ -n "$redact_args" ]]; then
+  INVOCATION_WITH_REDACTION="${target}"
+else
+  INVOCATION_WITH_REDACTION="${target} ${args}"
+fi
+
 # Print command being run to file that can be uploaded
 echo "python buildscripts/install_bazel.py" > bazel-invocation.txt
-echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${args} ${target}" >> bazel-invocation.txt
+echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >> bazel-invocation.txt
 
 # Run bazel command, retrying up to five times
 MAX_ATTEMPTS=5
 for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do
-  eval $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${args} ${target} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 60
+  eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10
   if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >> bazel_output.log 2>&1; fi
   $BAZEL_BINARY shutdown
 done
 
-$python ./buildscripts/simple_report.py --test-name "bazel run ${args} ${target}" --log-file bazel_output.log --exit-code $RET
+$python ./buildscripts/simple_report.py --test-name "bazel run ${INVOCATION_WITH_REDACTION}" --log-file bazel_output.log --exit-code $RET
 exit $RET