Skip to content

Commit 3e61217

Browse files
zackwintermdbMongoDB Bot
authored and
MongoDB Bot
committed
SERVER-103931 Add task to build index for SourceGraph via scip-clang (#36830)
GitOrigin-RevId: 8cd77a3ec23d5dfa93db9e45dbf3cd2d4f114d5f
1 parent ad535f0 commit 3e61217

File tree

7 files changed

+115
-4
lines changed

7 files changed

+115
-4
lines changed

BUILD.bazel

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,3 +441,21 @@ mongo_install(
441441
"//src/mongo/replay:mongor",
442442
],
443443
)
444+
445+
sh_binary(
446+
name = "scip-clang",
447+
srcs = ["//bazel:run_under_working_dir.sh"],
448+
data = ["@scip-clang//file"],
449+
env = {
450+
"BINARY_PATH": "$(location @scip-clang//file)",
451+
},
452+
)
453+
454+
sh_binary(
455+
name = "scip-src",
456+
srcs = ["//bazel:run_under_working_dir.sh"],
457+
data = ["@scip-src//file"],
458+
env = {
459+
"BINARY_PATH": "$(location @scip-src//file)",
460+
},
461+
)

MODULE.bazel

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ use_repo(bazel_features_deps, "bazel_features_globals", "bazel_features_version"
1111

1212
http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
1313

14+
http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
15+
1416
http_archive(
1517
name = "bazel_clang_tidy",
1618
integrity = "sha256-A+TGGfuHdS7vWT20eyAEiA3u4YXFtuzsJDcpTEmDMS0=",
@@ -22,6 +24,30 @@ http_archive(
2224
] * 5,
2325
)
2426

27+
# SourceGraph indexer
28+
http_file(
29+
name = "scip-clang",
30+
executable = True,
31+
sha256 = "9fc7d1c309f8adee1b8663efa802efa7363ef22ee06f4711117535e406690689",
32+
urls = [
33+
# Implements retry by relisting each url multiple times to be used as a failover.
34+
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
35+
"https://github.com/sourcegraph/scip-clang/releases/download/v0.3.2/scip-clang-x86_64-linux",
36+
] * 5,
37+
)
38+
39+
# SourceGraph CLI
40+
http_file(
41+
name = "scip-src",
42+
executable = True,
43+
sha256 = "eb3d627c4dad56b9f5d06634b90384104db065d410c4b338fa5e6558be4f3ad9",
44+
urls = [
45+
# Implements retry by relisting each url multiple times to be used as a failover.
46+
# TODO(SERVER-86719): Re-implement http_archive to allow sleeping between retries
47+
"https://github.com/sourcegraph/src-cli/releases/download/6.4.0/src_linux_amd64",
48+
] * 5,
49+
)
50+
2551
bazel_dep(name = "platforms", version = "0.0.9")
2652
single_version_override(
2753
module_name = "platforms",

bazel/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"])
33
# Expose script for external usage through bazel.
44
exports_files([
55
"install_rules.py",
6+
"run_under_working_dir.sh",
67
])
78

89
sh_binary(

bazel/run_under_working_dir.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
RUNFILES_WORKING_DIRECTORY="$(pwd)"
6+
7+
if [ -z $BUILD_WORKING_DIRECTORY ]; then
8+
echo "ERROR: BUILD_WORKING_DIRECTORY was not set, was this run from bazel?"
9+
exit 1
10+
fi
11+
12+
cd $BUILD_WORKING_DIRECTORY
13+
14+
${RUNFILES_WORKING_DIRECTORY}/${BINARY_PATH} "${@:1}"

etc/evergreen_yml_components/tasks/compile_tasks.yml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,30 @@ tasks:
179179
--keep_going
180180
--build_tag_filters=${bazel_filters_for_cache_hydration}
181181
182+
- name: build_source_graph_index
183+
tags: ["assigned_to_jira_team_devprod_build", "auxiliary"]
184+
depends_on:
185+
- name: version_expansions_gen
186+
variant: generate-tasks-for-version
187+
commands:
188+
- func: "bazel compile"
189+
vars:
190+
targets: compiledb
191+
bazel_args: --intree_compdb
192+
- func: "bazel compile"
193+
vars:
194+
targets: //src/...
195+
bazel_args: --output_groups=compilation_outputs --build_enterprise=True --config=dbg --remote_download_outputs=all
196+
- func: "bazel run"
197+
vars:
198+
target: //:scip-clang -- --compdb-path ./compile_commands.json
199+
- func: "bazel run"
200+
vars:
201+
target: //:scip-src -- code-intel upload -file=index.scip
202+
args: -github-token=${SOURCE_GRAPH_GITHUB_CLONER_TOKEN}
203+
env: SRC_ENDPOINT=https://mongodb.sourcegraphcloud.com SRC_ACCESS_TOKEN=${SOURCE_GRAPH_API_ACCESS_TOKEN}
204+
redact_args: true
205+
182206
- name: hydrate_bazel_profile_opt
183207
tags:
184208
[
@@ -974,6 +998,12 @@ task_groups:
974998
- hydrate_bazel_profile_dbg_aubsan
975999
- hydrate_bazel_profile_dbg_tsan
9761000

1001+
- <<: *compile_task_group_template
1002+
name: build_source_graph_index_TG
1003+
max_hosts: -1
1004+
tasks:
1005+
- build_source_graph_index
1006+
9771007
- <<: *compile_bazel_task_group_template
9781008
name: compile_bazel_windows_TG
9791009
tasks:

etc/evergreen_yml_components/variants/ubuntu/test_dev_master_branch_only.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,17 @@ buildvariants:
4545
tasks:
4646
- name: hydrate_bazel_profile_TG
4747
- name: .clang_tidy
48+
49+
- name: &upload_source_graph_index upload_source_graph_index
50+
display_name: "~ Upload Source Graph Index"
51+
tags: ["source_graph_indexing"]
52+
cron: "30 * * * *" # Every hour, 30 min offset
53+
# SourceGraph's indexer requires a lot of CPU to execute in a reasonable amount of time
54+
run_on:
55+
- rhel93-xxlarge
56+
stepback: false
57+
expansions:
58+
compile_variant: *upload_source_graph_index
59+
evergreen_remote_exec: on
60+
tasks:
61+
- name: build_source_graph_index_TG

evergreen/bazel_run.sh

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
# Required environment variables:
55
# * ${target} - Build target
66
# * ${args} - Extra command line args to pass to "bazel run"
7+
# * ${env} - Env variable string to set (ex. ENV_VAR_ABC=123)
8+
# * ${redact_args} - If set, redact the args in the report
79

810
# Needed for evergreen scripts that use evergreen expansions and utility methods.
911
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
@@ -17,7 +19,7 @@ set -o verbose
1719
activate_venv
1820

1921
# Use `eval` to force evaluation of the environment variables in the echo statement:
20-
eval echo "Execution environment: Args: ${args} Target: ${target}"
22+
eval echo "Execution environment: Args: ${args} Target: ${target} Env: ${env} redact_args: ${redact_args}"
2123

2224
source ./evergreen/bazel_utility_functions.sh
2325
source ./evergreen/bazel_RBE_supported.sh
@@ -46,17 +48,23 @@ if [[ -f /etc/os-release ]]; then
4648
fi
4749
fi
4850

51+
if [[ -n "$redact_args" ]]; then
52+
INVOCATION_WITH_REDACTION="${target}"
53+
else
54+
INVOCATION_WITH_REDACTION="${target} ${args}"
55+
fi
56+
4957
# Print command being run to file that can be uploaded
5058
echo "python buildscripts/install_bazel.py" > bazel-invocation.txt
51-
echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${args} ${target}" >> bazel-invocation.txt
59+
echo "bazel run --verbose_failures ${bazel_compile_flags} ${task_compile_flags} ${LOCAL_ARG} ${INVOCATION_WITH_REDACTION}" >> bazel-invocation.txt
5260

5361
# Run bazel command, retrying up to five times
5462
MAX_ATTEMPTS=5
5563
for ((i = 1; i <= $MAX_ATTEMPTS; i++)); do
56-
eval $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${args} ${target} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 60
64+
eval $env $BAZEL_BINARY run --verbose_failures $LOCAL_ARG ${target} ${args} >> bazel_output.log 2>&1 && RET=0 && break || RET=$? && sleep 10
5765
if [ $i -lt $MAX_ATTEMPTS ]; then echo "Bazel failed to execute, retrying ($(($i + 1)) of $MAX_ATTEMPTS attempts)... " >> bazel_output.log 2>&1; fi
5866
$BAZEL_BINARY shutdown
5967
done
6068

61-
$python ./buildscripts/simple_report.py --test-name "bazel run ${args} ${target}" --log-file bazel_output.log --exit-code $RET
69+
$python ./buildscripts/simple_report.py --test-name "bazel run ${INVOCATION_WITH_REDACTION}" --log-file bazel_output.log --exit-code $RET
6270
exit $RET

0 commit comments

Comments
 (0)