Skip to content

Commit d3a226c

Browse files
authored
docs: Improve builds for Envoy and mobile (stamping) (#29887)
Signed-off-by: Ryan Northey <[email protected]>
1 parent f04ac9d commit d3a226c

File tree

13 files changed

+93
-117
lines changed

13 files changed

+93
-117
lines changed

api/CONTRIBUTING.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,19 @@ documentation.
2323
The documentation can be built locally in the root of https://github.com/envoyproxy/envoy via:
2424

2525
```
26-
docs/build.sh
26+
ci/do_ci.sh docs
2727
```
2828

2929
To skip configuration examples validation:
3030

3131
```
32-
SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh
32+
SPHINX_SKIP_CONFIG_VALIDATION=true ci/do_ci.sh docs
3333
```
3434

3535
Or to use a hermetic Docker container:
3636

3737
```
38-
./ci/run_envoy_docker.sh './ci/do_ci.sh docs'
38+
./ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
3939
```
4040

4141
This process builds RST documentation directly from the proto files, merges it with the static RST

bazel/BUILD

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,8 @@ genrule(
4141
outs = ["gnu_build_id.ldscript"],
4242
cmd = """
4343
echo --build-id=0x$$(
44-
grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\
45-
| sed 's/^BUILD_SCM_REVISION //') \\
44+
grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \
45+
| sed 's/^BUILD_SCM_REVISION //') \
4646
> $@
4747
""",
4848
# Undocumented attr to depend on workspace status files.
@@ -55,8 +55,8 @@ genrule(
5555
name = "raw_build_id",
5656
outs = ["raw_build_id.ldscript"],
5757
cmd = """
58-
grep BUILD_SCM_REVISION bazel-out/volatile-status.txt \\
59-
| sed 's/^BUILD_SCM_REVISION //' \\
58+
grep -E "^BUILD_SCM_REVISION" bazel-out/volatile-status.txt \
59+
| sed 's/^BUILD_SCM_REVISION //' \
6060
| tr -d '\\n' \\
6161
> $@
6262
""",

bazel/get_workspace_status

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,21 @@
2323
if [ -f SOURCE_VERSION ]
2424
then
2525
echo "BUILD_SCM_REVISION $(cat SOURCE_VERSION)"
26+
echo "ENVOY_BUILD_SCM_REVISION $(cat SOURCE_VERSION)"
2627
echo "STABLE_BUILD_SCM_REVISION $(cat SOURCE_VERSION)"
2728
echo "BUILD_SCM_STATUS Distribution"
2829
exit 0
2930
fi
3031

3132
if [[ -n "$BAZEL_FAKE_SCM_REVISION" ]]; then
3233
echo "BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION"
34+
echo "ENVOY_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION"
3335
echo "STABLE_BUILD_SCM_REVISION $BAZEL_FAKE_SCM_REVISION"
3436
else
3537
# The code below presents an implementation that works for git repository
3638
git_rev=$(git rev-parse HEAD) || exit 1
3739
echo "BUILD_SCM_REVISION ${git_rev}"
40+
echo "ENVOY_BUILD_SCM_REVISION ${git_rev}"
3841
echo "STABLE_BUILD_SCM_REVISION ${git_rev}"
3942
fi
4043

ci/do_ci.sh

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -648,7 +648,21 @@ case $CI_TARGET in
648648
setup_clang_toolchain
649649
echo "generating docs..."
650650
# Build docs.
651-
"${ENVOY_SRCDIR}/docs/build.sh"
651+
# We want the binary at the end
652+
BAZEL_BUILD_OPTIONS+=(--remote_download_toplevel)
653+
[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs
654+
rm -rf "${DOCS_OUTPUT_DIR}"
655+
mkdir -p "${DOCS_OUTPUT_DIR}"
656+
if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then
657+
export SPHINX_RUNNER_ARGS="-v warn"
658+
BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS")
659+
fi
660+
if [[ -n "${DOCS_BUILD_RST}" ]]; then
661+
bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst
662+
cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz
663+
fi
664+
bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:html
665+
tar -xzf bazel-bin/docs/html.tar.gz -C "$DOCS_OUTPUT_DIR"
652666
;;
653667

654668
docs-upload)

docs/BUILD

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,11 +246,15 @@ pkg_tar(
246246
genrule(
247247
name = "html_release",
248248
outs = ["html_release.tar.gz"],
249+
# BUILD_SHA must be set in release builds
250+
# The Envoy workspace will provide this on stamped builds. For external builds
251+
# you must either pass an env var or pass it through the workspace's status.
249252
cmd = """
250253
. $(location //bazel:volatile_env) \
254+
&& _BUILD_SHA=$${BUILD_DOCS_SHA:-$${ENVOY_BUILD_SCM_REVISION:-$${{BUILD_SCM_REVISION}}} \
251255
&& $(location //tools/docs:sphinx_runner) \
252256
$${SPHINX_RUNNER_ARGS:-} \
253-
--build_sha="$${BUILD_DOCS_SHA:-$${BUILD_SCM_REVISION}}" \
257+
--build_sha="$$_BUILD_SHA" \
254258
--docs_tag="$${BUILD_DOCS_TAG:-}" \
255259
--version_file=$(location //:VERSION.txt) \
256260
--descriptor_path=$(location @envoy_api//:v3_proto_set) \
@@ -290,5 +294,5 @@ genrule(
290294

291295
alias(
292296
name = "docs",
293-
actual = ":html",
297+
actual = ":html_release",
294298
)

docs/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,14 @@ In both cases, the generated output can be found in `generated/docs`.
99
If you have an [existing Envoy development environment](https://github.com/envoyproxy/envoy/tree/main/bazel#quick-start-bazel-build-for-developers), you should have the necessary dependencies and requirements and be able to build the documentation directly.
1010

1111
```bash
12-
./docs/build.sh
12+
./ci/do_ci.sh docs
1313
```
1414

1515
By default configuration examples are going to be validated during build. To disable validation,
1616
set `SPHINX_SKIP_CONFIG_VALIDATION` environment variable to `true`:
1717

1818
```bash
19-
SPHINX_SKIP_CONFIG_VALIDATION=true docs/build.sh
19+
SPHINX_SKIP_CONFIG_VALIDATION=true ./ci/do_ci.sh docs
2020
```
2121

2222
## Using the Docker build container to build the documentation
@@ -27,7 +27,7 @@ image that is used in continuous integration.
2727
This can be done as follows:
2828

2929
```
30-
./ci/run_envoy_docker.sh 'docs/build.sh'
30+
./ci/run_envoy_docker.sh './ci/do_ci.sh docs'
3131
```
3232

3333
To use this method you will need a minimum of 4-5GB of disk space available to accommodate the build image.

docs/build.sh

Lines changed: 3 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,5 @@
11
#!/usr/bin/env bash
22

3-
# set SPHINX_SKIP_CONFIG_VALIDATION environment variable to true to skip
4-
# validation of configuration examples
5-
6-
set -e
7-
8-
9-
if [[ ! $(command -v bazel) ]]; then
10-
# shellcheck disable=SC2016
11-
echo 'ERROR: bazel must be installed and available in "$PATH" to build docs' >&2
12-
exit 1
13-
fi
14-
15-
VERSION="$(cat VERSION.txt)"
16-
MAIN_BRANCH="refs/heads/main"
17-
DEV_VERSION_REGEX="-dev$"
18-
19-
# default is to build html only
20-
BUILD_TYPE=html
21-
22-
if [[ "$VERSION" =~ $DEV_VERSION_REGEX ]]; then
23-
if [[ "$CI_BRANCH" == "$MAIN_BRANCH" ]]; then
24-
# no need to build html, just rst
25-
BUILD_TYPE=rst
26-
fi
27-
else
28-
export BUILD_DOCS_TAG="v${VERSION}"
29-
echo "BUILD AZP RELEASE BRANCH ${BUILD_DOCS_TAG}"
30-
BAZEL_BUILD_OPTIONS+=("--action_env=BUILD_DOCS_TAG")
31-
fi
32-
33-
# This is for local RBE setup, should be no-op for builds without RBE setting in bazelrc files.
34-
IFS=" " read -ra BAZEL_BUILD_OPTIONS <<< "${BAZEL_BUILD_OPTION_LIST:-}"
35-
IFS=" " read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTION_LIST:-}"
36-
37-
# We want the binary at the end
38-
BAZEL_BUILD_OPTIONS+=(--remote_download_toplevel)
39-
40-
if [[ -n "${CI_TARGET_BRANCH}" ]] || [[ -n "${SPHINX_QUIET}" ]]; then
41-
export SPHINX_RUNNER_ARGS="-v warn"
42-
BAZEL_BUILD_OPTIONS+=("--action_env=SPHINX_RUNNER_ARGS")
43-
fi
44-
45-
# Building html/rst is determined by then needs of CI but can be overridden in dev.
46-
if [[ "${BUILD_TYPE}" == "html" ]] || [[ -n "${DOCS_BUILD_HTML}" ]]; then
47-
BUILD_HTML=1
48-
BUILD_HTML_TARGET="//docs"
49-
BUILD_HTML_TARBALL="bazel-bin/docs/html.tar.gz"
50-
if [[ -n "${CI_BRANCH}" ]] || [[ -n "${DOCS_BUILD_RELEASE}" ]]; then
51-
# CI build - use git sha
52-
BUILD_HTML_TARGET="//docs:html_release"
53-
BUILD_HTML_TARBALL="bazel-bin/docs/html_release.tar.gz"
54-
fi
55-
fi
56-
if [[ "${BUILD_TYPE}" == "rst" ]] || [[ -n "${DOCS_BUILD_RST}" ]]; then
57-
BUILD_RST=1
58-
fi
59-
60-
# Build html/rst
61-
if [[ -n "${BUILD_RST}" ]]; then
62-
bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //docs:rst
63-
fi
64-
if [[ -n "${BUILD_HTML}" ]]; then
65-
bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" "$BUILD_HTML_TARGET"
66-
fi
67-
68-
[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs
69-
rm -rf "${DOCS_OUTPUT_DIR}"
70-
mkdir -p "${DOCS_OUTPUT_DIR}"
71-
72-
# Save html/rst to output directory
73-
if [[ -n "${BUILD_HTML}" ]]; then
74-
tar -xzf "$BUILD_HTML_TARBALL" -C "$DOCS_OUTPUT_DIR"
75-
fi
76-
if [[ -n "${BUILD_RST}" ]]; then
77-
cp bazel-bin/docs/rst.tar.gz "$DOCS_OUTPUT_DIR"/envoy-docs-rst.tar.gz
78-
fi
3+
# shellcheck disable=SC2016
4+
echo 'This script has been removed. Please use `ci/do_ci.sh docs` instead' >&2
5+
exit 1

mobile/BUILD

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ licenses(["notice"]) # Apache 2
1515

1616
envoy_py_namespace()
1717

18+
exports_files(["VERSION"])
19+
1820
alias(
1921
name = "ios_xcframework",
2022
actual = "//library/swift:Envoy",

mobile/docs/BUILD

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,42 @@ pkg_tar(
6464
)
6565

6666
genrule(
67-
name = "docs",
68-
outs = ["docs.tar.gz"],
67+
name = "html_release",
68+
outs = ["html_release.tar.gz"],
69+
cmd = """
70+
. $(location @envoy//bazel:volatile_env) \
71+
&& VERSION_NUMBER="$$(cat $(location //:VERSION))" \
72+
&& export ENVOY_BLOB_SHA=$${BUILD_SHA:-$${ENVOY_BUILD_SCM_REVISION:-$${{BUILD_SCM_REVISION}}} \
73+
&& export ENVOY_DOCS_VERSION_STRING="$${VERSION_NUMBER}"-"$${ENVOY_BLOB_SHA:0:6}" \
74+
&& export ENVOY_DOCS_RELEASE_LEVEL=pre-release \
75+
&& mkdir -p build \
76+
&& tar xf $(location :rst) -C build \
77+
&& $(location :sphinx) \
78+
-W \
79+
--keep-going \
80+
-b html \
81+
build \
82+
output \
83+
&& tar czf $@ -C output .
84+
""",
85+
stamp = 1,
86+
tools = [
87+
":rst",
88+
":sphinx",
89+
"//:VERSION",
90+
"@envoy//bazel:volatile_env",
91+
],
92+
)
93+
94+
genrule(
95+
name = "html",
96+
outs = ["html.tar.gz"],
6997
cmd = """
7098
mkdir -p build \
99+
&& VERSION_NUMBER="$$(cat $(location //:VERSION))" \
100+
&& export ENVOY_BLOB_SHA="$${BUILD_SHA:-UNKNOWN}" \
101+
&& export ENVOY_DOCS_VERSION_STRING="$${VERSION_NUMBER}"-"$${ENVOY_BLOB_SHA:0:6}" \
102+
&& export ENVOY_DOCS_RELEASE_LEVEL=pre-release \
71103
&& tar xf $(location :rst) -C build \
72104
&& $(location :sphinx) \
73105
-W \
@@ -80,5 +112,11 @@ genrule(
80112
tools = [
81113
":rst",
82114
":sphinx",
115+
"//:VERSION",
83116
],
84117
)
118+
119+
alias(
120+
name = "docs",
121+
actual = ":html_release",
122+
)

mobile/docs/build.sh

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,9 @@ then
1919
# Check the version_history.rst contains current release version.
2020
grep --fixed-strings "$VERSION_NUMBER" docs/root/intro/version_history.rst \
2121
|| (echo "Git tag not found in version_history.rst" && exit 1)
22-
23-
# Now that we now there is a match, we can use the tag.
24-
export ENVOY_DOCS_VERSION_STRING="tag-$GITHUB_REF_NAME"
25-
export ENVOY_DOCS_RELEASE_LEVEL=tagged
26-
export ENVOY_BLOB_SHA="$GITHUB_REF_NAME"
22+
DOCS_TARGET=//docs
2723
else
28-
BUILD_SHA=$(git rev-parse HEAD)
29-
export ENVOY_DOCS_VERSION_STRING="${VERSION_NUMBER}"-"${BUILD_SHA:0:6}"
30-
export ENVOY_DOCS_RELEASE_LEVEL=pre-release
31-
export ENVOY_BLOB_SHA="$BUILD_SHA"
24+
DOCS_TARGET=//docs:html
3225
fi
3326

3427
[[ -z "${DOCS_OUTPUT_DIR}" ]] && DOCS_OUTPUT_DIR=generated/docs
@@ -37,10 +30,6 @@ rm -rf "${DOCS_OUTPUT_DIR}"
3730
mkdir -p "${DOCS_OUTPUT_DIR}"
3831
DOCS_OUTPUT_DIR="$(realpath "$DOCS_OUTPUT_DIR")"
3932

40-
./bazelw build \
41-
--action_env=ENVOY_BLOB_SHA \
42-
--action_env=ENVOY_DOCS_RELEASE_LEVEL \
43-
--action_env=ENVOY_DOCS_VERSION_STRING \
44-
//docs
33+
./bazelw build "$DOCS_TARGET"
4534

46-
tar xf bazel-bin/docs/docs.tar.gz -C "$DOCS_OUTPUT_DIR" .
35+
tar xf bazel-bin/docs/html.tar.gz -C "$DOCS_OUTPUT_DIR" .

mobile/docs/conf.py

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,11 @@ def setup(app):
4848
app.add_directive('substitution-code-block', SubstitutionCodeBlock)
4949

5050

51-
if not os.environ.get('ENVOY_DOCS_RELEASE_LEVEL'):
51+
if not (release_level := os.environ.get('ENVOY_DOCS_RELEASE_LEVEL')):
5252
raise Exception("ENVOY_DOCS_RELEASE_LEVEL env var must be defined")
5353

54-
release_level = os.environ['ENVOY_DOCS_RELEASE_LEVEL']
55-
blob_sha = os.environ['ENVOY_BLOB_SHA']
54+
if not (blob_sha := os.environ.get("ENVOY_BLOB_SHA")):
55+
raise Exception("ENVOY_BLOB_SHA env var must be defined")
5656

5757
# If extensions (or modules to document with autodoc) are in another directory,
5858
# add these directories to sys.path here. If the directory is relative to the
@@ -103,13 +103,12 @@ def setup(app):
103103
# |version| and |release|, also used in various other places throughout the
104104
# built documents.
105105

106-
if not os.environ.get('ENVOY_DOCS_VERSION_STRING'):
106+
# The short X.Y version.
107+
if not (version := os.environ.get("ENVOY_DOCS_VERSION_STRING")):
107108
raise Exception("ENVOY_DOCS_VERSION_STRING env var must be defined")
108109

109-
# The short X.Y version.
110-
version = os.environ['ENVOY_DOCS_VERSION_STRING']
111110
# The full version, including alpha/beta/rc tags.
112-
release = os.environ['ENVOY_DOCS_VERSION_STRING']
111+
release = version
113112

114113
# The language for content autogenerated by Sphinx. Refer to documentation
115114
# for a list of supported languages.

tools/base/requirements.in

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ envoy.dependency.check>=0.1.7
1515
envoy.distribution.release>=0.0.9
1616
envoy.distribution.repo>=0.0.8
1717
envoy.distribution.verify>=0.0.11
18-
envoy.docs.sphinx_runner>=0.2.7
18+
envoy.docs.sphinx_runner>=0.2.9
1919
envoy.gpg.identity>=0.1.1
2020
envoy.gpg.sign>=0.2.0
2121
flake8>=6

tools/base/requirements.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -477,9 +477,9 @@ envoy-distribution-verify==0.0.11 \
477477
envoy-docker-utils==0.0.2 \
478478
--hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76
479479
# via envoy-distribution-distrotest
480-
envoy-docs-sphinx-runner==0.2.8 \
481-
--hash=sha256:129db23430fd3fae3cf62fda74b9479c6880698fef46f895058c98f1f3cf8e20 \
482-
--hash=sha256:736cc88874bdf42778cec02648fcbd82971154d38618b3699e17c049bdec74c9
480+
envoy-docs-sphinx-runner==0.2.9 \
481+
--hash=sha256:1fa789b1d29ea929df67b07e5ca910d62e2057cd229719725030889da53b1a09 \
482+
--hash=sha256:4bfa1946104e263471d522b47d683e127124a5ad47334d69de4aea0eac282576
483483
# via -r requirements.in
484484
envoy-github-abstract==0.0.22 \
485485
--hash=sha256:2dd65e2f247a4947d0198b295c82716c13162e30c433b7625c27d59eee7bcf78 \

0 commit comments

Comments
 (0)