Introduce version and build info stamping. (#4054)

This adds a defined Carbon version to the Bazel build and codebase that
can be used both to implement features like version checks and to report
a meaningful version on the command line. This replaces a hard-coded
string and a TODO in the driver.

As part of this, it adds support for defining the version in Bazel, and
special build flags for overriding relevant parts such as the
pre-release marker used. The exact structure and meaning of our version
string, including the pre-release parts, is implemented here in line
with the draft proposal:

https://docs.google.com/document/d/11S5VAPe5Pm_BZPlajWrqDDVr9qc7-7tS2VshqO0wWkk/edit?resourcekey=0-2YFC9Uvl4puuDnWlr2MmYw

This also introduces a workspace status command to the repository to
extract the git commit SHA and other information when building, and the
logic to stamp that into binaries as part of the version string when
useful. The technique used leverages weak symbols with whole archive
linking to allow a link-time override of unstamped data with stamped
data in the leaf executable. This makes building with `--stamp` a
reasonable default, especially for development builds. The CI system is
explicitly opted out of this as there it has no benefit.

Last but not least, all of these are wired into the install rules so
that we build installable packages with the version number in a
conventional place in the directory and filename.

---------

Co-authored-by: Jon Ross-Perkins <[email protected]>

Author: chandlerc

.bazelrc

@@ -2,6 +2,22 @@
 # Exceptions. See /LICENSE for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+# Ensure all builds have Carbon's workspace status attached. We have carefully
+# factored the stamping done by this to avoid excessive build performance impact
+# and so enable stamping with it by default. CI and systems especially dependent
+# on caching should explicitly use `--nostamp`.
+build --workspace_status_command=./scripts/workspace_status.py
+build --stamp
+
+# Provide aliases for configuring the release and pre-release version being
+# built. For documentation of these flags, see //bazel/version/BUILD.
+build --flag_alias=release=//bazel/version:release
+build --flag_alias=pre_release=//bazel/version:pre_release
+build --flag_alias=rc_number=//bazel/version:rc_number
+build --flag_alias=beta_number=//bazel/version:beta_number
+build --flag_alias=alpha_number=//bazel/version:alpha_number
+build --flag_alias=nightly_date=//bazel/version:nightly_date
+
 # Support running clang-tidy with:
 #   bazel build --config=clang-tidy -k //...
 # See: https://github.com/erenon/bazel_clang_tidy

.github/workflows/tests.yaml

@@ -294,6 +294,9 @@ jobs:
           # https://discord.com/channels/655572317891461132/707150492370862090/1151605725576056934
           build --jobs=32
 
+          # Avoid any cache impact from build stamping in CI.
+          build --nostamp
+
           # General build options.
           build --verbose_failures
           test --test_output=errors

bazel/version/BUILD

@@ -0,0 +1,97 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "int_flag", "string_flag")
+
+package(default_visibility = ["//toolchain/install:__pkg__"])
+
+exports_files(["gen_tmpl.py"])
+
+# Several flags are provided for customizing the exact version used for the
+# build of Carbon. Each of these is documented here, but rather than using the
+# label-based names in Bazel invocations (`bazel build --//bazel/version:flag`)
+# we suggest using the flag aliases provided in the project's `.bazelrc` and we
+# document the flags using those aliases. The aliases match the local flag names
+# here.
+#
+# For more details on the versioning scheme used by Carbon, see:
+# - https://docs.google.com/document/d/11S5VAPe5Pm_BZPlajWrqDDVr9qc7-7tS2VshqO0wWkk/edit?resourcekey=0-2YFC9Uvl4puuDnWlr2MmYw
+#   TODO: Replace with path to the markdown once this lands.
+#
+# First, we provide a flag to enable a release version: `--release`. It is
+# disabled by default, and if enabled it must be the only version flag used.
+bool_flag(
+    name = "release",
+    build_setting_default = False,
+)
+
+# A `--pre_release=KIND` flag where `KIND` must be one of:
+# - `rc` -- a release candidate version.
+#    Example: `--pre_release=rc --rc_number=2`
+# - `beta` -- a beta version.
+#    Example: `--pre_release=beta --beta_number=2`
+# - `alpha` -- an alpha version.
+#    Example: `--pre_release=alpha --alpha_number=2`
+# - `nightly -- a nightly version.
+#    Example: `--pre_release=nightly --nightly_date=2024.06.17`
+# - `dev` -- the default, a development build.
+#    Example: `--pre_release=dev`
+#
+# This flag cannot be used along with `--release`, and for all but the `dev`
+# kind must be combined with one of the below flags to specify further details
+# of the version.
+string_flag(
+    name = "pre_release",
+    build_setting_default = "dev",
+    values = [
+        "rc",
+        "beta",
+        "alpha",
+        "nightly",
+        "dev",
+    ],
+)
+
+# `--rc_number=N` sets the release candidate number to `N`. Requires `--pre_release=rc`.
+int_flag(
+    name = "rc_number",
+    build_setting_default = -1,
+)
+
+# When `--pre_release=beta` is used, `--beta_number=N` set's the beta
+# pre-release number to `N`.
+#
+# An error if used without `--pre_release=beta`.
+int_flag(
+    name = "beta_number",
+    build_setting_default = -1,
+)
+
+# When `--pre_release=alpha` is used, `--alpha_number=N` set's the alpha
+# pre-release number to `N`.
+#
+# An error if used without `--pre_release=alpha`.
+int_flag(
+    name = "alpha_number",
+    build_setting_default = -1,
+)
+
+# When `--pre_release=nightly` is used, `--nightly_date=YYYY.MM.DD` set's the
+# nightly build date. The value for this flag must be a string with the exact
+# format of `YYYY.MM.DD`.
+#
+# An error if used without `--pre_release=nightly`.
+string_flag(
+    name = "nightly_date",
+    build_setting_default = "",
+)
+
+# A config setting to observe the value of the `--stamp` command line flag
+# within starlark with a macro and `select`. This is a workaround suggested for
+# a Bazel issue: https://github.com/bazelbuild/bazel/issues/11164
+config_setting(
+    name = "internal_stamp_flag_detect",
+    values = {"stamp": "1"},
+    visibility = ["//visibility:public"],
+)

bazel/version/compute_version.bzl

@@ -0,0 +1,69 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""Compute the version string."""
+
+load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
+load("//:version_base.bzl", "version_base")
+
+def _validate_nightly_date(date):
+    date_components = date.split(".", 2)
+    if len(date_components) != 3:
+        fail("Must provide a nightly date in 'YYYY.MM.DD' format, found '{}'.".format(date))
+    year = date_components[0]
+    if len(year) != 4 or not year.isdigit():
+        fail("The nightly date year was not a sequence of four digits.")
+    month = date_components[1]
+    if len(month) != 2 or not month.isdigit():
+        fail("The nightly date month was not a sequence of two digits.")
+    day = date_components[2]
+    if len(day) != 2 or not day.isdigit():
+        fail("The nightly date day was not a sequence of two digits.")
+
+def compute_version(ctx):
+    """Compute the version string.
+
+    Args:
+        ctx: The context for a rule computing the version.
+
+    Returns:
+        The version string.
+    """
+    version = version_base
+
+    # See if we need to append a pre-release suffix to the version.
+    #
+    # TODO: We should more fully check for erroneous combinations of flags here
+    # to help ensure users don't get surprising results.
+    if not ctx.attr._release_flag[BuildSettingInfo].value:
+        pre_release = ctx.attr._pre_release_flag[BuildSettingInfo].value
+        pre_release_numbers = {
+            "alpha": ctx.attr._alpha_number_flag[BuildSettingInfo].value,
+            "beta": ctx.attr._beta_number_flag[BuildSettingInfo].value,
+            "rc": ctx.attr._rc_number_flag[BuildSettingInfo].value,
+        }
+        if pre_release in pre_release_numbers:
+            number = pre_release_numbers[pre_release]
+            if number < 0:
+                fail("Must provide a non-negative {} number when building that pre-release.".format(pre_release))
+            version += "-{0}.{1}".format(pre_release, number)
+        elif pre_release == "nightly":
+            date = ctx.attr._nightly_date_flag[BuildSettingInfo].value
+            _validate_nightly_date(date)
+            version += "-0.nightly.{}".format(date)
+        elif pre_release == "dev":
+            version += "-0.dev"
+        else:
+            fail("Invalid pre-release flag: " + pre_release)
+
+    return version
+
+VERSION_ATTRS = {
+    "_alpha_number_flag": attr.label(default = ":alpha_number"),
+    "_beta_number_flag": attr.label(default = ":beta_number"),
+    "_nightly_date_flag": attr.label(default = ":nightly_date"),
+    "_pre_release_flag": attr.label(default = ":pre_release"),
+    "_rc_number_flag": attr.label(default = ":rc_number"),
+    "_release_flag": attr.label(default = ":release"),
+}

bazel/version/gen_tmpl.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+"""Generate a file from a template, substituting the provided key/value pairs.
+
+The file format should match Python's `string.Template` substitution rules:
+- `$$` for a literal `$`
+- `$identifier` for some key `identifier` to be substituted
+- `${identifier}` when adjacent text would be interpreted as part of the
+  identifier.
+
+The keys must be strings that are valid identifiers: `[_A-Za-z][_A-Za-z0-9]*`
+
+The values may not contain newlines or any vertical whitespace.
+
+The initial key/value pairs are read from the command line using repeated
+`--substitute=KEY=DEFAULT-VALUE` flags.
+
+Updated values for those keys will be read from any files provided to the
+`--status-file` flag. This flag can be given multiple times and the values will
+be read and updated from the files in order, meaning the last file's value will
+win. New keys are never read from these files. The file format parsed is Bazel's
+[status file format](https://bazel.build/docs/user-manual#workspace-status):
+each line is a single entry starting with a key using only characters `[_A-Z]`,
+one space character, and the rest of the line is the value. To assist with using
+Bazel status files, if the key parsed from the file begins with `STABLE_`, that
+prefix is removed. Any keys which are present in the substitutions provided on
+the command line will have their value updated with the string read from the
+file.
+"""
+
+__copyright__ = """
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""
+
+import argparse
+import sys
+from pathlib import Path
+from string import Template
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument(
+        "--template",
+        metavar="FILE",
+        type=Path,
+        required=True,
+        help="The template source file to use.",
+    )
+    parser.add_argument(
+        "--output",
+        metavar="FILE",
+        type=Path,
+        required=True,
+        help="The output source file to produce.",
+    )
+    parser.add_argument(
+        "--substitution",
+        metavar="KEY=DEFAULT-VALUE",
+        action="append",
+        help="A substitution that should be supported and its default value.",
+    )
+    parser.add_argument(
+        "--status-file",
+        metavar="FILE",
+        type=Path,
+        action="append",
+        default=[],
+        help="A file of key/value updates in Bazel's status file format.",
+    )
+    parser.add_argument("-v", "--verbose", action="store_true")
+    args = parser.parse_args()
+
+    # Collect the supported substitutions from the command line.
+    substitutions = {}
+    for substitution_arg in args.substitution:
+        key, value = substitution_arg.split("=", 1)
+        substitutions.update({key: value})
+
+    # Read either of the two status files provided to build up substitutions,
+    # with the stable file last so its values override any duplicates.
+    for status_file in args.status_file:
+        if args.verbose:
+            print(f"Reading status file: {status_file}", file=sys.stderr)
+        for line in status_file.open():
+            # Remove line endings.
+            line = line.rstrip("\r\n")
+            # Exactly matches our pattern
+            (key, value) = line.split(" ", 1)
+            key = key.removeprefix("STABLE_")
+            if key in substitutions:
+                if args.verbose:
+                    print(f"Parsed: '{key}': '{value}'", file=sys.stderr)
+                substitutions.update({key: value})
+
+    if args.verbose:
+        print(f"Reading template file: {args.template}", file=sys.stderr)
+    with open(args.template) as template_file:
+        template = template_file.read()
+
+    result = Template(template).substitute(substitutions)
+
+    if args.verbose:
+        print(f"Writing output file: {args.output}", file=sys.stderr)
+    with open(args.output, mode="w") as output_file:
+        output_file.write(result)
+
+
+if __name__ == "__main__":
+    main()