Skip to content

MGET support for Redis protocol with OSS Cluster slot-aware routing #152

MGET support for Redis protocol with OSS Cluster slot-aware routing

MGET support for Redis protocol with OSS Cluster slot-aware routing #152

Workflow file for this run

name: Build and Publish RPM Packages
on:
push:
branches:
- master
- main
paths-ignore:
- '**.md'
- 'debian/**'
pull_request:
branches:
- master
- main
paths-ignore:
- '**.md'
- 'debian/**'
release:
types: [published]
workflow_dispatch:
inputs:
tag_name:
description: "Release tag name for testing (e.g., 2.1.2)"
required: false
default: ""
# Default to least-privilege; jobs that need write opt in below.
permissions:
contents: read
actions: read
jobs:
build-srpm:
runs-on: ubuntu-latest
container: rockylinux:9
steps:
- name: Install git and basic tools
run: |
dnf install -y git rpm-build rpmdevtools rsync
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- name: Setup RPM build tree
run: rpmdev-setuptree
- name: Get version
id: version
# Pass user-controlled inputs through env: rather than ${{ }} expansion
# so they cannot inject shell syntax. configure.ac is editable by
# forked-PR authors; AC_INIT() containing backticks or $() would
# otherwise be evaluated when the version flows back through
# `${{ steps.version.outputs.version }}` in later steps.
env:
INPUT_TAG: ${{ github.event.inputs.tag_name }}
RELEASE_TAG: ${{ github.event.release.tag_name }}
run: |
set -eu
# Extract version from configure.ac (source of truth)
SOURCE_VERSION=$(awk -F'[(),]' '/AC_INIT/ {gsub(/ /, "", $3); print $3}' configure.ac)
if [ -n "$INPUT_TAG" ]; then
VERSION="$INPUT_TAG"
elif [ -n "$RELEASE_TAG" ]; then
VERSION="$RELEASE_TAG"
else
VERSION="$SOURCE_VERSION"
fi
# Defense in depth: reject anything that is not a simple semver-ish
# token. Blocks backticks, $(...), /, &, spaces, etc. before the
# value is reused in sed replacements or file paths.
case "$VERSION" in
""|*[!A-Za-z0-9._-]*)
echo "::error::Invalid version string: $VERSION"
exit 1
;;
esac
# Hard-fail on releases if the tag and configure.ac drift apart —
# publishing RPMs labeled with a tag that doesn't match the source
# version is worse than failing the build.
if [ -n "$RELEASE_TAG" ] && [ "$VERSION" != "$SOURCE_VERSION" ]; then
echo "::error::Release tag ($VERSION) does not match configure.ac version ($SOURCE_VERSION). Update configure.ac or retag the release."
exit 1
fi
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "Building version: $VERSION (source: $SOURCE_VERSION)"
- name: Create source tarball
env:
VERSION: ${{ steps.version.outputs.version }}
run: |
mkdir -p "memtier-benchmark-$VERSION"
rsync -a --exclude='.git' --exclude='memtier-benchmark-*' . "memtier-benchmark-$VERSION/"
tar czf "$HOME/rpmbuild/SOURCES/memtier-benchmark-$VERSION.tar.gz" "memtier-benchmark-$VERSION"
- name: Update spec file version
env:
VERSION: ${{ steps.version.outputs.version }}
run: |
sed -i "s/^Version:.*/Version: $VERSION/" rpm/memtier-benchmark.spec
cp rpm/memtier-benchmark.spec ~/rpmbuild/SPECS/
- name: Build SRPM
run: |
rpmbuild -bs ~/rpmbuild/SPECS/memtier-benchmark.spec
- name: Upload SRPM artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: srpm
path: ~/rpmbuild/SRPMS/*.src.rpm
retention-days: 5
build-rpm:
needs: build-srpm
# Needs contents: write only to attach RPMs as release assets (release event).
permissions:
contents: write
strategy:
fail-fast: false
matrix:
include:
# x86_64 builds
- distro: el8
image: rockylinux:8
arch: x86_64
runner: ubuntu-latest
- distro: el9
image: rockylinux:9
arch: x86_64
runner: ubuntu-latest
- distro: el10
image: rockylinux/rockylinux:10
arch: x86_64
runner: ubuntu-latest
- distro: amzn2023
image: amazonlinux:2023
arch: x86_64
runner: ubuntu-latest
# arm64 builds
- distro: el8
image: rockylinux:8
arch: aarch64
runner: ubuntu-24.04-arm
- distro: el9
image: rockylinux:9
arch: aarch64
runner: ubuntu-24.04-arm
- distro: el10
image: rockylinux/rockylinux:10
arch: aarch64
runner: ubuntu-24.04-arm
- distro: amzn2023
image: amazonlinux:2023
arch: aarch64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
container: ${{ matrix.image }}
name: Build RPM (${{ matrix.distro }}-${{ matrix.arch }})
steps:
- name: Install build dependencies
run: |
if command -v dnf &> /dev/null; then
dnf install -y rpm-build gcc-c++ make autoconf automake libtool \
libevent-devel openssl-devel zlib-devel pkgconfig
else
yum install -y rpm-build gcc-c++ make autoconf automake libtool \
libevent-devel openssl-devel zlib-devel pkgconfig
fi
- name: Download SRPM
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: srpm
- name: Install SRPM
# SRPM is produced by the prior job from this repo's own spec;
# re-installing it here unpacks sources and the spec into ~/rpmbuild/.
run: rpm -ivh *.src.rpm
- name: Build RPM
run: |
rpmbuild -bb ~/rpmbuild/SPECS/memtier-benchmark.spec
- name: Static check with rpmlint (advisory)
continue-on-error: true
run: |
# rpmlint lives in different repos across distros; try a few then skip if unavailable.
if command -v dnf &>/dev/null; then
dnf install -y rpmlint \
|| dnf install -y --enablerepo=crb rpmlint \
|| dnf install -y --enablerepo=powertools rpmlint \
|| dnf install -y --enablerepo=epel rpmlint \
|| echo "rpmlint unavailable on ${{ matrix.distro }}"
fi
if command -v rpmlint &>/dev/null; then
find ~/rpmbuild/RPMS -name '*.rpm' -type f -print0 | xargs -0 rpmlint || true
fi
- name: Upload RPM artifact
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: rpm-${{ matrix.distro }}-${{ matrix.arch }}
path: ~/rpmbuild/RPMS/**/*.rpm
retention-days: 5
- name: Upload as release assets
if: github.event_name == 'release'
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
with:
files: ~/rpmbuild/RPMS/**/*.rpm
smoke-test-packages:
needs: build-rpm
# Run on PRs too: the smoke test exercises install + repo-resolved install,
# which is exactly the user-facing path we want to protect against
# regressions before merging.
strategy:
fail-fast: false
matrix:
include:
# x86_64 tests
- distro: el8
image: rockylinux:8
arch: x86_64
runner: ubuntu-latest
- distro: el9
image: rockylinux:9
arch: x86_64
runner: ubuntu-latest
- distro: el10
image: rockylinux/rockylinux:10
arch: x86_64
runner: ubuntu-latest
- distro: amzn2023
image: amazonlinux:2023
arch: x86_64
runner: ubuntu-latest
# arm64 tests
- distro: el8
image: rockylinux:8
arch: aarch64
runner: ubuntu-24.04-arm
- distro: el9
image: rockylinux:9
arch: aarch64
runner: ubuntu-24.04-arm
- distro: el10
image: rockylinux/rockylinux:10
arch: aarch64
runner: ubuntu-24.04-arm
- distro: amzn2023
image: amazonlinux:2023
arch: aarch64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
container: ${{ matrix.image }}
name: Smoke test (${{ matrix.distro }}-${{ matrix.arch }})
steps:
- name: Install minimal shell tools
# Minimal Rocky 8/10 and AL2023 base images ship without findutils
# (and sometimes without diff/which). Install up-front so the rest of
# the job can use `find` etc.
run: |
if command -v dnf >/dev/null 2>&1; then
dnf install -y findutils
else
yum install -y findutils
fi
- name: Download RPM
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: rpm-${{ matrix.distro }}-${{ matrix.arch }}
path: rpms
- name: Install RPM directly (sanity check)
run: |
# First pass: install the file directly to confirm the RPM itself is sound.
RPM_FILE=$(find ./rpms -name "*.rpm" -type f ! -name "*debug*" | head -1)
if [ -z "$RPM_FILE" ]; then
echo "::error::No RPM file found in ./rpms"
find ./rpms -type f
exit 1
fi
echo "Installing: $RPM_FILE"
# Rely on the package's auto-deps to pull libevent/openssl — manually
# pre-installing them would mask a missing Requires:.
if command -v dnf &> /dev/null; then
dnf install -y "$RPM_FILE"
else
yum install -y "$RPM_FILE"
fi
- name: Verify direct install
run: |
# --version is the binary-loads check. Don't run --help here:
# memtier_benchmark exits non-zero on --help, which aborts `sh -e`.
memtier_benchmark --version
# Man page is referenced from the spec %files. Query the RPM
# rather than testing fixed paths — el10 uses zstd (.zst) instead
# of gzip (.gz), and other distros may diverge again later.
rpm -ql memtier-benchmark | grep -qE "/man1/memtier_benchmark\.1(\..+)?$"
# Bash completion file must be syntactically loadable.
bash -n /usr/share/bash-completion/completions/memtier_benchmark
echo "✓ direct install (${{ matrix.distro }}-${{ matrix.arch }}) verified"
- name: Test install via local YUM repo (mirrors README path layout)
run: |
# This step mirrors what users actually do: install via repo metadata
# using the same baseurl pattern documented in README.md. It catches
# createrepo / Requires / baseurl bugs that direct-file install misses,
# including $releasever-vs-distro mismatches like the AL2023 case.
set -euo pipefail
if command -v dnf &>/dev/null; then
dnf remove -y memtier-benchmark
dnf install -y createrepo_c || dnf install -y createrepo
else
yum remove -y memtier-benchmark
yum install -y createrepo_c || yum install -y createrepo
fi
DISTRO="${{ matrix.distro }}"
ARCH="${{ matrix.arch }}"
REPO_ROOT=/tmp/localrepo
mkdir -p "$REPO_ROOT/$DISTRO/$ARCH"
find ./rpms -name '*.rpm' -type f ! -name '*debug*' \
-exec cp {} "$REPO_ROOT/$DISTRO/$ARCH/" \;
if command -v createrepo_c &>/dev/null; then
createrepo_c "$REPO_ROOT/$DISTRO/$ARCH"
else
createrepo "$REPO_ROOT/$DISTRO/$ARCH"
fi
# Match the README baseurl pattern. \$releasever and \$basearch are
# passed through to dnf; only $REPO_ROOT is shell-expanded here.
if [ "$DISTRO" = "amzn2023" ]; then
BASEURL="file://$REPO_ROOT/amzn2023/\$basearch"
else
BASEURL="file://$REPO_ROOT/el\$releasever/\$basearch"
fi
cat > /etc/yum.repos.d/local-test.repo <<EOF
[local-test]
name=Local Test
baseurl=$BASEURL
enabled=1
gpgcheck=0
EOF
if command -v dnf &>/dev/null; then
dnf install -y memtier-benchmark
else
yum install -y memtier-benchmark
fi
memtier_benchmark --version
echo "✓ repo install (${{ matrix.distro }}-${{ matrix.arch }}) verified via $BASEURL"
publish-to-yum:
runs-on: ubuntu-latest
needs: smoke-test-packages
environment: build
permissions:
contents: read
# Only publish on actual releases (not prereleases or workflow_dispatch)
if: github.event_name == 'release' && github.event.release.prerelease == false
steps:
- name: Setup RPM Signing key
# Same key used for the DEB pipeline (APT_SIGNING_KEY); name retained
# for parity with existing secret naming.
run: |
mkdir -m 0700 -p ~/.gnupg
echo "$APT_SIGNING_KEY" | gpg --batch --import
env:
APT_SIGNING_KEY: ${{ secrets.APT_SIGNING_KEY }}
- name: Download all RPM artifacts
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
pattern: rpm-*
path: rpms
merge-multiple: false
- name: Install createrepo and verify AWS CLI
run: |
sudo apt-get update
# awscli is no longer in apt on Ubuntu 24.04 (noble); the GitHub-hosted
# ubuntu-latest runner ships AWS CLI v2 pre-installed at
# /usr/local/bin/aws, so we only need to install the rpm tooling here.
sudo apt-get install -y createrepo-c rpm
aws --version
- name: Sign RPMs
run: |
set -euo pipefail
# Derive the signing identifier from the imported key rather than
# hard-coding a UID. The DEB pipeline relies on "first key in the
# keyring" semantics; we mirror that by selecting the first secret
# key's fingerprint, which works regardless of which UID(s) the
# APT_SIGNING_KEY secret happens to carry.
SIGN_FPR=$(gpg --list-secret-keys --with-colons \
| awk -F: '/^fpr:/{print $10; exit}')
if [ -z "$SIGN_FPR" ]; then
echo "::error::No secret key found in keyring; APT_SIGNING_KEY import failed?"
exit 1
fi
echo "Signing with key: $SIGN_FPR"
echo "%_gpg_name $SIGN_FPR" >> ~/.rpmmacros
# Use find instead of globstar (not enabled by default in bash)
find rpms -name "*.rpm" -type f | while read -r rpm_file; do
echo "Signing: $rpm_file"
rpm --addsign "$rpm_file"
done
- name: Verify RPM signatures
# Fail fast if signing silently produced unsigned/invalid RPMs.
run: |
set -euo pipefail
# Import the public half of whatever key is in the keyring (avoids
# coupling to a specific UID). Empty output here means the earlier
# APT_SIGNING_KEY import didn't actually populate the keyring.
gpg --export --armor > /tmp/redis-pub.asc
if [ ! -s /tmp/redis-pub.asc ]; then
echo "::error::Public key export is empty — APT_SIGNING_KEY import likely failed"
exit 1
fi
sudo rpm --import /tmp/redis-pub.asc
bad=0
while read -r rpm_file; do
# `rpm --addsign` on rpm >= 4.13 produces a V4 header signature,
# stored in `%{RSAHEADER}` / `%{DSAHEADER}`. The legacy `%{SIGPGP}`
# tag is `(none)` on V4-signed packages, so querying it falsely
# reports modern RPMs as unsigned. Use `rpm -Kv` instead — it
# prints a per-component verification line (`Header V4 RSA/SHA512
# Signature, key ID xxxx: OK`) and exits 0 only when every
# component checks out against the imported key, covering both
# V3 (legacy SIGPGP) and V4 (RSAHEADER/DSAHEADER) signatures.
# `|| true` so a non-zero rpm exit (unsigned / NOKEY / BAD) does
# not abort the script under `set -e` before our grep-based
# diagnostics can run. We classify the result via the captured
# output below, not via the exit code.
verify_output=$(rpm -Kv "$rpm_file" 2>&1 || true)
# `rpm -Kv` emits one line per signature/digest component:
# Header V4 RSA/SHA512 Signature, key ID xxxx: <verdict> (signature)
# Header SHA256 digest: <verdict> (header digest)
# Payload SHA256 digest: <verdict> (payload digest)
# where <verdict> is OK, NOKEY, BAD, or NOTTRUSTED.
# The V4 header signature only covers the header — payload tampering
# surfaces only as a Payload digest BAD, NOT as a signature BAD — so
# any BAD/NOTTRUSTED *anywhere* in the output must hard-fail.
# `rpm --addsign` (the prior step) is the source of truth for "RPM is
# signed" — its non-zero exit fails the workflow before this verify
# step runs. NOKEY on the signature line surfaces a key-management
# glitch in the gpg→rpm round-trip on a fresh runner (typically
# subkey export not picked up by `rpm --import`); it does NOT mean
# the package is unsigned, and treating it as a publish-blocking
# error has wedged the YUM publish twice in 2.3.1.
# Classification in priority order:
# 1. BAD or NOTTRUSTED anywhere -> hard-fail (corrupted/untrusted)
# 2. no V3/V4 signature header -> hard-fail (signing didn't run)
# 3. NOKEY on signature line -> warn (key not in CI keyring)
# 4. otherwise -> ok
if echo "$verify_output" | grep -qwE 'BAD|NOTTRUSTED'; then
echo "::error::RPM has corrupted signature/digest or untrusted key: $rpm_file"
echo "$verify_output"
bad=$((bad + 1))
continue
fi
if ! echo "$verify_output" | grep -qE 'Header V[34] (RSA|DSA)/.*: (OK|NOKEY)'; then
echo "::error::RPM unsigned (no V3/V4 signature header found): $rpm_file"
echo "$verify_output"
bad=$((bad + 1))
continue
fi
if echo "$verify_output" | grep -qE 'Header V[34] (RSA|DSA)/.*: NOKEY'; then
echo "::warning::RPM signed but public key not in CI keyring (NOKEY): $rpm_file"
fi
echo "✓ signed: $rpm_file"
done < <(find rpms -name "*.rpm" -type f)
if [ "$bad" -gt 0 ]; then
echo "::error::$bad RPM(s) failed signature verification"
exit 1
fi
- name: Sync existing repo from S3
run: |
set -euo pipefail
mkdir -p ./repo
# Distinguish "bucket path is empty (first-ever publish)" from "sync
# failed for any other reason". A blanket `|| echo` masks transient
# errors and lets the later `--delete` upload wipe prior releases.
if aws s3 ls "s3://${{ secrets.APT_S3_BUCKET }}/rpm/" >/dev/null 2>&1; then
# Path exists with at least one object — a sync failure now is real.
aws s3 sync "s3://${{ secrets.APT_S3_BUCKET }}/rpm" ./repo
else
echo "No existing rpm/ prefix in s3://${{ secrets.APT_S3_BUCKET }}, starting fresh"
fi
env:
AWS_ACCESS_KEY_ID: ${{ secrets.APT_S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.APT_S3_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.APT_S3_REGION }}
- name: Organize RPMs by distro and architecture
run: |
# Artifact names are rpm-{distro}-{arch}, e.g., rpm-el9-x86_64
for artifact_dir in rpms/rpm-*; do
artifact_name=$(basename "$artifact_dir")
# Extract distro and arch from artifact name (rpm-el9-x86_64 -> el9, x86_64)
distro=$(echo "$artifact_name" | sed 's/rpm-//' | rev | cut -d'-' -f2- | rev)
arch=$(echo "$artifact_name" | rev | cut -d'-' -f1 | rev)
mkdir -p "repo/$distro/$arch"
# Use find instead of globstar
find "$artifact_dir" -name "*.rpm" -type f -exec cp {} "repo/$distro/$arch/" \;
echo "Organized: $artifact_name -> repo/$distro/$arch/"
done
- name: Verify RPMs were organized
run: |
RPM_COUNT=$(find repo -name "*.rpm" -type f | wc -l)
if [ "$RPM_COUNT" -eq 0 ]; then
echo "::error::No RPMs found in repo directory. Aborting to prevent wiping S3."
exit 1
fi
echo "Found $RPM_COUNT RPMs to publish"
- name: Create/Update YUM repository metadata
run: |
for distro_dir in repo/*/; do
for arch_dir in "$distro_dir"*/; do
if [ -d "$arch_dir" ] && ls "$arch_dir"/*.rpm 1>/dev/null 2>&1; then
echo "Creating repo metadata for: $arch_dir"
createrepo_c --update "$arch_dir"
fi
done
done
- name: Upload repo to S3
run: |
# Use --delete to keep repo clean, but we verified RPMs exist above
aws s3 sync ./repo "s3://${{ secrets.APT_S3_BUCKET }}/rpm" --delete
env:
AWS_ACCESS_KEY_ID: ${{ secrets.APT_S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.APT_S3_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.APT_S3_REGION }}
- name: List uploaded packages
run: |
echo "📦 Packages uploaded to YUM repository:"
find repo -name "*.rpm" -type f