ci: Allow for multiple icount benchmarks in the same run

We don't actually need this for now, but eventually it would be nice to
run icount benchmarks on multiple targets. Start tagging artifact names
with the architecture, and allow passing `--tag` to `ci-util.py` in
order to retrieve the correct one.
This commit is contained in:
Trevor Gross 2025-05-29 21:22:47 +00:00
parent 8edaa6e5c8
commit 4d325e8b2c
3 changed files with 37 additions and 8 deletions

View file

@ -195,8 +195,14 @@ jobs:
benchmarks:
name: Benchmarks
runs-on: ubuntu-24.04
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
include:
- target: x86_64-unknown-linux-gnu
os: ubuntu-24.04
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@master
with:
@ -215,12 +221,14 @@ jobs:
cargo binstall -y iai-callgrind-runner --version "$iai_version"
sudo apt-get install valgrind
- uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.target }}
- name: Run icount benchmarks
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: ./ci/bench-icount.sh
run: ./ci/bench-icount.sh ${{ matrix.target }}
- name: Upload the benchmark baseline
uses: actions/upload-artifact@v4

View file

@ -2,10 +2,21 @@
set -eux
target="${1:-}"
if [ -z "$target" ]; then
host_target=$(rustc -vV | awk '/^host/ { print $2 }')
echo "Defaulted to host target $host_target"
target="$host_target"
fi
iai_home="iai-home"
# Use the arch as a tag to disambiguate artifacts
tag="$(echo "$target" | cut -d'-' -f1)"
# Download the baseline from master
./ci/ci-util.py locate-baseline --download --extract
./ci/ci-util.py locate-baseline --download --extract --tag "$tag"
# Run benchmarks once
function run_icount_benchmarks() {
@ -44,6 +55,7 @@ function run_icount_benchmarks() {
# If this is for a pull request, ignore regressions if specified.
./ci/ci-util.py check-regressions --home "$iai_home" --allow-pr-override "$PR_NUMBER"
else
# Disregard regressions after merge
./ci/ci-util.py check-regressions --home "$iai_home" || true
fi
}
@ -53,6 +65,6 @@ run_icount_benchmarks --features force-soft-floats -- --save-baseline=softfloat
run_icount_benchmarks -- --save-baseline=hardfloat
# Name and tar the new baseline
name="baseline-icount-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
name="baseline-icount-$tag-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
echo "BASELINE_NAME=$name" >>"$GITHUB_ENV"
tar cJf "$name.tar.xz" "$iai_home"

View file

@ -28,11 +28,14 @@ USAGE = cleandoc(
Calculate a matrix of which functions had source change, print that as
a JSON object.
locate-baseline [--download] [--extract]
locate-baseline [--download] [--extract] [--tag TAG]
Locate the most recent benchmark baseline available in CI and, if flags
specify, download and extract it. Never exits with nonzero status if
downloading fails.
`--tag` can be specified to look for artifacts with a specific tag, such as
for a specific architecture.
Note that `--extract` will overwrite files in `iai-home`.
check-regressions [--home iai-home] [--allow-pr-override pr_number]
@ -50,7 +53,7 @@ REPO_ROOT = Path(__file__).parent.parent
GIT = ["git", "-C", REPO_ROOT]
DEFAULT_BRANCH = "master"
WORKFLOW_NAME = "CI" # Workflow that generates the benchmark artifacts
ARTIFACT_GLOB = "baseline-icount*"
ARTIFACT_PREFIX = "baseline-icount*"
# Place this in a PR body to skip regression checks (must be at the start of a line).
REGRESSION_DIRECTIVE = "ci: allow-regressions"
# Place this in a PR body to skip extensive tests
@ -278,6 +281,7 @@ def locate_baseline(flags: list[str]) -> None:
download = False
extract = False
tag = ""
while len(flags) > 0:
match flags[0]:
@ -285,6 +289,9 @@ def locate_baseline(flags: list[str]) -> None:
download = True
case "--extract":
extract = True
case "--tag":
tag = flags[1]
flags = flags[1:]
case _:
eprint(USAGE)
exit(1)
@ -333,8 +340,10 @@ def locate_baseline(flags: list[str]) -> None:
eprint("skipping download step")
return
artifact_glob = f"{ARTIFACT_PREFIX}{f"-{tag}" if tag else ""}*"
sp.run(
["gh", "run", "download", str(job_id), f"--pattern={ARTIFACT_GLOB}"],
["gh", "run", "download", str(job_id), f"--pattern={artifact_glob}"],
check=False,
)
@ -344,7 +353,7 @@ def locate_baseline(flags: list[str]) -> None:
# Find the baseline with the most recent timestamp. GH downloads the files to e.g.
# `some-dirname/some-dirname.tar.xz`, so just glob the whole thing together.
candidate_baselines = glob(f"{ARTIFACT_GLOB}/{ARTIFACT_GLOB}")
candidate_baselines = glob(f"{artifact_glob}/{artifact_glob}")
if len(candidate_baselines) == 0:
eprint("no possible baseline directories found")
return