ci: Allow for multiple icount benchmarks in the same run
We don't actually need this for now, but eventually it would be nice to run icount benchmarks on multiple targets. Start tagging artifact names with the architecture, and allow passing `--tag` to `ci-util.py` in order to retrieve the correct one.
This commit is contained in:
parent
8edaa6e5c8
commit
4d325e8b2c
3 changed files with 37 additions and 8 deletions
|
|
@ -195,8 +195,14 @@ jobs:
|
|||
|
||||
benchmarks:
|
||||
name: Benchmarks
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
os: ubuntu-24.04
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@master
|
||||
with:
|
||||
|
|
@ -215,12 +221,14 @@ jobs:
|
|||
cargo binstall -y iai-callgrind-runner --version "$iai_version"
|
||||
sudo apt-get install valgrind
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
key: ${{ matrix.target }}
|
||||
|
||||
- name: Run icount benchmarks
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
run: ./ci/bench-icount.sh
|
||||
run: ./ci/bench-icount.sh ${{ matrix.target }}
|
||||
|
||||
- name: Upload the benchmark baseline
|
||||
uses: actions/upload-artifact@v4
|
||||
|
|
|
|||
|
|
@ -2,10 +2,21 @@
|
|||
|
||||
set -eux
|
||||
|
||||
target="${1:-}"
|
||||
|
||||
if [ -z "$target" ]; then
|
||||
host_target=$(rustc -vV | awk '/^host/ { print $2 }')
|
||||
echo "Defaulted to host target $host_target"
|
||||
target="$host_target"
|
||||
fi
|
||||
|
||||
iai_home="iai-home"
|
||||
|
||||
# Use the arch as a tag to disambiguate artifacts
|
||||
tag="$(echo "$target" | cut -d'-' -f1)"
|
||||
|
||||
# Download the baseline from master
|
||||
./ci/ci-util.py locate-baseline --download --extract
|
||||
./ci/ci-util.py locate-baseline --download --extract --tag "$tag"
|
||||
|
||||
# Run benchmarks once
|
||||
function run_icount_benchmarks() {
|
||||
|
|
@ -44,6 +55,7 @@ function run_icount_benchmarks() {
|
|||
# If this is for a pull request, ignore regressions if specified.
|
||||
./ci/ci-util.py check-regressions --home "$iai_home" --allow-pr-override "$PR_NUMBER"
|
||||
else
|
||||
# Disregard regressions after merge
|
||||
./ci/ci-util.py check-regressions --home "$iai_home" || true
|
||||
fi
|
||||
}
|
||||
|
|
@ -53,6 +65,6 @@ run_icount_benchmarks --features force-soft-floats -- --save-baseline=softfloat
|
|||
run_icount_benchmarks -- --save-baseline=hardfloat
|
||||
|
||||
# Name and tar the new baseline
|
||||
name="baseline-icount-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
|
||||
name="baseline-icount-$tag-$(date -u +'%Y%m%d%H%M')-${GITHUB_SHA:0:12}"
|
||||
echo "BASELINE_NAME=$name" >>"$GITHUB_ENV"
|
||||
tar cJf "$name.tar.xz" "$iai_home"
|
||||
|
|
|
|||
|
|
@ -28,11 +28,14 @@ USAGE = cleandoc(
|
|||
Calculate a matrix of which functions had source change, print that as
|
||||
a JSON object.
|
||||
|
||||
locate-baseline [--download] [--extract]
|
||||
locate-baseline [--download] [--extract] [--tag TAG]
|
||||
Locate the most recent benchmark baseline available in CI and, if flags
|
||||
specify, download and extract it. Never exits with nonzero status if
|
||||
downloading fails.
|
||||
|
||||
`--tag` can be specified to look for artifacts with a specific tag, such as
|
||||
for a specific architecture.
|
||||
|
||||
Note that `--extract` will overwrite files in `iai-home`.
|
||||
|
||||
check-regressions [--home iai-home] [--allow-pr-override pr_number]
|
||||
|
|
@ -50,7 +53,7 @@ REPO_ROOT = Path(__file__).parent.parent
|
|||
GIT = ["git", "-C", REPO_ROOT]
|
||||
DEFAULT_BRANCH = "master"
|
||||
WORKFLOW_NAME = "CI" # Workflow that generates the benchmark artifacts
|
||||
ARTIFACT_GLOB = "baseline-icount*"
|
||||
ARTIFACT_PREFIX = "baseline-icount*"
|
||||
# Place this in a PR body to skip regression checks (must be at the start of a line).
|
||||
REGRESSION_DIRECTIVE = "ci: allow-regressions"
|
||||
# Place this in a PR body to skip extensive tests
|
||||
|
|
@ -278,6 +281,7 @@ def locate_baseline(flags: list[str]) -> None:
|
|||
|
||||
download = False
|
||||
extract = False
|
||||
tag = ""
|
||||
|
||||
while len(flags) > 0:
|
||||
match flags[0]:
|
||||
|
|
@ -285,6 +289,9 @@ def locate_baseline(flags: list[str]) -> None:
|
|||
download = True
|
||||
case "--extract":
|
||||
extract = True
|
||||
case "--tag":
|
||||
tag = flags[1]
|
||||
flags = flags[1:]
|
||||
case _:
|
||||
eprint(USAGE)
|
||||
exit(1)
|
||||
|
|
@ -333,8 +340,10 @@ def locate_baseline(flags: list[str]) -> None:
|
|||
eprint("skipping download step")
|
||||
return
|
||||
|
||||
artifact_glob = f"{ARTIFACT_PREFIX}{f"-{tag}" if tag else ""}*"
|
||||
|
||||
sp.run(
|
||||
["gh", "run", "download", str(job_id), f"--pattern={ARTIFACT_GLOB}"],
|
||||
["gh", "run", "download", str(job_id), f"--pattern={artifact_glob}"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
|
@ -344,7 +353,7 @@ def locate_baseline(flags: list[str]) -> None:
|
|||
|
||||
# Find the baseline with the most recent timestamp. GH downloads the files to e.g.
|
||||
# `some-dirname/some-dirname.tar.xz`, so just glob the whole thing together.
|
||||
candidate_baselines = glob(f"{ARTIFACT_GLOB}/{ARTIFACT_GLOB}")
|
||||
candidate_baselines = glob(f"{artifact_glob}/{artifact_glob}")
|
||||
if len(candidate_baselines) == 0:
|
||||
eprint("no possible baseline directories found")
|
||||
return
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue