Perform PR and try builds dynamically

src/ci/github-actions/calculate-job-matrix.py

@@ -7,19 +7,53 @@ be executed on CI.
 It reads job definitions from `src/ci/github-actions/jobs.yml`
 and filters them based on the event that happened on CI.
 
-Currently, it only supports PR builds.
+Currently, it only supports PR and try builds.
 """
 
 import json
+import os
+import sys
 from pathlib import Path
+from typing import List, Dict
 
 import yaml
 
 JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
 
 
+def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
+    for job in jobs:
+        job["name"] = f"{prefix} - {job['image']}"
+    return jobs
+
+
 if __name__ == "__main__":
+    github_ctx = json.loads(os.environ["GITHUB_CTX"])
+
     with open(JOBS_YAML_PATH) as f:
-        jobs = yaml.safe_load(f)
-    job_output = jobs["pr"]
-    print(f"jobs={json.dumps(job_output)}")
+        data = yaml.safe_load(f)
+
+    event_name = github_ctx["event_name"]
+    ref = github_ctx["ref"]
+    repository = github_ctx["repository"]
+
+    old_bors_try_build = (
+        ref in ("refs/heads/try", "refs/heads/try-perf") and
+        repository == "rust-lang-ci/rust"
+    )
+    new_bors_try_build = (
+        ref == "refs/heads/automation/bors/try" and
+        repository == "rust-lang/rust"
+    )
+    try_build = old_bors_try_build or new_bors_try_build
+
+    jobs = []
+    # Pull request CI jobs. Their name is 'PR - <image>'
+    if event_name == "pull_request":
+        jobs = name_jobs(data["pr"], "PR")
+    # Try builds
+    elif event_name == "push" and try_build:
+        jobs = name_jobs(data["try"], "try")
+
+    print(f"Output:\n{json.dumps(jobs, indent=4)}", file=sys.stderr)
+    print(f"jobs={json.dumps(jobs)}")

src/ci/github-actions/ci.yml

@@ -351,17 +351,25 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
+        env:
+          GITHUB_CTX: ${{ toJSON(github) }}
         run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
-  pr:
+  # This is either a PR or a try job, decided by calculate-job-matrix.py
+  job:
     <<: *base-ci-job
-    name: PR - ${{ matrix.name }}
+    name: ${{ matrix.name }}
     needs: [ calculate_matrix ]
     env:
-      <<: [*shared-ci-variables, *public-variables]
-      PR_CI_JOB: 1
-    if: github.event_name == 'pull_request'
-    continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
+      CI_JOB_NAME: ${{ matrix.image }}
+      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SCCACHE_BUCKET: rust-lang-ci-sccache2
+      TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+      CACHE_DOMAIN: ci-caches.rust-lang.org
+    continue-on-error: ${{ matrix.continue_on_error || false }}
     strategy:
       matrix:
         # Check the `calculate_matrix` job to see how is the matrix defined.
@@ -751,23 +759,6 @@ jobs:
               SCRIPT: python x.py dist bootstrap --include-default-paths
             <<: *job-windows-8c
 
-  try:
-    <<: *base-ci-job
-    name: try - ${{ matrix.name }}
-    env:
-      DIST_TRY_BUILD: 1
-      <<: [*shared-ci-variables, *prod-variables]
-    if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
-    strategy:
-      matrix:
-        include:
-          - &dist-x86_64-linux
-            name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-16c
-
-
   master:
     name: master
     runs-on: ubuntu-latest
@@ -791,11 +782,11 @@ jobs:
   # build completed, as there is no practical way to detect when a workflow is
   # successful listening to webhooks only.
   try-success:
-    needs: [try]
+    needs: [ job ]
     if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
     <<: *base-success-job
   try-failure:
-    needs: [try]
+    needs: [ job ]
     if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
     <<: *base-failure-job
   auto-success:

src/ci/github-actions/jobs.yml

@@ -2,7 +2,7 @@
 # dynamically in CI from ci.yml.
 # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
 # modify this file.
-shared_defs:
+runners:
   - &base-job
     env: { }
 
@@ -37,14 +37,49 @@ shared_defs:
   - &job-aarch64-linux
     os: [ self-hosted, ARM64, linux ]
 
+envs:
+  - &prod-variables
+    DEPLOY_BUCKET: rust-lang-ci2
+    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
+    TOOLSTATE_PUBLISH: 1
+    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
+    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
+    # rotate them in a single branch while keeping the old key in another
+    # branch, which wouldn't be possible if the key was named with the kind
+    # (caches, artifacts...).
+    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
+    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
+    AWS_REGION: us-west-1
+
+  - &pr-env
+    PR_CI_JOB: 1
+
+# Pull request jobs
 pr:
-  - name: mingw-check
+  - image: mingw-check
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: mingw-check-tidy
+  - image: mingw-check-tidy
+    continue_on_error: true
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: x86_64-gnu-llvm-17
+  - image: x86_64-gnu-llvm-17
     env:
       ENABLE_GCC_CODEGEN: "1"
+      <<: *pr-env
     <<: *job-linux-16c
-  - name: x86_64-gnu-tools
+  - image: x86_64-gnu-tools
+    env:
+      <<: *pr-env
+    <<: *job-linux-16c
+
+# Try build jobs
+try:
+  - image: dist-x86_64-linux
+    env:
+      DIST_TRY_BUILD: 1
+      CODEGEN_BACKENDS: llvm,cranelift
+      <<: *prod-variables
     <<: *job-linux-16c