user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rollup merge of #66569 - pietroalbini:gha-1, r=alexcrichton

Browse source 
GitHub Actions: preparations, part 1

This PR adds the first batch of commits in preparation for GitHub Actions:

* Added GitHub Actions support in `src/ci/shared.sh` and bootstrap.
* Addded a `setup-environment.sh` script which guesses and sets the `DEPLOY`, `DEPLOY_ALT` and `IMAGE` environment variables automatically, to reduce the verbosity of the CI configuration.

This PR does **not** yet add any builders on GitHub Actions.

r? @alexcrichton
This commit is contained in:
Pietro Albini 2019-11-25 15:05:21 +01:00 • committed by GitHub
commit c75044f463
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 154 additions and 248 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/bootstrap/util.rs
View file

@ -262,6 +262,8 @@ pub enum CiEnv {
None,
/// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds.
AzurePipelines,
/// The GitHub Actions environment, for Linux (including Docker), Windows and macOS builds.
GitHubActions,
}
impl CiEnv {
@ -269,6 +271,8 @@ impl CiEnv {
pub fn current() -> CiEnv {
if env::var("TF_BUILD").ok().map_or(false, |e| &*e == "True") {
CiEnv::AzurePipelines
} else if env::var("GITHUB_ACTIONS").ok().map_or(false, |e| &*e == "true") {
CiEnv::GitHubActions
} else {
CiEnv::None
}

168
src/ci/azure-pipelines/auto.yml
View file

@ -19,136 +19,46 @@ jobs:
strategy:
matrix:
x86_64-gnu-llvm-6.0:
IMAGE: x86_64-gnu-llvm-6.0
RUST_BACKTRACE: 1
dist-x86_64-linux:
IMAGE: dist-x86_64-linux
DEPLOY: 1
# "alternate" deployments, these are "nightlies" but have LLVM assertions
# turned on, they're deployed to a different location primarily for
# additional testing.
dist-x86_64-linux: {}
dist-x86_64-linux-alt:
IMAGE: dist-x86_64-linux
DEPLOY_ALT: 1
# Linux builders, remaining docker images
arm-android:
IMAGE: arm-android
armhf-gnu:
IMAGE: armhf-gnu
dist-various-1:
IMAGE: dist-various-1
DEPLOY: 1
dist-various-2:
IMAGE: dist-various-2
DEPLOY: 1
dist-aarch64-linux:
IMAGE: dist-aarch64-linux
DEPLOY: 1
dist-android:
IMAGE: dist-android
DEPLOY: 1
dist-arm-linux:
IMAGE: dist-arm-linux
DEPLOY: 1
dist-armhf-linux:
IMAGE: dist-armhf-linux
DEPLOY: 1
dist-armv7-linux:
IMAGE: dist-armv7-linux
DEPLOY: 1
dist-i586-gnu-i586-i686-musl:
IMAGE: dist-i586-gnu-i586-i686-musl
DEPLOY: 1
dist-i686-freebsd:
IMAGE: dist-i686-freebsd
DEPLOY: 1
dist-i686-linux:
IMAGE: dist-i686-linux
DEPLOY: 1
dist-mips-linux:
IMAGE: dist-mips-linux
DEPLOY: 1
dist-mips64-linux:
IMAGE: dist-mips64-linux
DEPLOY: 1
dist-mips64el-linux:
IMAGE: dist-mips64el-linux
DEPLOY: 1
dist-mipsel-linux:
IMAGE: dist-mipsel-linux
DEPLOY: 1
dist-powerpc-linux:
IMAGE: dist-powerpc-linux
DEPLOY: 1
dist-powerpc64-linux:
IMAGE: dist-powerpc64-linux
DEPLOY: 1
dist-powerpc64le-linux:
IMAGE: dist-powerpc64le-linux
DEPLOY: 1
dist-s390x-linux:
IMAGE: dist-s390x-linux
DEPLOY: 1
dist-x86_64-freebsd:
IMAGE: dist-x86_64-freebsd
DEPLOY: 1
dist-x86_64-musl:
IMAGE: dist-x86_64-musl
DEPLOY: 1
dist-x86_64-netbsd:
IMAGE: dist-x86_64-netbsd
DEPLOY: 1
i686-gnu:
IMAGE: i686-gnu
i686-gnu-nopt:
IMAGE: i686-gnu-nopt
test-various:
IMAGE: test-various
wasm32:
IMAGE: wasm32
x86_64-gnu:
IMAGE: x86_64-gnu
x86_64-gnu-full-bootstrap:
IMAGE: x86_64-gnu-full-bootstrap
x86_64-gnu-aux:
IMAGE: x86_64-gnu-aux
arm-android: {}
armhf-gnu: {}
dist-various-1: {}
dist-various-2: {}
dist-aarch64-linux: {}
dist-android: {}
dist-arm-linux: {}
dist-armhf-linux: {}
dist-armv7-linux: {}
dist-i586-gnu-i586-i686-musl: {}
dist-i686-freebsd: {}
dist-i686-linux: {}
dist-mips-linux: {}
dist-mips64-linux: {}
dist-mips64el-linux: {}
dist-mipsel-linux: {}
dist-powerpc-linux: {}
dist-powerpc64-linux: {}
dist-powerpc64le-linux: {}
dist-s390x-linux: {}
dist-x86_64-freebsd: {}
dist-x86_64-musl: {}
dist-x86_64-netbsd: {}
i686-gnu: {}
i686-gnu-nopt: {}
test-various: {}
wasm32: {}
x86_64-gnu: {}
x86_64-gnu-full-bootstrap: {}
x86_64-gnu-aux: {}
x86_64-gnu-tools:
IMAGE: x86_64-gnu-tools
DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
x86_64-gnu-debug:
IMAGE: x86_64-gnu-debug
x86_64-gnu-nopt:
IMAGE: x86_64-gnu-nopt
x86_64-gnu-distcheck:
IMAGE: x86_64-gnu-distcheck
mingw-check:
IMAGE: mingw-check
x86_64-gnu-debug: {}
x86_64-gnu-nopt: {}
x86_64-gnu-distcheck: {}
mingw-check: {}
- job: macOS
timeoutInMinutes: 600
@ -176,7 +86,6 @@ jobs:
dist-x86_64-apple:
SCRIPT: ./x.py dist
RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
DEPLOY: 1
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
@ -186,7 +95,6 @@ jobs:
dist-x86_64-apple-alt:
SCRIPT: ./x.py dist
RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
DEPLOY_ALT: 1
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
@ -204,7 +112,6 @@ jobs:
dist-i686-apple:
SCRIPT: ./x.py dist
RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --set rust.jemalloc
DEPLOY: 1
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
MACOSX_DEPLOYMENT_TARGET: 10.7
NO_LLVM_ASSERTIONS: 1
@ -304,7 +211,6 @@ jobs:
--enable-profiler
SCRIPT: python x.py dist
DIST_REQUIRE_ALL_TOOLS: 1
DEPLOY: 1
dist-i686-msvc:
RUST_CONFIGURE_ARGS: >-
--build=i686-pc-windows-msvc
@ -313,22 +219,18 @@ jobs:
--enable-profiler
SCRIPT: python x.py dist
DIST_REQUIRE_ALL_TOOLS: 1
DEPLOY: 1
dist-i686-mingw:
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
SCRIPT: python x.py dist
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
DEPLOY: 1
dist-x86_64-mingw:
SCRIPT: python x.py dist
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
CUSTOM_MINGW: 1
DIST_REQUIRE_ALL_TOOLS: 1
DEPLOY: 1
# "alternate" deployment, see .travis.yml for more info
dist-x86_64-msvc-alt:
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
SCRIPT: python x.py dist
DEPLOY_ALT: 1

7
src/ci/azure-pipelines/pr.yml
View file

@ -18,10 +18,7 @@ jobs:
- template: steps/run.yml
strategy:
matrix:
x86_64-gnu-llvm-6.0:
IMAGE: x86_64-gnu-llvm-6.0
mingw-check:
IMAGE: mingw-check
x86_64-gnu-llvm-6.0: {}
mingw-check: {}
x86_64-gnu-tools:
IMAGE: x86_64-gnu-tools
CI_ONLY_WHEN_SUBMODULES_CHANGED: 1

34
src/ci/azure-pipelines/steps/run.yml
View file

@ -28,6 +28,9 @@ steps:
- checkout: self
fetchDepth: 2
- bash: src/ci/scripts/setup-environment.sh
displayName: Setup environment
- bash: src/ci/scripts/should-skip-this.sh
displayName: Decide whether to run this job
@ -38,38 +41,26 @@ steps:
displayName: Show the current environment
- bash: src/ci/scripts/install-sccache.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Install sccache
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/install-clang.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Install clang
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/switch-xcode.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Switch to Xcode 9.3
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/install-wix.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Install wix
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/install-innosetup.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Install InnoSetup
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/windows-symlink-build-dir.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Ensure the build happens on C:\ instead of D:\
condition: and(succeeded(), not(variables.SKIP_JOB))
@ -78,35 +69,22 @@ steps:
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/install-msys2.sh
env:
AGENT_OS: $(Agent.OS)
SYSTEM_WORKFOLDER: $(System.Workfolder)
displayName: Install msys2
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/install-msys2-packages.sh
env:
AGENT_OS: $(Agent.OS)
SYSTEM_WORKFOLDER: $(System.Workfolder)
displayName: Install msys2 packages
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/install-mingw.sh
env:
AGENT_OS: $(Agent.OS)
SYSTEM_WORKFOLDER: $(System.Workfolder)
displayName: Install MinGW
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/install-ninja.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Install ninja
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/enable-docker-ipv6.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Enable IPv6 on Docker
condition: and(succeeded(), not(variables.SKIP_JOB))
@ -120,22 +98,16 @@ steps:
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/checkout-submodules.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Checkout submodules
condition: and(succeeded(), not(variables.SKIP_JOB))
- bash: src/ci/scripts/verify-line-endings.sh
env:
AGENT_OS: $(Agent.OS)
displayName: Verify line endings
condition: and(succeeded(), not(variables.SKIP_JOB))
# Ensure the `aws` CLI is installed so we can deploy later on, cache docker
# images, etc.
- bash: src/ci/scripts/install-awscli.sh
env:
AGENT_OS: $(Agent.OS)
condition: and(succeeded(), not(variables.SKIP_JOB))
displayName: Install awscli

73
src/ci/azure-pipelines/try.yml
View file

@ -6,72 +6,19 @@ variables:
- group: prod-credentials
jobs:
- job: Linux
- job: Windows
timeoutInMinutes: 600
pool:
vmImage: ubuntu-16.04
vmImage: 'vs2017-win2016'
steps:
- template: steps/run.yml
strategy:
matrix:
dist-x86_64-linux:
IMAGE: dist-x86_64-linux
DEPLOY: 1
dist-x86_64-linux-alt:
IMAGE: dist-x86_64-linux
DEPLOY_ALT: 1
# The macOS and Windows builds here are currently disabled due to them not being
# overly necessary on `try` builds. We also don't actually have anything that
# consumes the artifacts currently. Perhaps one day we can reenable, but for now
# it helps free up capacity on Azure.
# - job: macOS
# timeoutInMinutes: 600
# pool:
# vmImage: macos-10.13
# steps:
# - template: steps/run.yml
# strategy:
# matrix:
# dist-x86_64-apple:
# SCRIPT: ./x.py dist
# RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
# DEPLOY: 1
# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
# MACOSX_DEPLOYMENT_TARGET: 10.7
# NO_LLVM_ASSERTIONS: 1
# NO_DEBUG_ASSERTIONS: 1
# DIST_REQUIRE_ALL_TOOLS: 1
#
# dist-x86_64-apple-alt:
# SCRIPT: ./x.py dist
# RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
# DEPLOY_ALT: 1
# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
# MACOSX_DEPLOYMENT_TARGET: 10.7
# NO_LLVM_ASSERTIONS: 1
# NO_DEBUG_ASSERTIONS: 1
#
# - job: Windows
# timeoutInMinutes: 600
# pool:
# vmImage: 'vs2017-win2016'
# steps:
# - template: steps/run.yml
# strategy:
# matrix:
# dist-x86_64-msvc:
# RUST_CONFIGURE_ARGS: >
# --build=x86_64-pc-windows-msvc
# --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
# --enable-full-tools
# --enable-profiler
# SCRIPT: python x.py dist
# DIST_REQUIRE_ALL_TOOLS: 1
# DEPLOY: 1
#
# dist-x86_64-msvc-alt:
# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
# SCRIPT: python x.py dist
# DEPLOY_ALT: 1
dist-x86_64-msvc:
RUST_CONFIGURE_ARGS: >-
--build=x86_64-pc-windows-msvc
--target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py dist
DIST_REQUIRE_ALL_TOOLS: 1

2
src/ci/docker/run.sh
View file

@ -172,6 +172,8 @@ docker \
--env CI \
--env TF_BUILD \
--env BUILD_SOURCEBRANCHNAME \
--env GITHUB_ACTIONS \
--env GITHUB_REF \
--env TOOLSTATE_REPO_ACCESS_TOKEN \
--env TOOLSTATE_REPO \
--env TOOLSTATE_PUBLISH \

4
src/ci/run.sh
View file

@ -23,9 +23,7 @@ fi
ci_dir=`cd $(dirname $0) && pwd`
source "$ci_dir/shared.sh"
branch_name=$(getCIBranch)
if [ ! isCI ] || [ "$branch_name" = "auto" ] || [ "$branch_name" = "try" ]; then
if [ ! isCI ] || isCiBranch auto || isCiBranch beta; then
RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
fi

2
src/ci/scripts/install-mingw.sh
View file

@ -52,7 +52,7 @@ if isWindows; then
if [[ "${CUSTOM_MINGW-0}" -ne 1 ]]; then
pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \
mingw-w64-$arch-gcc mingw-w64-$arch-python2
ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${bits}/bin"
ciCommandAddPath "$(ciCheckoutPath)/msys2/mingw${bits}/bin"
else
mingw_dir="mingw${bits}"

6
src/ci/scripts/install-msys2.sh
View file

@ -12,8 +12,8 @@ IFS=$'\n\t'
source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
if isWindows; then
choco install msys2 --params="/InstallDir:${SYSTEM_WORKFOLDER}/msys2 /NoPath" -y --no-progress
mkdir -p "${SYSTEM_WORKFOLDER}/msys2/home/${USERNAME}"
choco install msys2 --params="/InstallDir:$(ciCheckoutPath)/msys2 /NoPath" -y --no-progress
mkdir -p "$(ciCheckoutPath)/msys2/home/${USERNAME}"
ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/usr/bin"
ciCommandAddPath "$(ciCheckoutPath)/msys2/usr/bin"
fi

31
src/ci/scripts/setup-environment.sh Executable file
View file

@ -0,0 +1,31 @@
#!/bin/bash
# This script guesses some environment variables based on the builder name and
# the current platform, to reduce the amount of variables defined in the CI
# configuration.
set -euo pipefail
IFS=$'\n\t'
source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
# Builders starting with `dist-` are dist builders, but if they also end with
# `-alt` they are alternate dist builders.
if [[ "${CI_JOB_NAME}" = dist-* ]]; then
if [[ "${CI_JOB_NAME}" = *-alt ]]; then
echo "alternate dist builder detected, setting DEPLOY_ALT=1"
ciCommandSetEnv DEPLOY_ALT 1
else
echo "normal dist builder detected, setting DEPLOY=1"
ciCommandSetEnv DEPLOY 1
fi
fi
# All the Linux builds happen inside Docker.
if isLinux; then
if [[ -z "${IMAGE+x}" ]]; then
echo "linux builder detected, using docker to run the build"
ciCommandSetEnv IMAGE "${CI_JOB_NAME}"
else
echo "a custom docker image is already set"
fi
fi

71
src/ci/shared.sh
View file

@ -27,27 +27,66 @@ function retry {
}
function isCI {
[ "$CI" = "true" ] || [ "$TF_BUILD" = "True" ]
[[ "${CI-false}" = "true" ]] || isAzurePipelines || isGitHubActions
}
function isAzurePipelines {
[[ "${TF_BUILD-False}" = "True" ]]
}
function isGitHubActions {
[[ "${GITHUB_ACTIONS-false}" = "true" ]]
}
function isMacOS {
[ "$AGENT_OS" = "Darwin" ]
[[ "${OSTYPE}" = "darwin"* ]]
}
function isWindows {
[ "$AGENT_OS" = "Windows_NT" ]
[[ "${OSTYPE}" = "cygwin" ]] || [[ "${OSTYPE}" = "msys" ]]
}
function isLinux {
[ "$AGENT_OS" = "Linux" ]
[[ "${OSTYPE}" = "linux-gnu" ]]
}
function getCIBranch {
echo "$BUILD_SOURCEBRANCHNAME"
function isCiBranch {
if [[ $# -ne 1 ]]; then
echo "usage: $0 <branch-name>"
exit 1
fi
name="$1"
if isAzurePipelines; then
[[ "${BUILD_SOURCEBRANCHNAME}" = "${name}" ]]
elif isGitHubActions; then
[[ "${GITHUB_REF}" = "refs/heads/${name}" ]]
else
echo "isCiBranch only works inside CI!"
exit 1
fi
}
function ciCommit {
echo "${BUILD_SOURCEVERSION}"
if isAzurePipelines; then
echo "${BUILD_SOURCEVERSION}"
elif isGitHubActions; then
echo "${GITHUB_SHA}"
else
echo "ciCommit only works inside CI!"
exit 1
fi
}
function ciCheckoutPath {
if isAzurePipelines; then
echo "${SYSTEM_WORKFOLDER}"
elif isGitHubActions; then
echo "${GITHUB_WORKSPACE}"
else
echo "ciCheckoutPath only works inside CI!"
exit 1
fi
}
function ciCommandAddPath {
@ -57,7 +96,14 @@ function ciCommandAddPath {
fi
path="$1"
echo "##vso[task.prependpath]${path}"
if isAzurePipelines; then
echo "##vso[task.prependpath]${path}"
elif isGitHubActions; then
echo "::add-path::${value}"
else
echo "ciCommandAddPath only works inside CI!"
exit 1
fi
}
function ciCommandSetEnv {
@ -68,5 +114,12 @@ function ciCommandSetEnv {
name="$1"
value="$2"
echo "##vso[task.setvariable variable=${name}]${value}"
if isAzurePipelines; then
echo "##vso[task.setvariable variable=${name}]${value}"
elif isGitHubActions; then
echo "::set-env name=${name}::${value}"
else
echo "ciCommandSetEnv only works inside CI!"
exit 1
fi
}