user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #66681 - Mark-Simulacrum:toolstate-into-bootstrap, r=pietroalbini

Browse source 
Move toolstate checking into bootstrap

This intends no functional changes, merely translates the spread of shell/python into Rust.

One problematic area that I'd like to avoid but wasn't quite able to figure out how is the master branch script which is still in bash/python -- I cared less about that since it is orthogonal to the actual checking that we're doing, though as-is we're duplicating some code across Rust and that script.

r? @kennytm or maybe @pietroalbini
This commit is contained in:
bors 2019-12-10 16:50:45 +00:00
commit 7dbfb0a8ca
10 changed files with 445 additions and 266 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/bootstrap/builder.rs
View file

@ -368,6 +368,7 @@ impl<'a> Builder<'a> {
check::Rustdoc
),
Kind::Test => describe!(
crate::toolstate::ToolStateCheck,
test::Tidy,
test::Ui,
test::CompileFail,

27
src/bootstrap/lib.rs
View file

@ -169,7 +169,6 @@ mod job {
pub use crate::config::Config;
use crate::flags::Subcommand;
use crate::cache::{Interned, INTERNER};
use crate::toolstate::ToolState;
const LLVM_TOOLS: &[&str] = &[
"llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility
@ -1074,32 +1073,6 @@ impl Build {
}
}
/// Updates the actual toolstate of a tool.
///
/// The toolstates are saved to the file specified by the key
/// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be
/// done. The file is updated immediately after this function completes.
pub fn save_toolstate(&self, tool: &str, state: ToolState) {
if let Some(ref path) = self.config.save_toolstates {
if let Some(parent) = path.parent() {
// Ensure the parent directory always exists
t!(std::fs::create_dir_all(parent));
}
let mut file = t!(fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(path));
let mut current_toolstates: HashMap<Box<str>, ToolState> =
serde_json::from_reader(&mut file).unwrap_or_default();
current_toolstates.insert(tool.into(), state);
t!(file.seek(SeekFrom::Start(0)));
t!(file.set_len(0));
t!(serde_json::to_writer(file, &current_toolstates));
}
}
fn in_tree_crates(&self, root: &str) -> Vec<&Crate> {
let mut ret = Vec::new();
let mut list = vec![INTERNER.intern_str(root)];

407
src/bootstrap/toolstate.rs
View file

@ -1,4 +1,28 @@
use serde::{Deserialize, Serialize};
use build_helper::t;
use std::time;
use std::fs;
use std::io::{Seek, SeekFrom};
use std::collections::HashMap;
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
use std::fmt;
use std::process::Command;
use std::path::PathBuf;
use std::env;
// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then.
const BETA_WEEK_START: u64 = 35;
#[cfg(linux)]
const OS: Option<&str> = Some("linux");
#[cfg(windows)]
const OS: Option<&str> = Some("windows");
#[cfg(all(not(linux), not(windows)))]
const OS: Option<&str> = None;
type ToolstateData = HashMap<Box<str>, ToolState>;
#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
@ -12,9 +36,392 @@ pub enum ToolState {
BuildFail = 0,
}
impl fmt::Display for ToolState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", match self {
ToolState::TestFail => "test-fail",
ToolState::TestPass => "test-pass",
ToolState::BuildFail => "build-fail",
})
}
}
impl Default for ToolState {
fn default() -> Self {
// err on the safe side
ToolState::BuildFail
}
}
/// Number of days after the last promotion of beta.
/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens.
/// The Wednesday after this has value 0.
/// We track this value to prevent regressing tools in the last week of the 6-week cycle.
fn days_since_beta_promotion() -> u64 {
let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed());
(since_epoch.as_secs() / 86400 - 20) % 42
}
// These tools must test-pass on the beta/stable channels.
//
// On the nightly channel, their build step must be attempted, but they may not
// be able to build successfully.
static STABLE_TOOLS: &[(&str, &str)] = &[
("book", "src/doc/book"),
("nomicon", "src/doc/nomicon"),
("reference", "src/doc/reference"),
("rust-by-example", "src/doc/rust-by-example"),
("edition-guide", "src/doc/edition-guide"),
("rls", "src/tools/rls"),
("rustfmt", "src/tools/rustfmt"),
("clippy-driver", "src/tools/clippy"),
];
// These tools are permitted to not build on the beta/stable channels.
//
// We do require that we checked whether they build or not on the tools builder,
// though, as otherwise we will be unable to file an issue if they start
// failing.
static NIGHTLY_TOOLS: &[(&str, &str)] = &[
("miri", "src/tools/miri"),
("embedded-book", "src/doc/embedded-book"),
("rustc-guide", "src/doc/rustc-guide"),
];
fn print_error(tool: &str, submodule: &str) {
eprintln!("");
eprintln!("We detected that this PR updated '{}', but its tests failed.", tool);
eprintln!("");
eprintln!("If you do intend to update '{}', please check the error messages above and", tool);
eprintln!("commit another update.");
eprintln!("");
eprintln!("If you do NOT intend to update '{}', please ensure you did not accidentally", tool);
eprintln!("change the submodule at '{}'. You may ask your reviewer for the", submodule);
eprintln!("proper steps.");
std::process::exit(3);
}
fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) {
// Changed files
let output = std::process::Command::new("git")
.arg("diff")
.arg("--name-status")
.arg("HEAD")
.arg("HEAD^")
.output();
let output = match output {
Ok(o) => o,
Err(e) => {
eprintln!("Failed to get changed files: {:?}", e);
std::process::exit(1);
}
};
let output = t!(String::from_utf8(output.stdout));
for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) {
let changed = output.lines().any(|l| {
l.starts_with("M") && l.ends_with(submodule)
});
eprintln!("Verifying status of {}...", tool);
if !changed {
continue;
}
eprintln!("This PR updated '{}', verifying if status is 'test-pass'...", submodule);
if toolstates[*tool] != ToolState::TestPass {
print_error(tool, submodule);
}
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct ToolStateCheck;
impl Step for ToolStateCheck {
type Output = ();
/// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
///
/// This tool in `src/tools` will verify the validity of all our links in the
/// documentation to ensure we don't have a bunch of dead ones.
fn run(self, builder: &Builder<'_>) {
if builder.config.dry_run {
return;
}
let days_since_beta_promotion = days_since_beta_promotion();
let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START;
let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable");
let toolstates = builder.toolstates();
let mut did_error = false;
for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) {
if !toolstates.contains_key(*tool) {
did_error = true;
eprintln!("error: Tool `{}` was not recorded in tool state.", tool);
}
}
if did_error {
std::process::exit(1);
}
check_changed_files(&toolstates);
for (tool, _) in STABLE_TOOLS.iter() {
let state = toolstates[*tool];
if state != ToolState::TestPass {
if !is_nightly {
did_error = true;
eprintln!("error: Tool `{}` should be test-pass but is {}", tool, state);
} else if in_beta_week {
did_error = true;
eprintln!("error: Tool `{}` should be test-pass but is {} during beta week.",
tool, state);
}
}
}
if did_error {
std::process::exit(1);
}
if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() {
commit_toolstate_change(&toolstates, in_beta_week);
}
}
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("check-tools")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(ToolStateCheck);
}
}
impl Builder<'_> {
fn toolstates(&self) -> HashMap<Box<str>, ToolState> {
if let Some(ref path) = self.config.save_toolstates {
if let Some(parent) = path.parent() {
// Ensure the parent directory always exists
t!(std::fs::create_dir_all(parent));
}
let mut file = t!(fs::OpenOptions::new()
.create(true)
.write(true)
.read(true)
.open(path));
serde_json::from_reader(&mut file).unwrap_or_default()
} else {
Default::default()
}
}
/// Updates the actual toolstate of a tool.
///
/// The toolstates are saved to the file specified by the key
/// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be
/// done. The file is updated immediately after this function completes.
pub fn save_toolstate(&self, tool: &str, state: ToolState) {
if let Some(ref path) = self.config.save_toolstates {
if let Some(parent) = path.parent() {
// Ensure the parent directory always exists
t!(std::fs::create_dir_all(parent));
}
let mut file = t!(fs::OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(path));
let mut current_toolstates: HashMap<Box<str>, ToolState> =
serde_json::from_reader(&mut file).unwrap_or_default();
current_toolstates.insert(tool.into(), state);
t!(file.seek(SeekFrom::Start(0)));
t!(file.set_len(0));
t!(serde_json::to_writer(file, &current_toolstates));
}
}
}
/// This function `commit_toolstate_change` provides functionality for pushing a change
/// to the `rust-toolstate` repository.
///
/// The function relies on a GitHub bot user, which should have a Personal access
/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for
/// some reason you need to change the token, please update the Azure Pipelines
/// variable group.
///
/// 1. Generate a new Personal access token:
///
/// * Login to the bot account, and go to Settings -> Developer settings ->
/// Personal access tokens
/// * Click "Generate new token"
/// * Enable the "public_repo" permission, then click "Generate token"
/// * Copy the generated token (should be a 40-digit hexadecimal number).
/// Save it somewhere secure, as the token would be gone once you leave
/// the page.
///
/// 2. Update the variable group in Azure Pipelines
///
/// * Ping a member of the infrastructure team to do this.
///
/// 4. Replace the email address below if the bot account identity is changed
///
/// * See <https://help.github.com/articles/about-commit-email-addresses/>
/// if a private email by GitHub is wanted.
fn commit_toolstate_change(
current_toolstate: &ToolstateData,
in_beta_week: bool,
) {
fn git_config(key: &str, value: &str) {
let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status();
let success = match status {
Ok(s) => s.success(),
Err(_) => false,
};
if !success {
panic!("git config key={} value={} successful (status: {:?})", key, value, status);
}
}
// If changing anything here, then please check that src/ci/publish_toolstate.sh is up to date
// as well.
git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com");
git_config("user.name", "Rust Toolstate Update");
git_config("credential.helper", "store");
let credential = format!(
"https://{}:x-oauth-basic@github.com\n",
t!(env::var("TOOLSTATE_REPO_ACCESS_TOKEN")),
);
let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials");
t!(fs::write(&git_credential_path, credential));
let status = Command::new("git").arg("clone")
.arg("--depth=1")
.arg(t!(env::var("TOOLSTATE_REPO")))
.status();
let success = match status {
Ok(s) => s.success(),
Err(_) => false,
};
if !success {
panic!("git clone successful (status: {:?})", status);
}
let old_toolstate = t!(fs::read("rust-toolstate/_data/latest.json"));
let old_toolstate: Vec<RepoState> = t!(serde_json::from_slice(&old_toolstate));
let message = format!("({} CI update)", OS.expect("linux/windows only"));
let mut success = false;
for _ in 1..=5 {
// Update the toolstate results (the new commit-to-toolstate mapping) in the toolstate repo.
change_toolstate(&current_toolstate, &old_toolstate, in_beta_week);
// `git commit` failing means nothing to commit.
let status = t!(Command::new("git")
.current_dir("rust-toolstate")
.arg("commit")
.arg("-a")
.arg("-m")
.arg(&message)
.status());
if !status.success() {
success = true;
break;
}
let status = t!(Command::new("git")
.current_dir("rust-toolstate")
.arg("push")
.arg("origin")
.arg("master")
.status());
// If we successfully push, exit.
if status.success() {
success = true;
break;
}
eprintln!("Sleeping for 3 seconds before retrying push");
std::thread::sleep(std::time::Duration::from_secs(3));
let status = t!(Command::new("git")
.current_dir("rust-toolstate")
.arg("fetch")
.arg("origin")
.arg("master")
.status());
assert!(status.success());
let status = t!(Command::new("git")
.current_dir("rust-toolstate")
.arg("reset")
.arg("--hard")
.arg("origin/master")
.status());
assert!(status.success());
}
if !success {
panic!("Failed to update toolstate repository with new data");
}
}
fn change_toolstate(
current_toolstate: &ToolstateData,
old_toolstate: &[RepoState],
in_beta_week: bool,
) {
let mut regressed = false;
for repo_state in old_toolstate {
let tool = &repo_state.tool;
let state = if cfg!(linux) {
&repo_state.linux
} else if cfg!(windows) {
&repo_state.windows
} else {
unimplemented!()
};
let new_state = current_toolstate[tool.as_str()];
if new_state != *state {
eprintln!("The state of `{}` has changed from `{}` to `{}`", tool, state, new_state);
if (new_state as u8) < (*state as u8) {
if !["rustc-guide", "miri", "embedded-book"].contains(&tool.as_str()) {
regressed = true;
}
}
}
}
if regressed && in_beta_week {
std::process::exit(1);
}
let commit = t!(std::process::Command::new("git")
.arg("rev-parse")
.arg("HEAD")
.output());
let commit = t!(String::from_utf8(commit.stdout));
let toolstate_serialized = t!(serde_json::to_string(&current_toolstate));
let history_path = format!("rust-toolstate/history/{}.tsv", OS.expect("linux/windows only"));
let mut file = t!(fs::read_to_string(&history_path));
let end_of_first_line = file.find('\n').unwrap();
file.insert_str(end_of_first_line, &format!("{}\t{}\n", commit, toolstate_serialized));
t!(fs::write(&history_path, file));
}
#[derive(Debug, Serialize, Deserialize)]
struct RepoState {
tool: String,
windows: ToolState,
linux: ToolState,
commit: String,
datetime: String,
}

2
src/ci/azure-pipelines/auto.yml
View file

@ -163,7 +163,7 @@ jobs:
NO_LLVM_ASSERTIONS: 1
# MSVC tools tests
x86_64-msvc-tools:
SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
# 32/64-bit MinGW builds.

5
src/ci/azure-pipelines/master.yml
View file

@ -16,10 +16,7 @@ steps:
- checkout: self
fetchDepth: 2
- script: |
export MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
. src/ci/docker/x86_64-gnu-tools/repo.sh
commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN"
- script: src/ci/publish_toolstate.sh
displayName: Publish toolstate
env:
TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN)

4
src/ci/docker/x86_64-gnu-tools/Dockerfile
View file

@ -17,9 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
COPY x86_64-gnu-tools/checkregression.py /tmp/
COPY x86_64-gnu-tools/checktools.sh /tmp/
COPY x86_64-gnu-tools/repo.sh /tmp/
# Run rustbook with `linkcheck` feature enabled
ENV CHECK_LINKS 1
@ -27,4 +25,4 @@ ENV CHECK_LINKS 1
ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--save-toolstates=/tmp/toolstate/toolstates.json
ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstate/toolstates.json linux
ENV SCRIPT /tmp/checktools.sh ../x.py

48
src/ci/docker/x86_64-gnu-tools/checkregression.py
View file

@ -1,48 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
## This script has two purposes: detect any tool that *regressed*, which is used
## during the week before the beta branches to reject PRs; and detect any tool
## that *changed* to see if we need to update the toolstate repo.
import sys
import json
# Regressions for these tools during the beta cutoff week do not cause failure.
# See `status_check` in `checktools.sh` for tools that have to pass on the
# beta/stable branches.
REGRESSION_OK = ["rustc-guide", "miri", "embedded-book"]
if __name__ == '__main__':
os_name = sys.argv[1]
toolstate_file = sys.argv[2]
current_state = sys.argv[3]
verb = sys.argv[4] # 'regressed' or 'changed'
with open(toolstate_file, 'r') as f:
toolstate = json.load(f)
with open(current_state, 'r') as f:
current = json.load(f)
regressed = False
for cur in current:
tool = cur['tool']
state = cur[os_name]
new_state = toolstate.get(tool, '')
if verb == 'regressed':
updated = new_state < state
elif verb == 'changed':
updated = new_state != state
else:
print('Unknown verb {}'.format(updated))
sys.exit(2)
if updated:
print(
'The state of "{}" has {} from "{}" to "{}"'
.format(tool, verb, state, new_state)
)
if not (verb == 'regressed' and tool in REGRESSION_OK):
regressed = True
if regressed:
sys.exit(1)

116
src/ci/docker/x86_64-gnu-tools/checktools.sh
View file

@ -3,18 +3,6 @@
set -eu
X_PY="$1"
TOOLSTATE_FILE="$(realpath -m $2)"
OS="$3"
COMMIT="$(git rev-parse HEAD)"
CHANGED_FILES="$(git diff --name-status HEAD HEAD^)"
SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))"
# ^ Number of days after the last promotion of beta.
# Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens.
# The Wednesday after this has value 0.
# We track this value to prevent regressing tools in the last week of the 6-week cycle.
mkdir -p "$(dirname $TOOLSTATE_FILE)"
touch "$TOOLSTATE_FILE"
# Try to test all the tools and store the build/test success in the TOOLSTATE_FILE
@ -34,106 +22,4 @@ python2.7 "$X_PY" test --no-fail-fast \
set -e
cat "$TOOLSTATE_FILE"
echo
# This function checks if a particular tool is *not* in status "test-pass".
check_tool_failed() {
grep -vq '"'"$1"'":"test-pass"' "$TOOLSTATE_FILE"
}
# This function checks that if a tool's submodule changed, the tool's state must improve
verify_submodule_changed() {
echo "Verifying status of $1..."
if echo "$CHANGED_FILES" | grep -q "^M[[:blank:]]$2$"; then
echo "This PR updated '$2', verifying if status is 'test-pass'..."
if check_tool_failed "$1"; then
echo
echo "⚠️ We detected that this PR updated '$1', but its tests failed."
echo
echo "If you do intend to update '$1', please check the error messages above and"
echo "commit another update."
echo
echo "If you do NOT intend to update '$1', please ensure you did not accidentally"
echo "change the submodule at '$2'. You may ask your reviewer for the"
echo "proper steps."
exit 3
fi
fi
}
# deduplicates the submodule check and the assertion that on beta some tools MUST be passing.
# $1 should be "submodule_changed" to only check tools that got changed by this PR,
# or "beta_required" to check all tools that have $2 set to "beta".
check_dispatch() {
if [ "$1" = submodule_changed ]; then
# ignore $2 (branch id)
verify_submodule_changed $3 $4
elif [ "$2" = beta ]; then
echo "Requiring test passing for $3..."
if check_tool_failed "$3"; then
exit 4
fi
fi
}
# List all tools here.
# This function gets called with "submodule_changed" for each PR that changed a submodule,
# and with "beta_required" for each PR that lands on beta/stable.
# The purpose of this function is to *reject* PRs if a tool is not "test-pass" and
# (a) the tool's submodule has been updated, or (b) we landed on beta/stable and the
# tool has to "test-pass" on that branch.
status_check() {
check_dispatch $1 beta book src/doc/book
check_dispatch $1 beta nomicon src/doc/nomicon
check_dispatch $1 beta reference src/doc/reference
check_dispatch $1 beta rust-by-example src/doc/rust-by-example
check_dispatch $1 beta edition-guide src/doc/edition-guide
check_dispatch $1 beta rls src/tools/rls
check_dispatch $1 beta rustfmt src/tools/rustfmt
check_dispatch $1 beta clippy-driver src/tools/clippy
# These tools are not required on the beta/stable branches, but they *do* cause
# PRs to fail if a submodule update does not fix them.
# They will still cause failure during the beta cutoff week, unless `checkregression.py`
# exempts them from that.
check_dispatch $1 nightly miri src/tools/miri
check_dispatch $1 nightly embedded-book src/doc/embedded-book
check_dispatch $1 nightly rustc-guide src/doc/rustc-guide
}
# If this PR is intended to update one of these tools, do not let the build pass
# when they do not test-pass.
status_check "submodule_changed"
CHECK_NOT="$(readlink -f "$(dirname $0)/checkregression.py")"
# This callback is called by `commit_toolstate_change`, see `repo.sh`.
change_toolstate() {
# only update the history
if python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" changed; then
echo 'Toolstate is not changed. Not updating.'
else
if [ $SIX_WEEK_CYCLE -ge 35 ]; then
# Reject any regressions during the week before beta cutoff.
python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" regressed
fi
sed -i "1 a\\
$COMMIT\t$(cat "$TOOLSTATE_FILE")
" "history/$OS.tsv"
fi
}
if [ "$RUST_RELEASE_CHANNEL" = nightly ]; then
if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then
. "$(dirname $0)/repo.sh"
MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
echo "($OS CI update)" > "$MESSAGE_FILE"
commit_toolstate_change "$MESSAGE_FILE" change_toolstate
rm -f "$MESSAGE_FILE"
fi
exit 0
fi
# abort compilation if an important tool doesn't build
# (this code is reachable if not on the nightly channel)
status_check "beta_required"
python2.7 "$X_PY" test check-tools

68
src/ci/docker/x86_64-gnu-tools/repo.sh
View file

@ -1,68 +0,0 @@
#!/bin/sh
# This file provides the function `commit_toolstate_change` for pushing a change
# to the `rust-toolstate` repository.
#
# The function relies on a GitHub bot user, which should have a Personal access
# token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for
# some reason you need to change the token, please update the Azure Pipelines
# variable group.
#
# 1. Generate a new Personal access token:
#
# * Login to the bot account, and go to Settings -> Developer settings ->
# Personal access tokens
# * Click "Generate new token"
# * Enable the "public_repo" permission, then click "Generate token"
# * Copy the generated token (should be a 40-digit hexadecimal number).
# Save it somewhere secure, as the token would be gone once you leave
# the page.
#
# 2. Update the variable group in Azure Pipelines
#
# * Ping a member of the infrastructure team to do this.
#
# 4. Replace the email address below if the bot account identity is changed
#
# * See <https://help.github.com/articles/about-commit-email-addresses/>
# if a private email by GitHub is wanted.
commit_toolstate_change() {
OLDFLAGS="$-"
set -eu
git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com'
git config --global user.name 'Rust Toolstate Update'
git config --global credential.helper store
printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \
> "$HOME/.git-credentials"
git clone --depth=1 $TOOLSTATE_REPO
cd rust-toolstate
FAILURE=1
MESSAGE_FILE="$1"
shift
for RETRY_COUNT in 1 2 3 4 5; do
# Call the callback.
# - If we are in the `auto` branch (pre-landing), this is called from `checktools.sh` and
# the callback is `change_toolstate` in that file. The purpose of this is to publish the
# test results (the new commit-to-toolstate mapping) in the toolstate repo.
# - If we are in the `master` branch (post-landing), this is called by the CI pipeline
# and the callback is `src/tools/publish_toolstate.py`. The purpose is to publish
# the new "current" toolstate in the toolstate repo.
"$@"
# `git commit` failing means nothing to commit.
FAILURE=0
git commit -a -F "$MESSAGE_FILE" || break
# On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering.
git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1)
FAILURE=1
git fetch origin master
git reset --hard origin/master
done
cd ..
set +eu
set "-$OLDFLAGS"
return $FAILURE
}

33
src/ci/publish_toolstate.sh Normal file
View file

@ -0,0 +1,33 @@
#!/bin/sh
set -eu
# The following lines are also found in src/bootstrap/toolstate.rs,
# so if updating here, please also update that file.
export MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com'
git config --global user.name 'Rust Toolstate Update'
git config --global credential.helper store
printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \
> "$HOME/.git-credentials"
git clone --depth=1 $TOOLSTATE_REPO
cd rust-toolstate
FAILURE=1
for RETRY_COUNT in 1 2 3 4 5; do
# The purpose is to publish the new "current" toolstate in the toolstate repo.
"$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \
"$(git log --format=%s -n1 HEAD)" \
"$MESSAGE_FILE" \
"$TOOLSTATE_REPO_ACCESS_TOKEN"
# `git commit` failing means nothing to commit.
FAILURE=0
git commit -a -F "$MESSAGE_FILE" || break
# On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering.
git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1)
FAILURE=1
git fetch origin master
git reset --hard origin/master
done