user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #4490 from Kobzol/use-josh-sync

Browse source 
Use `josh-sync` instead of `miri-script` for Josh synchronization
This commit is contained in:
Ralf Jung 2025-07-23 20:36:59 +00:00 • committed by GitHub
commit cbd1557d02
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 42 additions and 355 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/tools/miri/.github/workflows/ci.yml vendored
View file

@ -155,35 +155,48 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 256 # get a bit more of the history
- name: install josh-proxy
run: cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04
- name: install josh-sync
run: cargo +stable install --locked --git https://github.com/rust-lang/josh-sync
- name: setup bot git name and email
run: |
git config --global user.name 'The Miri Cronjob Bot'
git config --global user.email 'miri@cron.bot'
- name: Install nightly toolchain
run: rustup toolchain install nightly --profile minimal
- name: get changes from rustc
run: ./miri rustc-pull
- name: Install rustup-toolchain-install-master
run: cargo install -f rustup-toolchain-install-master
- name: format changes (if any)
run: |
./miri toolchain
./miri fmt --check || (./miri fmt && git commit -am "fmt")
- name: Push changes to a branch and create PR
run: |
# `git diff --exit-code` "succeeds" if the diff is empty.
if git diff --exit-code HEAD^; then echo "Nothing changed in rustc, skipping PR"; exit 0; fi
# The diff is non-empty, create a PR.
# Make it easier to see what happens.
set -x
# Temporarily disable early exit to examine the status code of rustc-josh-sync
set +e
rustc-josh-sync pull
exitcode=$?
set -e
# If there were no changes to pull, rustc-josh-sync returns status code 2.
# In that case, skip the rest of the job.
if [ $exitcode -eq 2 ]; then
echo "Nothing changed in rustc, skipping PR"
exit 0
elif [ $exitcode -ne 0 ]; then
# If return code was not 0 or 2, rustc-josh-sync actually failed
echo "error: rustc-josh-sync failed"
exit ${exitcode}
fi
# Format changes
./miri toolchain
./miri fmt --check || (./miri fmt && git commit -am "fmt")
# Create a PR
BRANCH="rustup-$(date -u +%Y-%m-%d)"
git switch -c $BRANCH
git push -u origin $BRANCH
gh pr create -B master --title 'Automatic Rustup' --body 'Please close and re-open this PR to trigger CI, then enable auto-merge.'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ZULIP_BOT_EMAIL: ${{ secrets.ZULIP_BOT_EMAIL }}
ZULIP_API_TOKEN: ${{ secrets.ZULIP_API_TOKEN }}
cron-fail-notify:
name: cronjob failure notification

14
src/tools/miri/CONTRIBUTING.md
View file

@ -297,14 +297,14 @@ You can also directly run Miri on a Rust source file:
## Advanced topic: Syncing with the rustc repo
We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit changes between the
We use the [`josh-sync`](https://github.com/rust-lang/josh-sync) tool to transmit changes between the
rustc and Miri repositories. You can install it as follows:
```sh
cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04
cargo install --locked --git https://github.com/rust-lang/josh-sync
```
Josh will automatically be started and stopped by `./miri`.
The commands below will automatically install and manage the [Josh](https://github.com/josh-project/josh) proxy that performs the actual work.
### Importing changes from the rustc repo
@ -312,10 +312,12 @@ Josh will automatically be started and stopped by `./miri`.
We assume we start on an up-to-date master branch in the Miri repo.
1) First, create a branch for the pull, e.g. `git checkout -b rustup`
2) Then run the following:
```sh
# Fetch and merge rustc side of the history. Takes ca 5 min the first time.
# This will also update the `rustc-version` file.
./miri rustc-pull
rustc-josh-sync pull
# Update local toolchain and apply formatting.
./miri toolchain && ./miri fmt
git commit -am "rustup"
@ -328,12 +330,12 @@ needed.
### Exporting changes to the rustc repo
We will use the josh proxy to push to your fork of rustc. Run the following in the Miri repo,
We will use the `josh-sync` tool to push to your fork of rustc. Run the following in the Miri repo,
assuming we are on an up-to-date master branch:
```sh
# Push the Miri changes to your rustc fork (substitute your github handle for YOUR_NAME).
./miri rustc-push YOUR_NAME miri
rustc-josh-sync push miri YOUR_NAME
```
This will create a new branch called `miri` in your fork, and the output should include a link that

2
src/tools/miri/josh-sync.toml Normal file
View file

@ -0,0 +1,2 @@
repo = "miri"
filter = ":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri"

90
src/tools/miri/miri-script/Cargo.lock
View file

@ -116,27 +116,6 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
[[package]]
name = "directories"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.60.2",
]
[[package]]
name = "dunce"
version = "1.0.5"
@ -165,17 +144,6 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "getrandom"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.1+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.3.3"
@ -185,7 +153,7 @@ dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi 0.14.2+wasi-0.2.4",
"wasi",
]
[[package]]
@ -221,16 +189,6 @@ version = "0.2.174"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
[[package]]
name = "libredox"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
dependencies = [
"bitflags",
"libc",
]
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
@ -249,7 +207,6 @@ version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"directories",
"dunce",
"itertools",
"path_macro",
@ -275,12 +232,6 @@ version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "path_macro"
version = "1.0.0"
@ -311,17 +262,6 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "redox_users"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b"
dependencies = [
"getrandom 0.2.16",
"libredox",
"thiserror",
]
[[package]]
name = "rustc_version"
version = "0.4.1"
@ -427,32 +367,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
dependencies = [
"fastrand",
"getrandom 0.3.3",
"getrandom",
"once_cell",
"rustix",
"windows-sys 0.59.0",
]
[[package]]
name = "thiserror"
version = "2.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "2.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "unicode-ident"
version = "1.0.18"
@ -475,12 +395,6 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasi"
version = "0.14.2+wasi-0.2.4"

1
src/tools/miri/miri-script/Cargo.toml
View file

@ -22,7 +22,6 @@ anyhow = "1.0"
xshell = "0.2.6"
rustc_version = "0.4"
dunce = "1.0.4"
directories = "6"
serde = "1"
serde_json = "1"
serde_derive = "1"

229
src/tools/miri/miri-script/src/commands.rs
View file

@ -2,11 +2,9 @@ use std::collections::BTreeMap;
use std::ffi::{OsStr, OsString};
use std::fmt::Write as _;
use std::fs::{self, File};
use std::io::{self, BufRead, BufReader, BufWriter, Write as _};
use std::ops::Not;
use std::io::{self, BufRead, BufReader, BufWriter};
use std::path::PathBuf;
use std::time::Duration;
use std::{env, net, process};
use std::{env, process};
use anyhow::{Context, Result, anyhow, bail};
use path_macro::path;
@ -18,11 +16,6 @@ use xshell::{Shell, cmd};
use crate::Command;
use crate::util::*;
/// Used for rustc syncs.
const JOSH_FILTER: &str =
":rev(75dd959a3a40eb5b4574f8d2e23aa6efbeb33573:prefix=src/tools/miri):/src/tools/miri";
const JOSH_PORT: u16 = 42042;
impl MiriEnv {
/// Prepares the environment: builds miri and cargo-miri and a sysroot.
/// Returns the location of the sysroot.
@ -99,66 +92,6 @@ impl Command {
Ok(())
}
fn start_josh() -> Result<impl Drop> {
// Determine cache directory.
let local_dir = {
let user_dirs =
directories::ProjectDirs::from("org", "rust-lang", "miri-josh").unwrap();
user_dirs.cache_dir().to_owned()
};
// Start josh, silencing its output.
let mut cmd = process::Command::new("josh-proxy");
cmd.arg("--local").arg(local_dir);
cmd.arg("--remote").arg("https://github.com");
cmd.arg("--port").arg(JOSH_PORT.to_string());
cmd.arg("--no-background");
cmd.stdout(process::Stdio::null());
cmd.stderr(process::Stdio::null());
let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
// Create a wrapper that stops it on drop.
struct Josh(process::Child);
impl Drop for Josh {
fn drop(&mut self) {
#[cfg(unix)]
{
// Try to gracefully shut it down.
process::Command::new("kill")
.args(["-s", "INT", &self.0.id().to_string()])
.output()
.expect("failed to SIGINT josh-proxy");
// Sadly there is no "wait with timeout"... so we just give it some time to finish.
std::thread::sleep(Duration::from_millis(100));
// Now hopefully it is gone.
if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() {
return;
}
}
// If that didn't work (or we're not on Unix), kill it hard.
eprintln!(
"I have to kill josh-proxy the hard way, let's hope this does not break anything."
);
self.0.kill().expect("failed to SIGKILL josh-proxy");
}
}
// Wait until the port is open. We try every 10ms until 1s passed.
for _ in 0..100 {
// This will generally fail immediately when the port is still closed.
let josh_ready = net::TcpStream::connect_timeout(
&net::SocketAddr::from(([127, 0, 0, 1], JOSH_PORT)),
Duration::from_millis(1),
);
if josh_ready.is_ok() {
return Ok(Josh(josh));
}
// Not ready yet.
std::thread::sleep(Duration::from_millis(10));
}
bail!("Even after waiting for 1s, josh-proxy is still not available.")
}
pub fn exec(self) -> Result<()> {
// First, and crucially only once, run the auto-actions -- but not for all commands.
match &self {
@ -170,11 +103,7 @@ impl Command {
| Command::Fmt { .. }
| Command::Doc { .. }
| Command::Clippy { .. } => Self::auto_actions()?,
| Command::Toolchain { .. }
| Command::Bench { .. }
| Command::RustcPull { .. }
| Command::RustcPush { .. }
| Command::Squash => {}
| Command::Toolchain { .. } | Command::Bench { .. } | Command::Squash => {}
}
// Then run the actual command.
match self {
@ -191,8 +120,6 @@ impl Command {
Command::Bench { target, no_install, save_baseline, load_baseline, benches } =>
Self::bench(target, no_install, save_baseline, load_baseline, benches),
Command::Toolchain { flags } => Self::toolchain(flags),
Command::RustcPull { commit } => Self::rustc_pull(commit.clone()),
Command::RustcPush { github_user, branch } => Self::rustc_push(github_user, branch),
Command::Squash => Self::squash(),
}
}
@ -233,156 +160,6 @@ impl Command {
Ok(())
}
fn rustc_pull(commit: Option<String>) -> Result<()> {
let sh = Shell::new()?;
sh.change_dir(miri_dir()?);
let commit = commit.map(Result::Ok).unwrap_or_else(|| {
let rust_repo_head =
cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?;
rust_repo_head
.split_whitespace()
.next()
.map(|front| front.trim().to_owned())
.ok_or_else(|| anyhow!("Could not obtain Rust repo HEAD from remote."))
})?;
// Make sure the repo is clean.
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
bail!("working directory must be clean before running `./miri rustc-pull`");
}
// Make sure josh is running.
let josh = Self::start_josh()?;
let josh_url =
format!("http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git");
// Update rust-version file. As a separate commit, since making it part of
// the merge has confused the heck out of josh in the past.
// We pass `--no-verify` to avoid running git hooks like `./miri fmt` that could in turn
// trigger auto-actions.
// We do this before the merge so that if there are merge conflicts, we have
// the right rust-version file while resolving them.
sh.write_file("rust-version", format!("{commit}\n"))?;
const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rustc";
cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}")
.run()
.context("FAILED to commit rust-version file, something went wrong")?;
// Fetch given rustc commit.
cmd!(sh, "git fetch {josh_url}")
.run()
.inspect_err(|_| {
// Try to un-do the previous `git commit`, to leave the repo in the state we found it.
cmd!(sh, "git reset --hard HEAD^")
.run()
.expect("FAILED to clean up again after failed `git fetch`, sorry for that");
})
.context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?;
// This should not add any new root commits. So count those before and after merging.
let num_roots = || -> Result<u32> {
Ok(cmd!(sh, "git rev-list HEAD --max-parents=0 --count")
.read()
.context("failed to determine the number of root commits")?
.parse::<u32>()?)
};
let num_roots_before = num_roots()?;
// Merge the fetched commit.
const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc";
cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
.run()
.context("FAILED to merge new commits, something went wrong")?;
// Check that the number of roots did not increase.
if num_roots()? != num_roots_before {
bail!("Josh created a new root commit. This is probably not the history you want.");
}
drop(josh);
Ok(())
}
fn rustc_push(github_user: String, branch: String) -> Result<()> {
let sh = Shell::new()?;
sh.change_dir(miri_dir()?);
let base = sh.read_file("rust-version")?.trim().to_owned();
// Make sure the repo is clean.
if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() {
bail!("working directory must be clean before running `./miri rustc-push`");
}
// Make sure josh is running.
let josh = Self::start_josh()?;
let josh_url =
format!("http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git");
// Find a repo we can do our preparation in.
if let Ok(rustc_git) = env::var("RUSTC_GIT") {
// If rustc_git is `Some`, we'll use an existing fork for the branch updates.
sh.change_dir(rustc_git);
} else {
// Otherwise, do this in the local Miri repo.
println!(
"This will pull a copy of the rust-lang/rust history into this Miri checkout, growing it by about 1GB."
);
print!(
"To avoid that, abort now and set the `RUSTC_GIT` environment variable to an existing rustc checkout. Proceed? [y/N] "
);
std::io::stdout().flush()?;
let mut answer = String::new();
std::io::stdin().read_line(&mut answer)?;
if answer.trim().to_lowercase() != "y" {
std::process::exit(1);
}
};
// Prepare the branch. Pushing works much better if we use as base exactly
// the commit that we pulled from last time, so we use the `rust-version`
// file to find out which commit that would be.
println!("Preparing {github_user}/rust (base: {base})...");
if cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}")
.ignore_stderr()
.read()
.is_ok()
{
println!(
"The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again."
);
std::process::exit(1);
}
cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}")
.ignore_stdout()
.ignore_stderr() // silence the "create GitHub PR" message
.run()?;
println!();
// Do the actual push.
sh.change_dir(miri_dir()?);
println!("Pushing miri changes...");
cmd!(sh, "git push {josh_url} HEAD:{branch}").run()?;
println!();
// Do a round-trip check to make sure the push worked as expected.
cmd!(sh, "git fetch {josh_url} {branch}").ignore_stderr().read()?;
let head = cmd!(sh, "git rev-parse HEAD").read()?;
let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?;
if head != fetch_head {
bail!(
"Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\
Expected {head}, got {fetch_head}."
);
}
println!(
"Confirmed that the push round-trips back to Miri properly. Please create a rustc PR:"
);
println!(
// Open PR with `subtree update` title to silence the `no-merges` triagebot check
// See https://github.com/rust-lang/rust/pull/114157
" https://github.com/rust-lang/rust/compare/{github_user}:{branch}?quick_pull=1&title=Miri+subtree+update&body=r?+@ghost"
);
drop(josh);
Ok(())
}
fn squash() -> Result<()> {
let sh = Shell::new()?;
sh.change_dir(miri_dir()?);

22
src/tools/miri/miri-script/src/main.rs
View file

@ -142,25 +142,6 @@ pub enum Command {
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
flags: Vec<String>,
},
/// Pull and merge Miri changes from the rustc repo.
///
/// The fetched commit is stored in the `rust-version` file, so the next `./miri toolchain` will
/// install the rustc that just got pulled.
RustcPull {
/// The commit to fetch (default: latest rustc commit).
commit: Option<String>,
},
/// Push Miri changes back to the rustc repo.
///
/// This will pull a copy of the rustc history into the Miri repo, unless you set the RUSTC_GIT
/// env var to an existing clone of the rustc repo.
RustcPush {
/// The Github user that owns the rustc fork to which we should push.
github_user: String,
/// The branch to push to.
#[arg(default_value = "miri-sync")]
branch: String,
},
/// Squash the commits of the current feature branch into one.
Squash,
}
@ -184,8 +165,7 @@ impl Command {
flags.extend(remainder);
Ok(())
}
Self::Bench { .. } | Self::RustcPull { .. } | Self::RustcPush { .. } | Self::Squash =>
bail!("unexpected \"--\" found in arguments"),
Self::Bench { .. } | Self::Squash => bail!("unexpected \"--\" found in arguments"),
}
}
}