user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove josh-sync crate

Browse source
This commit is contained in:
Jakub Beránek 2025-07-04 22:15:27 +02:00
parent 016bc61312
commit 2f20628da2
4 changed files with 0 additions and 455 deletions
Download patch file Download diff file Expand all files Collapse all files

1
library/compiler-builtins/Cargo.toml
View file

@ -3,7 +3,6 @@ resolver = "2"
members = [
"builtins-shim",
"builtins-test",
"crates/josh-sync",
"crates/libm-macros",
"crates/musl-math-sys",
"crates/panic-handler",

8
library/compiler-builtins/crates/josh-sync/Cargo.toml
View file

@ -1,8 +0,0 @@
[package]
name = "josh-sync"
edition = "2024"
publish = false
[dependencies]
directories = "6.0.0"
regex-lite = "0.1.6"

45
library/compiler-builtins/crates/josh-sync/src/main.rs
View file

@ -1,45 +0,0 @@
use std::io::{Read, Write};
use std::process::exit;
use std::{env, io};
use crate::sync::{GitSync, Josh};
mod sync;
const USAGE: &str = r#"Utility for synchroniing compiler-builtins with rust-lang/rust
Usage:
josh-sync rustc-pull
Pull from rust-lang/rust to compiler-builtins. Creates a commit
updating the version file, followed by a merge commit.
josh-sync rustc-push GITHUB_USERNAME [BRANCH]
Create a branch off of rust-lang/rust updating compiler-builtins.
"#;
fn main() {
let sync = GitSync::from_current_dir();
// Collect args, then recollect as str refs so we can match on them
let args: Vec<_> = env::args().collect();
let args: Vec<&str> = args.iter().map(String::as_str).collect();
match args.as_slice()[1..] {
["rustc-pull"] => sync.rustc_pull(None),
["rustc-push", github_user, branch] => sync.rustc_push(github_user, Some(branch)),
["rustc-push", github_user] => sync.rustc_push(github_user, None),
["start-josh"] => {
let _josh = Josh::start();
println!("press enter to stop");
io::stdout().flush().unwrap();
let _ = io::stdin().read(&mut [0u8]).unwrap();
}
_ => {
println!("{USAGE}");
exit(1);
}
}
}

401
library/compiler-builtins/crates/josh-sync/src/sync.rs
View file

@ -1,401 +0,0 @@
use std::borrow::Cow;
use std::net::{SocketAddr, TcpStream};
use std::process::{Command, Stdio, exit};
use std::time::Duration;
use std::{env, fs, process, thread};
use regex_lite::Regex;
const JOSH_PORT: u16 = 42042;
const DEFAULT_PR_BRANCH: &str = "update-builtins";
pub struct GitSync {
upstream_repo: String,
upstream_ref: String,
upstream_url: String,
josh_filter: String,
josh_url_base: String,
}
/// This code was adapted from the miri repository, via the rustc-dev-guide
/// (<https://github.com/rust-lang/rustc-dev-guide/tree/c51adbd12d/josh-sync>)
impl GitSync {
pub fn from_current_dir() -> Self {
let upstream_repo =
env::var("UPSTREAM_ORG").unwrap_or_else(|_| "rust-lang".to_owned()) + "/rust";
Self {
upstream_url: format!("https://github.com/{upstream_repo}"),
upstream_repo,
upstream_ref: env::var("UPSTREAM_REF").unwrap_or_else(|_| "HEAD".to_owned()),
josh_filter: ":/library/compiler-builtins".to_owned(),
josh_url_base: format!("http://localhost:{JOSH_PORT}"),
}
}
/// Pull from rust-lang/rust to compiler-builtins.
pub fn rustc_pull(&self, commit: Option<String>) {
let Self {
upstream_ref,
upstream_url,
upstream_repo,
..
} = self;
let new_upstream_base = commit.unwrap_or_else(|| {
let out = check_output(["git", "ls-remote", upstream_url, upstream_ref]);
out.split_whitespace()
.next()
.unwrap_or_else(|| panic!("could not split output: '{out}'"))
.to_owned()
});
ensure_clean();
// Make sure josh is running.
let _josh = Josh::start();
let josh_url_filtered = self.josh_url(
&self.upstream_repo,
Some(&new_upstream_base),
Some(&self.josh_filter),
);
let previous_upstream_base = fs::read_to_string("rust-version")
.expect("failed to read `rust-version`")
.trim()
.to_string();
assert_ne!(previous_upstream_base, new_upstream_base, "nothing to pull");
let orig_head = check_output(["git", "rev-parse", "HEAD"]);
println!("original upstream base: {previous_upstream_base}");
println!("new upstream base: {new_upstream_base}");
println!("original HEAD: {orig_head}");
// Fetch the latest upstream HEAD so we can get a summary. Use the Josh URL for caching.
run([
"git",
"fetch",
&self.josh_url(&self.upstream_repo, Some(&new_upstream_base), Some(":/")),
&new_upstream_base,
"--depth=1",
]);
let new_summary = check_output(["git", "log", "-1", "--format=%h %s", &new_upstream_base]);
let new_summary = replace_references(&new_summary, &self.upstream_repo);
// Update rust-version file. As a separate commit, since making it part of
// the merge has confused the heck out of josh in the past.
// We pass `--no-verify` to avoid running git hooks.
// We do this before the merge so that if there are merge conflicts, we have
// the right rust-version file while resolving them.
fs::write("rust-version", format!("{new_upstream_base}\n"))
.expect("failed to write rust-version");
let prep_message = format!(
"Update the upstream Rust version\n\n\
To prepare for merging from {upstream_repo}, set the version file to:\n\n \
{new_summary}\n\
",
);
run([
"git",
"commit",
"rust-version",
"--no-verify",
"-m",
&prep_message,
]);
// Fetch given rustc commit.
run(["git", "fetch", &josh_url_filtered]);
let incoming_ref = check_output(["git", "rev-parse", "FETCH_HEAD"]);
println!("incoming ref: {incoming_ref}");
let merge_message = format!(
"Merge ref '{upstream_head_short}{filter}' from {upstream_url}\n\n\
Pull recent changes from {upstream_repo} via Josh.\n\n\
Upstream ref: {new_upstream_base}\n\
Filtered ref: {incoming_ref}\n\
",
upstream_head_short = &new_upstream_base[..12],
filter = self.josh_filter
);
// This should not add any new root commits. So count those before and after merging.
let num_roots = || -> u32 {
let out = check_output(["git", "rev-list", "HEAD", "--max-parents=0", "--count"]);
out.trim()
.parse::<u32>()
.unwrap_or_else(|e| panic!("failed to parse `{out}`: {e}"))
};
let num_roots_before = num_roots();
let pre_merge_sha = check_output(["git", "rev-parse", "HEAD"]);
println!("pre-merge HEAD: {pre_merge_sha}");
// Merge the fetched commit.
run([
"git",
"merge",
"FETCH_HEAD",
"--no-verify",
"--no-ff",
"-m",
&merge_message,
]);
let current_sha = check_output(["git", "rev-parse", "HEAD"]);
if current_sha == pre_merge_sha {
run(["git", "reset", "--hard", &orig_head]);
eprintln!(
"No merge was performed, no changes to pull were found. \
Rolled back the preparation commit."
);
exit(1);
}
// Check that the number of roots did not increase.
assert_eq!(
num_roots(),
num_roots_before,
"Josh created a new root commit. This is probably not the history you want."
);
}
/// Construct an update to rust-lang/rust from compiler-builtins.
pub fn rustc_push(&self, github_user: &str, branch: Option<&str>) {
let Self {
josh_filter,
upstream_url,
..
} = self;
let branch = branch.unwrap_or(DEFAULT_PR_BRANCH);
let josh_url = self.josh_url(&format!("{github_user}/rust"), None, Some(josh_filter));
let user_upstream_url = format!("git@github.com:{github_user}/rust.git");
let Ok(rustc_git) = env::var("RUSTC_GIT") else {
panic!("the RUSTC_GIT environment variable must be set to a rust-lang/rust checkout")
};
ensure_clean();
let base = fs::read_to_string("rust-version")
.expect("failed to read `rust-version`")
.trim()
.to_string();
// Make sure josh is running.
let _josh = Josh::start();
// Prepare the branch. Pushing works much better if we use as base exactly
// the commit that we pulled from last time, so we use the `rust-version`
// file to find out which commit that would be.
println!("Preparing {github_user}/rust (base: {base})...");
if Command::new("git")
.args(["-C", &rustc_git, "fetch", &user_upstream_url, branch])
.output() // capture output
.expect("could not run fetch")
.status
.success()
{
panic!(
"The branch '{branch}' seems to already exist in '{user_upstream_url}'. \
Please delete it and try again."
);
}
run(["git", "-C", &rustc_git, "fetch", upstream_url, &base]);
run_cfg("git", |c| {
c.args([
"-C",
&rustc_git,
"push",
&user_upstream_url,
&format!("{base}:refs/heads/{branch}"),
])
.stdout(Stdio::null())
.stderr(Stdio::null()) // silence the "create GitHub PR" message
});
println!("pushed PR branch");
// Do the actual push.
println!("Pushing changes...");
run(["git", "push", &josh_url, &format!("HEAD:{branch}")]);
println!();
// Do a round-trip check to make sure the push worked as expected.
run(["git", "fetch", &josh_url, branch]);
let head = check_output(["git", "rev-parse", "HEAD"]);
let fetch_head = check_output(["git", "rev-parse", "FETCH_HEAD"]);
assert_eq!(
head, fetch_head,
"Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\
Expected {head}, got {fetch_head}."
);
println!(
"Confirmed that the push round-trips back to compiler-builtins properly. Please \
create a rustc PR:"
);
// Open PR with `subtree update` title to silence the `no-merges` triagebot check
println!(
" {upstream_url}/compare/{github_user}:{branch}?quick_pull=1\
&title=Update%20the%20%60compiler-builtins%60%20subtree\
&body=Update%20the%20Josh%20subtree%20to%20https%3A%2F%2Fgithub.com%2Frust-lang%2F\
compiler-builtins%2Fcommit%2F{head_short}.%0A%0Ar%3F%20%40ghost",
head_short = &head[..12],
);
}
/// Construct a url to the local Josh server with (optionally)
fn josh_url(&self, repo: &str, rev: Option<&str>, filter: Option<&str>) -> String {
format!(
"{base}/{repo}.git{at}{rev}{filter}{filt_git}",
base = self.josh_url_base,
at = if rev.is_some() { "@" } else { "" },
rev = rev.unwrap_or_default(),
filter = filter.unwrap_or_default(),
filt_git = if filter.is_some() { ".git" } else { "" }
)
}
}
/// Fail if there are files that need to be checked in.
fn ensure_clean() {
let read = check_output(["git", "status", "--untracked-files=no", "--porcelain"]);
assert!(
read.is_empty(),
"working directory must be clean before performing rustc pull"
);
}
/* Helpers for running commands with logged invocations */
/// Run a command from an array, passing its output through.
fn run<'a, Args: AsRef<[&'a str]>>(l: Args) {
let l = l.as_ref();
run_cfg(l[0], |c| c.args(&l[1..]));
}
/// Run a command from an array, collecting its output.
fn check_output<'a, Args: AsRef<[&'a str]>>(l: Args) -> String {
let l = l.as_ref();
check_output_cfg(l[0], |c| c.args(&l[1..]))
}
/// [`run`] with configuration.
fn run_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) {
// self.read(l.as_ref());
check_output_cfg(prog, |c| f(c.stdout(Stdio::inherit())));
}
/// [`read`] with configuration. All shell helpers print the command and pass stderr.
fn check_output_cfg(prog: &str, f: impl FnOnce(&mut Command) -> &mut Command) -> String {
let mut cmd = Command::new(prog);
cmd.stderr(Stdio::inherit());
f(&mut cmd);
eprintln!("+ {cmd:?}");
let out = cmd.output().expect("command failed");
assert!(out.status.success());
String::from_utf8(out.stdout.trim_ascii().to_vec()).expect("non-UTF8 output")
}
/// Replace `#1234`-style issue/PR references with `repo#1234` to ensure links work across
/// repositories.
fn replace_references<'a>(s: &'a str, repo: &str) -> Cow<'a, str> {
let re = Regex::new(r"\B(?P<id>#\d+)\b").unwrap();
re.replace(s, &format!("{repo}$id"))
}
/// Create a wrapper that stops Josh on drop.
pub struct Josh(process::Child);
impl Josh {
pub fn start() -> Self {
// Determine cache directory.
let user_dirs =
directories::ProjectDirs::from("org", "rust-lang", "rustc-compiler-builtins-josh")
.unwrap();
let local_dir = user_dirs.cache_dir().to_owned();
// Start josh, silencing its output.
#[expect(clippy::zombie_processes, reason = "clippy can't handle the loop")]
let josh = process::Command::new("josh-proxy")
.arg("--local")
.arg(local_dir)
.args([
"--remote=https://github.com",
&format!("--port={JOSH_PORT}"),
"--no-background",
])
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.expect("failed to start josh-proxy, make sure it is installed");
// Wait until the port is open. We try every 10ms until 1s passed.
for _ in 0..100 {
// This will generally fail immediately when the port is still closed.
let addr = SocketAddr::from(([127, 0, 0, 1], JOSH_PORT));
let josh_ready = TcpStream::connect_timeout(&addr, Duration::from_millis(1));
if josh_ready.is_ok() {
println!("josh up and running");
return Josh(josh);
}
// Not ready yet.
thread::sleep(Duration::from_millis(10));
}
panic!("Even after waiting for 1s, josh-proxy is still not available.")
}
}
impl Drop for Josh {
fn drop(&mut self) {
if cfg!(unix) {
// Try to gracefully shut it down.
Command::new("kill")
.args(["-s", "INT", &self.0.id().to_string()])
.output()
.expect("failed to SIGINT josh-proxy");
// Sadly there is no "wait with timeout"... so we just give it some time to finish.
thread::sleep(Duration::from_millis(100));
// Now hopefully it is gone.
if self
.0
.try_wait()
.expect("failed to wait for josh-proxy")
.is_some()
{
return;
}
}
// If that didn't work (or we're not on Unix), kill it hard.
eprintln!(
"I have to kill josh-proxy the hard way, let's hope this does not \
break anything."
);
self.0.kill().expect("failed to SIGKILL josh-proxy");
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_replace() {
assert_eq!(replace_references("#1234", "r-l/rust"), "r-l/rust#1234");
assert_eq!(replace_references("#1234x", "r-l/rust"), "#1234x");
assert_eq!(
replace_references("merge #1234", "r-l/rust"),
"merge r-l/rust#1234"
);
assert_eq!(
replace_references("foo/bar#1234", "r-l/rust"),
"foo/bar#1234"
);
}
}