Merge pull request #20280 from Kobzol/josh-sync

Switch to using josh-sync

Cargo.lock

@@ -395,15 +395,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "directories"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d"
-dependencies = [
- "dirs-sys",
-]
-
 [[package]]
 name = "dirs"
 version = "6.0.0"
@@ -3105,7 +3096,6 @@ name = "xtask"
 version = "0.1.0"
 dependencies = [
  "anyhow",
- "directories",
  "edition",
  "either",
  "flate2",
@@ -3113,7 +3103,6 @@ dependencies = [
  "proc-macro2",
  "quote",
  "stdx",
- "time",
  "ungrammar",
  "write-json",
  "xflags",

docs/book/src/contributing/README.md

@@ -252,18 +252,8 @@ Release steps:
 4. Commit & push the changelog.
 5. Run `cargo xtask publish-release-notes <CHANGELOG>` -- this will convert the changelog entry in AsciiDoc to Markdown and update the body of GitHub Releases entry.
 6. Tweet.
-7. Make a new branch and run `cargo xtask rustc-pull`, open a PR, and merge it.
+7. Perform a subtree [pull](#performing-a-pull).
-   This will pull any changes from `rust-lang/rust` into `rust-analyzer`.
+8. Perform a subtree [push](#performing-a-push).
-8. Switch to `master`, pull, then run `cargo xtask rustc-push --rust-path ../rust-rust-analyzer --rust-fork matklad/rust`.
-   Replace `matklad/rust` with your own fork of `rust-lang/rust`.
-   You can use the token to authenticate when you get prompted for a password, since `josh` will push over HTTPS, not SSH.
-   This will push the `rust-analyzer` changes to your fork.
-   You can then open a PR against `rust-lang/rust`.
-
-Note: besides the `rust-rust-analyzer` clone, the Josh cache (stored under `~/.cache/rust-analyzer-josh`) will contain a bare clone of `rust-lang/rust`.
-This currently takes about 3.5 GB.
-
-This [HackMD](https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg) has details about how `josh` syncs work.
 
 If the GitHub Actions release fails because of a transient problem like a timeout, you can re-run the job from the Actions console.
 If it fails because of something that needs to be fixed, remove the release tag (if needed), fix the problem, then start over.
@@ -288,3 +278,43 @@ There are two sets of people with extra permissions:
   If you don't feel like reviewing for whatever reason, someone else will pick the review up (but please speak up if you don't feel like it)!
 * The [rust-lang](https://github.com/rust-lang) team [t-rust-analyzer-contributors]([https://github.com/orgs/rust-analyzer/teams/triage](https://github.com/rust-lang/team/blob/master/teams/rust-analyzer-contributors.toml)).
   This team has general triaging permissions allowing to label, close and re-open issues.
+
+## Synchronizing subtree changes
+`rust-analyzer` is a [josh](https://josh-project.github.io/josh/intro.html) subtree of the [rust-lang/rust](https://github.com/rust-lang/rust)
+repository. We use the [rustc-josh-sync](https://github.com/rust-lang/josh-sync) tool to perform synchronization between these two
+repositories. You can find documentation of the tool [here](https://github.com/rust-lang/josh-sync).
+
+You can install the synchronization tool using the following commands:
+```
+cargo install --locked --git https://github.com/rust-lang/josh-sync
+```
+
+Both pulls (synchronizing changes from rust-lang/rust into rust-analyzer) and pushes (synchronizing
+changes from rust-analyzer into rust-lang/rust) are performed from this repository.
+changes from rust-analyzer to rust-lang/rust) are performed from this repository.
+
+Usually we first perform a pull, wait for it to be merged, and then perform a push.
+
+### Performing a pull
+1) Checkout a new branch that will be used to create a PR against rust-analyzer
+2) Run the pull command
+    ```
+    rustc-josh-sync pull
+    ```
+3) Push the branch to your fork of `rust-analyzer` and create a PR
+  - If you have the `gh` CLI installed, `rustc-josh-sync` can create the PR for you.
+
+### Performing a push
+
+Wait for the previous pull to be merged.
+
+1) Switch to `master` and pull
+2) Run the push command to create a branch named `<branch-name>` in a `rustc` fork under the `<gh-username>` account
+    ```
+    rustc-josh-sync push <branch-name> <gh-username>
+    ```
+   - The push will ask you to download a checkout of the `rust-lang/rust` repository.
+   - If you get prompted for a password, see [this](https://github.com/rust-lang/josh-sync?tab=readme-ov-file#git-peculiarities).
+3) Create a PR from `<branch-name>` into `rust-lang/rust`
+
+> Besides the `rust` checkout, the Josh cache (stored under `~/.cache/rustc-josh`) will contain a bare clone of `rust-lang/rust`. This currently takes several GBs.

xtask/Cargo.toml

@@ -8,7 +8,6 @@ rust-version.workspace = true
 
 [dependencies]
 anyhow.workspace = true
-directories = "6.0"
 flate2 = "1.1.2"
 write-json = "0.1.4"
 xshell.workspace = true

xtask/src/flags.rs

@@ -59,20 +59,6 @@ xflags::xflags! {
             optional --dry-run
         }
 
-        cmd rustc-pull {
-            /// rustc commit to pull.
-            optional --commit refspec: String
-        }
-
-        cmd rustc-push {
-            /// rust local path, e.g. `../rust-rust-analyzer`.
-            required --rust-path rust_path: String
-            /// rust fork name, e.g.  `matklad/rust`.
-            required --rust-fork rust_fork: String
-            /// branch name.
-            optional --branch branch: String
-        }
-
         cmd dist {
             /// Use mimalloc allocator for server
             optional --mimalloc
@@ -121,8 +107,6 @@ pub enum XtaskCmd {
     Install(Install),
     FuzzTests(FuzzTests),
     Release(Release),
-    RustcPull(RustcPull),
-    RustcPush(RustcPush),
     Dist(Dist),
     PublishReleaseNotes(PublishReleaseNotes),
     Metrics(Metrics),
@@ -151,18 +135,6 @@ pub struct Release {
     pub dry_run: bool,
 }
 
-#[derive(Debug)]
-pub struct RustcPull {
-    pub commit: Option<String>,
-}
-
-#[derive(Debug)]
-pub struct RustcPush {
-    pub rust_path: String,
-    pub rust_fork: String,
-    pub branch: Option<String>,
-}
-
 #[derive(Debug)]
 pub struct Dist {
     pub mimalloc: bool,

xtask/src/main.rs

@@ -42,8 +42,6 @@ fn main() -> anyhow::Result<()> {
         flags::XtaskCmd::Install(cmd) => cmd.run(sh),
         flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh),
         flags::XtaskCmd::Release(cmd) => cmd.run(sh),
-        flags::XtaskCmd::RustcPull(cmd) => cmd.run(sh),
-        flags::XtaskCmd::RustcPush(cmd) => cmd.run(sh),
         flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
         flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh),
         flags::XtaskCmd::Metrics(cmd) => cmd.run(sh),

xtask/src/release.rs

@@ -1,12 +1,5 @@
 mod changelog;
 
-use std::process::{Command, Stdio};
-use std::thread;
-use std::time::Duration;
-
-use anyhow::{Context as _, bail};
-use directories::ProjectDirs;
-use stdx::JodChild;
 use xshell::{Shell, cmd};
 
 use crate::{date_iso, flags, is_release_tag, project_root};
@@ -59,171 +52,3 @@ impl flags::Release {
         Ok(())
     }
 }
-
-// git sync implementation adapted from https://github.com/rust-lang/miri/blob/62039ac/miri-script/src/commands.rs
-impl flags::RustcPull {
-    pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
-        sh.change_dir(project_root());
-        let commit = self.commit.map(Result::Ok).unwrap_or_else(|| {
-            let rust_repo_head =
-                cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?;
-            rust_repo_head
-                .split_whitespace()
-                .next()
-                .map(|front| front.trim().to_owned())
-                .ok_or_else(|| anyhow::format_err!("Could not obtain Rust repo HEAD from remote."))
-        })?;
-        // Make sure the repo is clean.
-        if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
-            bail!("working directory must be clean before running `cargo xtask pull`");
-        }
-        // This should not add any new root commits. So count those before and after merging.
-        let num_roots = || -> anyhow::Result<u32> {
-            Ok(cmd!(sh, "git rev-list HEAD --max-parents=0 --count")
-                .read()
-                .context("failed to determine the number of root commits")?
-                .parse::<u32>()?)
-        };
-        let num_roots_before = num_roots()?;
-        // Make sure josh is running.
-        let josh = start_josh()?;
-
-        // Update rust-version file. As a separate commit, since making it part of
-        // the merge has confused the heck out of josh in the past.
-        // We pass `--no-verify` to avoid running any git hooks that might exist,
-        // in case they dirty the repository.
-        sh.write_file("rust-version", format!("{commit}\n"))?;
-        const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rust-lang/rust";
-        cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}")
-            .run()
-            .context("FAILED to commit rust-version file, something went wrong")?;
-
-        // Fetch given rustc commit.
-        cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
-            .run()
-            .inspect_err(|_| {
-                // Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
-                cmd!(sh, "git reset --hard HEAD^")
-                    .run()
-                    .expect("FAILED to clean up again after failed `git fetch`, sorry for that");
-            })
-            .context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?;
-
-        // Merge the fetched commit.
-        const MERGE_COMMIT_MESSAGE: &str = "Merge from rust-lang/rust";
-        cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
-            .run()
-            .context("FAILED to merge new commits, something went wrong")?;
-
-        // Check that the number of roots did not increase.
-        if num_roots()? != num_roots_before {
-            bail!("Josh created a new root commit. This is probably not the history you want.");
-        }
-
-        drop(josh);
-        Ok(())
-    }
-}
-
-impl flags::RustcPush {
-    pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
-        let branch = self.branch.as_deref().unwrap_or("sync-from-ra");
-        let rust_path = self.rust_path;
-        let rust_fork = self.rust_fork;
-
-        sh.change_dir(project_root());
-        let base = sh.read_file("rust-version")?.trim().to_owned();
-        // Make sure the repo is clean.
-        if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
-            bail!("working directory must be clean before running `cargo xtask push`");
-        }
-        // Make sure josh is running.
-        let josh = start_josh()?;
-
-        // Find a repo we can do our preparation in.
-        sh.change_dir(rust_path);
-
-        // Prepare the branch. Pushing works much better if we use as base exactly
-        // the commit that we pulled from last time, so we use the `rust-version`
-        // file to find out which commit that would be.
-        println!("Preparing {rust_fork} (base: {base})...");
-        if cmd!(sh, "git fetch https://github.com/{rust_fork} {branch}")
-            .ignore_stderr()
-            .read()
-            .is_ok()
-        {
-            bail!(
-                "The branch `{branch}` seems to already exist in `https://github.com/{rust_fork}`. Please delete it and try again."
-            );
-        }
-        cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
-        cmd!(sh, "git push https://github.com/{rust_fork} {base}:refs/heads/{branch}")
-            .ignore_stdout()
-            .ignore_stderr() // silence the "create GitHub PR" message
-            .run()?;
-        println!();
-
-        // Do the actual push.
-        sh.change_dir(project_root());
-        println!("Pushing rust-analyzer changes...");
-        cmd!(
-            sh,
-            "git push http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git HEAD:{branch}"
-        )
-        .run()?;
-        println!();
-
-        // Do a round-trip check to make sure the push worked as expected.
-        cmd!(
-            sh,
-            "git fetch http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git {branch}"
-        )
-        .ignore_stderr()
-        .read()?;
-        let head = cmd!(sh, "git rev-parse HEAD").read()?;
-        let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?;
-        if head != fetch_head {
-            bail!(
-                "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\
-                Expected {head}, got {fetch_head}."
-            );
-        }
-        println!(
-            "Confirmed that the push round-trips back to rust-analyzer properly. Please create a rustc PR:"
-        );
-        // https://github.com/github-linguist/linguist/compare/master...octocat:linguist:master
-        let fork_path = rust_fork.replace('/', ":");
-        println!(
-            "    https://github.com/rust-lang/rust/compare/{fork_path}:{branch}?quick_pull=1&title=Subtree+update+of+rust-analyzer&body=r?+@ghost"
-        );
-
-        drop(josh);
-        Ok(())
-    }
-}
-
-/// Used for rustc syncs.
-const JOSH_FILTER: &str = ":rev(55d9a533b309119c8acd13061581b43ae8840823:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer";
-const JOSH_PORT: &str = "42042";
-
-fn start_josh() -> anyhow::Result<impl Drop> {
-    // Determine cache directory.
-    let local_dir = {
-        let user_dirs = ProjectDirs::from("org", "rust-lang", "rust-analyzer-josh").unwrap();
-        user_dirs.cache_dir().to_owned()
-    };
-
-    // Start josh, silencing its output.
-    let mut cmd = Command::new("josh-proxy");
-    cmd.arg("--local").arg(local_dir);
-    cmd.arg("--remote").arg("https://github.com");
-    cmd.arg("--port").arg(JOSH_PORT);
-    cmd.arg("--no-background");
-    cmd.stdout(Stdio::null());
-    cmd.stderr(Stdio::null());
-    let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
-    // Give it some time so hopefully the port is open. (100ms was not enough.)
-    thread::sleep(Duration::from_millis(200));
-
-    Ok(JodChild(josh))
-}