Auto merge of #8834 - Gankra:rust-src-vendor, r=ehuss

Check if rust-src contains a vendor dir, and patch it in

This is the cargo side of https://github.com/rust-lang/wg-cargo-std-aware/issues/23

Note that this design naively assumes there is only one version of each package. It does not robustly verify this, and will presumably just cryptically fail to resolve dependencies.

See https://github.com/rust-lang/rust/pull/78790 for the other half of this change.

src/cargo/core/compiler/standard_lib.rs

@@ -7,9 +7,10 @@ use crate::core::resolver::features::{FeaturesFor, ResolvedFeatures};
 use crate::core::resolver::{HasDevUnits, ResolveOpts};
 use crate::core::{Dependency, PackageId, PackageSet, Resolve, SourceId, Workspace};
 use crate::ops::{self, Packages};
-use crate::util::errors::CargoResult;
+use crate::util::errors::{CargoResult, CargoResultExt};
 use std::collections::{HashMap, HashSet};
 use std::env;
+use std::fs;
 use std::path::PathBuf;
 
 /// Parse the `-Zbuild-std` flag.
@@ -38,28 +39,48 @@ pub fn resolve_std<'cfg>(
     crates: &[String],
 ) -> CargoResult<(PackageSet<'cfg>, Resolve, ResolvedFeatures)> {
     let src_path = detect_sysroot_src_path(target_data)?;
-    let to_patch = [
+
-        "rustc-std-workspace-core",
+    // Special std packages should be pulled from `library/` and should be
-        "rustc-std-workspace-alloc",
+    // prefixed with `rustc-std-workspace-` in certain places.
-        "rustc-std-workspace-std",
+    let libs_prefix = "library/";
-    ];
+    let special_std_prefix = "rustc-std-workspace-";
-    let patches = to_patch
+    let libs_path = src_path.join(libs_prefix);
-        .iter()
+
-        .map(|&name| {
+    // Crates in rust-src to build. libtest is in some sense the "root" package
-            let source_path = SourceId::for_path(&src_path.join("library").join(name))?;
+    // of std, as nothing else depends on it, so it must be explicitly added.
-            let dep = Dependency::parse_no_deprecated(name, None, source_path)?;
+    let mut members = vec![format!("{}test", libs_prefix)];
+
+    // If rust-src contains a "vendor" directory, then patch in all the crates it contains.
+    let vendor_path = src_path.join("vendor");
+    let vendor_dir = fs::read_dir(&vendor_path)
+        .chain_err(|| format!("could not read vendor path {}", vendor_path.display()))?;
+    let patches = vendor_dir
+        .into_iter()
+        .map(|entry| {
+            let entry = entry?;
+            let name = entry
+                .file_name()
+                .into_string()
+                .map_err(|_| anyhow::anyhow!("package name wasn't utf8"))?;
+
+            // Remap the rustc-std-workspace crates to the actual rust-src libraries
+            let path = if let Some(real_name) = name.strip_prefix(special_std_prefix) {
+                // Record this crate as something to build in the workspace
+                members.push(format!("{}{}", libs_prefix, real_name));
+                libs_path.join(&name)
+            } else {
+                entry.path()
+            };
+            let source_path = SourceId::for_path(&path)?;
+            let dep = Dependency::parse_no_deprecated(&name, None, source_path)?;
             Ok(dep)
         })
-        .collect::<CargoResult<Vec<_>>>()?;
+        .collect::<CargoResult<Vec<_>>>()
+        .chain_err(|| "failed to generate vendor patches")?;
+
     let crates_io_url = crate::sources::CRATES_IO_INDEX.parse().unwrap();
     let mut patch = HashMap::new();
     patch.insert(crates_io_url, patches);
-    let members = vec![
-        String::from("library/std"),
-        String::from("library/core"),
-        String::from("library/alloc"),
-        String::from("library/test"),
-    ];
     let ws_config = crate::core::WorkspaceConfig::Root(crate::core::WorkspaceRootConfig::new(
         &src_path,
         &Some(members),

tests/testsuite/mock-std/library/test/Cargo.toml

@@ -10,6 +10,7 @@ std = { path = "../std" }
 panic_unwind = { path = "../panic_unwind" }
 compiler_builtins = { path = "../compiler_builtins" }
 registry-dep-using-std = { version = "*", features = ['mockbuild'] }
+registry-dep-only-used-by-test = { version = "*" }
 
 [features]
 panic-unwind = []

tests/testsuite/mock-std/library/test/src/lib.rs

@@ -7,4 +7,5 @@ extern crate test;
 pub use test::*;
 
 pub fn custom_api() {
+    registry_dep_only_used_by_test::wow_testing_is_so_easy();
 }

tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "registry-dep-only-used-by-test"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+
+[features]

tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/src/lib.rs

@@ -0,0 +1,2 @@
+pub fn wow_testing_is_so_easy() {
+}

tests/testsuite/mock-std/vendor/registry-dep-using-alloc/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "registry-dep-using-alloc"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-alloc = { version = "*", optional = true }
+rustc-std-workspace-core = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-alloc", "rustc-std-workspace-core"]

tests/testsuite/mock-std/vendor/registry-dep-using-alloc/src/lib.rs

@@ -0,0 +1,9 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+    core::custom_api();
+    alloc::custom_api();
+}

tests/testsuite/mock-std/vendor/registry-dep-using-core/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "registry-dep-using-core"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-core = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-core"]

tests/testsuite/mock-std/vendor/registry-dep-using-core/src/lib.rs

@@ -0,0 +1,8 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+    core::custom_api();
+}

tests/testsuite/mock-std/vendor/registry-dep-using-std/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "registry-dep-using-std"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-std = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-std"]

tests/testsuite/mock-std/vendor/registry-dep-using-std/src/lib.rs

@@ -0,0 +1,8 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+    std::custom_api();
+}

tests/testsuite/mock-std/vendor/rustc-std-workspace-alloc/Cargo.toml

@@ -0,0 +1 @@
+this file shouldn't be read 

tests/testsuite/mock-std/vendor/rustc-std-workspace-core/Cargo.toml

@@ -0,0 +1 @@
+this file shouldn't be read 

tests/testsuite/mock-std/vendor/rustc-std-workspace-std/Cargo.toml

@@ -0,0 +1 @@
+this file shouldn't be read 

tests/testsuite/standard_lib.rs

@@ -27,71 +27,18 @@ fn setup() -> Option<Setup> {
         return None;
     }
 
-    // Our mock sysroot requires a few packages from crates.io, so make sure
+    // Register a version of one of the std dependencies that doesn't compile.
-    // they're "published" to crates.io. Also edit their code a bit to make sure
+    // This ensures that the mock-std's vendor is actually being used.
-    // that they have access to our custom crates with custom apis.
     Package::new("registry-dep-using-core", "1.0.0")
         .file(
             "src/lib.rs",
             "
-                #![no_std]
+               don't compile me bro!!
-
-                #[cfg(feature = \"mockbuild\")]
-                pub fn custom_api() {
-                }
-
-                #[cfg(not(feature = \"mockbuild\"))]
-                pub fn non_sysroot_api() {
-                    core::custom_api();
-                }
             ",
         )
         .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
         .feature("mockbuild", &["rustc-std-workspace-core"])
         .publish();
-    Package::new("registry-dep-using-alloc", "1.0.0")
-        .file(
-            "src/lib.rs",
-            "
-                #![no_std]
-
-                extern crate alloc;
-
-                #[cfg(feature = \"mockbuild\")]
-                pub fn custom_api() {
-                }
-
-                #[cfg(not(feature = \"mockbuild\"))]
-                pub fn non_sysroot_api() {
-                    core::custom_api();
-                    alloc::custom_api();
-                }
-            ",
-        )
-        .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
-        .add_dep(Dependency::new("rustc-std-workspace-alloc", "*").optional(true))
-        .feature(
-            "mockbuild",
-            &["rustc-std-workspace-core", "rustc-std-workspace-alloc"],
-        )
-        .publish();
-    Package::new("registry-dep-using-std", "1.0.0")
-        .file(
-            "src/lib.rs",
-            "
-                #[cfg(feature = \"mockbuild\")]
-                pub fn custom_api() {
-                }
-
-                #[cfg(not(feature = \"mockbuild\"))]
-                pub fn non_sysroot_api() {
-                    std::custom_api();
-                }
-            ",
-        )
-        .add_dep(Dependency::new("rustc-std-workspace-std", "*").optional(true))
-        .feature("mockbuild", &["rustc-std-workspace-std"])
-        .publish();
 
     let p = ProjectBuilder::new(paths::root().join("rustc-wrapper"))
         .file(
@@ -335,6 +282,81 @@ fn depend_same_as_std() {
         None => return,
     };
 
+    // Our mock sysroot requires a few packages from crates.io, so make sure
+    // they're "published" to crates.io. Also edit their code a bit to make sure
+    // that they have access to our custom crates with custom apis.
+    Package::new("registry-dep-using-core", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                #![no_std]
+
+                #[cfg(feature = \"mockbuild\")]
+                pub fn custom_api() {
+                }
+
+                #[cfg(not(feature = \"mockbuild\"))]
+                pub fn non_sysroot_api() {
+                    core::custom_api();
+                }
+            ",
+        )
+        .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
+        .feature("mockbuild", &["rustc-std-workspace-core"])
+        .publish();
+    Package::new("registry-dep-using-alloc", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                #![no_std]
+
+                extern crate alloc;
+
+                #[cfg(feature = \"mockbuild\")]
+                pub fn custom_api() {
+                }
+
+                #[cfg(not(feature = \"mockbuild\"))]
+                pub fn non_sysroot_api() {
+                    core::custom_api();
+                    alloc::custom_api();
+                }
+            ",
+        )
+        .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
+        .add_dep(Dependency::new("rustc-std-workspace-alloc", "*").optional(true))
+        .feature(
+            "mockbuild",
+            &["rustc-std-workspace-core", "rustc-std-workspace-alloc"],
+        )
+        .publish();
+    Package::new("registry-dep-using-std", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                #[cfg(feature = \"mockbuild\")]
+                pub fn custom_api() {
+                }
+
+                #[cfg(not(feature = \"mockbuild\"))]
+                pub fn non_sysroot_api() {
+                    std::custom_api();
+                }
+            ",
+        )
+        .add_dep(Dependency::new("rustc-std-workspace-std", "*").optional(true))
+        .feature("mockbuild", &["rustc-std-workspace-std"])
+        .publish();
+    Package::new("registry-dep-only-used-by-test", "1.0.0")
+        .file(
+            "src/lib.rs",
+            "
+                pub fn wow_testing_is_so_easy() {
+                }
+            ",
+        )
+        .publish();
+
     let p = project()
         .file(
             "src/lib.rs",