From 3842d8e6f20067f716c99ba39f4d9f2c8abe84f2 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 22 Apr 2019 17:54:27 -0700 Subject: [PATCH] Import the cargo-vendor subcommand into Cargo This commit imports the external [alexcrichton/cargo-vendor repository][repo] into Cargo itself. This means it will no longer be necessary to install the `cargo-vendor` subcommand in order to vendor dependencies. Additionally it'll always support the latest feature set of Cargo as it'll be built into Cargo! All tests were imported as part of this commit, but not all features were imported. Some flags have been left out that were added later in the lifetime of `cargo vendor` which seem like they're more questionable to stabilize. I'm hoping that they can have separate PRs adding their implementation here, and we can make a decision of their stabilization at a later date. The current man page for `cargo vendor -h` will look like: cargo-vendor Vendor all dependencies for a project locally USAGE: cargo vendor [OPTIONS] [--] [path] OPTIONS: -q, --quiet No output printed to stdout --manifest-path Path to Cargo.toml --no-delete Don't delete older crates in the vendor directory -s, --sync ... Additional `Cargo.toml` to sync and vendor --respect-source-config Respect `[source]` config in `.cargo/config` -v, --verbose Use verbose output (-vv very verbose/build.rs output) --color Coloring: auto, always, never --frozen Require Cargo.lock and cache are up to date --locked Require Cargo.lock is up to date -Z ... Unstable (nightly-only) flags to Cargo, see 'cargo -Z help' for details -h, --help Prints help information ARGS: Where to vendor crates (`vendor` by default) This cargo subcommand will vendor all crates.io and git dependencies for a project into the specified directory at ``. After this command completes the vendor directory specified by `` will contain all remote sources from dependencies specified. Additionally manifest beyond the default one can be specified with the `-s` option. The `cargo vendor` command will also print out the configuration necessary to use the vendored sources, which when needed is then encoded into `.cargo/config`. Since this change is not importing 100% of the functionality of the existing `cargo vendor` this change does run a risk of being a breaking change for any folks using such functionality. Executing `cargo vendor` will favor the built-in command rather than an external subcommand, causing unimplemented features to become errors about flag usage. [repo]: https://github.com/alexcrichton/cargo-vendor --- src/bin/cargo/commands/mod.rs | 3 + src/bin/cargo/commands/vendor.rs | 123 +++++ src/cargo/lib.rs | 44 +- src/cargo/ops/mod.rs | 2 + src/cargo/ops/vendor.rs | 327 +++++++++++++ src/cargo/sources/directory.rs | 21 +- src/cargo/sources/registry/local.rs | 17 +- src/cargo/sources/registry/remote.rs | 5 +- src/cargo/util/config.rs | 7 + src/cargo/util/sha256.rs | 31 +- src/doc/man/cargo-vendor.adoc | 75 +++ src/doc/man/generated/cargo-metadata.html | 2 +- src/doc/man/generated/cargo-vendor.html | 207 ++++++++ src/doc/src/SUMMARY.md | 1 + src/doc/src/commands/cargo-vendor.md | 4 + src/doc/src/reference/source-replacement.md | 8 +- src/etc/man/cargo-vendor.1 | 223 +++++++++ src/etc/man/cargo.1 | 2 +- tests/testsuite/main.rs | 1 + tests/testsuite/support/registry.rs | 4 +- tests/testsuite/vendor.rs | 508 ++++++++++++++++++++ 21 files changed, 1554 insertions(+), 61 deletions(-) create mode 100644 src/bin/cargo/commands/vendor.rs create mode 100644 src/cargo/ops/vendor.rs create mode 100644 src/doc/man/cargo-vendor.adoc create mode 100644 src/doc/man/generated/cargo-vendor.html create mode 100644 src/doc/src/commands/cargo-vendor.md create mode 100644 src/etc/man/cargo-vendor.1 create mode 100644 tests/testsuite/vendor.rs diff --git a/src/bin/cargo/commands/mod.rs b/src/bin/cargo/commands/mod.rs index 860c78de6..16e02774d 100644 --- a/src/bin/cargo/commands/mod.rs +++ b/src/bin/cargo/commands/mod.rs @@ -30,6 +30,7 @@ pub fn builtin() -> Vec { test::cli(), uninstall::cli(), update::cli(), + vendor::cli(), verify_project::cli(), version::cli(), yank::cli(), @@ -66,6 +67,7 @@ pub fn builtin_exec(cmd: &str) -> Option) -> Cli "test" => test::exec, "uninstall" => uninstall::exec, "update" => update::exec, + "vendor" => vendor::exec, "verify-project" => verify_project::exec, "version" => version::exec, "yank" => yank::exec, @@ -102,6 +104,7 @@ pub mod search; pub mod test; pub mod uninstall; pub mod update; +pub mod vendor; pub mod verify_project; pub mod version; pub mod yank; diff --git a/src/bin/cargo/commands/vendor.rs b/src/bin/cargo/commands/vendor.rs new file mode 100644 index 000000000..509a5c2b2 --- /dev/null +++ b/src/bin/cargo/commands/vendor.rs @@ -0,0 +1,123 @@ +use crate::command_prelude::*; +use cargo::ops; +use std::path::PathBuf; + +pub fn cli() -> App { + subcommand("vendor") + .about("Vendor all dependencies for a project locally") + .arg(opt("quiet", "No output printed to stdout").short("q")) + .arg_manifest_path() + .arg(Arg::with_name("path").help("Where to vendor crates (`vendor` by default)")) + .arg( + Arg::with_name("no-delete") + .long("no-delete") + .help("Don't delete older crates in the vendor directory"), + ) + .arg( + Arg::with_name("tomls") + .short("s") + .long("sync") + .help("Additional `Cargo.toml` to sync and vendor") + .value_name("TOML") + .multiple(true), + ) + .arg( + Arg::with_name("respect-source-config") + .long("respect-source-config") + .help("Respect `[source]` config in `.cargo/config`") + .multiple(true), + ) + .arg( + Arg::with_name("no-merge-sources") + .long("no-merge-sources") + .hidden(true), + ) + .arg( + Arg::with_name("relative-path") + .long("relative-path") + .hidden(true), + ) + .arg( + Arg::with_name("only-git-deps") + .long("only-git-deps") + .hidden(true), + ) + .arg( + Arg::with_name("explicit-version") + .short("-x") + .long("explicit-version") + .hidden(true), + ) + .arg( + Arg::with_name("disallow-duplicates") + .long("disallow-duplicates") + .hidden(true), + ) + .after_help( + "\ +This cargo subcommand will vendor all crates.io and git dependencies for a +project into the specified directory at ``. After this command completes +the vendor directory specified by `` will contain all remote sources from +dependencies specified. Additional manifests beyond the default one can be +specified with the `-s` option. + +The `cargo vendor` command will also print out the configuration necessary +to use the vendored sources, which you will need to add to `.cargo/config`. +", + ) +} + +pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult { + // We're doing the vendoring operation outselves, so we don't actually want + // to respect any of the `source` configuration in Cargo itself. That's + // intended for other consumers of Cargo, but we want to go straight to the + // source, e.g. crates.io, to fetch crates. + if !args.is_present("respect-source-config") { + config.values_mut()?.remove("source"); + } + + // When we moved `cargo vendor` into Cargo itself we didn't stabilize a few + // flags, so try to provide a helpful error message in that case to enusre + // that users currently using the flag aren't tripped up. + let crates_io_cargo_vendor_flag = if args.is_present("no-merge-sources") { + Some("--no-merge-sources") + } else if args.is_present("relative-path") { + Some("--relative-path") + } else if args.is_present("only-git-deps") { + Some("--only-git-deps") + } else if args.is_present("explicit-version") { + Some("--explicit-version") + } else if args.is_present("disallow-duplicates") { + Some("--disallow-duplicates") + } else { + None + }; + if let Some(flag) = crates_io_cargo_vendor_flag { + return Err(failure::format_err!("\ +the crates.io `cargo vendor` command has now been merged into Cargo itself +and does not support the flag `{}` currently; to continue using the flag you +can execute `cargo-vendor vendor ...`, and if you would like to see this flag +supported in Cargo itself please feel free to file an issue at +https://github.com/rust-lang/cargo/issues/new +", flag).into()); + } + + let ws = args.workspace(config)?; + let path = args + .value_of_os("path") + .map(|val| PathBuf::from(val.to_os_string())) + .unwrap_or(PathBuf::from("vendor")); + ops::vendor( + &ws, + &ops::VendorOptions { + no_delete: args.is_present("no-delete"), + destination: &path, + extra: args + .values_of_os("tomls") + .unwrap_or_default() + .map(|s| PathBuf::from(s.to_os_string())) + .collect(), + }, + )?; + Ok(()) +} diff --git a/src/cargo/lib.rs b/src/cargo/lib.rs index 16e67a458..b91905d6c 100644 --- a/src/cargo/lib.rs +++ b/src/cargo/lib.rs @@ -26,6 +26,7 @@ #![allow(clippy::unneeded_field_pattern)] use std::fmt; +use std::io; use failure::Error; use log::debug; @@ -147,23 +148,38 @@ fn handle_cause(cargo_err: &Error, shell: &mut Shell) -> bool { drop(writeln!(shell.err(), " {}", error)); } + fn print_stderror_causes(error: &dyn std::error::Error, shell: &mut Shell) { + let mut cur = std::error::Error::source(error); + while let Some(err) = cur { + print(&err.to_string(), shell); + cur = std::error::Error::source(err); + } + } + let verbose = shell.verbosity(); - if verbose == Verbose { - // The first error has already been printed to the shell. - // Print all remaining errors. - for err in cargo_err.iter_causes() { - print(&err.to_string(), shell); - } - } else { - // The first error has already been printed to the shell. - // Print remaining errors until one marked as `Internal` appears. - for err in cargo_err.iter_causes() { - if err.downcast_ref::().is_some() { - return false; - } + // The first error has already been printed to the shell. + for err in cargo_err.iter_causes() { - print(&err.to_string(), shell); + // If we're not in verbose mode then print remaining errors until one + // marked as `Internal` appears. + if verbose != Verbose && err.downcast_ref::().is_some() { + return false; + } + + print(&err.to_string(), shell); + + // Using the `failure` crate currently means that when using + // `iter_causes` we're only iterating over the `failure` causes, but + // this doesn't include the causes from the standard library `Error` + // trait. We don't have a great way of getting an `&dyn Error` from a + // `&dyn Fail`, so we currently just special case a few errors that are + // known to maybe have causes and we try to print them here. + // + // Note that this isn't an exhaustive match since causes for + // `std::error::Error` aren't the most common thing in the world. + if let Some(io) = err.downcast_ref::() { + print_stderror_causes(io, shell); } } diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs index 997ead71f..9e40062b2 100644 --- a/src/cargo/ops/mod.rs +++ b/src/cargo/ops/mod.rs @@ -26,6 +26,7 @@ pub use self::resolve::{ add_overrides, get_resolved_packages, resolve_with_previous, resolve_ws, resolve_ws_precisely, resolve_ws_with_method, }; +pub use self::vendor::{vendor, VendorOptions}; mod cargo_clean; mod cargo_compile; @@ -46,3 +47,4 @@ mod fix; mod lockfile; mod registry; mod resolve; +mod vendor; diff --git a/src/cargo/ops/vendor.rs b/src/cargo/ops/vendor.rs new file mode 100644 index 000000000..fccee9699 --- /dev/null +++ b/src/cargo/ops/vendor.rs @@ -0,0 +1,327 @@ +use crate::core::shell::Verbosity; +use crate::core::{GitReference, Workspace}; +use crate::ops; +use crate::sources::path::PathSource; +use crate::util::Sha256; +use crate::util::{paths, CargoResult, CargoResultExt, Config}; +use failure::bail; +use serde::Serialize; +use std::collections::HashSet; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +pub struct VendorOptions<'a> { + pub no_delete: bool, + pub destination: &'a Path, + pub extra: Vec, +} + +pub fn vendor(ws: &Workspace<'_>, opts: &VendorOptions<'_>) -> CargoResult<()> { + let mut extra_workspaces = Vec::new(); + for extra in opts.extra.iter() { + let extra = ws.config().cwd().join(extra); + let ws = Workspace::new(&extra, ws.config())?; + extra_workspaces.push(ws); + } + let workspaces = extra_workspaces.iter().chain(Some(ws)).collect::>(); + let vendor_config = + sync(ws.config(), &workspaces, opts).chain_err(|| format!("failed to sync"))?; + + let shell = ws.config().shell(); + if shell.verbosity() != Verbosity::Quiet { + eprint!("To use vendored sources, add this to your .cargo/config for this project:\n\n"); + print!("{}", &toml::to_string(&vendor_config).unwrap()); + } + + Ok(()) +} + +#[derive(Serialize)] +struct VendorConfig { + source: BTreeMap, +} + +#[derive(Serialize)] +#[serde(rename_all = "lowercase", untagged)] +enum VendorSource { + Directory { + directory: PathBuf, + }, + Registry { + registry: Option, + #[serde(rename = "replace-with")] + replace_with: String, + }, + Git { + git: String, + branch: Option, + tag: Option, + rev: Option, + #[serde(rename = "replace-with")] + replace_with: String, + }, +} + +fn sync( + config: &Config, + workspaces: &[&Workspace<'_>], + opts: &VendorOptions<'_>, +) -> CargoResult { + let canonical_destination = opts.destination.canonicalize(); + let canonical_destination = canonical_destination + .as_ref() + .map(|p| &**p) + .unwrap_or(opts.destination); + + fs::create_dir_all(&canonical_destination)?; + let mut to_remove = HashSet::new(); + if !opts.no_delete { + for entry in canonical_destination.read_dir()? { + let entry = entry?; + to_remove.insert(entry.path()); + } + } + + // First up attempt to work around rust-lang/cargo#5956. Apparently build + // artifacts sprout up in Cargo's global cache for whatever reason, although + // it's unsure what tool is causing these issues at this time. For now we + // apply a heavy-hammer approach which is to delete Cargo's unpacked version + // of each crate to start off with. After we do this we'll re-resolve and + // redownload again, which should trigger Cargo to re-extract all the + // crates. + // + // Note that errors are largely ignored here as this is a best-effort + // attempt. If anything fails here we basically just move on to the next + // crate to work with. + for ws in workspaces { + let (packages, resolve) = + ops::resolve_ws(&ws).chain_err(|| "failed to load pkg lockfile")?; + + packages + .get_many(resolve.iter()) + .chain_err(|| "failed to download packages")?; + + for pkg in resolve.iter() { + // Don't delete actual source code! + if pkg.source_id().is_path() { + if let Ok(path) = pkg.source_id().url().to_file_path() { + if let Ok(path) = path.canonicalize() { + to_remove.remove(&path); + } + } + continue; + } + if pkg.source_id().is_git() { + continue; + } + if let Ok(pkg) = packages.get_one(pkg) { + drop(fs::remove_dir_all(pkg.manifest_path().parent().unwrap())); + } + } + } + + let mut checksums = HashMap::new(); + let mut ids = BTreeMap::new(); + + // Next up let's actually download all crates and start storing internal + // tables about them. + for ws in workspaces { + let (packages, resolve) = + ops::resolve_ws(&ws).chain_err(|| "failed to load pkg lockfile")?; + + packages + .get_many(resolve.iter()) + .chain_err(|| "failed to download packages")?; + + for pkg in resolve.iter() { + // No need to vendor path crates since they're already in the + // repository + if pkg.source_id().is_path() { + continue; + } + ids.insert( + pkg.clone(), + packages + .get_one(pkg) + .chain_err(|| "failed to fetch package")? + .clone(), + ); + + checksums.insert(pkg.clone(), resolve.checksums().get(&pkg).cloned()); + } + } + + let mut versions = HashMap::new(); + for id in ids.keys() { + let map = versions.entry(id.name()).or_insert_with(BTreeMap::default); + if let Some(prev) = map.get(&id.version()) { + bail!( + "found duplicate version of package `{} v{}` \ + vendored from two sources:\n\ + \n\ + \tsource 1: {}\n\ + \tsource 2: {}", + id.name(), + id.version(), + prev, + id.source_id() + ); + } + map.insert(id.version(), id.source_id()); + } + + let mut sources = BTreeSet::new(); + for (id, pkg) in ids.iter() { + // Next up, copy it to the vendor directory + let src = pkg + .manifest_path() + .parent() + .expect("manifest_path should point to a file"); + let max_version = *versions[&id.name()].iter().rev().next().unwrap().0; + let dir_has_version_suffix = id.version() != max_version; + let dst_name = if dir_has_version_suffix { + // Eg vendor/futures-0.1.13 + format!("{}-{}", id.name(), id.version()) + } else { + // Eg vendor/futures + id.name().to_string() + }; + + sources.insert(id.source_id()); + let dst = canonical_destination.join(&dst_name); + to_remove.remove(&dst); + let cksum = dst.join(".cargo-checksum.json"); + if dir_has_version_suffix && cksum.exists() { + // Always re-copy directory without version suffix in case the version changed + continue; + } + + config.shell().status( + "Vendoring", + &format!("{} ({}) to {}", id, src.to_string_lossy(), dst.display()), + )?; + + let _ = fs::remove_dir_all(&dst); + let pathsource = PathSource::new(&src, id.source_id(), config); + let paths = pathsource.list_files(&pkg)?; + let mut map = BTreeMap::new(); + cp_sources(&src, &paths, &dst, &mut map) + .chain_err(|| format!("failed to copy over vendored sources for: {}", id))?; + + // Finally, emit the metadata about this package + let json = serde_json::json!({ + "package": checksums.get(id), + "files": map, + }); + + File::create(&cksum)?.write_all(json.to_string().as_bytes())?; + } + + for path in to_remove { + if path.is_dir() { + paths::remove_dir_all(&path)?; + } else { + paths::remove_file(&path)?; + } + } + + // add our vendored source + let mut config = BTreeMap::new(); + + let merged_source_name = "vendored-sources"; + config.insert( + merged_source_name.to_string(), + VendorSource::Directory { + directory: canonical_destination.to_path_buf(), + }, + ); + + // replace original sources with vendor + for source_id in sources { + let name = if source_id.is_default_registry() { + "crates-io".to_string() + } else { + source_id.url().to_string() + }; + + let source = if source_id.is_default_registry() { + VendorSource::Registry { + registry: None, + replace_with: merged_source_name.to_string(), + } + } else if source_id.is_git() { + let mut branch = None; + let mut tag = None; + let mut rev = None; + if let Some(reference) = source_id.git_reference() { + match *reference { + GitReference::Branch(ref b) => branch = Some(b.clone()), + GitReference::Tag(ref t) => tag = Some(t.clone()), + GitReference::Rev(ref r) => rev = Some(r.clone()), + } + } + VendorSource::Git { + git: source_id.url().to_string(), + branch, + tag, + rev, + replace_with: merged_source_name.to_string(), + } + } else { + panic!("Invalid source ID: {}", source_id) + }; + config.insert(name, source); + } + + Ok(VendorConfig { source: config }) +} + +fn cp_sources( + src: &Path, + paths: &[PathBuf], + dst: &Path, + cksums: &mut BTreeMap, +) -> CargoResult<()> { + for p in paths { + let relative = p.strip_prefix(&src).unwrap(); + + match relative.to_str() { + // Skip git config files as they're not relevant to builds most of + // the time and if we respect them (e.g. in git) then it'll + // probably mess with the checksums when a vendor dir is checked + // into someone else's source control + Some(".gitattributes") | Some(".gitignore") | Some(".git") => continue, + + // Temporary Cargo files + Some(".cargo-ok") => continue, + + // Skip patch-style orig/rej files. Published crates on crates.io + // have `Cargo.toml.orig` which we don't want to use here and + // otherwise these are rarely used as part of the build process. + Some(filename) => { + if filename.ends_with(".orig") || filename.ends_with(".rej") { + continue; + } + } + _ => {} + }; + + // Join pathname components individually to make sure that the joined + // path uses the correct directory separators everywhere, since + // `relative` may use Unix-style and `dst` may require Windows-style + // backslashes. + let dst = relative + .iter() + .fold(dst.to_owned(), |acc, component| acc.join(&component)); + + fs::create_dir_all(dst.parent().unwrap())?; + + fs::copy(&p, &dst) + .chain_err(|| format!("failed to copy `{}` to `{}`", p.display(), dst.display()))?; + let cksum = Sha256::new().update_path(dst)?.finish_hex(); + cksums.insert(relative.to_str().unwrap().replace("\\", "/"), cksum); + } + Ok(()) +} diff --git a/src/cargo/sources/directory.rs b/src/cargo/sources/directory.rs index aab31383c..06f668f4d 100644 --- a/src/cargo/sources/directory.rs +++ b/src/cargo/sources/directory.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; use std::fmt::{self, Debug, Formatter}; -use std::fs::File; -use std::io::Read; use std::path::{Path, PathBuf}; use serde::Deserialize; @@ -170,23 +168,12 @@ impl<'cfg> Source for DirectorySource<'cfg> { None => failure::bail!("failed to find entry for `{}` in directory source", id), }; - let mut buf = [0; 16 * 1024]; for (file, cksum) in cksum.files.iter() { - let mut h = Sha256::new(); let file = pkg.root().join(file); - - (|| -> CargoResult<()> { - let mut f = File::open(&file)?; - loop { - match f.read(&mut buf)? { - 0 => return Ok(()), - n => h.update(&buf[..n]), - } - } - })() - .chain_err(|| format!("failed to calculate checksum of: {}", file.display()))?; - - let actual = hex::encode(h.finish()); + let actual = Sha256::new() + .update_path(&file) + .chain_err(|| format!("failed to calculate checksum of: {}", file.display()))? + .finish_hex(); if &*actual != cksum { failure::bail!( "\ diff --git a/src/cargo/sources/registry/local.rs b/src/cargo/sources/registry/local.rs index d2f6dc936..35c8db483 100644 --- a/src/cargo/sources/registry/local.rs +++ b/src/cargo/sources/registry/local.rs @@ -1,9 +1,8 @@ use crate::core::{InternedString, PackageId}; use crate::sources::registry::{MaybeLock, RegistryConfig, RegistryData}; -use crate::util::errors::{CargoResult, CargoResultExt}; +use crate::util::errors::CargoResult; use crate::util::paths; use crate::util::{Config, Filesystem, Sha256}; -use hex; use std::fs::File; use std::io::prelude::*; use std::io::SeekFrom; @@ -99,18 +98,8 @@ impl<'cfg> RegistryData for LocalRegistry<'cfg> { // We don't actually need to download anything per-se, we just need to // verify the checksum matches the .crate file itself. - let mut state = Sha256::new(); - let mut buf = [0; 64 * 1024]; - loop { - let n = crate_file - .read(&mut buf) - .chain_err(|| format!("failed to read `{}`", path.display()))?; - if n == 0 { - break; - } - state.update(&buf[..n]); - } - if hex::encode(state.finish()) != checksum { + let actual = Sha256::new().update_file(&crate_file)?.finish_hex(); + if actual != checksum { failure::bail!("failed to verify the checksum of `{}`", pkg) } diff --git a/src/cargo/sources/registry/remote.rs b/src/cargo/sources/registry/remote.rs index 1abae42e9..c44964432 100644 --- a/src/cargo/sources/registry/remote.rs +++ b/src/cargo/sources/registry/remote.rs @@ -281,9 +281,8 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> { data: &[u8], ) -> CargoResult { // Verify what we just downloaded - let mut state = Sha256::new(); - state.update(data); - if hex::encode(state.finish()) != checksum { + let actual = Sha256::new().update(data).finish_hex(); + if actual != checksum { failure::bail!("failed to verify the checksum of `{}`", pkg) } diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index 54ee81704..9409277f6 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -294,6 +294,13 @@ impl Config { self.values.try_borrow_with(|| self.load_values()) } + pub fn values_mut(&mut self) -> CargoResult<&mut HashMap> { + match self.values.borrow_mut() { + Some(map) => Ok(map), + None => failure::bail!("config values not loaded yet"), + } + } + // Note: this is used by RLS, not Cargo. pub fn set_values(&self, values: HashMap) -> CargoResult<()> { if self.values.borrow().is_some() { diff --git a/src/cargo/util/sha256.rs b/src/cargo/util/sha256.rs index 32d04ae93..a8f762c15 100644 --- a/src/cargo/util/sha256.rs +++ b/src/cargo/util/sha256.rs @@ -1,6 +1,9 @@ use self::crypto_hash::{Algorithm, Hasher}; +use std::fs::File; use crypto_hash; -use std::io::Write; +use std::io::{self, Write, Read}; +use std::path::Path; +use crate::util::{CargoResult, CargoResultExt}; pub struct Sha256(Hasher); @@ -10,8 +13,28 @@ impl Sha256 { Sha256(hasher) } - pub fn update(&mut self, bytes: &[u8]) { + pub fn update(&mut self, bytes: &[u8]) -> &mut Sha256 { let _ = self.0.write_all(bytes); + self + } + + pub fn update_file(&mut self, mut file: &File) -> io::Result<&mut Sha256> { + let mut buf = [0; 64 * 1024]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break Ok(self); + } + self.update(&buf[..n]); + } + } + + pub fn update_path>(&mut self, path: P) -> CargoResult<&mut Sha256> { + let path = path.as_ref(); + let file = File::open(path)?; + self.update_file(&file) + .chain_err(|| format!("failed to read `{}`", path.display()))?; + Ok(self) } pub fn finish(&mut self) -> [u8; 32] { @@ -20,6 +43,10 @@ impl Sha256 { ret.copy_from_slice(&data[..]); ret } + + pub fn finish_hex(&mut self) -> String { + hex::encode(self.finish()) + } } impl Default for Sha256 { diff --git a/src/doc/man/cargo-vendor.adoc b/src/doc/man/cargo-vendor.adoc new file mode 100644 index 000000000..7f52ec815 --- /dev/null +++ b/src/doc/man/cargo-vendor.adoc @@ -0,0 +1,75 @@ += cargo-vendor(1) +:idprefix: cargo_vendor_ +:doctype: manpage + +== NAME + +cargo-vendor - Vendor all dependencies locally + +== SYNOPSIS + +`cargo vendor [_OPTIONS_] [_PATH_]` + +== DESCRIPTION + +This cargo subcommand will vendor all crates.io and git dependencies for a +project into the specified directory at ``. After this command completes +the vendor directory specified by `` will contain all remote sources from +dependencies specified. Additional manifests beyond the default one can be +specified with the `-s` option. + +The `cargo vendor` command will also print out the configuration necessary +to use the vendored sources, which you will need to add to `.cargo/config`. + +== OPTIONS + +=== Owner Options + +*-s* _MANIFEST_:: +*--sync* _MANIFEST_:: + Specify extra `Cargo.toml` manifests to workspaces which should also be + vendored and synced to the output. + +*--no-delete*:: + Don't delete the "vendor" directory when vendoring, but rather keep all + existing contents of the vendor directory + +*--respect-source-config*:: + Instead of ignoring `[source]` configuration by default in `.cargo/config` + read it and use it when downloading crates from crates.io, for example + +=== Manifest Options + +include::options-manifest-path.adoc[] + +=== Display Options + +include::options-display.adoc[] + +=== Common Options + +include::options-common.adoc[] + +include::options-locked.adoc[] + +include::section-environment.adoc[] + +include::section-exit-status.adoc[] + +== EXAMPLES + +. Vendor all dependencies into a local "vendor" folder + + cargo vendor + +. Vendor all dependencies into a local "third-part/vendor" folder + + cargo vendor third-party/vendor + +. Vendor the current workspace as well as another to "vendor" + + cargo vendor -s ../path/to/Cargo.toml + +== SEE ALSO +man:cargo[1] + diff --git a/src/doc/man/generated/cargo-metadata.html b/src/doc/man/generated/cargo-metadata.html index 25665e319..007f7ff81 100644 --- a/src/doc/man/generated/cargo-metadata.html +++ b/src/doc/man/generated/cargo-metadata.html @@ -35,7 +35,7 @@ for a Rust API for reading the metadata.

-
{
+
{
     /* Array of all packages in the workspace.
        It also includes all feature-enabled dependencies unless --no-deps is used.
     */
diff --git a/src/doc/man/generated/cargo-vendor.html b/src/doc/man/generated/cargo-vendor.html
new file mode 100644
index 000000000..87ead7624
--- /dev/null
+++ b/src/doc/man/generated/cargo-vendor.html
@@ -0,0 +1,207 @@
+
NAME

+

+
cargo-vendor - Vendor all dependencies locally

+

+

+
SYNOPSIS

+

+

+
cargo vendor [OPTIONS] [PATH]

+

+

+

+

+
DESCRIPTION

+

+

+
This cargo subcommand will vendor all crates.io and git dependencies for a
+project into the specified directory at <path>. After this command completes
+the vendor directory specified by <path> will contain all remote sources from
+dependencies specified. Additional manifests beyond the default one can be
+specified with the -s option.

+

+

+
The cargo vendor command will also print out the configuration necessary
+to use the vendored sources, which you will need to add to .cargo/config.

+

+

+

+

+
OPTIONS

+

+

+
Owner Options

+

+

+
-s MANIFEST

+
--sync MANIFEST

+

+
Specify extra Cargo.toml manifests to workspaces which should also be
+vendored and synced to the output.

+

+
--no-delete

+

+
Don’t delete the "vendor" directory when vendoring, but rather keep all
+existing contents of the vendor directory

+

+
--respect-source-config

+

+
Instead of ignoring [source] configuration by default in .cargo/config
+read it and use it when downloading crates from crates.io, for example

+

+

+

+

+

+
Manifest Options

+

+

+
--manifest-path PATH

+

+
Path to the Cargo.toml file. By default, Cargo searches in the current
+directory or any parent directory for the Cargo.toml file.

+

+

+

+

+

+
Display Options

+

+

+
-v

+
--verbose

+

+
Use verbose output. May be specified twice for "very verbose" output which
+includes extra output such as dependency warnings and build script output.
+May also be specified with the term.verbose
+config value.

+

+
-q

+
--quiet

+

+
No output printed to stdout.

+

+
--color WHEN

+

+
Control when colored output is used. Valid values:

+

+
    
+
  • 
+
    auto (default): Automatically detect if color support is available on the
+terminal.
    
+
    • 
+
  • 
+
    always: Always display colors.
    
+
    • 
+
  • 
+
    never: Never display colors.
    
+
    • 
+

+

+

+
May also be specified with the term.color
+config value.

+

+

+

+

+

+

+
Common Options

+

+

+
-h

+
--help

+

+
Prints help information.

+

+
-Z FLAG…​

+

+
Unstable (nightly-only) flags to Cargo. Run cargo -Z help for
+details.

+

+
--frozen

+
--locked

+

+
Either of these flags requires that the Cargo.lock file is
+up-to-date. If the lock file is missing, or it needs to be updated, Cargo will
+exit with an error. The --frozen flag also prevents Cargo from
+attempting to access the network to determine if it is out-of-date.

+

+
These may be used in environments where you want to assert that the
+Cargo.lock file is up-to-date (such as a CI build) or want to avoid network
+access.

+

+

+

+

+

+

+

+

+
ENVIRONMENT

+

+

+
See the reference for
+details on environment variables that Cargo reads.

+

+

+

+

+
Exit Status

+

+

+

+
0

+

+
Cargo succeeded.

+

+
101

+

+
Cargo failed to complete.

+

+

+

+

+

+

+
EXAMPLES

+

+

+
    
+
  1. 
+
    Vendor all dependencies into a local "vendor" folder
    
+
    
+
    
+
    cargo vendor
    
+
    
+
    
+
    2. 
+
  2. 
+
    Vendor all dependencies into a local "third-part/vendor" folder
    
+
    
+
    
+
    cargo vendor third-party/vendor
    
+
    
+
    
+
    3. 
+
  3. 
+
    Vendor the current workspace as well as another to "vendor"
    
+
    
+
    
+
    cargo vendor -s ../path/to/Cargo.toml
    
+
    
+
    
+
    4. 
+

+

+

+

+

+
SEE ALSO

+

+

+
cargo(1)

+

+

+

\ No newline at end of file
diff --git a/src/doc/src/SUMMARY.md b/src/doc/src/SUMMARY.md
index 3a69c0a31..4943d2512 100644
--- a/src/doc/src/SUMMARY.md
+++ b/src/doc/src/SUMMARY.md
@@ -49,6 +49,7 @@
         * [metadata](commands/cargo-metadata.md)
         * [pkgid](commands/cargo-pkgid.md)
         * [update](commands/cargo-update.md)
+        * [vendor](commands/cargo-vendor.md)
         * [verify-project](commands/cargo-verify-project.md)
     * [Package Commands](commands/package-commands.md)
         * [init](commands/cargo-init.md)
diff --git a/src/doc/src/commands/cargo-vendor.md b/src/doc/src/commands/cargo-vendor.md
new file mode 100644
index 000000000..1e0f333fe
--- /dev/null
+++ b/src/doc/src/commands/cargo-vendor.md
@@ -0,0 +1,4 @@
+# cargo vendor
+{{#include command-common.html}}
+{{#include ../../man/generated/cargo-vendor.html}}
+
diff --git a/src/doc/src/reference/source-replacement.md b/src/doc/src/reference/source-replacement.md
index d7e23fa99..802f41d62 100644
--- a/src/doc/src/reference/source-replacement.md
+++ b/src/doc/src/reference/source-replacement.md
@@ -107,12 +107,8 @@ the crates that are present).
 
 A "directory source" is similar to a local registry source where it contains a
 number of crates available on the local filesystem, suitable for vendoring
-dependencies. Also like local registries, directory sources can primarily be
-managed by an external subcommand, [`cargo-vendor`][cargo-vendor],
-[available on crates.io][cargo-vendor] and can be
-installed with `cargo install cargo-vendor`.
-
-[cargo-vendor]: https://crates.io/crates/cargo-vendor
+dependencies. Directory sources are primarily managed the `cargo vendor`
+subcommand.
 
 Directory sources are distinct from local registries though in that they contain
 the unpacked version of `*.crate` files, making it more suitable in some
diff --git a/src/etc/man/cargo-vendor.1 b/src/etc/man/cargo-vendor.1
new file mode 100644
index 000000000..3937cc949
--- /dev/null
+++ b/src/etc/man/cargo-vendor.1
@@ -0,0 +1,223 @@
+'\" t
+.\"     Title: cargo-vendor
+.\"    Author: [see the "AUTHOR(S)" section]
+.\" Generator: Asciidoctor 2.0.8
+.\"      Date: 2019-04-29
+.\"    Manual: \ \&
+.\"    Source: \ \&
+.\"  Language: English
+.\"
+.TH "CARGO\-VENDOR" "1" "2019-04-29" "\ \&" "\ \&"
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
+.ss \n[.ss] 0
+.nh
+.ad l
+.de URL
+\fI\\$2\fP <\\$1>\\$3
+..
+.als MTO URL
+.if \n[.g] \{\
+.  mso www.tmac
+.  am URL
+.    ad l
+.  .
+.  am MTO
+.    ad l
+.  .
+.  LINKSTYLE blue R < >
+.\}
+.SH "NAME"
+cargo\-vendor \- Vendor all dependencies locally
+.SH "SYNOPSIS"
+.sp
+\fBcargo vendor [\fIOPTIONS\fP] [\fIPATH\fP]\fP
+.SH "DESCRIPTION"
+.sp
+This cargo subcommand will vendor all crates.io and git dependencies for a
+project into the specified directory at \fB\fP. After this command completes
+the vendor directory specified by \fB\fP will contain all remote sources from
+dependencies specified. Additional manifests beyond the default one can be
+specified with the \fB\-s\fP option.
+.sp
+The \fBcargo vendor\fP command will also print out the configuration necessary
+to use the vendored sources, which you will need to add to \fB.cargo/config\fP.
+.SH "OPTIONS"
+.SS "Owner Options"
+.sp
+\fB\-s\fP \fIMANIFEST\fP, \fB\-\-sync\fP \fIMANIFEST\fP
+.RS 4
+Specify extra \fBCargo.toml\fP manifests to workspaces which should also be
+vendored and synced to the output.
+.RE
+.sp
+\fB\-\-no\-delete\fP
+.RS 4
+Don\(cqt delete the "vendor" directory when vendoring, but rather keep all
+existing contents of the vendor directory
+.RE
+.sp
+\fB\-\-respect\-source\-config\fP
+.RS 4
+Instead of ignoring \fB[source]\fP configuration by default in \fB.cargo/config\fP
+read it and use it when downloading crates from crates.io, for example
+.RE
+.SS "Manifest Options"
+.sp
+\fB\-\-manifest\-path\fP \fIPATH\fP
+.RS 4
+Path to the \fBCargo.toml\fP file. By default, Cargo searches in the current
+directory or any parent directory for the \fBCargo.toml\fP file.
+.RE
+.SS "Display Options"
+.sp
+\fB\-v\fP, \fB\-\-verbose\fP
+.RS 4
+Use verbose output. May be specified twice for "very verbose" output which
+includes extra output such as dependency warnings and build script output.
+May also be specified with the \fBterm.verbose\fP
+.URL "https://doc.rust\-lang.org/cargo/reference/config.html" "config value" "."
+.RE
+.sp
+\fB\-q\fP, \fB\-\-quiet\fP
+.RS 4
+No output printed to stdout.
+.RE
+.sp
+\fB\-\-color\fP \fIWHEN\fP
+.RS 4
+Control when colored output is used. Valid values:
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.  sp -1
+.  IP \(bu 2.3
+.\}
+\fBauto\fP (default): Automatically detect if color support is available on the
+terminal.
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.  sp -1
+.  IP \(bu 2.3
+.\}
+\fBalways\fP: Always display colors.
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.  sp -1
+.  IP \(bu 2.3
+.\}
+\fBnever\fP: Never display colors.
+.RE
+.sp
+May also be specified with the \fBterm.color\fP
+.URL "https://doc.rust\-lang.org/cargo/reference/config.html" "config value" "."
+.RE
+.SS "Common Options"
+.sp
+\fB\-h\fP, \fB\-\-help\fP
+.RS 4
+Prints help information.
+.RE
+.sp
+\fB\-Z\fP \fIFLAG\fP...
+.RS 4
+Unstable (nightly\-only) flags to Cargo. Run \fBcargo \-Z help\fP for
+details.
+.RE
+.sp
+\fB\-\-frozen\fP, \fB\-\-locked\fP
+.RS 4
+Either of these flags requires that the \fBCargo.lock\fP file is
+up\-to\-date. If the lock file is missing, or it needs to be updated, Cargo will
+exit with an error. The \fB\-\-frozen\fP flag also prevents Cargo from
+attempting to access the network to determine if it is out\-of\-date.
+.sp
+These may be used in environments where you want to assert that the
+\fBCargo.lock\fP file is up\-to\-date (such as a CI build) or want to avoid network
+access.
+.RE
+.SH "ENVIRONMENT"
+.sp
+See \c
+.URL "https://doc.rust\-lang.org/cargo/reference/environment\-variables.html" "the reference" " "
+for
+details on environment variables that Cargo reads.
+.SH "EXIT STATUS"
+.sp
+0
+.RS 4
+Cargo succeeded.
+.RE
+.sp
+101
+.RS 4
+Cargo failed to complete.
+.RE
+.SH "EXAMPLES"
+.sp
+.RS 4
+.ie n \{\
+\h'-04' 1.\h'+01'\c
+.\}
+.el \{\
+.  sp -1
+.  IP " 1." 4.2
+.\}
+Vendor all dependencies into a local "vendor" folder
+.sp
+.if n .RS 4
+.nf
+cargo vendor
+.fi
+.if n .RE
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04' 2.\h'+01'\c
+.\}
+.el \{\
+.  sp -1
+.  IP " 2." 4.2
+.\}
+Vendor all dependencies into a local "third\-part/vendor" folder
+.sp
+.if n .RS 4
+.nf
+cargo vendor third\-party/vendor
+.fi
+.if n .RE
+.RE
+.sp
+.RS 4
+.ie n \{\
+\h'-04' 3.\h'+01'\c
+.\}
+.el \{\
+.  sp -1
+.  IP " 3." 4.2
+.\}
+Vendor the current workspace as well as another to "vendor"
+.sp
+.if n .RS 4
+.nf
+cargo vendor \-s ../path/to/Cargo.toml
+.fi
+.if n .RE
+.RE
+.SH "SEE ALSO"
+.sp
+\fBcargo\fP(1)
\ No newline at end of file
diff --git a/src/etc/man/cargo.1 b/src/etc/man/cargo.1
index f715eea6f..757510051 100644
--- a/src/etc/man/cargo.1
+++ b/src/etc/man/cargo.1
@@ -487,4 +487,4 @@ See \c
 for issues.
 .SH "SEE ALSO"
 .sp
-\fBrustc\fP(1), \fBrustdoc\fP(1)
\ No newline at end of file
+\fBrustc\fP(1), \fBrustdoc\fP(1)
diff --git a/tests/testsuite/main.rs b/tests/testsuite/main.rs
index 5dcf126ec..031e5c9ce 100644
--- a/tests/testsuite/main.rs
+++ b/tests/testsuite/main.rs
@@ -88,6 +88,7 @@ mod small_fd_limits;
 mod test;
 mod tool_paths;
 mod update;
+mod vendor;
 mod verify_project;
 mod version;
 mod warn_on_failure;
diff --git a/tests/testsuite/support/registry.rs b/tests/testsuite/support/registry.rs
index 023ece913..709f748ce 100644
--- a/tests/testsuite/support/registry.rs
+++ b/tests/testsuite/support/registry.rs
@@ -538,9 +538,7 @@ impl Package {
 }
 
 pub fn cksum(s: &[u8]) -> String {
-    let mut sha = Sha256::new();
-    sha.update(s);
-    hex::encode(&sha.finish())
+    Sha256::new().update(s).finish_hex()
 }
 
 impl Dependency {
diff --git a/tests/testsuite/vendor.rs b/tests/testsuite/vendor.rs
new file mode 100644
index 000000000..aa5e93470
--- /dev/null
+++ b/tests/testsuite/vendor.rs
@@ -0,0 +1,508 @@
+use crate::support::git;
+use crate::support::registry::Package;
+use crate::support::{basic_lib_manifest, project, Project};
+
+#[test]
+fn vendor_simple() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                log = "0.3.5"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    Package::new("log", "0.3.5").publish();
+
+    p.cargo("vendor --respect-source-config").run();
+    let lock = p.read_file("vendor/log/Cargo.toml");
+    assert!(lock.contains("version = \"0.3.5\""));
+
+    add_vendor_config(&p);
+    p.cargo("build").run();
+}
+
+fn add_vendor_config(p: &Project) {
+    p.change_file(
+        ".cargo/config",
+        r#"
+            [source.crates-io]
+            replace-with = 'vendor'
+
+            [source.vendor]
+            directory = 'vendor'
+        "#,
+    );
+}
+
+#[test]
+fn two_versions() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                bitflags = "0.8.0"
+                bar = { path = "bar" }
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .file(
+            "bar/Cargo.toml",
+            r#"
+                [package]
+                name = "bar"
+                version = "0.1.0"
+
+                [dependencies]
+                bitflags = "0.7.0"
+            "#,
+        )
+        .file("bar/src/lib.rs", "")
+        .build();
+
+    Package::new("bitflags", "0.7.0").publish();
+    Package::new("bitflags", "0.8.0").publish();
+
+    p.cargo("vendor --respect-source-config").run();
+
+    let lock = p.read_file("vendor/bitflags/Cargo.toml");
+    assert!(lock.contains("version = \"0.8.0\""));
+    let lock = p.read_file("vendor/bitflags-0.7.0/Cargo.toml");
+    assert!(lock.contains("version = \"0.7.0\""));
+
+    add_vendor_config(&p);
+    p.cargo("build").run();
+}
+
+#[test]
+fn help() {
+    let p = project().build();
+    p.cargo("vendor -h").run();
+}
+
+#[test]
+fn update_versions() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                bitflags = "0.7.0"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    Package::new("bitflags", "0.7.0").publish();
+    Package::new("bitflags", "0.8.0").publish();
+
+    p.cargo("vendor --respect-source-config").run();
+
+    let lock = p.read_file("vendor/bitflags/Cargo.toml");
+    assert!(lock.contains("version = \"0.7.0\""));
+
+    p.change_file(
+        "Cargo.toml",
+        r#"
+            [package]
+            name = "foo"
+            version = "0.1.0"
+
+            [dependencies]
+            bitflags = "0.8.0"
+        "#,
+    );
+    p.cargo("vendor --respect-source-config").run();
+
+    let lock = p.read_file("vendor/bitflags/Cargo.toml");
+    assert!(lock.contains("version = \"0.8.0\""));
+}
+
+#[test]
+fn two_lockfiles() {
+    let p = project()
+        .no_manifest()
+        .file(
+            "foo/Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                bitflags = "=0.7.0"
+            "#,
+        )
+        .file("foo/src/lib.rs", "")
+        .file(
+            "bar/Cargo.toml",
+            r#"
+                [package]
+                name = "bar"
+                version = "0.1.0"
+
+                [dependencies]
+                bitflags = "=0.8.0"
+            "#,
+        )
+        .file("bar/src/lib.rs", "")
+        .build();
+
+    Package::new("bitflags", "0.7.0").publish();
+    Package::new("bitflags", "0.8.0").publish();
+
+    p.cargo("vendor --respect-source-config -s bar/Cargo.toml --manifest-path foo/Cargo.toml")
+        .run();
+
+    let lock = p.read_file("vendor/bitflags/Cargo.toml");
+    assert!(lock.contains("version = \"0.8.0\""));
+    let lock = p.read_file("vendor/bitflags-0.7.0/Cargo.toml");
+    assert!(lock.contains("version = \"0.7.0\""));
+
+    add_vendor_config(&p);
+    p.cargo("build").cwd("foo").run();
+    p.cargo("build").cwd("bar").run();
+}
+
+#[test]
+fn delete_old_crates() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                bitflags = "=0.7.0"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    Package::new("bitflags", "0.7.0").publish();
+    Package::new("log", "0.3.5").publish();
+
+    p.cargo("vendor --respect-source-config").run();
+    p.read_file("vendor/bitflags/Cargo.toml");
+
+    p.change_file(
+        "Cargo.toml",
+        r#"
+            [package]
+            name = "foo"
+            version = "0.1.0"
+
+            [dependencies]
+            log = "=0.3.5"
+        "#,
+    );
+
+    p.cargo("vendor --respect-source-config").run();
+    let lock = p.read_file("vendor/log/Cargo.toml");
+    assert!(lock.contains("version = \"0.3.5\""));
+    assert!(!p.root().join("vendor/bitflags/Cargo.toml").exists());
+}
+
+#[test]
+fn ignore_files() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                url = "1.4.1"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    Package::new("url", "1.4.1")
+        .file("src/lib.rs", "")
+        .file("foo.orig", "")
+        .file(".gitignore", "")
+        .file(".gitattributes", "")
+        .file("foo.rej", "")
+        .publish();
+
+    p.cargo("vendor --respect-source-config").run();
+    let csum = p.read_file("vendor/url/.cargo-checksum.json");
+    assert!(!csum.contains("foo.orig"));
+    assert!(!csum.contains(".gitignore"));
+    assert!(!csum.contains(".gitattributes"));
+    assert!(!csum.contains(".cargo-ok"));
+    assert!(!csum.contains("foo.rej"));
+}
+
+#[test]
+fn included_files_only() {
+    let git = git::new("a", |p| {
+        p.file("Cargo.toml", &basic_lib_manifest("a"))
+            .file("src/lib.rs", "")
+            .file(".gitignore", "a")
+            .file("a/b.md", "")
+    })
+    .unwrap();
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            &format!(
+                r#"
+                    [package]
+                    name = "foo"
+                    version = "0.1.0"
+
+                    [dependencies]
+                    a = {{ git = '{}' }}
+                "#,
+                git.url()
+            ),
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    p.cargo("vendor --respect-source-config").run();
+    let csum = p.read_file("vendor/a/.cargo-checksum.json");
+    assert!(!csum.contains("a/b.md"));
+}
+
+#[test]
+fn dependent_crates_in_crates() {
+    let git = git::new("a", |p| {
+        p.file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "a"
+                version = "0.1.0"
+
+                [dependencies]
+                b = { path = 'b' }
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .file("b/Cargo.toml", &basic_lib_manifest("b"))
+        .file("b/src/lib.rs", "")
+    })
+    .unwrap();
+    let p = project()
+        .file(
+            "Cargo.toml",
+            &format!(
+                r#"
+                    [package]
+                    name = "foo"
+                    version = "0.1.0"
+
+                    [dependencies]
+                    a = {{ git = '{}' }}
+                "#,
+                git.url()
+            ),
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    p.cargo("vendor --respect-source-config").run();
+    p.read_file("vendor/a/.cargo-checksum.json");
+    p.read_file("vendor/b/.cargo-checksum.json");
+}
+
+#[test]
+fn vendoring_git_crates() {
+    let git = git::new("git", |p| {
+        p.file("Cargo.toml", &basic_lib_manifest("serde_derive"))
+            .file("src/lib.rs", "")
+            .file("src/wut.rs", "")
+    })
+    .unwrap();
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            &format!(
+                r#"
+                    [package]
+                    name = "foo"
+                    version = "0.1.0"
+
+                    [dependencies.serde]
+                    version = "0.5.0"
+
+                    [dependencies.serde_derive]
+                    version = "0.5.0"
+
+                    [patch.crates-io]
+                    serde_derive = {{ git = '{}' }}
+                "#,
+                git.url()
+            ),
+        )
+        .file("src/lib.rs", "")
+        .build();
+    Package::new("serde", "0.5.0")
+        .dep("serde_derive", "0.5")
+        .publish();
+    Package::new("serde_derive", "0.5.0").publish();
+
+    p.cargo("vendor --respect-source-config").run();
+    p.read_file("vendor/serde_derive/src/wut.rs");
+
+    add_vendor_config(&p);
+    p.cargo("build").run();
+}
+
+#[test]
+fn git_simple() {
+    let git = git::new("git", |p| {
+        p.file("Cargo.toml", &basic_lib_manifest("a"))
+            .file("src/lib.rs", "")
+    })
+    .unwrap();
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            &format!(
+                r#"
+                    [package]
+                    name = "foo"
+                    version = "0.1.0"
+
+                    [dependencies]
+                    a = {{ git = '{}' }}
+                "#,
+                git.url()
+            ),
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    p.cargo("vendor --respect-source-config").run();
+    let csum = p.read_file("vendor/a/.cargo-checksum.json");
+    assert!(csum.contains("\"package\":null"));
+}
+
+#[test]
+fn git_duplicate() {
+    let git = git::new("a", |p| {
+        p.file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "a"
+                version = "0.1.0"
+
+                [dependencies]
+                b = { path = 'b' }
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .file("b/Cargo.toml", &basic_lib_manifest("b"))
+        .file("b/src/lib.rs", "")
+    })
+    .unwrap();
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            &format!(
+                r#"
+                    [package]
+                    name = "foo"
+                    version = "0.1.0"
+
+                    [dependencies]
+                    a = {{ git = '{}' }}
+                    b = '0.5.0'
+
+                "#,
+                git.url()
+            ),
+        )
+        .file("src/lib.rs", "")
+        .build();
+    Package::new("b", "0.5.0").publish();
+
+    p.cargo("vendor --respect-source-config")
+        .with_stderr(
+            "\
+[UPDATING] [..]
+[UPDATING] [..]
+[DOWNLOADING] [..]
+[DOWNLOADED] [..]
+error: failed to sync
+
+Caused by:
+  found duplicate version of package `b v0.5.0` vendored from two sources:
+
+source 1: [..]
+source 2: [..]
+",
+        )
+        .with_status(101)
+        .run();
+}
+
+#[test]
+fn depend_on_vendor_dir_not_deleted() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                libc = "0.2.30"
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .build();
+
+    Package::new("libc", "0.2.30").publish();
+
+    p.cargo("vendor --respect-source-config").run();
+    assert!(p.root().join("vendor/libc").is_dir());
+
+    p.change_file(
+        "Cargo.toml",
+        r#"
+            [package]
+            name = "foo"
+            version = "0.1.0"
+
+            [dependencies]
+            libc = "0.2.30"
+
+            [patch.crates-io]
+            libc = { path = 'vendor/libc' }
+        "#,
+    );
+
+    p.cargo("vendor --respect-source-config").run();
+    assert!(p.root().join("vendor/libc").is_dir());
+}