Parse rmeta directives coming from rustc

This commit updates Cargo to process JSON directives emitted by rustc
when we're pipelining compilation. In this mode Cargo will attempt to
start subsequent compilations of crates as soon as possible, fully
completing the features of pipelined compilations in Cargo!

src/cargo/core/compiler/mod.rs

@@ -231,12 +231,16 @@ fn rustc<'a, 'cfg>(
     // present it if we can.
     let extract_rendered_errors = if rmeta_produced {
         match cx.bcx.build_config.message_format {
-            MessageFormat::Json => false,
+            MessageFormat::Json => {
+                rustc.arg("-Zemit-directives");
+                false
+            }
             MessageFormat::Human => {
                 rustc
                     .arg("--error-format=json")
                     .arg("--json-rendered=termcolor")
-                    .arg("-Zunstable-options");
+                    .arg("-Zunstable-options")
+                    .arg("-Zemit-directives");
                 true
             }
 
@@ -315,32 +319,20 @@ fn rustc<'a, 'cfg>(
                 mode,
                 &mut |line| on_stdout_line(state, line, package_id, &target),
                 &mut |line| {
-                    on_stderr_line(state, line, package_id, &target, extract_rendered_errors)
+                    on_stderr_line(
+                        state,
+                        line,
+                        package_id,
+                        &target,
+                        extract_rendered_errors,
+                        rmeta_produced,
+                    )
                 },
             )
             .map_err(internal_if_simple_exit_code)
             .chain_err(|| format!("Could not compile `{}`.", name))?;
         }
 
-        // FIXME(rust-lang/rust#58465): this is the whole point of "pipelined
-        // compilation" in Cargo.  We want to, here in this unit, call
-        // `finish_rmeta` as soon as we can which indicates that the metadata
-        // file is emitted by rustc and ready to go. This will start dependency
-        // compilations as soon as possible.
-        //
-        // The compiler doesn't currently actually implement the ability to let
-        // us know, however, when the metadata file is ready to go. It actually
-        // today produces the file very late in compilation, far later than it
-        // would otherwise be able to do!
-        //
-        // In any case this is all covered by the issue above. This is just a
-        // marker for "yes we unconditionally do this today but tomorrow we
-        // should actually read what rustc is doing and execute this at an
-        // appropriate time, ideally long before rustc finishes completely".
-        if rmeta_produced {
-            state.rmeta_produced();
-        }
-
         if do_rename && real_name != crate_name {
             let dst = &outputs[0].path;
             let src = dst.with_file_name(
@@ -698,7 +690,7 @@ fn rustdoc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoResult
         rustdoc
             .exec_with_streaming(
                 &mut |line| on_stdout_line(state, line, package_id, &target),
-                &mut |line| on_stderr_line(state, line, package_id, &target, false),
+                &mut |line| on_stderr_line(state, line, package_id, &target, false, false),
                 false,
             )
             .chain_err(|| format!("Could not document `{}`.", name))?;
@@ -1094,6 +1086,7 @@ fn on_stderr_line(
     package_id: PackageId,
     target: &Target,
     extract_rendered_errors: bool,
+    look_for_metadata_directive: bool,
 ) -> CargoResult<()> {
     // We primarily want to use this function to process JSON messages from
     // rustc. The compiler should always print one JSON message per line, and
@@ -1124,6 +1117,29 @@ fn on_stderr_line(
         }
     }
 
+    // In some modes of execution we will execute rustc with `-Z
+    // emit-directives` to look for metadata files being produced. When this
+    // happens we may be able to start subsequent compilations more quickly than
+    // waiting for an entire compile to finish, possibly using more parallelism
+    // available to complete a compilation session more quickly.
+    //
+    // In these cases look for a matching directive and inform Cargo internally
+    // that a metadata file has been produced.
+    if look_for_metadata_directive {
+        #[derive(serde::Deserialize)]
+        struct CompilerDirective {
+            directive: String,
+        }
+        if let Ok(directive) = serde_json::from_str::<CompilerDirective>(compiler_message.get()) {
+            log::trace!("found directive from rustc: `{}`", directive.directive);
+            if directive.directive.starts_with("metadata file written") {
+                log::debug!("looks like metadata finished early!");
+                state.rmeta_produced();
+                return Ok(())
+            }
+        }
+    }
+
     // And failing all that above we should have a legitimate JSON diagnostic
     // from the compiler, so wrap it in an external Cargo JSON message
     // indicating which package it came from and then emit it.

tests/testsuite/build.rs

@@ -4572,6 +4572,10 @@ Caused by:
 
 #[test]
 fn tricky_pipelining() {
+    if !crate::support::is_nightly() {
+        return;
+    }
+
     let foo = project()
         .file(
             "Cargo.toml",

tests/testsuite/build_script.rs

@@ -2219,7 +2219,6 @@ fn diamond_passes_args_only_once() {
         .build();
 
     p.cargo("build -v")
-        .env("CARGO_BUILD_PIPELINING", "true")
         .with_stderr(
             "\
 [COMPILING] c v0.5.0 ([..]
@@ -2231,7 +2230,7 @@ fn diamond_passes_args_only_once() {
 [COMPILING] a v0.5.0 ([..]
 [RUNNING] `rustc [..]`
 [COMPILING] foo v0.5.0 ([..]
-[RUNNING] `[..]rmeta -L native=test`
+[RUNNING] `[..]rlib -L native=test`
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
 ",
         )