rustdoc-json: Postcard output

Cargo.lock

@@ -630,6 +630,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "cobs"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"
+
 [[package]]
 name = "collect-license-metadata"
 version = "0.1.0"
@@ -1096,6 +1102,18 @@ dependencies = [
  "stable_deref_trait",
 ]
 
+[[package]]
+name = "embedded-io"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
+
+[[package]]
+name = "embedded-io"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
+
 [[package]]
 name = "ena"
 version = "0.14.3"
@@ -1992,8 +2010,10 @@ name = "jsondoclint"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "camino",
  "clap",
  "fs-err",
+ "postcard",
  "rustc-hash 2.1.1",
  "rustdoc-json-types",
  "serde",
@@ -2821,6 +2841,18 @@ dependencies = [
  "portable-atomic",
 ]
 
+[[package]]
+name = "postcard"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "170a2601f67cc9dba8edd8c4870b15f71a6a2dc196daec8c83f72b59dff628a8"
+dependencies = [
+ "cobs",
+ "embedded-io 0.4.0",
+ "embedded-io 0.6.1",
+ "serde",
+]
+
 [[package]]
 name = "potential_utf"
 version = "0.1.2"
@@ -4661,6 +4693,7 @@ dependencies = [
  "indexmap",
  "itertools",
  "minifier",
+ "postcard",
  "pulldown-cmark-escape",
  "regex",
  "rustdoc-json-types",

src/librustdoc/Cargo.toml

@@ -25,6 +25,7 @@ tracing = "0.1"
 tracing-tree = "0.3.0"
 threadpool = "1.8.1"
 unicode-segmentation = "1.9"
+postcard = { version = "1.1.1", default-features = false, features = ["use-std"] }
 
 [dependencies.tracing-subscriber]
 version = "0.3.3"

src/librustdoc/config.rs

@@ -31,14 +31,15 @@ use crate::{html, opts, theme};
 #[derive(Clone, Copy, PartialEq, Eq, Debug, Default)]
 pub(crate) enum OutputFormat {
     Json,
+    Postcard,
     #[default]
     Html,
     Doctest,
 }
 
 impl OutputFormat {
     pub(crate) fn is_json(&self) -> bool {
-        matches!(self, OutputFormat::Json)
+        matches!(self, OutputFormat::Json | OutputFormat::Postcard)
     }
 }
 
@@ -50,6 +51,7 @@ impl TryFrom<&str> for OutputFormat {
             "json" => Ok(OutputFormat::Json),
             "html" => Ok(OutputFormat::Html),
             "doctest" => Ok(OutputFormat::Doctest),
+            "postcard" => Ok(OutputFormat::Postcard),
             _ => Err(format!("unknown output format `{value}`")),
         }
     }
@@ -305,6 +307,8 @@ pub(crate) struct RenderOptions {
     pub(crate) parts_out_dir: Option<PathToParts>,
     /// disable minification of CSS/JS
     pub(crate) disable_minification: bool,
+
+    pub(crate) output_format: OutputFormat,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -488,7 +492,7 @@ impl Options {
             // If `-Zunstable-options` is used, nothing to check after this point.
             (_, false, true) => {}
             (None | Some(OutputFormat::Html), false, _) => {}
-            (Some(OutputFormat::Json), false, false) => {
+            (Some(OutputFormat::Json | OutputFormat::Postcard), false, false) => {
                 dcx.fatal(
                     "the -Z unstable-options flag must be passed to enable --output-format for documentation generation (see https://github.com/rust-lang/rust/issues/76578)",
                 );
@@ -886,6 +890,7 @@ impl Options {
             include_parts_dir,
             parts_out_dir,
             disable_minification,
+            output_format,
         };
         Some((input, options, render_options))
     }

src/librustdoc/json/mod.rs

@@ -37,6 +37,12 @@ use crate::formats::cache::Cache;
 use crate::json::conversions::IntoJson;
 use crate::{clean, try_err};
 
+#[derive(Clone, Copy, Debug)]
+pub(crate) enum OutputFormat {
+    Json,
+    Postcard,
+}
+
 pub(crate) struct JsonRenderer<'tcx> {
     tcx: TyCtxt<'tcx>,
     /// A mapping of IDs that contains all local items for this crate which gets output as a top
@@ -49,6 +55,7 @@ pub(crate) struct JsonRenderer<'tcx> {
     cache: Rc<Cache>,
     imported_items: DefIdSet,
     id_interner: RefCell<ids::IdInterner>,
+    output_format: OutputFormat,
 }
 
 impl<'tcx> JsonRenderer<'tcx> {
@@ -114,10 +121,30 @@ impl<'tcx> JsonRenderer<'tcx> {
         path: &str,
     ) -> Result<(), Error> {
         self.sess().time("rustdoc_json_serialize_and_write", || {
-            try_err!(
-                serde_json::ser::to_writer(&mut writer, &output_crate).map_err(|e| e.to_string()),
-                path
-            );
+            match self.output_format {
+                OutputFormat::Json => {
+                    try_err!(
+                        serde_json::ser::to_writer(&mut writer, &output_crate)
+                            .map_err(|e| e.to_string()),
+                        path
+                    );
+                }
+                OutputFormat::Postcard => {
+                    let output = (
+                        rustdoc_json_types::postcard::Header {
+                            magic: rustdoc_json_types::postcard::MAGIC,
+                            format_version: rustdoc_json_types::FORMAT_VERSION,
+                        },
+                        output_crate,
+                    );
+
+                    try_err!(
+                        postcard::to_io(&output, &mut writer).map_err(|e| e.to_string()),
+                        path
+                    );
+                }
+            }
+
             try_err!(writer.flush(), path);
             Ok(())
         })
@@ -201,6 +228,13 @@ impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> {
                 out_dir: if options.output_to_stdout { None } else { Some(options.output) },
                 cache: Rc::new(cache),
                 imported_items,
+                output_format: match options.output_format {
+                    crate::config::OutputFormat::Json => OutputFormat::Json,
+                    crate::config::OutputFormat::Postcard => OutputFormat::Postcard,
+                    crate::config::OutputFormat::Html | crate::config::OutputFormat::Doctest => {
+                        unreachable!()
+                    }
+                },
                 id_interner: Default::default(),
             },
             krate,
@@ -366,7 +400,10 @@ impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> {
 
             let mut p = out_dir.clone();
             p.push(output_crate.index.get(&output_crate.root).unwrap().name.clone().unwrap());
-            p.set_extension("json");
+            p.set_extension(match self.output_format {
+                OutputFormat::Json => "json",
+                OutputFormat::Postcard => "postcard",
+            });
 
             self.serialize_and_write(
                 output_crate,

src/librustdoc/lib.rs

@@ -922,9 +922,10 @@ fn main_args(early_dcx: &mut EarlyDiagCtxt, at_args: &[String]) {
                 config::OutputFormat::Html => sess.time("render_html", || {
                     run_renderer::<html::render::Context<'_>>(krate, render_opts, cache, tcx)
                 }),
-                config::OutputFormat::Json => sess.time("render_json", || {
-                    run_renderer::<json::JsonRenderer<'_>>(krate, render_opts, cache, tcx)
-                }),
+                config::OutputFormat::Json | config::OutputFormat::Postcard => sess
+                    .time("render_json", || {
+                        run_renderer::<json::JsonRenderer<'_>>(krate, render_opts, cache, tcx)
+                    }),
                 // Already handled above with doctest runners.
                 config::OutputFormat::Doctest => unreachable!(),
             }

src/rustdoc-json-types/lib.rs

@@ -32,6 +32,19 @@ pub type FxHashMap<K, V> = HashMap<K, V>; // re-export for use in src/librustdoc
 /// Consuming code should assert that this value matches the format version(s) that it supports.
 pub const FORMAT_VERSION: u32 = 46;
 
+pub mod postcard {
+
+    pub type Magic = [u8; 22];
+    pub const MAGIC: Magic = *b"\x00\xFFRustdocJsonPostcard\xFF";
+
+    #[derive(Clone, Debug, PartialEq, Eq, serde_derive::Serialize, serde_derive::Deserialize)]
+    pub struct Header {
+        // Order here matters
+        pub magic: Magic,
+        pub format_version: u32,
+    }
+}
+
 /// The root of the emitted JSON blob.
 ///
 /// It contains all type/documentation information

src/rustdoc-json-types/tests.rs

@@ -38,3 +38,11 @@ fn test_union_info_roundtrip() {
     let decoded: ItemEnum = bincode::deserialize(&encoded).unwrap();
     assert_eq!(u, decoded);
 }
+
+#[test]
+fn magic_never_dies() {
+    // Extra check to make sure that the postcard magic header never changes.
+    // Don't change this value
+    assert_eq!(crate::postcard::MAGIC, *b"\x00\xFFRustdocJsonPostcard\xFF");
+    // Don't change that value
+}

src/tools/compiletest/src/runtest.rs

@@ -245,6 +245,13 @@ enum Emit {
     LinkArgsAsm,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum DocKind {
+    Html,
+    Json,
+    Postcard,
+}
+
 impl<'test> TestCx<'test> {
     /// Code executed for each revision in turn (or, if there are no
     /// revisions, exactly once, with revision == None).
@@ -861,8 +868,15 @@ impl<'test> TestCx<'test> {
 
     /// `root_out_dir` and `root_testpaths` refer to the parameters of the actual test being run.
     /// Auxiliaries, no matter how deep, have the same root_out_dir and root_testpaths.
-    fn document(&self, root_out_dir: &Utf8Path, root_testpaths: &TestPaths) -> ProcRes {
+    fn document(
+        &self,
+        root_out_dir: &Utf8Path,
+        root_testpaths: &TestPaths,
+        kind: DocKind,
+    ) -> ProcRes {
         if self.props.build_aux_docs {
+            assert_eq!(kind, DocKind::Html, "build-aux-docs doesn't make sense for rustdoc json");
+
             for rel_ab in &self.props.aux.builds {
                 let aux_testpaths = self.compute_aux_test_paths(root_testpaths, rel_ab);
                 let props_for_aux =
@@ -877,7 +891,7 @@ impl<'test> TestCx<'test> {
                 create_dir_all(aux_cx.output_base_dir()).unwrap();
                 // use root_testpaths here, because aux-builds should have the
                 // same --out-dir and auxiliary directory.
-                let auxres = aux_cx.document(&root_out_dir, root_testpaths);
+                let auxres = aux_cx.document(&root_out_dir, root_testpaths, kind);
                 if !auxres.status.success() {
                     return auxres;
                 }
@@ -922,8 +936,14 @@ impl<'test> TestCx<'test> {
             .args(&self.props.compile_flags)
             .args(&self.props.doc_flags);
 
-        if self.config.mode == RustdocJson {
-            rustdoc.arg("--output-format").arg("json").arg("-Zunstable-options");
+        match kind {
+            DocKind::Html => {}
+            DocKind::Json => {
+                rustdoc.arg("--output-format").arg("json").arg("-Zunstable-options");
+            }
+            DocKind::Postcard => {
+                rustdoc.arg("--output-format").arg("postcard").arg("-Zunstable-options");
+            }
         }
 
         if let Some(ref linker) = self.config.target_linker {
@@ -1992,7 +2012,7 @@ impl<'test> TestCx<'test> {
         let aux_dir = new_rustdoc.aux_output_dir();
         new_rustdoc.build_all_auxiliary(&new_rustdoc.testpaths, &aux_dir, &mut rustc);
 
-        let proc_res = new_rustdoc.document(&compare_dir, &new_rustdoc.testpaths);
+        let proc_res = new_rustdoc.document(&compare_dir, &new_rustdoc.testpaths, DocKind::Html);
         if !proc_res.status.success() {
             eprintln!("failed to run nightly rustdoc");
             return;

src/tools/compiletest/src/runtest/js_doc.rs

@@ -1,13 +1,13 @@
 use std::process::Command;
 
-use super::TestCx;
+use super::{DocKind, TestCx};
 
 impl TestCx<'_> {
     pub(super) fn run_rustdoc_js_test(&self) {
         if let Some(nodejs) = &self.config.nodejs {
             let out_dir = self.output_base_dir();
 
-            self.document(&out_dir, &self.testpaths);
+            self.document(&out_dir, &self.testpaths, DocKind::Html);
 
             let file_stem = self.testpaths.file.file_stem().expect("no file stem");
             let res = self.run_command_to_procres(

src/tools/compiletest/src/runtest/rustdoc.rs

@@ -1,6 +1,7 @@
 use std::process::Command;
 
 use super::{TestCx, remove_and_create_dir_all};
+use crate::runtest::DocKind;
 
 impl TestCx<'_> {
     pub(super) fn run_rustdoc_test(&self) {
@@ -11,7 +12,7 @@ impl TestCx<'_> {
             panic!("failed to remove and recreate output directory `{out_dir}`: {e}")
         });
 
-        let proc_res = self.document(&out_dir, &self.testpaths);
+        let proc_res = self.document(&out_dir, &self.testpaths, DocKind::Html);
         if !proc_res.status.success() {
             self.fatal_proc_rec("rustdoc failed!", &proc_res);
         }

src/tools/compiletest/src/runtest/rustdoc_json.rs

@@ -1,6 +1,7 @@
 use std::process::Command;
 
 use super::{TestCx, remove_and_create_dir_all};
+use crate::runtest::DocKind;
 
 impl TestCx<'_> {
     pub(super) fn run_rustdoc_json_test(&self) {
@@ -13,7 +14,11 @@ impl TestCx<'_> {
             panic!("failed to remove and recreate output directory `{out_dir}`: {e}")
         });
 
-        let proc_res = self.document(&out_dir, &self.testpaths);
+        let proc_res = self.document(&out_dir, &self.testpaths, DocKind::Json);
+        if !proc_res.status.success() {
+            self.fatal_proc_rec("rustdoc failed!", &proc_res);
+        }
+        let proc_res = self.document(&out_dir, &self.testpaths, DocKind::Postcard);
         if !proc_res.status.success() {
             self.fatal_proc_rec("rustdoc failed!", &proc_res);
         }
@@ -35,11 +40,11 @@ impl TestCx<'_> {
             })
         }
 
-        let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap());
-        json_out.set_extension("json");
+        let postcard_out = json_out.with_extension("postcard");
 
         let res = self.run_command_to_procres(
-            Command::new(self.config.jsondoclint_path.as_ref().unwrap()).arg(&json_out),
+            Command::new(self.config.jsondoclint_path.as_ref().unwrap())
+                .args([&json_out, &postcard_out]),
         );
 
         if !res.status.success() {