Merge of #626 - Support zstd-compressed ELF sections.

zstd has been introduced as an alternative to zlib for the compression
of debug sections.[^0] Toolchain support is widely present at this time
but lack of support in backtrace is a severe limitation on using this
feature in Rust programs.

This uses a Rust reimplementation of zstd (the ruzstd crate). This has
the benefit of simplifying the build process, but this crate is less
used and admittedly slower than the zstd crate that binds to the C
libzstd.

[^0]: https://maskray.me/blog/2022-09-09-zstd-compressed-debug-sections

Author: workingjubilee

.github/workflows/main.yml

@@ -22,6 +22,12 @@ jobs:
             rust: beta
           - os: ubuntu-20.04
             rust: nightly
+          - os: ubuntu-24.04
+            rust: stable
+          - os: ubuntu-24.04
+            rust: beta
+          - os: ubuntu-24.04
+            rust: nightly
           - os: macos-latest
             rust: stable
           - os: macos-latest
@@ -55,6 +61,12 @@ jobs:
         shell: bash
         if: contains(matrix.rust, 'i686')
 
+      # Starting with Ubuntu 22.04 libc6-dbg is needed.
+      - name: Install libc debug info
+        run: sudo apt-get install -y libc6-dbg
+        shell: bash
+        if: contains(matrix.os, 'ubuntu-24.04')
+
       - name: Enable collapse_debuginfo based on version
         run: echo RUSTFLAGS="--cfg dbginfo=\"collapsible\" $RUSTFLAGS" >> $GITHUB_ENV
         shell: bash
@@ -80,6 +92,11 @@ jobs:
         if: contains(matrix.os, 'ubuntu')
         env:
           RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zlib"
+      - run: cargo test
+        if: contains(matrix.os, 'ubuntu-24.04') ||
+            (contains(matrix.os, 'ubuntu') && contains(matrix.rust, 'nightly'))
+        env:
+          RUSTFLAGS: "-C link-arg=-Wl,--compress-debug-sections=zstd"
 
       # Test that, on macOS, packed/unpacked debuginfo both work
       - run: cargo clean && cargo test
@@ -248,7 +265,7 @@ jobs:
         with:
           submodules: true
       - name: Install Rust
-        run: rustup update 1.65.0 --no-self-update && rustup default 1.65.0
+        run: rustup update 1.73.0 --no-self-update && rustup default 1.73.0
       - run: cargo build
 
   miri:

Cargo.lock

@@ -43,6 +43,7 @@ windows-targets = "0.52.6"
 
 [target.'cfg(not(all(windows, target_env = "msvc", not(target_vendor = "uwp"))))'.dependencies]
 miniz_oxide = { version = "0.8", default-features = false }
+ruzstd = { version = "0.7.2", default-features = false }
 addr2line = { version = "0.24.0", default-features = false }
 libc = { version = "0.2.156", default-features = false }
 

Cargo.toml

@@ -18,6 +18,7 @@ libc = { version = "0.2.156", default-features = false }
 
 [target.'cfg(not(all(windows, target_env = "msvc", not(target_vendor = "uwp"))))'.dependencies]
 miniz_oxide = { version = "0.8", optional = true, default-features = false }
+ruzstd = { version = "0.7.2", optional = true, default-features = false }
 addr2line = { version = "0.24.0", optional = true, default-features = false }
 
 [target.'cfg(not(all(windows, target_env = "msvc", not(target_vendor = "uwp"))))'.dependencies.object]
@@ -31,7 +32,7 @@ windows-targets = "0.52.6"
 
 [features]
 default = ['backtrace']
-backtrace = ['addr2line', 'miniz_oxide', 'object']
+backtrace = ['addr2line', 'miniz_oxide', 'object', 'ruzstd']
 std = []
 
 [lints.rust]

crates/as-if-std/Cargo.toml

@@ -9,7 +9,9 @@ use super::{gimli, Context, Endian, EndianSlice, Mapping, Stash, Vec};
 use alloc::sync::Arc;
 use core::convert::{TryFrom, TryInto};
 use core::str;
-use object::elf::{ELFCOMPRESS_ZLIB, ELF_NOTE_GNU, NT_GNU_BUILD_ID, SHF_COMPRESSED};
+use object::elf::{
+    ELFCOMPRESS_ZLIB, ELFCOMPRESS_ZSTD, ELF_NOTE_GNU, NT_GNU_BUILD_ID, SHF_COMPRESSED,
+};
 use object::read::elf::{CompressionHeader, FileHeader, SectionHeader, SectionTable, Sym};
 use object::read::StringTable;
 use object::{BigEndian, Bytes, NativeEndian};
@@ -213,22 +215,30 @@ impl<'a> Object<'a> {
             let mut data = Bytes(section.data(self.endian, self.data).ok()?);
 
             // Check for DWARF-standard (gABI) compression, i.e., as generated
-            // by ld's `--compress-debug-sections=zlib-gabi` flag.
+            // by ld's `--compress-debug-sections=zlib-gabi` and
+            // `--compress-debug-sections=zstd` flags.
             let flags: u64 = section.sh_flags(self.endian).into();
             if (flags & u64::from(SHF_COMPRESSED)) == 0 {
                 // Not compressed.
                 return Some(data.0);
             }
 
             let header = data.read::<<Elf as FileHeader>::CompressionHeader>().ok()?;
-            if header.ch_type(self.endian) != ELFCOMPRESS_ZLIB {
-                // Zlib compression is the only known type.
-                return None;
+            match header.ch_type(self.endian) {
+                ELFCOMPRESS_ZLIB => {
+                    let size = usize::try_from(header.ch_size(self.endian)).ok()?;
+                    let buf = stash.allocate(size);
+                    decompress_zlib(data.0, buf)?;
+                    return Some(buf);
+                }
+                ELFCOMPRESS_ZSTD => {
+                    let size = usize::try_from(header.ch_size(self.endian)).ok()?;
+                    let buf = stash.allocate(size);
+                    decompress_zstd(data.0, buf)?;
+                    return Some(buf);
+                }
+                _ => return None, // Unknown compression type.
             }
-            let size = usize::try_from(header.ch_size(self.endian)).ok()?;
-            let buf = stash.allocate(size);
-            decompress_zlib(data.0, buf)?;
-            return Some(buf);
         }
 
         // Check for the nonstandard GNU compression format, i.e., as generated
@@ -347,6 +357,40 @@ fn decompress_zlib(input: &[u8], output: &mut [u8]) -> Option<()> {
     }
 }
 
+fn decompress_zstd(mut input: &[u8], mut output: &mut [u8]) -> Option<()> {
+    use ruzstd::frame::ReadFrameHeaderError;
+    use ruzstd::frame_decoder::FrameDecoderError;
+    use ruzstd::io::Read;
+
+    while !input.is_empty() {
+        let mut decoder = match ruzstd::StreamingDecoder::new(&mut input) {
+            Ok(decoder) => decoder,
+            Err(FrameDecoderError::ReadFrameHeaderError(ReadFrameHeaderError::SkipFrame {
+                length,
+                ..
+            })) => {
+                input = &input.get(length as usize..)?;
+                continue;
+            }
+            Err(_) => return None,
+        };
+        loop {
+            let bytes_written = decoder.read(output).ok()?;
+            if bytes_written == 0 {
+                break;
+            }
+            output = &mut output[bytes_written..];
+        }
+    }
+
+    if !output.is_empty() {
+        // Lengths didn't match, something is wrong.
+        return None;
+    }
+
+    Some(())
+}
+
 const DEBUG_PATH: &[u8] = b"/usr/lib/debug";
 
 fn debug_path_exists() -> bool {