fix(cksum): reject absurdly large SHAKE --length instead of aborting

src/uucore/src/lib/features/checksum/mod.rs

@@ -540,7 +540,7 @@ pub fn digest_reader<T: Read>(
     let output_size = io::copy(reader, &mut digest_writer)? as usize;
     digest_writer.finalize();
 
-    Ok((digest.result(), output_size))
+    Ok((digest.result()?, output_size))
 }
 
 pub enum BlakeLength<'s> {

src/uucore/src/lib/features/sum.rs

@@ -67,13 +67,24 @@ pub trait Digest {
         self.output_bits().div_ceil(8)
     }
 
-    fn result(&mut self) -> DigestOutput {
-        let mut buf: Vec<u8> = vec![0; self.output_bytes()];
+    fn result(&mut self) -> io::Result<DigestOutput> {
+        let mut buf: Vec<u8> = Vec::new();
+        try_reserve_zeroed(&mut buf, self.output_bytes())?;
         self.hash_finalize(&mut buf);
-        DigestOutput::Vec(buf)
+        Ok(DigestOutput::Vec(buf))
     }
 }
 
+/// Grows `buf` to `len` zero bytes, returning an [`io::Error`] with
+/// [`io::ErrorKind::OutOfMemory`] instead of aborting the process if the
+/// allocation can't be satisfied (e.g. an absurdly large `--length`, #12869).
+fn try_reserve_zeroed(buf: &mut Vec<u8>, len: usize) -> io::Result<()> {
+    buf.try_reserve_exact(len)
+        .map_err(|e| io::Error::new(io::ErrorKind::OutOfMemory, e))?;
+    buf.resize(len, 0);
+    Ok(())
+}
+
 /// first element of the tuple is the blake2b state
 /// second is the number of output bits
 pub struct Blake2b {
@@ -246,12 +257,12 @@ impl Digest for Crc {
         out.copy_from_slice(&self.digest.finalize().to_ne_bytes());
     }
 
-    fn result(&mut self) -> DigestOutput {
+    fn result(&mut self) -> io::Result<DigestOutput> {
         let mut out: [u8; 8] = [0; 8];
         self.hash_finalize(&mut out);
 
         let x = u64::from_ne_bytes(out);
-        DigestOutput::Crc((x & (u32::MAX as u64)) as u32)
+        Ok(DigestOutput::Crc((x & (u32::MAX as u64)) as u32))
     }
 
     fn reset(&mut self) {
@@ -297,10 +308,10 @@ impl Digest for CRC32B {
         32
     }
 
-    fn result(&mut self) -> DigestOutput {
+    fn result(&mut self) -> io::Result<DigestOutput> {
         let mut out = [0; 4];
         self.hash_finalize(&mut out);
-        DigestOutput::Crc(u32::from_be_bytes(out))
+        Ok(DigestOutput::Crc(u32::from_be_bytes(out)))
     }
 }
 
@@ -321,10 +332,10 @@ impl Digest for Bsd {
         out.copy_from_slice(&self.state.to_ne_bytes());
     }
 
-    fn result(&mut self) -> DigestOutput {
+    fn result(&mut self) -> io::Result<DigestOutput> {
         let mut out = [0; 2];
         self.hash_finalize(&mut out);
-        DigestOutput::U16(self.state)
+        Ok(DigestOutput::U16(self.state))
     }
 
     fn reset(&mut self) {
@@ -354,10 +365,10 @@ impl Digest for SysV {
         out.copy_from_slice(&(self.state as u16).to_ne_bytes());
     }
 
-    fn result(&mut self) -> DigestOutput {
+    fn result(&mut self) -> io::Result<DigestOutput> {
         let mut out = [0; 2];
         self.hash_finalize(&mut out);
-        DigestOutput::U16((self.state & (u16::MAX as u32)) as u16)
+        Ok(DigestOutput::U16((self.state & (u16::MAX as u32)) as u16))
     }
 
     fn reset(&mut self) {
@@ -442,10 +453,11 @@ macro_rules! impl_digest_shake {
                 self.bit_size
             }
 
-            fn result(&mut self) -> DigestOutput {
-                let mut bytes = vec![0; self.output_bits().div_ceil(8)];
+            fn result(&mut self) -> io::Result<DigestOutput> {
+                let mut bytes = Vec::new();
+                try_reserve_zeroed(&mut bytes, self.output_bits().div_ceil(8))?;
                 self.hash_finalize(&mut bytes);
-                DigestOutput::Vec(bytes)
+                Ok(DigestOutput::Vec(bytes))
             }
         }
     };

tests/by-util/test_cksum.rs

@@ -3315,6 +3315,22 @@ fn test_check_shake256_no_length() {
         .stderr_only("cksum: 'standard input': no properly formatted checksum lines found\n");
 }
 
+#[test]
+fn test_shake_extremely_large_length_does_not_abort() {
+    // Regression test for #12869: an absurdly large `--length` used to
+    // trigger an unguarded allocation that aborts the process instead of
+    // returning a normal error.
+    new_ucmd!()
+        .args(&[
+            "--algorithm",
+            "shake128",
+            "--length",
+            "10011111117721172727",
+        ])
+        .pipe_in("")
+        .fails();
+}
+
 #[template]
 #[rstest]
 #[case::no_length(