Heap Out-of-Bounds Write in libhevc Encoder

[foxllb]

libhevc.tar.gz

Summary

If a caller enables decoded picture hash SEI in the libhevc encoder, the
stock hevcenc path can perform a heap out-of-bounds write while generating
suffix SEI data.

Details

The root cause is an incorrect bitstream space calculation when
emulation-prevention bytes may be inserted.

In encoder/ihevce_bitstream.c, ihevce_put_bits() checks space as if a
32-bit flush can consume only 4 bytes:

if((u4_strm_buf_offset + (WORD_SIZE >> 3)) >= u4_max_strm_size)
{
    return (IHEVCE_BITSTREAM_BUFFER_OVERFLOW);
}

for(i = WORD_SIZE; i > 0; i -= 8)
{
    UWORD8 u1_next_byte = (u4_cur_word >> (i - 8)) & 0xFF;
    PUTBYTE_EPB(pu1_strm_buf, u4_strm_buf_offset, u1_next_byte, zero_run);
}

The actual write path uses PUTBYTE_EPB() from encoder/ihevce_bitstream.h:

#define PUTBYTE_EPB(ptr, off, byte, zero_run) \
    { \
        if(INSERT_EPB(zero_run, byte)) \
        { \
            ptr[off] = EPB_BYTE; \
            off++; \
            zero_run = 0; \
        } \
        ptr[off] = byte; \
        off++; \
        zero_run = byte ? 0 : zero_run + 1; \
    }

This means a 32-bit flush may consume up to 6 bytes, not 4. The source
already documents that this corner case is not handled.

After the stream offset has been advanced too far, the next SEI NAL start code
write in encoder/ihevce_bitstream.c performs the heap out-of-bounds write:

if((u4_strm_buf_offset + 4) > ps_bitstrm->u4_max_strm_size)
{
    return (IHEVCE_BITSTREAM_BUFFER_OVERFLOW);
}

if(insert_leading_zero_8bits)
{
    pu1_strm_buf[u4_strm_buf_offset] = 0x00;
    u4_strm_buf_offset++;
}

pu1_strm_buf[u4_strm_buf_offset] = 0x00;
u4_strm_buf_offset++;

pu1_strm_buf[u4_strm_buf_offset] = 0x00;
u4_strm_buf_offset++;

pu1_strm_buf[u4_strm_buf_offset] = 0x01;
u4_strm_buf_offset++;

The public sample encoder accepts --sei_hash_flags and stores it into
i4_decoded_pic_hash_sei_flag:

case SEI_HASH_FLAGS:
    sscanf(value, "%d", &i4_value);
    ps_static_prms->s_out_strm_prms.i4_decoded_pic_hash_sei_flag = i4_value;
    break;

When the PoC sets this option to 3, suffix hash SEI generation reaches the
start-code write through encoder/ihevce_encode_header_sei_vui.c:

/* Insert Start Code */
return_status = ihevce_put_nal_start_code_prefix(ps_bitstrm, 1);

By default, the plugin keeps this feature disabled:

ps_params->s_out_strm_prms.i4_decoded_pic_hash_sei_flag = 0;

So the issue requires an explicit caller-controlled encoder option, but once it
is enabled, it is reachable through the stock hevcenc interface.

PoC

Repository root:

cd /root/code/libhevc

Run the default PoC:

bash test/poc/run_asan_heap_oob_write.sh

Run the alternate PoC:

bash test/poc/run_asan_heap_oob_write.sh test/poc/hevcenc-512-4f-hash3-qp24-p3-intra.cfg

The helper script:

1. Configures an ASan build in build-asan-poc/
2. Passes -DLIBHEVC_ENABLE_GTEST=OFF
3. Builds the stock hevcenc target
4. Runs hevcenc from the repository root with the PoC config

PoC files:

• test/poc/run_asan_heap_oob_write.sh
• test/poc/hevcenc-512-4f-hash3.cfg
• test/poc/hevcenc-512-4f-hash3-qp24-p3-intra.cfg
• test/poc/api-crash-512x512-4f-hash3.yuv

Expected result:

• ASan reports heap-buffer-overflow
• the top write is in ihevce_put_nal_start_code_prefix()

Representative ASan output:

=================================================================
ERROR: AddressSanitizer: heap-buffer-overflow
WRITE of size 1 at the end of the output heap buffer
    #0 in ihevce_put_nal_start_code_prefix
    #1 in ihevce_generate_sei
    #2 in ihevce_entropy_encode_frame
...
SUMMARY: AddressSanitizer: heap-buffer-overflow in ihevce_put_nal_start_code_prefix

Impact

This is a heap out-of-bounds write in the libhevc encoder.

The directly impacted consumer is any integration that exposes the libhevc
encoding path and allows a caller to enable decoded picture hash SEI. The
current PoC proves the issue through the public hevcenc binary and its
documented configuration interface.

Patch

Patch file saved locally:

• libhevc_heap_oob_write_fix.patch

diff --git a/encoder/ihevce_bitstream.c b/encoder/ihevce_bitstream.c
index f48b0cd..4391412 100644
--- a/encoder/ihevce_bitstream.c
+++ b/encoder/ihevce_bitstream.c
@@ -61,6 +61,27 @@
 /*****************************************************************************/
 /* Function Definitions                                                      */
 /*****************************************************************************/
+static UWORD32 ihevce_get_worst_case_epb_bytes(WORD32 zero_run, UWORD32 u4_num_payload_bytes)
+{
+    UWORD32 i;
+    UWORD32 u4_num_epb_bytes = 0;
+
+    ASSERT((zero_run >= 0) && (zero_run <= EPB_ZERO_BYTES));
+
+    for(i = 0; i < u4_num_payload_bytes; i++)
+    {
+        if(EPB_ZERO_BYTES == zero_run)
+        {
+            u4_num_epb_bytes++;
+            zero_run = 0;
+        }
+
+        zero_run++;
+    }
+
+    return u4_num_epb_bytes;
+}
+
 /**
 ******************************************************************************
 *
@@ -177,14 +198,15 @@ IHEVCE_ERROR_T ihevce_put_bits(bitstrm_t *ps_bitstrm, UWORD32 u4_code_val, WORD3
         WORD32 zero_run = ps_bitstrm->i4_zero_bytes_run;
 
         UWORD8 *pu1_strm_buf = ps_bitstrm->pu1_strm_buffer;
+        UWORD32 u4_bytes_to_flush = WORD_SIZE >> 3;
+        UWORD32 u4_max_bytes_to_write;
 
         WORD32 i, rem_bits = (code_len - bits_left_in_cw);
 
-        /*********************************************************************/
-        /* Bitstream overflow check                                          */
-        /* NOTE: corner case of epb bytes (max 2 for 32bit word) not handled */
-        /*********************************************************************/
-        if((u4_strm_buf_offset + (WORD_SIZE >> 3)) >= u4_max_strm_size)
+        u4_max_bytes_to_write =
+            u4_bytes_to_flush + ihevce_get_worst_case_epb_bytes(zero_run, u4_bytes_to_flush);
+
+        if((u4_strm_buf_offset + u4_max_bytes_to_write) > u4_max_strm_size)
         {
             /* return without corrupting the buffer beyond its size */
             return (IHEVCE_BITSTREAM_BUFFER_OVERFLOW);
@@ -273,12 +295,13 @@ IHEVCE_ERROR_T ihevce_put_rbsp_trailing_bits(bitstrm_t *ps_bitstrm)
     UWORD32 u4_max_strm_size = ps_bitstrm->u4_max_strm_size;
     WORD32 zero_run = ps_bitstrm->i4_zero_bytes_run;
     UWORD8 *pu1_strm_buf = ps_bitstrm->pu1_strm_buffer;
+    UWORD32 u4_bytes_to_flush = (WORD_SIZE >> 3) - bytes_left_in_cw;
+    UWORD32 u4_max_bytes_to_write;
+
+    u4_max_bytes_to_write =
+        u4_bytes_to_flush + ihevce_get_worst_case_epb_bytes(zero_run, u4_bytes_to_flush);
 
-    /*********************************************************************/
-    /* Bitstream overflow check                                          */
-    /* NOTE: corner case of epb bytes (max 2 for 32bit word) not handled */
-    /*********************************************************************/
-    if((u4_strm_buf_offset + (WORD_SIZE >> 3) - bytes_left_in_cw) >= u4_max_strm_size)
+    if((u4_strm_buf_offset + u4_max_bytes_to_write) > u4_max_strm_size)
     {
         /* return without corrupting the buffer beyond its size */
         return (IHEVCE_BITSTREAM_BUFFER_OVERFLOW);
diff --git a/encoder/ihevce_entropy_interface.c b/encoder/ihevce_entropy_interface.c
index 6014f8d..0a20573 100644
--- a/encoder/ihevce_entropy_interface.c
+++ b/encoder/ihevce_entropy_interface.c
@@ -591,7 +591,12 @@ WORD32 ihevce_entropy_encode_frame(
             }
 
             /* Updating bytes generated and Updating strm_buffer pointer */
+            if(i4_bytes_generated > out_buf_size)
+            {
+                return (ret | IHEVCE_BITSTREAM_BUFFER_OVERFLOW);
+            }
             ps_curr_out->i4_bytes_generated += i4_bytes_generated;
+            out_buf_size -= i4_bytes_generated;
 
             /* Re-Initialize the bitstream engine after each tile or slice */
             ihevce_bitstrm_init(

[foxllb]

Has anyone handled this issue? @harishdm @ram-mohan

[ram-mohan]

Thank you for the bug. I will go through the change and submit it once every thing is ok.