Skip to content

fix scalefactor delta overflows to comply with AAC spec (ISO 14496-3) #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fabiangreffrath merged 2 commits into from
Apr 13, 2026
Merged

fix scalefactor delta overflows to comply with AAC spec (ISO 14496-3) #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6fa44fb
fix scalefactor delta overflows to comply with AAC spec (ISO 14496-3)
nschimme Apr 10, 2026
e4975fc
quantizer: Implement peak-clamping to prevent Huffman Book 11 overflow
nschimme Apr 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions libfaac/frame.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "channels.h"
#include "bitstream.h"
#include "filtbank.h"
#include "quantize.h"
#include "util.h"
#include "tns.h"
#include "stereo.h"
Expand Down Expand Up @@ -196,8 +197,8 @@ int FAACAPI faacEncSetConfiguration(faacEncHandle hpEncoder,
if (!config->quantqual)
{
config->quantqual = (faac_real)config->bitRate * hEncoder->numChannels / 1280;
if (config->quantqual > 100)
config->quantqual = (config->quantqual - 100) * 3.0 + 100;
if (config->quantqual > DEFQUAL)
config->quantqual = (config->quantqual - DEFQUAL) * 3.0 + DEFQUAL;
}
}

Expand Down
35 changes: 13 additions & 22 deletions libfaac/huff2.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ static int escape(int x, int *code)
int preflen = 0;
int base = 32;

if (x >= 8192)
if (x > MAX_HUFF_ESC_VAL)
{
fprintf(stderr, "%s(%d): x_quant >= 8192\n", __FILE__, __LINE__);
fprintf(stderr, "%s(%d): x_quant > %d\n", __FILE__, __LINE__, MAX_HUFF_ESC_VAL);
return 0;
}

Expand Down Expand Up @@ -430,7 +430,7 @@ int writesf(CoderInfo *coder, BitStream *stream, int write)

lastsf = coder->global_gain;
lastis = 0;
lastpns = coder->global_gain - 90;
lastpns = coder->global_gain - SF_PNS_OFFSET;

@nschimme nschimme Apr 13, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that SF_PNS_OFFSET was in the original code


// fixme: move range check to quantizer
for (cnt = 0; cnt < coder->bandcnt; cnt++)
Expand All @@ -440,18 +440,15 @@ int writesf(CoderInfo *coder, BitStream *stream, int write)
if ((book == HCB_INTENSITY) || (book== HCB_INTENSITY2))
{
diff = coder->sf[cnt] - lastis;
if (diff > 60)
diff = 60;
if (diff < -60)
diff = -60;
length = book12[60 + diff].len;
diff = clamp_sf_diff(diff);
length = book12[SF_DELTA + diff].len;

bits += length;

lastis += diff;

if (write)
PutBit(stream, book12[60 + diff].data, length);
PutBit(stream, book12[SF_DELTA + diff].data, length);
}
else if (book == HCB_PNS)
{
Expand All @@ -470,32 +467,26 @@ int writesf(CoderInfo *coder, BitStream *stream, int write)
continue;
}

if (diff > 60)
diff = 60;
if (diff < -60)
diff = -60;
diff = clamp_sf_diff(diff);

length = book12[60 + diff].len;
length = book12[SF_DELTA + diff].len;
bits += length;
lastpns += diff;

if (write)
PutBit(stream, book12[60 + diff].data, length);
PutBit(stream, book12[SF_DELTA + diff].data, length);
}
else if (book)
else if ((book != HCB_ZERO) && (book != HCB_NONE))
{
diff = coder->sf[cnt] - lastsf;
if (diff > 60)
diff = 60;
if (diff < -60)
diff = -60;
length = book12[60 + diff].len;
diff = clamp_sf_diff(diff);
length = book12[SF_DELTA + diff].len;

bits += length;
lastsf += diff;

if (write)
PutBit(stream, book12[60 + diff].data, length);
PutBit(stream, book12[SF_DELTA + diff].data, length);
}

}
Expand Down
35 changes: 35 additions & 0 deletions libfaac/huff2.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
****************************************************************************/

#ifndef HUFF2_H
#define HUFF2_H

#include "bitstream.h"

/* Huffman Codebooks */
enum {
HCB_ZERO = 0,
HCB_ESC = 11,
Expand All @@ -28,8 +32,39 @@ enum {
HCB_NONE
};

/* Maximum value representable by Huffman Book 11 escape sequences.
* Values >= 8192 would cause bitstream overflow/sync loss. */
#define MAX_HUFF_ESC_VAL 8191

/* Scalefactor Management */
enum {
/* Baseline scalefactor value used in bitstream */
SF_OFFSET = 100,
/* Minimum allowable scalefactor to prevent underflow */
SF_MIN = 10,
/* PNS predictor initialization offset (starts at floor) */
SF_PNS_OFFSET = SF_OFFSET - SF_MIN,
/* Max allowed difference between successive scalefactors (AAC spec) */
SF_DELTA = 60,
};

/**
* Restrict scalefactor delta to the spec-defined +/- SF_DELTA range.
* This ensures the delta remains valid for Book 12 Huffman encoding.
*/
static inline int clamp_sf_diff(int diff)
{
if (diff > SF_DELTA)
return SF_DELTA;
if (diff < -SF_DELTA)
return -SF_DELTA;
return diff;
}

int huffbook(CoderInfo *coderInfo,
int *qs /* quantized spectrum */,
int len);
int writebooks(CoderInfo *coder, BitStream *stream, int writeFlag);
int writesf(CoderInfo *coder, BitStream *bitStream, int writeFlag);

#endif /* HUFF2_H */
2 changes: 1 addition & 1 deletion libfaac/huffdata.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ hcode16_t book11[289] = {
{5,4},
};

hcode32_t book12[121] = {
hcode32_t book12[2 * SF_DELTA + 1] = {
{18,262120},{18,262118},{18,262119},{18,262117},{19,524277},{19,524273},{19,524269},{19,524278},
{19,524270},{19,524271},{19,524272},{19,524284},{19,524285},{19,524287},{19,524286},{19,524279},
{19,524280},{19,524283},{19,524281},{18,262116},{19,524282},{18,262115},{17,131055},{17,131056},
Expand Down
9 changes: 8 additions & 1 deletion libfaac/huffdata.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
****************************************************************************/

#ifndef HUFFDATA_H
#define HUFFDATA_H

#include "huff2.h"

#include <stdint.h>

typedef struct {
Expand All @@ -38,4 +43,6 @@ extern hcode16_t book08[64];
extern hcode16_t book09[169];
extern hcode16_t book10[169];
extern hcode16_t book11[289];
extern hcode32_t book12[121];
extern hcode32_t book12[2 * SF_DELTA + 1];

#endif /* HUFFDATA_H */
68 changes: 43 additions & 25 deletions libfaac/quantize.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ static void quantize_scalar(const faac_real * __restrict xr, int * __restrict xi
}

static QuantizeFunc qfunc = quantize_scalar;
static faac_real sfstep;
static faac_real max_quant_limit;

void QuantizeInit(void)
{
Expand All @@ -66,12 +68,19 @@ void QuantizeInit(void)
else
#endif
qfunc = quantize_scalar;

/* 2^0.25 (1.50515 dB) step from AAC specs */
sfstep = 1.0 / FAAC_LOG10(FAAC_SQRT(FAAC_SQRT(2.0)));

@fabiangreffrath fabiangreffrath Apr 13, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you declare them as static const in the file header?

@nschimme nschimme Apr 13, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately Clang still rejects this as its more standards compliant than GCC and in the C language, global or static variables must be initialized with a constant expression.


/* Inverse-quantizer headroom: ensures (x * gain)^0.75 + 0.4054 <= 8191.
* Pre-calculated to avoid redundant runtime power functions. */
max_quant_limit = FAAC_POW((faac_real)MAX_HUFF_ESC_VAL + 1.0 - MAGIC_NUMBER, 4.0/3.0);
}
#define NOISEFLOOR 0.4

// band sound masking
static void bmask(CoderInfo * __restrict coderInfo, faac_real * __restrict xr0, faac_real * __restrict bandqual,
faac_real * __restrict bandenrg, int gnum, faac_real quality)
faac_real * __restrict bandenrg, faac_real * __restrict bandmaxe, int gnum, faac_real quality)
{
int sfb, start, end, cnt;
int *cb_offset = coderInfo->sfb_offset;
Expand Down Expand Up @@ -130,6 +139,8 @@ static void bmask(CoderInfo * __restrict coderInfo, faac_real * __restrict xr0,
}
}
bandenrg[sfb] = avge;
/* Track peak magnitude to identify potential Huffman overflows. */
bandmaxe[sfb] = FAAC_SQRT(maxe);
maxe *= gsize;

#define NOISETONE 0.2
Expand Down Expand Up @@ -166,21 +177,16 @@ static void qlevel(CoderInfo * __restrict coderInfo,
const faac_real * __restrict xr0,
const faac_real * __restrict bandqual,
const faac_real * __restrict bandenrg,
const faac_real * __restrict bandmaxe,
int gnum,
int pnslevel
)
{
int sb;
#if !defined(__clang__) && defined(__GNUC__) && (GCC_VERSION >= 40600)
/* 2^0.25 (1.50515 dB) step from AAC specs */
static const faac_real sfstep = 1.0 / FAAC_LOG10(FAAC_SQRT(FAAC_SQRT(2.0)));
#else
static const faac_real sfstep = 20 / 1.50515;
#endif
int gsize = coderInfo->groups.len[gnum];
faac_real pnsthr = 0.1 * pnslevel;

for (sb = 0; sb < coderInfo->sfbn; sb++)
for (sb = 0; sb < coderInfo->sfbn && coderInfo->bandcnt < MAX_SCFAC_BANDS; sb++)
{
faac_real sfacfix;
int sfac;
Expand Down Expand Up @@ -220,11 +226,25 @@ static void qlevel(CoderInfo * __restrict coderInfo,
}

sfac = FAAC_LRINT(FAAC_LOG10(bandqual[sb] / rmsx) * sfstep);
if ((SF_OFFSET - sfac) < 10)

if ((SF_OFFSET - sfac) < SF_MIN)
sfacfix = 0.0;
else
{
sfacfix = FAAC_POW(10, sfac / sfstep);

/* Bitstream saturation check: if gain * peak exceeds the Huffman limit,
* clamp gain and re-sync the integer scalefactor to prevent overflow. */
if (sfacfix * bandmaxe[sb] > max_quant_limit)
{
sfacfix = max_quant_limit / bandmaxe[sb];
sfac = (int)FAAC_FLOOR(FAAC_LOG10(sfacfix) * sfstep);
/* Re-derive gain from the floored scalefactor to ensure bit-exact
* sync with the decoder's inverse quantizer. */
sfacfix = FAAC_POW(10, sfac / sfstep);
}
}

end -= start;
xi = xitab;
if (sfacfix <= 0.0)
Expand All @@ -249,6 +269,7 @@ int BlocQuant(CoderInfo * __restrict coder, faac_real * __restrict xr, AACQuantC
{
faac_real bandlvl[MAX_SCFAC_BANDS];
faac_real bandenrg[MAX_SCFAC_BANDS];
faac_real bandmaxe[MAX_SCFAC_BANDS];
int cnt;
faac_real *gxr;

Expand All @@ -264,9 +285,9 @@ int BlocQuant(CoderInfo * __restrict coder, faac_real * __restrict xr, AACQuantC
gxr = xr;
for (cnt = 0; cnt < coder->groups.n; cnt++)
{
bmask(coder, gxr, bandlvl, bandenrg, cnt,
bmask(coder, gxr, bandlvl, bandenrg, bandmaxe, cnt,
(faac_real)aacquantCfg->quality/DEFQUAL);
qlevel(coder, gxr, bandlvl, bandenrg, cnt, aacquantCfg->pnslevel);
qlevel(coder, gxr, bandlvl, bandenrg, bandmaxe, cnt, aacquantCfg->pnslevel);
gxr += coder->groups.len[cnt] * BLOCK_LEN_SHORT;
}

Expand All @@ -285,31 +306,28 @@ int BlocQuant(CoderInfo * __restrict coder, faac_real * __restrict xr, AACQuantC

lastsf = coder->global_gain;
lastis = 0;
// fixme: move SF range check to quantizer
int lastpns = coder->global_gain - SF_PNS_OFFSET;
for (cnt = 0; cnt < coder->bandcnt; cnt++)
{
int book = coder->book[cnt];
if ((book == HCB_INTENSITY) || (book == HCB_INTENSITY2))
{
int diff = coder->sf[cnt] - lastis;

if (diff < -60)
diff = -60;
if (diff > 60)
diff = 60;

diff = clamp_sf_diff(diff);
lastis += diff;
coder->sf[cnt] = lastis;
}
else if (book == HCB_ESC)
else if (book == HCB_PNS)
{
int diff = coder->sf[cnt] - lastpns;
diff = clamp_sf_diff(diff);
lastpns += diff;
coder->sf[cnt] = lastpns;
}
else if ((book != HCB_ZERO) && (book != HCB_NONE))
{
int diff = coder->sf[cnt] - lastsf;

if (diff < -60)
diff = -60;
if (diff > 60)
diff = 60;

diff = clamp_sf_diff(diff);
lastsf += diff;
coder->sf[cnt] = lastsf;
}
Expand Down
1 change: 0 additions & 1 deletion libfaac/quantize.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ enum {
MAXQUAL = 5000,
MAXQUALADTS = MAXQUAL,
MINQUAL = 10,
SF_OFFSET = 100,
};

int BlocQuant(CoderInfo *coderInfo, faac_real *xr, AACQuantCfg *aacquantCfg);
Expand Down
Loading