Program received signal SIGILL, Illegal instruction. at jfif_encode in jfif.c:695,747

[ambrosecm]

Desctiption

When I used the jfif_encode function to handle a specific input, a segmentation fault (SEGV) was triggered at jfif_encode in jfif.c:695,747

ffjpeg/src/jfif.c

Lines 659 to 749 in caade60

	void* jfif_encode(BMP *pb)
	{
	JFIF *jfif = NULL;
	void *bs = NULL;
	int jw, jh;
	int *yuv_datbuf[3] = {0};
	int *ydst, *udst, *vdst;
	int *isrc, *idst;
	BYTE *bsrc;
	int du[64]= {0};
	int dc[4 ]= {0};
	int i, j, m, n;
	int failed = 1;
	// check input params
	if (!pb) {
	printf("invalid input params !\n");
	return NULL;
	}
	// allocate jfif context
	jfif = calloc(1, sizeof(JFIF));
	if (!jfif) return NULL;
	// init dct module
	init_dct_module();
	// init jfif context
	jfif->width = pb->width;
	jfif->height = pb->height;
	jfif->pqtab[0] = malloc(64*sizeof(int));
	jfif->pqtab[1] = malloc(64*sizeof(int));
	jfif->phcac[0] = calloc(1, sizeof(HUFCODEC));
	jfif->phcac[1] = calloc(1, sizeof(HUFCODEC));
	jfif->phcdc[0] = calloc(1, sizeof(HUFCODEC));
	jfif->phcdc[1] = calloc(1, sizeof(HUFCODEC));
	jfif->datalen = jfif->width * jfif->height * 2;
	jfif->databuf = malloc(jfif->datalen);
	if ( !jfif->pqtab[0] || !jfif->pqtab[1]
	|| !jfif->phcac[0] || !jfif->phcac[1]
	|| !jfif->phcdc[0] || !jfif->phcdc[1]
	|| !jfif->databuf ) {
	goto done;
	}
	// init qtab
	memcpy(jfif->pqtab[0], STD_QUANT_TAB_LUMIN, 64*sizeof(int));
	memcpy(jfif->pqtab[1], STD_QUANT_TAB_CHROM, 64*sizeof(int));
	// open bit stream
	bs = bitstr_open(jfif->databuf, "mem", jfif->datalen);
	if (!bs) {
	printf("failed to open bitstr for jfif_decode !");
	goto done;
	}
	// init huffman codec
	memcpy(jfif->phcac[0]->huftab, STD_HUFTAB_LUMIN_AC, sizeof(STD_HUFTAB_LUMIN_AC));
	memcpy(jfif->phcac[1]->huftab, STD_HUFTAB_CHROM_AC, sizeof(STD_HUFTAB_CHROM_AC));
	memcpy(jfif->phcdc[0]->huftab, STD_HUFTAB_LUMIN_DC, sizeof(STD_HUFTAB_LUMIN_DC));
	memcpy(jfif->phcdc[1]->huftab, STD_HUFTAB_CHROM_DC, sizeof(STD_HUFTAB_CHROM_DC));
	jfif->phcac[0]->output = bs; huffman_encode_init(jfif->phcac[0], 1);
	jfif->phcac[1]->output = bs; huffman_encode_init(jfif->phcac[1], 1);
	jfif->phcdc[0]->output = bs; huffman_encode_init(jfif->phcdc[0], 1);
	jfif->phcdc[1]->output = bs; huffman_encode_init(jfif->phcdc[1], 1);
	// init comp_num & comp_info
	jfif->comp_num = 3;
	jfif->comp_info[0].id = 1;
	jfif->comp_info[0].samp_factor_v = 2;
	jfif->comp_info[0].samp_factor_h = 2;
	jfif->comp_info[0].qtab_idx = 0;
	jfif->comp_info[0].htab_idx_ac = 0;
	jfif->comp_info[0].htab_idx_dc = 0;
	jfif->comp_info[1].id = 2;
	jfif->comp_info[1].samp_factor_v = 1;
	jfif->comp_info[1].samp_factor_h = 1;
	jfif->comp_info[1].qtab_idx = 1;
	jfif->comp_info[1].htab_idx_ac = 1;
	jfif->comp_info[1].htab_idx_dc = 1;
	jfif->comp_info[2].id = 3;
	jfif->comp_info[2].samp_factor_v = 1;
	jfif->comp_info[2].samp_factor_h = 1;
	jfif->comp_info[2].qtab_idx = 1;
	jfif->comp_info[2].htab_idx_ac = 1;
	jfif->comp_info[2].htab_idx_dc = 1;
	// init jw & jw, init yuv data buffer
	jw = ALIGN(pb->width , 16);
	jh = ALIGN(pb->height, 16);
	yuv_datbuf[0] = calloc(1, sizeof(int) * jw * jh / 1);

The main reason is that the parameter pb passed into jfif_encode is not checked and the member integer of pb is out of bounds, resulting in an Illegal instruction.

Test Environment

Ubuntu 22.04.1, 64bit
ffjpeg(master caade60)
program source file

How to trigger

Download the poc1 file ,poc2 file, program and run the following cmd:

 $ ./jfif_encode_du ./poc1
 $ ./jfif_encode_du ./poc2

Detail

GDB report

(gdb) r
Starting program: /home/ambrose/vsproject/HIMFuzz/harness/output/ffjpeg_deepseek24/crashes/jfif.c/jfif_encode/jfif_encode_du/jfif_encode_du output/default/crashes/id:000000,sig:04,src:000000,time:108180,execs:630,op:havoc,rep:8
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

Program received signal SIGILL, Illegal instruction.
0x00005555556873c7 in jfif_encode (pb=0x7bfff5b09020) at /home/ambrose/vsproject/TestLib/ffjpeg/src/jfif.c:695
695         jfif->datalen  = jfif->width * jfif->height * 2;
(gdb) bt
#0  0x00005555556873c7 in jfif_encode (pb=0x7bfff5b09020) at /home/ambrose/vsproject/TestLib/ffjpeg/src/jfif.c:695
#1  0x00005555556960f1 in main (argc=2, argv=0x7fffffffdad8)
    at output/ffjpeg_deepseek24/harness/code/jfif.c/jfif_encode/jfif_encode_du.c:27
(gdb) p jfif->width
$1 = 1577058306
(gdb) p jfif->height
$2 = 1

(gdb) r
Starting program: /home/ambrose/vsproject/HIMFuzz/harness/output/ffjpeg_deepseek24/crashes/jfif.c/jfif_encode/jfif_encode_du/jfif_encode_du output/default/crashes/id:000001,sig:04,src:000000,time:1639213,execs:10086,op:havoc,rep:11
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

Program received signal SIGILL, Illegal instruction.
0x000055555568cab4 in jfif_encode (pb=0x7bfff5b09020) at /home/ambrose/vsproject/TestLib/ffjpeg/src/jfif.c:747
747         jw = ALIGN(pb->width , 16);
(gdb) p pb->width
$1 = 2147483647