dnxhdenc: exact slice sizes in fast rate control; fail a picture instead of aborting FFmpeg 7.1.4 (unchanged on master as of 2026-09-27). dnxhd_encode_fast() sizes every slice from per-macroblock bit counts measured at the picture qscale Q and at Q + 1. When moving every macroblock to Q + 1 still leaves the worst-case padding (+31 bits per row) over budget, its `goto retry` pass raised macroblocks to Q + 2 and beyond while keeping their Q + 1 bit count (and subtracting the Q -> Q + 1 saving a second time). DNx bit counts are not monotonic in qscale, so such a macroblock can code larger than its estimate; the slice then overruns its allocation and flush_put_bits() hits av_assert0 (put_bits.h:150), aborting the host process. Seen with DNxHR LB at 640x360 on noisy content (PME, 2026-09-27). - Fast rate control only ever codes a macroblock at Q or Q + 1. If a full pass at Q + 1 does not fit, the base moves to Q + 1 and Q + 2 is measured; past qmax the picture fails with EINVAL ("increase qmax") like the RDO path. - mb_qscale is uint16_t: qscale may reach qmax (1024); uint8_t wrapped it. - Backstop: each slice is coded into a per-thread scratch buffer (with margin) and copied into the packet only when it fits its allocation; an oversized slice fails the picture with AVERROR_BUG instead of overrunning. --- a/libavcodec/dnxhdenc.h +++ b/libavcodec/dnxhdenc.h @@ -53,6 +53,9 @@ uint8_t *msip; ///< Macroblock Scan Indexes Payload uint32_t *slice_size; uint32_t *slice_offs; + int *slice_ret; ///< per-slice result of dnxhd_encode_thread + uint8_t *slice_scratch; ///< per-thread buffer a slice is coded into + unsigned slice_scratch_size; struct DNXHDEncContext *thread[MAX_THREADS]; @@ -98,7 +101,7 @@ unsigned lambda; uint32_t *mb_bits; - uint8_t *mb_qscale; + uint16_t *mb_qscale; ///< up to qmax (1024), so not uint8_t RCCMPEntry *mb_cmp; RCCMPEntry *mb_cmp_tmp; --- a/libavcodec/dnxhdenc.c +++ b/libavcodec/dnxhdenc.c @@ -493,7 +493,8 @@ if (!FF_ALLOCZ_TYPED_ARRAY(ctx->slice_size, ctx->m.mb_height) || !FF_ALLOCZ_TYPED_ARRAY(ctx->slice_offs, ctx->m.mb_height) || !FF_ALLOCZ_TYPED_ARRAY(ctx->mb_bits, ctx->m.mb_num) || - !FF_ALLOCZ_TYPED_ARRAY(ctx->mb_qscale, ctx->m.mb_num)) + !FF_ALLOCZ_TYPED_ARRAY(ctx->mb_qscale, ctx->m.mb_num) || + !FF_ALLOCZ_TYPED_ARRAY(ctx->slice_ret, ctx->m.mb_height)) return AVERROR(ENOMEM); if (avctx->active_thread_type == FF_THREAD_SLICE) { @@ -514,9 +515,22 @@ ctx->thread[i] = av_memdup(ctx, sizeof(DNXHDEncContext)); if (!ctx->thread[i]) return AVERROR(ENOMEM); + ctx->thread[i]->slice_scratch = NULL; } } + /* A slice is coded into a private buffer and copied into the packet only + * once it is known to fit its allocation. The margin (well above the + * largest macroblock) keeps the bit writer from reaching the end of the + * buffer while an oversized slice is being detected. */ + for (i = 0; i < FFMAX(avctx->thread_count, 1) && ctx->thread[i]; i++) { + DNXHDEncContext *t = ctx->thread[i]; + t->slice_scratch_size = ctx->coding_unit_size + 16384; + t->slice_scratch = av_malloc(t->slice_scratch_size); + if (!t->slice_scratch) + return AVERROR(ENOMEM); + } + return 0; } @@ -878,9 +892,10 @@ DNXHDEncContext *ctx = avctx->priv_data; PutBitContext pb0, *const pb = &pb0; int mb_y = jobnr, mb_x; + const uint64_t slice_bits = (uint64_t)ctx->slice_size[jobnr] * 8; + uint8_t *dst = (uint8_t *)arg + ctx->data_offset + ctx->slice_offs[jobnr]; ctx = ctx->thread[threadnr]; - init_put_bits(pb, (uint8_t *)arg + ctx->data_offset + ctx->slice_offs[jobnr], - ctx->slice_size[jobnr]); + init_put_bits(pb, ctx->slice_scratch, ctx->slice_scratch_size); ctx->m.last_dc[0] = ctx->m.last_dc[1] = @@ -904,9 +919,18 @@ dnxhd_encode_block(pb, ctx, block, last_index, n); } + /* Rate control sized this slice from per-macroblock bit counts; a + * slice that comes out larger must fail the picture, not overrun. */ + if (put_bits_count(pb) > slice_bits) { + av_log(avctx, AV_LOG_ERROR, + "slice %d exceeds its %"PRIu64"-bit allocation\n", + jobnr, slice_bits); + return AVERROR_BUG; + } } flush_put_bits(pb); - memset(put_bits_ptr(pb), 0, put_bytes_left(pb, 0)); + memcpy(dst, ctx->slice_scratch, put_bytes_output(pb)); + memset(dst + put_bytes_output(pb), 0, slice_bits / 8 - put_bytes_output(pb)); return 0; } @@ -1188,49 +1212,59 @@ static int dnxhd_encode_fast(AVCodecContext *avctx, DNXHDEncContext *ctx) { - int max_bits = 0; int ret, x, y; if ((ret = dnxhd_find_qscale(ctx)) < 0) return ret; - for (y = 0; y < ctx->m.mb_height; y++) { - for (x = 0; x < ctx->m.mb_width; x++) { - int mb = y * ctx->m.mb_width + x; - int rc = (ctx->qscale * ctx->m.mb_num ) + mb; - int delta_bits; - ctx->mb_qscale[mb] = ctx->qscale; - ctx->mb_bits[mb] = ctx->mb_rc[rc].bits; - max_bits += ctx->mb_rc[rc].bits; - if (!RC_VARIANCE) { - delta_bits = ctx->mb_rc[rc].bits - - ctx->mb_rc[rc + ctx->m.mb_num].bits; - ctx->mb_cmp[mb].mb = mb; - ctx->mb_cmp[mb].value = - delta_bits ? ((ctx->mb_rc[rc].ssd - - ctx->mb_rc[rc + ctx->m.mb_num].ssd) * 100) / - delta_bits - : INT_MIN; // avoid increasing qscale + for (;;) { + int max_bits = 0; + for (y = 0; y < ctx->m.mb_height; y++) { + for (x = 0; x < ctx->m.mb_width; x++) { + int mb = y * ctx->m.mb_width + x; + int rc = (ctx->qscale * ctx->m.mb_num ) + mb; + int delta_bits; + ctx->mb_qscale[mb] = ctx->qscale; + ctx->mb_bits[mb] = ctx->mb_rc[rc].bits; + max_bits += ctx->mb_rc[rc].bits; + if (!RC_VARIANCE) { + delta_bits = ctx->mb_rc[rc].bits - + ctx->mb_rc[rc + ctx->m.mb_num].bits; + ctx->mb_cmp[mb].mb = mb; + ctx->mb_cmp[mb].value = + delta_bits ? ((ctx->mb_rc[rc].ssd - + ctx->mb_rc[rc + ctx->m.mb_num].ssd) * 100) / + delta_bits + : INT_MIN; // avoid increasing qscale + } } + max_bits += 31; // worst padding } - max_bits += 31; // worst padding - } - if (!ret) { + if (ret) + break; if (RC_VARIANCE) avctx->execute2(avctx, dnxhd_mb_var_thread, NULL, NULL, ctx->m.mb_height); radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num); -retry: for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) { int mb = ctx->mb_cmp[x].mb; int rc = (ctx->qscale * ctx->m.mb_num ) + mb; max_bits -= ctx->mb_rc[rc].bits - ctx->mb_rc[rc + ctx->m.mb_num].bits; - if (ctx->mb_qscale[mb] < 255) - ctx->mb_qscale[mb]++; + ctx->mb_qscale[mb] = ctx->qscale + 1; ctx->mb_bits[mb] = ctx->mb_rc[rc + ctx->m.mb_num].bits; } - - if (max_bits > ctx->frame_bits) - goto retry; + if (max_bits <= ctx->frame_bits) + break; + /* Every macroblock is at qscale + 1 and the worst-case padding still + * does not fit. Raising single macroblocks further would code them at + * a qscale whose bit count was never measured for this picture (bit + * counts are not monotonic in qscale), so the slice sizes would be + * wrong. Move the base up instead and measure the next step. */ + if (ctx->qscale + 2 >= avctx->qmax) + return AVERROR(EINVAL); + ctx->qscale += 2; + avctx->execute2(avctx, dnxhd_calc_bits_thread, + NULL, NULL, ctx->m.mb_height); + ctx->qscale -= 1; } return 0; } @@ -1292,7 +1326,10 @@ av_assert1(!(ctx->slice_size[i] & 3)); } - avctx->execute2(avctx, dnxhd_encode_thread, buf, NULL, ctx->m.mb_height); + avctx->execute2(avctx, dnxhd_encode_thread, buf, ctx->slice_ret, ctx->m.mb_height); + for (i = 0; i < ctx->m.mb_height; i++) + if (ctx->slice_ret[i] < 0) + return ctx->slice_ret[i]; av_assert1(ctx->data_offset + offset + 4 <= ctx->coding_unit_size); memset(buf + ctx->data_offset + offset, 0, @@ -1330,16 +1367,18 @@ av_freep(&ctx->mb_cmp_tmp); av_freep(&ctx->slice_size); av_freep(&ctx->slice_offs); + av_freep(&ctx->slice_ret); av_freep(&ctx->qmatrix_c); av_freep(&ctx->qmatrix_l); av_freep(&ctx->qmatrix_c16); av_freep(&ctx->qmatrix_l16); - if (ctx->thread[1]) { - for (i = 1; i < avctx->thread_count; i++) - av_freep(&ctx->thread[i]); + for (i = 1; i < MAX_THREADS && ctx->thread[i]; i++) { + av_freep(&ctx->thread[i]->slice_scratch); + av_freep(&ctx->thread[i]); } + av_freep(&ctx->slice_scratch); return 0; }