FFMPEG C Library: Encoding h264 stream into Matroska .mkv container creates corrupt files

Above is the related question, and the answer of allocating memory for extradata of codec context works.

But on MacOS, with the encoder videotoolbox, this approach creates corrupted video file for any container, including mp4 and mkv. But what it does better for mkv is that at least the corrupted file is not 0 size.

I was wondering what is the correct way to encode stream to mkv on MacOS?

Below is my code initiating the outputcodec

    auto path = previewPath.toStdString(); // Convert file path to standard string

    // qInfo() << "[test0306] output path is " << _filePath;

    // Allocate the output media context
    avformat_alloc_output_context2(&_outputFormat, NULL, NULL, path.c_str());
    if (!_outputFormat) {
        printf("Could not deduce output format from file extension: using MPEG.n");
        avformat_alloc_output_context2(&_outputFormat, NULL, "mp4", path.c_str());
    }
    if (!_outputFormat) {
        emit cacheError(previewPath, "Could not create output contextn");
        return;
    }

    // Find the video encoder
    auto codecName = metaData["encoderId"].toString().toStdString();
    const AVCodec *codec = avcodec_find_encoder_by_name(codecName.c_str());
    qDebug() << "[test0307] this is the meta data encoder info: " << codecName;

    if (!codec) {
        qDebug() << "Cannot find encoder by name";
        emit cacheError(previewPath, "Cannot find encoder by name");
        return;
    }

    // Set up the sar, time base and frame rate from metadata
    // AVRational timeBase = {metaData["timeBaseNum"].toInt(), metaData["timeBaseDen"].toInt()};
    AVRational timeBase = {metaData["outputTimeBaseNum"].toInt(), metaData["outputTimeBaseDen"].toInt()};
    // AVRational framerate = {metaData["framerateNum"].toInt(), metaData["framerateDen"].toInt()};
    AVRational framerate = av_d2q(metaData["outputFps"].toDouble(), 100000);
    AVRational sar = {metaData["sarNum"].toInt(), metaData["sarDen"].toInt()};
    AVRational avg_frame_rate = {metaData["avg_frame_rate_num"].toInt(), metaData["avg_frame_rate_den"].toInt()};
    AVRational r_frame_rate = {metaData["r_frame_rate_num"].toInt(), metaData["r_frame_rate_den"].toInt()};

    qInfo() << "Diskcache is gonna cache video on fps" << metaData["outputFps"].toDouble();
    // Create a new stream in the output file
    AVStream* outSt = avformat_new_stream(_outputFormat, codec);
    if (!outSt) {
        emit cacheError(previewPath, "Failed to create output streamn");
        return;
    }
    // outSt->time_base = timeBase;
    // outSt->id = _outputFormat->nb_streams - 1;

    // Allocate the codec context for the encoder
    _outputCodecContext = avcodec_alloc_context3(codec);
    if (!_outputCodecContext) {
        emit cacheError(previewPath, "Could not alloc an encoding contextn");
        return;
    }

    // Configure the codec context
    _outputCodecContext->codec_id = codec->id;
    _outputCodecContext->codec_type = codec->type;
    // if(metaData["bitrate"].toInt() > 0){
    _outputCodecContext->bit_rate = metaData["bitrate"].toInt();
    // }
    // else{
    //     _outputCodecContext->bit_rate = 80000;
    // }
    _outputCodecContext->max_b_frames = 0;
    _outputCodecContext->width = metaData["outputWidth"].toInt();
    _outputCodecContext->height = metaData["outputHeight"].toInt();
    _outputCodecContext->framerate = framerate; // Set from metadata
    _outputCodecContext->time_base = timeBase;//timeBase; // Set from metadata
    _outputCodecContext->pix_fmt = AV_PIX_FMT_YUV420P; // img need yuv420p

    outSt->time_base           = _outputCodecContext->time_base;
    outSt->avg_frame_rate      = framerate;
    outSt->r_frame_rate        = framerate;
    outSt->disposition         = metaData["disposition"].toInt();
    outSt->discard             = static_cast<AVDiscard>(metaData["discard"].toInt());
    outSt->sample_aspect_ratio = sar;
    outSt->event_flags         = metaData["event_flags"].toInt();
    outSt->pts_wrap_bits       = metaData["pts_wrap_bits "].toInt();

    // // codecpar->extradata needed for MKV, this break mac playing back to preview
#ifdef Q_OS_WINDOWS
    _outputCodecContext->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    _outputCodecContext->extradata = (uint8_t*)av_mallocz(sizeof(int) * sizeof(int) * sizeof(int) / 2);
    _outputCodecContext->extradata_size = sizeof(int) * sizeof(int) * sizeof(int) / 2;
#elif defined(Q_OS_MACOS)

#endif


    // Open the codec
    AVDictionary *opt = NULL;
    int ret = avcodec_open2(_outputCodecContext, codec, &opt);
    av_dict_free(&opt);
    if (ret < 0) {
        emit cacheError(previewPath, "Could not open video codecn");
        return;
    }

    // Copy codec parameters from context to the stream
    ret = avcodec_parameters_from_context(outSt->codecpar, _outputCodecContext);
    if (ret < 0) {
        emit cacheError(previewPath, "Could not copy the stream parametersn");
        return;
    }

    // Print format details
    av_dump_format(_outputFormat, 0, path.c_str(), 1);

    // Open the output file
    ret = avio_open(&_outputFormat->pb, path.c_str(), AVIO_FLAG_WRITE);
    if (ret < 0) {
        emit cacheError(previewPath, "Could not open output filen");
        return;
    }

    // Write the stream header
    AVDictionary *params = NULL;
    //av_dict_set(&params, "movflags", "frag_keyframe+empty_moov+delay_moov+use_metadata_tags+write_colr", 0);
    ret = avformat_write_header(_outputFormat, NULL);
    if (ret < 0) {
        char err_buf[AV_ERROR_MAX_STRING_SIZE];  // Define a buffer for error strings.
        if (av_strerror(ret, err_buf, sizeof(err_buf)) == 0) {  // Safely get the error string.
            qInfo() << "write header error:" << err_buf;  // Now use the buffer for logging.
            emit cacheError(previewPath, "Error occurred when opening output file", ret);
        } else {
            qInfo() << "write header error: Unknown error with code" << ret;
            emit cacheError(previewPath, "Error occurred when opening output file: Unknown error", ret);
        }
        return;

    }

I have tried to limit this approach only on windows and waiting to find a solution to do it with videotoolbox on MacOS.