I created an AWS Lambda function in node.js 18 that is using a static, ver 7 build of FFmpeg located in a lambda layer. Unfortunately it’s just the ffmpeg build and doesn’t include ffprobe.

I have an mp4 audio file in one S3 bucket and a wav audio file in a second S3 bucket. I’m uploading the output file to a third S3 bucket.

Specs on the files (please let me know if any more info is needed)

Audio:
wav, 13kbps, aac (LC), 6:28 duration

Video:
mp4, 1280×720 resolution, 25 frame rate, h264 codec, 3:27 duration

Goal:
Create blank video to fill in the duration gaps so the full audio is covered before and after the mp4 video (using timestamps and duration). Strip the mp4 audio and use the wav audio only. Output should be an mp4 video with the wav audio playing over it and blank video for 27 seconds (based on timestamp) until mp4 video plays for 3:27, and then blank video to cover the rest of the audio until 6:28.

Actual Result:
An mp4 file with 3.554 frame rate and 10:06 duration.

import { S3Client, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
import { createWriteStream, createReadStream, promises as fsPromises } from 'fs';
import { exec } from 'child_process';
import { promisify } from 'util';
import { basename } from 'path';

const execAsync = promisify(exec);

const s3 = new S3Client({ region: 'us-east-1' });

async function downloadFileFromS3(bucket, key, downloadPath) {
    const getObjectParams = { Bucket: bucket, Key: key };
    const command = new GetObjectCommand(getObjectParams);
    const { Body } = await s3.send(command);
    return new Promise((resolve, reject) => {
        const fileStream = createWriteStream(downloadPath);
        Body.pipe(fileStream);
        Body.on('error', reject);
        fileStream.on('finish', resolve);
    });
}

async function uploadFileToS3(bucket, key, filePath) {
    const fileStream = createReadStream(filePath);
    const uploadParams = { Bucket: bucket, Key: key, Body: fileStream };
    try {
        await s3.send(new PutObjectCommand(uploadParams));
        console.log(`File uploaded successfully to ${bucket}/${key}`);
    } catch (err) {
        console.error("Error uploading file: ", err);
        throw new Error('Failed to upload file to S3');
    }
}

function parseDuration(durationStr) {
    const parts = durationStr.split(':');
    return parseInt(parts[0]) * 3600 + parseInt(parts[1]) * 60 + parseFloat(parts[2]);
}

export async function handler(event) {
    const videoBucket = "video-interaction-content";
    const videoKey = event.videoKey;
    const audioBucket = "audio-call-recordings";
    const audioKey = event.audioKey;
    const outputBucket = "synched-audio-video";
    const outputKey = `combined_${basename(videoKey, '.mp4')}.mp4`;

    const audioStartSeconds = new Date(event.audioStart).getTime() / 1000;
    const videoStartSeconds = new Date(event.videoStart).getTime() / 1000;
    const audioDurationSeconds = event.audioDuration / 1000;
    const timeDifference = audioStartSeconds - videoStartSeconds;

    try {
        const videoPath = `/tmp/${basename(videoKey)}`;
        const audioPath = `/tmp/${basename(audioKey)}`;
        await downloadFileFromS3(videoBucket, videoKey, videoPath);
        await downloadFileFromS3(audioBucket, audioKey, audioPath);

        //Initialize file list with video
        let filelist = [`file '${videoPath}'`];
        let totalVideoDuration = 0; // Initialize total video duration

        // Create first blank video if needed
        if (timeDifference < 0) {
            const blankVideoDuration = Math.abs(timeDifference);
            const blankVideoPath = `/tmp/blank_video.mp4`;
            await execAsync(`/opt/bin/ffmpeg -f lavfi -i color=c=black:s=1280x720:r=25 -c:v libx264 -t ${blankVideoDuration} ${blankVideoPath}`);
            //Add first blank video first in file list
            filelist.unshift(`file '${blankVideoPath}'`);
            totalVideoDuration += blankVideoDuration;
            console.log(`First blank video created with duration: ${blankVideoDuration} seconds`);
        }
        
        const videoInfo = await execAsync(`/opt/bin/ffmpeg -i ${videoPath} -f null -`);
        const videoDurationMatch = videoInfo.stderr.match(/Duration: ([d:.]+)/);
        const videoDuration = videoDurationMatch ? parseDuration(videoDurationMatch[1]) : 0;
        totalVideoDuration += videoDuration;

        // Calculate additional blank video duration
        const additionalBlankVideoDuration = audioDurationSeconds - totalVideoDuration;
        if (additionalBlankVideoDuration > 0) {
            const additionalBlankVideoPath = `/tmp/additional_blank_video.mp4`;
            await execAsync(`/opt/bin/ffmpeg -f lavfi -i color=c=black:s=1280x720:r=25 -c:v libx264 -t ${additionalBlankVideoDuration} ${additionalBlankVideoPath}`);
            //Add to the end of the file list
            filelist.push(`file '${additionalBlankVideoPath}'`);
            console.log(`Additional blank video created with duration: ${additionalBlankVideoDuration} seconds`);
        }

        // Create and write the file list to disk
        const concatFilePath = '/tmp/filelist.txt';
        await fsPromises.writeFile('/tmp/filelist.txt', filelist.join('n'));

        const extendedVideoPath = `/tmp/extended_${basename(videoKey)}`;
        //await execAsync(`/opt/bin/ffmpeg -f concat -safe 0 -i /tmp/filelist.txt -c copy ${extendedVideoPath}`);
        
        // Use -vsync vfr to adjust frame timing without full re-encoding
        await execAsync(`/opt/bin/ffmpeg -f concat -safe 0 -i ${concatFilePath} -c copy -vsync vfr ${extendedVideoPath}`);

        const outputPath = `/tmp/output_${basename(videoKey, '.mp4')}.mp4`;
        //await execAsync(`/opt/bin/ffmpeg -i ${extendedVideoPath} -i ${audioPath} -map 0:v:0 -map 1:a:0 -c:v copy -c:a aac -b:a 192k -shortest ${outputPath}`);

        await execAsync(`/opt/bin/ffmpeg -i ${extendedVideoPath} -i ${audioPath} -map 0:v:0 -map 1:a:0 -c:v copy -c:a aac -b:a 192k -shortest -r 25 ${outputPath}`);
        console.log('Video and audio have been merged successfully');

        await uploadFileToS3(outputBucket, outputKey, outputPath);
        console.log('File upload complete.');

        return { statusCode: 200, body: JSON.stringify('Video and audio have been merged successfully.') };
    } catch (error) {
        console.error('Error in Lambda function:', error);
        return { statusCode: 500, body: JSON.stringify('Failed to process video and audio.') };
    }
}

Attempts:
I’ve tried re-encoding the concatenated file but the lambda function times out. I hoped that by creating blank video with a 25fps and all the other specs from the original mp4, I wouldn’t have to re-encode the concatenated file. Obviously something is wrong, though. In the commented out code you can see I tried specifying 25 or not, and also tried -vsync and no -vsync. I’m new to FFmpeg so all tips are appreciated!