const videoElement = document.getElementById("video") as HTMLVideoElement;
let gotKey = false
const videoDecoder = new VideoDecoder({
output: (frame) => {
const canvas = document.createElement('canvas'); // Create a new canvas
const ctx = canvas.getContext('2d'); // Get the 2D rendering context
if (ctx) {
// Set the canvas dimensions to match the video frame
canvas.width = frame.displayWidth;
canvas.height = frame.displayHeight;
// Draw the frame onto the canvas
ctx.drawImage(frame, 0, 0);
// Append the canvas to your video element or container
if (videoElement.parentNode) {
videoElement.parentNode.insertBefore(canvas, videoElement.nextSibling);
}
// Optionally, add a class or style to the canvas for customization
canvas.style.border = "1px solid black";
}
frame.close(); // Release the video frame
},
error: (err) => console.error("VideoDecoder error:", err),
})
async function decodeH265Video(file: File) {
const arrayBuffer = await file.arrayBuffer();
const uint8Array = new Uint8Array(arrayBuffer);
// Split the file into NAL units
const nalUnits: Uint8Array[] = [];
let start = 0;
for (let i = 3; i < uint8Array.length - 3;) {
// Check for long start code (0x00000001)
if (uint8Array[i] === 0x01 && uint8Array[i - 1] === 0x00 && uint8Array[i - 2] === 0x00 && uint8Array[i - 3] === 0x00) {
if (start !== i - 3) {
nalUnits.push(uint8Array.slice(start, i - 3));
start = i - 3;
}
i += 2
continue
// Check for short start code (0x000001)
} else if (uint8Array[i-1] === 0x01 && uint8Array[i - 2] === 0x00 && uint8Array[i - 3] === 0x00) {
if (start !== i - 3) {
nalUnits.push(uint8Array.slice(start, i - 3));
start = i - 3;
}
}
i++
}
nalUnits.push(uint8Array.slice(start));
videoDecoder.addEventListener('error', (e) => console.error("Error event:", e));
videoDecoder.addEventListener('dequeue', () => console.log("Frame dequeued."));
let timestamp = 0; // Initialize the timestamp
// Decode NAL units
for (const nalUnit of nalUnits) {
let type: EncodedVideoChunkType = determineFrameType(nalUnit)
if (keys.length == 4) {
console.log(`Decoder state: ${videoDecoder.state}`);
console.log(`Initial queue size: ${videoDecoder.decodeQueueSize}`);
let merged = mergeUint8Arrays(keys)
const chunk = new self.EncodedVideoChunk({
type: type,
timestamp: timestamp,
data: merged,
});
videoDecoder.decode(chunk);
gotKey = true
keys = []
continue
}
if (sps && pps && vps) {
let result = await parseCodecInst.parseCodec(sps, pps, vps)
if (result.error) {
console.error("Failed to parse codec:", result.error);
}
if (result.data) {
const decoderSupport = await VideoDecoder.isConfigSupported(result.data)
if (decoderSupport.config && decoderSupport.supported) {
videoDecoder.configure(decoderSupport.config);
} else {
console.error("Decoder configuration is undefined.");
}
sps = pps = vps = null
continue
} else {
console.error("Failed to configure video decoder: result.data is undefined");
}
}
if (!gotKey) {
continue
}
try {
console.log(`Decoder state: ${videoDecoder.state}`);
console.log(`Initial queue size: ${videoDecoder.decodeQueueSize}`);
const chunk = new self.EncodedVideoChunk({
type: type,
timestamp: timestamp,
data: nalUnit,
});
videoDecoder.decode(chunk);
} catch (error) {
console.error("Failed to decode video chunk:", error);
}
timestamp += 1000 / 25; // Increment the timestamp by 1 frame at 30fps
}
}
function determineFrameType(nalUnit) {
// Locate and skip the start code (either 3 or 4 bytes)
let startCodeLength = 3; // Default to 3 bytes
if (nalUnit[0] === 0x00 && nalUnit[1] === 0x00 && nalUnit[2] === 0x01) {
startCodeLength = 3; // 0x00 00 01 (3 bytes)
} else if (nalUnit[0] === 0x00 && nalUnit[1] === 0x00 && nalUnit[2] === 0x00 && nalUnit[3] === 0x01) {
startCodeLength = 4; // 0x00 00 00 01 (4 bytes)
} else {
throw new Error("Invalid start code in NAL unit");
}
// Skip the start code to find the NAL header
const nalHeader = nalUnit[startCodeLength];
// Extract the NAL unit type from the NAL header
const nalType = (nalHeader & 0x7E) >> 1;
// Process the NAL unit type
if (nalType === 32) { // VPS (Video Parameter Set)
keys = [];
vps = nalUnit.slice(startCodeLength); // Exclude the start code
if (videoDecoder.state === "configured") {
videoDecoder.flush()
// videoDecoder.reset()
}
gotKey = false;
keys.push(nalUnit)
return "key";
} else if (nalType === 33) { // SPS (Sequence Parameter Set)
sps = nalUnit.slice(startCodeLength);
keys.push(nalUnit)
return "key";
} else if (nalType === 34) { // PPS (Picture Parameter Set)
pps = nalUnit.slice(startCodeLength);
keys.push(nalUnit)
return "key";
} else if (nalType === 19 || nalType === 20) { // IDR frames (keyframes)
keys.push(nalUnit)
return "key";
} else {
return "delta";
}
}
I’m using the above typescript code to decode a raw HEVC video in Annex B format in Chrome, but always get error ‘EncodingError: Decoder error’. First, I split the stream nal by nal then parse the vps sps pps from the nal and using wasm to parse it then got the description, it seems worked. second, I concatenate the vps sps pps and the first key frame in to one chunk and send it to decoder, then decode delta one by one but never got one output frame. I’m not familiar with Webcodecs API, so any help will be appreciated🙏