Im trying to render YUV AVFrame, that i get from camera using OpenTK, im creating a rectangle and trying to apply a texture to it, but it doesnt work.
Here is my window class
using OpenTK.Graphics.Egl;
using OpenTK.Graphics.OpenGL4;
using OpenTK.Windowing.Common;
using OpenTK.Windowing.Desktop;
using OpenTK.Windowing.GraphicsLibraryFramework;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace myFFmpeg
{
public class CameraWindow : GameWindow
{
private int vertexBufferHandle;
private int elementBufferHandle;
private int vertexArrayHandle;
private int frameNumber = 0;
private int yTex, uTex, vTex;
Shader shader;
Texture texture;
float[] vertices =
{
//Position | Texture coordinates
0.5f, 0.5f, 0.0f, 1.0f, 0.0f, // top right
0.5f, -0.5f, 0.0f, 1.0f, 1.0f, // bottom right
-0.5f, -0.5f, 0.0f, 0.0f, 1.0f, // bottom left
-0.5f, 0.5f, 0.0f, 0.0f, 0.0f // top left
};
private uint[] indices =
{
0, 1, 3, // first triangle
1, 2, 3 // second triangle
};
public CameraWindow(string title) : base(GameWindowSettings.Default, new NativeWindowSettings() { ClientSize = (1280, 720), Title = title }) { UpdateFrequency = 25; }
protected override void OnUpdateFrame(FrameEventArgs e)
{
base.OnUpdateFrame(e);
}
protected override void OnLoad()
{
GL.ClearColor(0.5f, 0.3f, 0.3f, 1.0f);
shader = new Shader(@"....shader.vert", @"....shader.frag");
texture = new Texture();
elementBufferHandle = GL.GenBuffer();
GL.BindBuffer(BufferTarget.ElementArrayBuffer, elementBufferHandle);
GL.BufferData(BufferTarget.ElementArrayBuffer, indices.Length * sizeof(uint), indices, BufferUsageHint.StaticDraw);
vertexBufferHandle = GL.GenBuffer();
GL.BindBuffer(BufferTarget.ArrayBuffer, vertexBufferHandle);
GL.BufferData(BufferTarget.ArrayBuffer, vertices.Length * sizeof(float), vertices, BufferUsageHint.StaticDraw);
GL.BindBuffer(BufferTarget.ArrayBuffer, 0);
vertexArrayHandle = GL.GenVertexArray();
GL.BindVertexArray(vertexArrayHandle);
GL.BindBuffer(BufferTarget.ArrayBuffer, vertexBufferHandle);
GL.VertexAttribPointer(0, 3, VertexAttribPointerType.Float, false, 5 * sizeof(float), 0);
GL.EnableVertexAttribArray(0);
int vertexShader = GL.CreateShader(ShaderType.VertexShader);
GL.ShaderSource(vertexShader, @"....shader.vert");
GL.CompileShader(vertexShader);
int fragmentShader = GL.CreateShader(ShaderType.FragmentShader);
GL.ShaderSource(fragmentShader, @"....shader.frag");
GL.CompileShader(fragmentShader);
int shaderProgram = GL.CreateProgram();
GL.AttachShader(shaderProgram, vertexShader);
GL.AttachShader(shaderProgram, fragmentShader);
GL.LinkProgram(shaderProgram);
int vertexPosLocation = GL.GetAttribLocation(shaderProgram, "vertexPos");
GL.EnableVertexAttribArray(vertexPosLocation);
GL.VertexAttribPointer(vertexPosLocation, 2, VertexAttribPointerType.Float, false, 4 * sizeof(float), 0);
int texCoordLocation = GL.GetAttribLocation(shaderProgram, "texCoord");
GL.EnableVertexAttribArray(texCoordLocation);
GL.VertexAttribPointer(texCoordLocation, 2, VertexAttribPointerType.Float, false, 4 * sizeof(float), 2 * sizeof(float));
GL.UseProgram(shaderProgram);
GL.ActiveTexture(TextureUnit.Texture0);
GL.BindTexture(TextureTarget.Texture2D, yTex);
GL.Uniform1(GL.GetUniformLocation(shaderProgram, "yTex"), 0);
GL.ActiveTexture(TextureUnit.Texture1);
GL.BindTexture(TextureTarget.Texture2D, uTex);
GL.Uniform1(GL.GetUniformLocation(shaderProgram, "uTex"), 1);
GL.ActiveTexture(TextureUnit.Texture2);
GL.BindTexture(TextureTarget.Texture2D, vTex);
GL.Uniform1(GL.GetUniformLocation(shaderProgram, "vTex"), 2);
GL.BindVertexArray(0);
//code
base.OnLoad();
}
protected override void OnUnload()
{
GL.BindBuffer(BufferTarget.ArrayBuffer, 0);
GL.DeleteBuffer(vertexBufferHandle);
GL.UseProgram(0);
shader.Dispose();
//code
base.OnUnload();
}
protected override void OnRenderFrame(FrameEventArgs e)
{
GL.Clear(ClearBufferMask.ColorBufferBit);
shader.Use();
texture.Use(frameNumber++);
GL.BindVertexArray(vertexArrayHandle);
GL.DrawElements(PrimitiveType.Triangles, indices.Length, DrawElementsType.UnsignedInt, indices);
Context.SwapBuffers();
base.OnRenderFrame(e);
}
protected override void OnFramebufferResize(FramebufferResizeEventArgs e)
{
base.OnFramebufferResize(e);
GL.Viewport(0, 0, e.Width, e.Height);
}
}
}
And my texture class:
using System;
using OpenTK;
using OpenTK.Graphics.OpenGL4;
using SkiaSharp;
using FFmpeg;
using SkiaSharp.Internals;
using StbImageSharp;
using FFmpeg.AutoGen;
using System.Threading;
namespace myFFmpeg
{
public class Texture
{
int Handle, yTex, uTex, vTex;
Program program = new Program();
public Texture()
{
Handle = GL.GenTexture();
}
public unsafe void Use(int frameNumber)
{
GL.BindTexture(TextureTarget.Texture2D, Handle);
// Generate textures only once (outside the loop)
if (yTex == 0)
{
GL.GenTextures(1, out yTex);
}
if (uTex == 0)
{
GL.GenTextures(1, out uTex);
}
if (vTex == 0)
{
GL.GenTextures(1, out vTex);
}
// Bind textures to specific units before rendering each frame
GL.ActiveTexture(TextureUnit.Texture0);
GL.BindTexture(TextureTarget.Texture2D, yTex);
GL.ActiveTexture(TextureUnit.Texture1);
GL.BindTexture(TextureTarget.Texture2D, uTex);
GL.ActiveTexture(TextureUnit.Texture2);
// Update textures with new frame data from FFmpeg
AVFrame frame = program.getFrame();
int width = frame.width;
int height = frame.height;
Console.BackgroundColor = ConsoleColor.White;
Console.ForegroundColor = ConsoleColor.Black;
Console.WriteLine((AVPixelFormat)frame.format);
Console.BackgroundColor = ConsoleColor.Black;
// Assuming YUV data is stored in separate planes (Y, U, V)
GL.BindTexture(TextureTarget.Texture2D, yTex);
GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Luminance, width, height, 0, PixelFormat.Luminance, PixelType.UnsignedByte, (IntPtr)frame.data[0]);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
GL.BindTexture(TextureTarget.Texture2D, uTex);
GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Luminance, width / 2, height / 2, 0, PixelFormat.Luminance, PixelType.UnsignedByte, (IntPtr)frame.data[1]);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMagFilter, (int)TextureMagFilter.Linear);
GL.BindTexture(TextureTarget.Texture2D, vTex);
GL.TexImage2D(TextureTarget.Texture2D, 0, PixelInternalFormat.Luminance, width / 2, height / 2, 0, PixelFormat.Luminance, PixelType.UnsignedByte, (IntPtr)frame.data[2]);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
GL.TexParameter(TextureTarget.Texture2D, TextureParameterName.TextureMinFilter, (int)TextureMinFilter.Linear);
}
}
}
And my shader class:
using OpenTK.Graphics.OpenGL4;
using System;
using System.IO;
namespace myFFmpeg
{
public class Shader : IDisposable
{
public int Handle { get; private set; }
public Shader(string vertexPath, string fragmentPath)
{
string vertexShaderSource = File.ReadAllText(vertexPath);
string fragmentShaderSource = File.ReadAllText(fragmentPath);
int vertexShader = GL.CreateShader(ShaderType.VertexShader);
GL.ShaderSource(vertexShader, vertexShaderSource);
GL.CompileShader(vertexShader);
CheckShaderCompilation(vertexShader);
int fragmentShader = GL.CreateShader(ShaderType.FragmentShader);
GL.ShaderSource(fragmentShader, fragmentShaderSource);
GL.CompileShader(fragmentShader);
CheckShaderCompilation(fragmentShader);
Handle = GL.CreateProgram();
GL.AttachShader(Handle, vertexShader);
GL.AttachShader(Handle, fragmentShader);
GL.LinkProgram(Handle);
CheckProgramLinking(Handle);
GL.DetachShader(Handle, vertexShader);
GL.DetachShader(Handle, fragmentShader);
GL.DeleteShader(vertexShader);
GL.DeleteShader(fragmentShader);
}
public void Use()
{
GL.UseProgram(Handle);
}
public int GetAttribLocation(string attribName)
{
return GL.GetAttribLocation(Handle, attribName);
}
public int GetUniformLocation(string uniformName)
{
return GL.GetUniformLocation(Handle, uniformName);
}
private void CheckShaderCompilation(int shader)
{
GL.GetShader(shader, ShaderParameter.CompileStatus, out int success);
if (success == 0)
{
string infoLog = GL.GetShaderInfoLog(shader);
throw new InvalidOperationException($"Shader compilation failed: {infoLog}");
}
}
private void CheckProgramLinking(int program)
{
GL.GetProgram(program, GetProgramParameterName.LinkStatus, out int success);
if (success == 0)
{
string infoLog = GL.GetProgramInfoLog(program);
throw new InvalidOperationException($"Program linking failed: {infoLog}");
}
}
public void Dispose()
{
GL.DeleteProgram(Handle);
}
}
}
Vert shader
#version 330 core
layout(location = 0) in vec3 vertexPos;
layout(location = 1) in vec2 texCoord;
out vec2 TexCoord;
void main()
{
gl_Position = vec4(vertexPos,1.0);
TexCoord = texCoord;
}
Frag shader
#version 330 core
in vec2 TexCoord;
out vec4 color;
uniform sampler2D yTex;
uniform sampler2D uTex;
uniform sampler2D vTex;
void main()
{
float y = texture(yTex, TexCoord).r;
float u = texture(uTex, TexCoord).r - 0.5;
float v = texture(vTex, TexCoord).r - 0.5;
// YUV to RGB conversion (BT.709)
float r = y + 1.5714 * v;
float g = y - 0.6486 * u - 0.3918 * v;
float b = y + 1.8556 * u;
color = vec4(r, g, b, 1.0);
}
I can provide more code, if needed..
I tried changing shaders, changing textures, getting frame using ffmpeg.av_hwframe_transfer_data(_receivedFrame, _pFrame, 0);
New contributor
dima2012 terminator is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.