I’m working to convert the SplaTV WebGL library to use ThreeJS and I’m struggling to get it converted. The code in the original repo is challenging to understand as it’s not documented and while I understand threejs, I’m still learning lower level WebGL. I’ve got the following:

import * as THREE from 'three';
import { OrbitControls } from "three/addons/controls/OrbitControls.js";

const vertexShader = `
  precision highp float;
  precision highp int;
  precision highp usampler2D;
  
  uniform usampler2D utexture;
  uniform mat4 projection;
  uniform mat4 view;
  uniform vec2 focal;
  uniform vec2 viewport;
  uniform float time;
  
  in vec2 position;
  in float index;
  
  out vec4 vColor;
  out vec2 vPosition;
  
  void main () {
      gl_Position = vec4(0.0, 0.0, 2.0, 1.0);

      uvec4 motion1 = texelFetch(utexture, ivec2((int(uint(index) & 0x3ffu) << 2) | 3, int(uint(index) >> 10)), 0);
      vec2 trbf = unpackHalf2x16(motion1.w);
      float dt = time - trbf.x;

      float topacity = exp(-1.0 * pow(dt / trbf.y, 2.0));
      if(topacity < 0.02) return;

      uvec4 motion0 = texelFetch(utexture, ivec2(((uint(index) & 0x3ffu) << 2) | 2u, uint(index) >> 10), 0);
      uvec4 static0 = texelFetch(utexture, ivec2(((uint(index) & 0x3ffu) << 2), uint(index) >> 10), 0);

      vec2 m0 = unpackHalf2x16(motion0.x), m1 = unpackHalf2x16(motion0.y), m2 = unpackHalf2x16(motion0.z), 
           m3 = unpackHalf2x16(motion0.w), m4 = unpackHalf2x16(motion1.x); 
      
      vec4 trot = vec4(unpackHalf2x16(motion1.y).xy, unpackHalf2x16(motion1.z).xy) * dt;
      vec3 tpos = (vec3(m0.xy, m1.x) * dt + vec3(m1.y, m2.xy) * dt*dt + vec3(m3.xy, m4.x) * dt*dt*dt);
      
      vec4 cam = view * vec4(uintBitsToFloat(static0.xyz) + tpos, 1);
      vec4 pos = projection * cam;
  
      float clip = 1.2 * pos.w;
      if (pos.z < -clip || pos.x < -clip || pos.x > clip || pos.y < -clip || pos.y > clip) return;
      uvec4 static1 = texelFetch(utexture, ivec2(((uint(index) & 0x3ffu) << 2) | 1u, uint(index) >> 10), 0);

      vec4 rot = vec4(unpackHalf2x16(static0.w).xy, unpackHalf2x16(static1.x).xy) + trot;
      vec3 scale = vec3(unpackHalf2x16(static1.y).xy, unpackHalf2x16(static1.z).x);
      rot /= sqrt(dot(rot, rot));
  
      mat3 S = mat3(scale.x, 0.0, 0.0, 0.0, scale.y, 0.0, 0.0, 0.0, scale.z);
      mat3 R = mat3(
        1.0 - 2.0 * (rot.z * rot.z + rot.w * rot.w), 2.0 * (rot.y * rot.z - rot.x * rot.w), 2.0 * (rot.y * rot.w + rot.x * rot.z),
        2.0 * (rot.y * rot.z + rot.x * rot.w), 1.0 - 2.0 * (rot.y * rot.y + rot.w * rot.w), 2.0 * (rot.z * rot.w - rot.x * rot.y),
        2.0 * (rot.y * rot.w - rot.x * rot.z), 2.0 * (rot.z * rot.w + rot.x * rot.y), 1.0 - 2.0 * (rot.y * rot.y + rot.z * rot.z));
      mat3 M = S * R;
      mat3 Vrk = 4.0 * transpose(M) * M;
      mat3 J = mat3(
          focal.x / cam.z, 0., -(focal.x * cam.x) / (cam.z * cam.z), 
          0., -focal.y / cam.z, (focal.y * cam.y) / (cam.z * cam.z), 
          0., 0., 0.
      );
  
      mat3 T = transpose(mat3(view)) * J;
      mat3 cov2d = transpose(T) * Vrk * T;
  
      float mid = (cov2d[0][0] + cov2d[1][1]) / 2.0;
      float radius = length(vec2((cov2d[0][0] - cov2d[1][1]) / 2.0, cov2d[0][1]));
      float lambda1 = mid + radius, lambda2 = mid - radius;
  
      if(lambda2 < 0.0) return;
      vec2 diagonalVector = normalize(vec2(cov2d[0][1], lambda1 - cov2d[0][0]));
      vec2 majorAxis = min(sqrt(2.0 * lambda1), 1024.0) * diagonalVector;
      vec2 minorAxis = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x);
      
      uint rgba = static1.w;
      vColor = 
        clamp(pos.z / pos.w + 1.0, 0.0, 1.0) * 
        vec4(1.0, 1.0, 1.0, topacity) *
        vec4(
          float((rgba & 0xffu)) / 255.0, 
          float((rgba >> 8) & 0xffu) / 255.0, 
          float((rgba >> 16) & 0xffu) / 255.0, 
          float((rgba >> 24) & 0xffu) / 255.0);

      vec2 vCenter = vec2(pos) / pos.w;
      gl_Position = vec4(
          vCenter 
          + position.x * majorAxis / viewport 
          + position.y * minorAxis / viewport, 0.0, 1.0);

      vPosition = position;
  }
`;

const fragmentShader = `
  precision highp float;
  
  in vec4 vColor;
  in vec2 vPosition;

  layout(location = 0) out vec4 fragColor;
  
  void main () {
      float A = -dot(vPosition, vPosition);
      if (A < -4.0) discard;
      float B = exp(A) * vColor.a;
      fragColor = vec4(B * vColor.rgb, B);
  }
`;

let vertexCount = 0;
const canvas = document.getElementById("canvas");

const worker = new Worker(
    URL.createObjectURL(
        new Blob(["(", createWorker.toString(), ")(self)"], {
            type: "application/javascript",
        })
    )
);

worker.onmessage = (e) => {
    if (e.data.depthIndex) {
        const { depthIndex, viewProj } = e.data;
        geometry.setAttribute("index", new THREE.BufferAttribute(depthIndex, 1));
        geometry.getAttribute("index").needsUpdate = true;
        vertexCount = e.data.vertexCount;
    }
};

// Create the scene, camera, and renderer
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
const renderer = new THREE.WebGLRenderer({
    canvas
});
renderer.setSize(window.innerWidth, window.innerHeight);

// Set camera position
camera.position.z = 5;

const controls = new OrbitControls(camera, renderer.domElement);
controls.enableDamping = true; // Smooth camera movement
controls.dampingFactor = 0.05; // Adjust damping for responsiveness

let texture = new THREE.DataTexture(new Uint32Array([0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff]), 2, 2, THREE.RGBAFormat, THREE.UnsignedIntType);
texture.needsUpdate = true;

// Define uniforms
const uniforms = {
    utexture: { value: texture },
    projection: { value: new THREE.Matrix4().identity() },
    view: { value: new THREE.Matrix4().identity() },
    focal: { value: new THREE.Vector2(1, 1) },
    viewport: { value: new THREE.Vector2(window.innerWidth, window.innerHeight) },
    time: { value: 0.0 }
};

// Create shader material
const material = new THREE.RawShaderMaterial({
    vertexShader: vertexShader,
    fragmentShader: fragmentShader,
    uniforms: uniforms,
    transparent: true,
    blending: THREE.CustomBlending,
    blendSrc: THREE.OneMinusDstAlphaFactor,
    blendDst: THREE.OneFactor,
    blendSrcAlpha: THREE.OneMinusDstAlphaFactor,
    blendDstAlpha: THREE.OneFactor,
    blendEquation: THREE.AddEquation,
    blendEquationAlpha: THREE.AddEquation,
    glslVersion: THREE.GLSL3
});

// Create geometry and mesh
let geometry = new THREE.BufferGeometry();
let triangleVertices = new Float32Array([-2, -2, 2, -2, 2, 2, -2, 2]);
geometry.setAttribute('position', new THREE.BufferAttribute(triangleVertices, 2));
geometry.setAttribute('index', new THREE.BufferAttribute(new Int32Array(vertexCount)));
const mesh = new THREE.Mesh(geometry, material);
scene.add(mesh);

// Test Cube
const boxMat = new THREE.MeshBasicMaterial( { color: 0x00ff00 } );
const boxGeo = new THREE.BoxGeometry( 1, 1, 1 );
const cube = new THREE.Mesh( boxGeo, boxMat );
scene.add( cube );

let lastVertexCount = -1;

const chunkHandler = (chunk, buffer, remaining, chunks) => {
    if (!remaining && chunk.type === "magic") {
        let intView = new Uint32Array(buffer);
        if (intView[0] !== 0x674b) throw new Error("This does not look like a splatv file");
        chunks.push({ size: intView[1], type: "chunks" });
    } else if (!remaining && chunk.type === "chunks") {
        for (let chunk of JSON.parse(new TextDecoder("utf-8").decode(buffer))) {
            chunks.push(chunk);
            if (chunk.type === "splat") {
                console.log("Received splat: " + remaining + " chunks: " + chunks.length);
            }
        }
    } else if (chunk.type === "splat") {
        if (vertexCount > lastVertexCount || remaining === 0) {
            lastVertexCount = vertexCount;
            worker.postMessage({ texture: new Float32Array(buffer), remaining: remaining });
            console.log("splat", remaining);

            const texdata = new Uint32Array(buffer);
            /// Create a DataTexture in Three.js
            texture = new THREE.DataTexture(texdata, chunk.texwidth, chunk.texheight, THREE.RGBAIntegerFormat, THREE.UnsignedIntType);
            
            // Set texture parameters
            texture.wrapS = THREE.ClampToEdgeWrapping;
            texture.wrapT = THREE.ClampToEdgeWrapping;
            texture.minFilter = THREE.NearestFilter;
            texture.magFilter = THREE.NearestFilter;
            
            material.uniforms.utexture.value = texture;
            texture.needsUpdate = true;
            material.uniformsNeedUpdate = true;
        }
    } else if (!remaining) {
        console.log("chunk", chunk, buffer);
    }
};

const req = await fetch("model.splatv", { mode: "cors", credentials: "omit" });
if (req.status != 200) throw new Error(req.status + " Unable to load " + req.url);

await readChunks(req.body.getReader(), [{ size: 8, type: "magic" }], chunkHandler);

// Animation loop
function animate() {

    const projectionMatrix = camera.projectionMatrix;

    // Retrieve the view matrix (inverse of the camera's world matrix)
    const viewMatrix = new THREE.Matrix4();
    viewMatrix.copy(camera.matrixWorld).invert();

    // Combine the projection and view matrices to get the view projection matrix
    const viewProjectionMatrix = new THREE.Matrix4();
    viewProjectionMatrix.multiplyMatrices(projectionMatrix, viewMatrix);
    worker.postMessage({ view: viewProjectionMatrix });
    geometry.needsUpdate = true;
    

    // Update time uniform
    uniforms.time.value += 0.05;
    if (vertexCount > 0) {
        uniforms.view.value = viewMatrix;
    }
    controls.update();
    renderer.render(scene, camera);
    texture.needsUpdate = true;
    
    requestAnimationFrame(animate);
}

animate();

async function readChunks(reader, chunks, handleChunk) {
    let chunk = chunks.shift();
    let buffer = new Uint8Array(chunk.size);
    let offset = 0;
    while (chunk) {
        let { done, value } = await reader.read();
        if (done) break;
        while (value.length + offset >= chunk.size) {
            buffer.set(value.subarray(0, chunk.size - offset), offset);
            value = value.subarray(chunk.size - offset);
            handleChunk(chunk, buffer.buffer, 0, chunks);
            chunk = chunks.shift();
            if (!chunk) break;
            buffer = new Uint8Array(chunk.size);
            offset = 0;
        }
        if (!chunk) break;
        buffer.set(value, offset);
        offset += value.length;
        handleChunk(chunk, buffer.buffer, buffer.byteLength - offset, chunks);
    }
    if (chunk) handleChunk(chunk, buffer.buffer, 0, chunks);
}

function createWorker(self) {
    let vertexCount = 0;
    let viewProj;
    let lastProj = [];
    let depthIndex = new Uint32Array();
    let positions;
    let lastVertexCount = -1;

    function runSort(viewProj) {
        if (!positions || !viewProj) return;
        if (lastVertexCount == vertexCount) {
            let dist = Math.hypot(...[2, 6, 10].map((k) => lastProj[k] - viewProj[k]));
            if (dist < 0.01) return;
        } else {
            lastVertexCount = vertexCount;
        }
        console.time("sort");
        let maxDepth = -Infinity;
        let minDepth = Infinity;
        let sizeList = new Int32Array(vertexCount);
        for (let i = 0; i < vertexCount; i++) {
            let depth =
                ((viewProj[2] * positions[3 * i + 0] + viewProj[6] * positions[3 * i + 1] + viewProj[10] * positions[3 * i + 2]) * 4096) | 0;
            sizeList[i] = depth;
            if (depth > maxDepth) maxDepth = depth;
            if (depth < minDepth) minDepth = depth;
        }

        // This is a 16 bit single-pass counting sort
        let depthInv = (256 * 256) / (maxDepth - minDepth);
        let counts0 = new Uint32Array(256 * 256);
        for (let i = 0; i < vertexCount; i++) {
            sizeList[i] = ((sizeList[i] - minDepth) * depthInv) | 0;
            counts0[sizeList[i]]++;
        }
        let starts0 = new Uint32Array(256 * 256);
        for (let i = 1; i < 256 * 256; i++) starts0[i] = starts0[i - 1] + counts0[i - 1];
        depthIndex = new Uint32Array(vertexCount);
        for (let i = 0; i < vertexCount; i++) depthIndex[starts0[sizeList[i]]++] = i;

        console.timeEnd("sort");
        lastProj = viewProj;
        self.postMessage({ depthIndex, viewProj, vertexCount }, [depthIndex.buffer]);
    }

    const throttledSort = () => {
        if (!sortRunning) {
            sortRunning = true;
            let lastView = viewProj;
            runSort(lastView);
            setTimeout(() => {
                sortRunning = false;
                if (lastView !== viewProj) {
                    throttledSort();
                }
            }, 0);
        }
    };

    let sortRunning;

    self.onmessage = (e) => {
        if (e.data.texture) {
            let texture = e.data.texture;
            vertexCount = Math.floor((texture.byteLength - e.data.remaining) / 4 / 16);
            positions = new Float32Array(vertexCount * 3);
            for (let i = 0; i < vertexCount; i++) {
                positions[3 * i + 0] = texture[16 * i + 0];
                positions[3 * i + 1] = texture[16 * i + 1];
                positions[3 * i + 2] = texture[16 * i + 2];
            }
            throttledSort();
        } else if (e.data.vertexCount) {
            vertexCount = e.data.vertexCount;
        } else if (e.data.view) {
            viewProj = e.data.view;
            throttledSort();
        } else if (e.data.ply) {
            vertexCount = 0;
            vertexCount = processPlyBuffer(e.data.ply);
        }
    };
}

I get the following errors in the console regarding the vertex shader:

I’m confused why the original repo passes a vec2 to the shader and it doesn’t seem like the vec2 position is even being updated but I could be missing where it’s happening. The index attribute values I pass to the shader using setAttribute doesn’t seem like it would have an effect if there are no positions but again the positions are a vec2 which doesn’t make sense to me. I believe I am passing the required uniforms and attributes as in the original repo but I’m not getting anything rendering to the screen.