So I am currently working on a project that is using @tensorflow-models/handpose to detect hand landmarks and then I am matching the x and y of the landmark 0 to a 3d model. However. I am using info to try to match the rotations of my hand wit the 3d object’s rotation. however what I can’t figure out is:
when for example I hold my hand vertically and then rotate it around itself the watch rotates with it, however when I do it horizontally the watch seems to rotate badly. also as you can figure out I think when i translate my hand from vertical to horizontal my watch(3D) object is also rotating around the z axis.
I am stuck with this for a while now and don’t know what to do. Any help would be really appreciated.
Here is the code:
<body>
<video id="video" autoplay playsinline></video>
<canvas id="output"></canvas>
<canvas id="three-canvas"></canvas>
<script>
const video = document.getElementById('video');
const canvas = document.getElementById('output');
const ctx = canvas.getContext('2d');
const threeCanvas = document.getElementById('three-canvas');
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
const renderer = new THREE.WebGLRenderer({ canvas: threeCanvas, alpha: true });
renderer.setSize(window.innerWidth, window.innerHeight);
camera.position.z = 10;
// Add lighting
const ambientLight = new THREE.AmbientLight(0x404040); // Soft white light
scene.add(ambientLight);
const directionalLight = new THREE.DirectionalLight(0xffffff, 1);
directionalLight.position.set(0, 1, 1).normalize();
scene.add(directionalLight);
// Load the watch model
const loader = new THREE.GLTFLoader();
const textureLoader = new THREE.TextureLoader();
let watchModel;
// Load the textures
const diffuseMap = textureLoader.load('{% static "textures/1/1_21068718-f0cmplp9luxiysskijb2s9xb-ExtraLarge.jpg" %}', function(texture) {
console.log("Diffuse map loaded successfully");
}, undefined, function(error) {
console.error("Error loading diffuse map:", error);
});
const roughnessMap = textureLoader.load('{% static "textures/1/1_Clay004_2K_Roughness.jpg" %}', function(texture) {
console.log("Roughness map loaded successfully");
}, undefined, function(error) {
console.error("Error loading roughness map:", error);
});
const bodyEmissionMap = textureLoader.load('{% static "textures/1/1_rolex_body_Emission.png" %}', function(texture) {
console.log("Body emission map loaded successfully");
}, undefined, function(error) {
console.error("Error loading body emission map:", error);
});
const dialEmissionMap = textureLoader.load('{% static "textures/1/1_rolex_dial_Emission.png" %}', function(texture) {
console.log("Dial emission map loaded successfully");
}, undefined, function(error) {
console.error("Error loading dial emission map:", error);
});
loader.load("{% static '1.glb' %}", function(gltf) {
watchModel = gltf.scene;
watchModel.traverse((child) => {
if (child.isMesh) {
child.material.map = diffuseMap;
child.material.roughnessMap = roughnessMap;
child.material.emissiveMap = bodyEmissionMap;
child.material.needsUpdate = true;
// Apply the dial emission map to the dial part of the model
if (child.name.includes("dial")) { // Assuming "dial" in the name for differentiation
child.material.emissiveMap = dialEmissionMap;
}
}
});
watchModel.scale.set(20, 20, 20); // Adjust the scale as needed
scene.add(watchModel);
}, undefined, function(error) {
console.error("Error loading GLTF model:", error);
});
async function setupCamera() {
const isMobile = /iPhone|iPad|iPod|Android/i.test(navigator.userAgent);
let constraints = {
video: {
facingMode: { exact: "environment" }
}
};
if (isMobile) {
constraints.video.width = { ideal: 1920 };
constraints.video.height = { ideal: 1080 };
}
try {
const stream = await navigator.mediaDevices.getUserMedia(constraints);
video.srcObject = stream;
} catch (error) {
if (error.name === 'OverconstrainedError') {
console.warn('OverconstrainedError: Trying to use default resolution constraints');
constraints = {
video: true
};
const stream = await navigator.mediaDevices.getUserMedia(constraints);
video.srcObject = stream;
} else {
throw error;
}
}
return new Promise((resolve) => {
video.onloadedmetadata = () => {
resolve(video);
};
});
}
async function main() {
await setupCamera();
video.play();
const net = await handpose.load();
console.log('Handpose model loaded.');
function resizeCanvas() {
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
renderer.setSize(window.innerWidth, window.innerHeight);
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
}
window.addEventListener('resize', resizeCanvas);
video.addEventListener('loadeddata', resizeCanvas);
resizeCanvas();
let lastFrameTime = 0;
const frameRate = 30; // Limit to 30 frames per second
function calculateAngle(v1, v2) {
return Math.atan2(v2.y - v1.y, v2.x - v1.x);
}
async function detectHands() {
const now = Date.now();
const elapsed = now - lastFrameTime;
if (elapsed > (1000 / frameRate)) {
lastFrameTime = now;
const predictions = await net.estimateHands(video);
ctx.clearRect(0, 0, canvas.width, canvas.height);
if (predictions.length > 0) {
predictions.forEach(prediction => {
const landmarks = prediction.landmarks;
const wrist = landmarks[0];
const middleFinger = landmarks[9]; // Use the base of the middle finger for orientation
const indexFinger = landmarks[5]; // Use the base of the index finger for rotation calculation
const x = (wrist[0] - video.videoWidth / 2) / 500; // Adjust the x offset as needed
const y = -(wrist[1] - video.videoHeight / 2) / 500; // Adjust the y offset as needed
const z = 0; // Adjust z scaling for visibility
if (watchModel) {
watchModel.position.set(x, y, z);
// Calculate the angle of rotation based on the wrist and middle finger
const angle = calculateAngle(
{ x: wrist[0], y: wrist[1] },
{ x: middleFinger[0], y: middleFinger[1] }
);
// Calculate the angle for hand rotation around the z-axis
const rotationAngle = calculateAngle(
{ x: indexFinger[0], y: indexFinger[1] },
{ x: middleFinger[0], y: middleFinger[1] }
);
// Apply the rotation
watchModel.rotation.set(Math.PI / 2, -Math.PI / 2 - angle, -rotationAngle);
}
ctx.strokeStyle = 'red';
ctx.lineWidth = 2;
ctx.fillStyle = 'red';
for (let i = 0; i < landmarks.length; i++) {
const [x, y] = landmarks[i];
ctx.beginPath();
ctx.arc(x, y, 5, 0, 2 * Math.PI);
ctx.fill();
}
const fingerConnections = [
[0, 1], [1, 2], [2, 3], [3, 4],
[0, 5], [5, 6], [6, 7], [7, 8],
[5, 9], [9, 10], [10, 11], [11, 12],
[9, 13], [13, 14], [14, 15], [15, 16],
[13, 17], [17, 18], [18, 19], [19, 20],
[0, 17]
];
ctx.beginPath();
fingerConnections.forEach(pair => {
const [start, end] = pair;
ctx.moveTo(landmarks[start][0], landmarks[start][1]);
ctx.lineTo(landmarks[end][0], landmarks[end][1]);
});
ctx.stroke();
});
}
}
renderer.render(scene, camera);
requestAnimationFrame(detectHands);
}
detectHands();
}
main();
</script>
</body>
Also you may see that the z position of the watch is 0. That’s also a problem I am not sure how to fix. by default the handpose model gives me the Z coordinates of the wrist landmark, however they are really small values(-0.000412). I tried scaling them by multiplying with a negative number but I still get bad results, the object still looks like it’s floating on my wrist.
Thank you!
Also I must mention I tried using rotation matrices and even quaternions but with no success. I get the same errors.
New contributor
sanoful is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.