I’m using media pipe to get the face landmarks of a face in a photo, then I get the 3d points of those landmarks and put them in 3d space in open3d.
I have the point cloud of the scene related to the image and with all the camera parameters(intrinsix and extrinsic), i use open3d to visualize the point cloud in the same view angle of the photo, then i took depth image of the point cloud.

import itertools
import numpy as np

import cv2
import mediapipe as mp
import open3d as o3d
from visualization import VisOpen3D

def loadcloud( path="", extension="", n=0):

    pcds = []

    for i in range(n):
        print("nnLoading point cloud %d.................................." % (i+1))
        if n>1:
            pcd = (o3d.io.read_point_cloud((path+"%d"+extension) % i, remove_nan_points=True, remove_infinite_points=True))
        else:
            pcd = (o3d.io.read_point_cloud(path+extension, remove_nan_points=True, remove_infinite_points=True))
        #pcd.scale(1.3, center=pcd.get_center())

        print(pcd)
        print("|________________________Punti_________________________|")
        print(np.asarray(pcd.points))
        print("|________________________Normals_______________________|")
        print(np.asarray(pcd.normals))
        print("|________________________Colori________________________|")
        print(np.asarray(pcd.colors))
        print("|______________________________________________________|")

        pcds.append(pcd)

    return pcds

if __name__ == "__main__":
    #img = cv2.imread("Set2/Fase0/Faccia1/001_00_image.png")

    pcd = loadcloud(path=("Set1/Fase0/Faccia1/001_03_cloud"), extension=".pcd", n=1)[0]
    #pcd = o3d.geometry.PointCloud.rotate(pcd, o3d.geometry.get_rotation_matrix_from_xyz((np.pi, 0, 0)))

    bounding_box0 = pcd.get_axis_aligned_bounding_box()
    bounding_box0.color = (0, 0, 1)

    sphere0 = o3d.geometry.TriangleMesh.create_sphere(radius=0.1)
    sphere0.translate([0, 0, 0])

    #o3d.visualization.draw_geometries([pcd, bounding_box0],mesh_show_back_face=True)
    h = 1080
    w = 1920

    window_visible = True
    vis = VisOpen3D(width=w, height=h, visible=window_visible)

    # point cloud
    vis.add_geometry(pcd)
    vis.load_view_point("view_point.json")


    # capture images
    depth = vis.capture_depth_float_buffer(show=True)
    image = vis.capture_screen_float_buffer(show=True)

    # save to file
    vis.capture_screen_image("capture_screen_image.png")
    vis.capture_depth_image("capture_depth_image.png")

    # Face Landmarks Acquisition
    #######################################
    img = cv2.imread("Set2/Fase0/Faccia1/001_00_image.png")
    mpDraw = mp.solutions.drawing_utils
    mpFaceMesh = mp.solutions.face_mesh
    faceMesh = mpFaceMesh.FaceMesh(max_num_faces=1, min_detection_confidence=0.5)
    drawSpec = mpDraw.DrawingSpec(thickness=1, circle_radius=2)
    points = []

    imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    results = faceMesh.process(imgRGB)
    if results.multi_face_landmarks:
        faceLms = results.multi_face_landmarks[0]
        mpDraw.draw_landmarks(img, faceLms, mpFaceMesh.FACEMESH_TESSELATION, drawSpec, drawSpec)
        LEFT_EYE_INDEXES = list(set(itertools.chain(*mpFaceMesh.FACEMESH_LEFT_EYE)))
        RIGHT_EYE_INDEXES = list(set(itertools.chain(*mpFaceMesh.FACEMESH_RIGHT_EYE)))
        lm = faceLms.landmark[0]

        print(f'FACE NUMBER: {1}')
        print('-----------------------')

        # outer face landmarks
        print(f'OUTER FACE LANDMARKS:')

        outer_x = []
        outer_y = []
        outer_z = []

        for OUTER_FACE_INDEX in mpFaceMesh.FACEMESH_TESSELATION:
            outer_x.append(faceLms.landmark[OUTER_FACE_INDEX[0]].x)
            outer_y.append(faceLms.landmark[OUTER_FACE_INDEX[0]].y)
            outer_z.append(faceLms.landmark[OUTER_FACE_INDEX[0]].z)

        points = []

        for i in range(len(outer_x)):
            points.append([outer_x[i], outer_y[i] / 2, outer_z[i]])
        """
        for i in range(len(outer_x)):
            points.append([outer_x[i], outer_y[i]/2, max(outer_z) * 2])
        """

        print(f'LEFT EYE LANDMARKS:')

        for LEFT_EYE_INDEX in LEFT_EYE_INDEXES[:2]:
            print(faceLms.landmark[LEFT_EYE_INDEX])

        print(f'RIGHT EYE LANDMARKS:')

        for RIGHT_EYE_INDEX in RIGHT_EYE_INDEXES[:2]:
            print(faceLms.landmark[RIGHT_EYE_INDEX])

        # PASTED CODE
        img_h, img_w, img_c = img.shape
        face_2d = []
        face_3d = []

        if results.multi_face_landmarks:
            for face_landmarks in results.multi_face_landmarks:
                for idx, lm in enumerate(face_landmarks.landmark):
                    if idx == 33 or idx == 263 or idx == 1 or idx == 61 or idx == 291 or idx == 199:
                        if idx == 1:
                            nose_2d = (lm.x * img_w, lm.y * img_h)
                            nose_3d = (lm.x * img_w, lm.y * img_h, lm.z * 3000)
                        x, y = int(lm.x * img_w), int(lm.y * img_h)

                        face_2d.append([x, y])
                        face_3d.append(([x, y, lm.z]))

        # Get 2d Coord
        face_2d = np.array(face_2d, dtype=np.float64)

        face_3d = np.array(face_3d, dtype=np.float64)

        focal_length = 1 * img_w

        cam_matrix = np.array([[focal_length, 0, img_h / 2],
                               [0, focal_length, img_w / 2],
                               [0, 0, 1]])
        distortion_matrix = np.zeros((4, 1), dtype=np.float64)

        success, rotation_vec, translation_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix, distortion_matrix)

        # getting rotational of face
        rmat, jac = cv2.Rodrigues(rotation_vec)

        angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)

        x = angles[0] * 360
        y = angles[1] * 360
        z = angles[2] * 360

        # here based on axis rot angle is calculated
        if y < -10:
            text = "Looking Left"
        elif y > 10:
            text = "Looking Right"
        elif x < -10:
            text = "Looking Down"
        elif x > 10:
            text = "Looking Up"
        else:
            text = "Forward"

        print("Face Rotation: ", text)

    # cv2.imshow("Image", img)
    # save img
    cv2.imwrite("face.jpg", img)
    ##############################################

    pcd1 = o3d.geometry.PointCloud()
    pcd1.points.extend(points)

    vis.add_geometry(pcd1)

    # draw camera
    if window_visible:
        vis.load_view_point("view_point.json")
        intrinsic = vis.get_view_point_intrinsics()
        extrinsic = vis.get_view_point_extrinsics()
        vis.draw_camera(intrinsic, extrinsic, scale=0.5, color=[0.8, 0.2, 0.8])
        vis.update_view_point(intrinsic, extrinsic)



    if window_visible:
        vis.load_view_point("view_point.json")
        vis.run()

    del vis

I can’t figure out how to translate the 3d face landmark in the correct potition.
How can I get the correct translation cord?