I’m trying to setup a 3D scanner using a single camera and a projector based on gray code structured light. The resulting pointclouds are heavily warped. ( see image below)

Warped point cloud
target point cloud: http://mesh.brown.edu/calibration/files/cartman-result.zip

The code to calibrate the system is based on this repo: https://github.com/kamino410/procam-calibration

The calibration of the setup has been verified using a separate tool: http://mesh.brown.edu/calibration/software.html
which successfully reconstructs images taken with our setup. The camera and projector intrinsics and exctrinsics obtained by the python code from the repo above closely match the calibration results from the tool above.

The decoding of the graycode encoded images is also verified. decoding the original images sent to the projector returns exact row and column indices and this was also verified using captured images.

Therefore, I suppose the issue lies with the triangulation.
I used this data to be able to compare results: http://mesh.brown.edu/calibration/files/cartman-model-images.zip

The code below is what i currently use to reconstruct the pointcloud. It can be used with this data: https://drive.google.com/file/d/1kl8CUDTTWEhQwL56JgqUTBApQwSgIef0/view?usp=sharing

import os
import cv2
import numpy as np

def get_cam_proj_pts(h_pixels, v_pixels, img_white):
    cam_w, cam_h = (1600, 1200)
    proj_w, proj_h = (720, 640)

    cam_pts = []
    proj_pts = []
    colors = []

    for i in range(cam_w):
        for j in range(cam_h):
            h_value = h_pixels[j, i]
            v_value = v_pixels[j, i]
            if h_value < 0 or v_value < 0:
                pass
            else:
                cam_pts.append([i, j])
                h_value = min(proj_w - 1, h_value)
                v_value = min(proj_h - 1, v_value)
                proj_pts.append([h_value, v_value])
                colors.append(img_white[j, i, :])

    cam_pts = np.array(cam_pts, dtype=np.float32)
    proj_pts = np.array(proj_pts, dtype=np.float32)
    colors_array = np.array(colors).astype(np.float64) / 255.0

    return cam_pts, proj_pts, colors_array


def triangulate(cam_mtx, cam_dist, proj_mtx, proj_dist, R, T, cam_pts, proj_pts):
    p1 = cam_mtx @ np.concatenate([np.eye(3), [[0], [0], [0]]], axis=-1)
    p2 = proj_mtx @ np.concatenate([R, T], axis=-1)

    points4d = cv2.triangulatePoints(p1, p2,
                                     cv2.undistortPoints(np.expand_dims(cam_pts, axis=1), cam_mtx, cam_dist),
                                     cv2.undistortPoints(np.expand_dims(proj_pts, axis=1), proj_mtx, proj_dist))
    points3d = points4d[:3] / points4d[3]

    return points3d


def main():
    # Declare folders to use for triangulation
    cal_result_folder = './cartman/calib_results_new/'
    gray_code_folder = './cartman/scan_1/'
    output_folder = './cartman/scan_1/point_clouds/'

    # Create folders if they don't exist
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    img_white = cv2.imread(os.path.join(gray_code_folder, 'cam_01.png'), cv2.IMREAD_COLOR)
    img_white = cv2.cvtColor(img_white, cv2.COLOR_BGR2RGB)

    # Load calibration data
    cam_mtx = np.load(os.path.join(cal_result_folder, 'cam_int.npy'))
    cam_dist = np.load(os.path.join(cal_result_folder, 'cam_dist.npy'))
    proj_mtx = np.load(os.path.join(cal_result_folder, 'proj_int.npy'))
    proj_dist = np.load(os.path.join(cal_result_folder, 'proj_dist.npy'))
    R = np.load(os.path.join(cal_result_folder, 'R.npy'))
    T = np.load(os.path.join(cal_result_folder, 'T.npy'))


    # Load decoded gray codes
    h_pixels = np.load(os.path.join(gray_code_folder + '/output/', 'h_pixels.npy'))
    v_pixels = np.load(os.path.join(gray_code_folder + '/output/', 'v_pixels.npy'))

    cam_pts, proj_pts, colors = get_cam_proj_pts(h_pixels, v_pixels, img_white)

    pts_3d = triangulate(cam_mtx, cam_dist, proj_mtx, proj_dist, R, T, cam_pts, proj_pts)

    from scanner.utils import visualize
    visualize.plot_point_cloud(pts_3d, colors, output_folder)
    print("done")


if __name__ == "__main__":
    main()

I tried to add R1 R2 and P1 P2 from cv2.stereoRectify to undistortPoints in any combination possible
which all lead to worse results.
I also tried converting the triangulation method from the Brown tool with even more warping.

After days of debugging, I’m quite lost so any insight in my potentials mistakes is welcome
thanks