I’ve been working on this issue for some time. I have 2 cameras (standard USB cameras) that I place 10cm apart from left to right. The intent is to get a depth map.
You can find the repo with all images & code I use here: https://github.com/ClemDFZ/stereo_vision
I followed all the steps to perform a stereo calibration with 45 images of a checkerboard held in various positions in front of the camera.
This calibration seems correct to me: when I remap the left and right images I get a nice result: Image of overlayed right and left images: raw images | after rectification
The raw images are not perfectly aligned but the rectified images are on the same horizontal axis.
The issue is with the disparity map: when I try this code:
rectified_left = cv2.remap(frame_left, stereoMapL_x, stereoMapL_y, cv2.INTER_LINEAR,cv2.BORDER_CONSTANT,0)
rectified_right = cv2.remap(frame_right, stereoMapR_x, stereoMapR_y, cv2.INTER_LINEAR,cv2.BORDER_CONSTANT,0)
rectified_pic = combine_images(rectified_left,rectified_right)
gray_left = cv2.cvtColor(rectified_left,cv2.COLOR_BGR2GRAY)
gray_right = cv2.cvtColor(rectified_right,cv2.COLOR_BGR2GRAY)
overlayed_base = overlay_images(frame_left,frame_right)
overlayed_rectified = overlay_images(rectified_left,rectified_right)
overlayed_combined = combine_images(overlayed_base,overlayed_rectified)
stereo = cv2.StereoSGBM_create(numDisparities=272, blockSize=5)
disparity = stereo.compute(gray_left,gray_right)
depth = cv2.reprojectImageTo3D(disparity,Q)
I get an aberrant disparity map.disparity map
The result with the depth map is even worse: depth map
Whether it is StereoSGBM_create with gray or RGB images, or StereoBM_create with gray images, the result is similar
I saw a guy (OpenCV PointCloud from Depth map) posting similar results. Apparently for him it was only a matter of image format. I have tried different formats with CV_16SC2 or CV_32F for the initUndistortRectifyMap in the calibration but that did not change much.
I’ve tried to change the different image type, different angles, different calibrations. I tried using the epipolar lines but did not manage to get much information on whether the calibration is right or no. I have also tried arrays of parameters for StereoSGBM with a lot of values of numDisparities, blockSize etc..
Did I miss something obvious ? It seems to me the calibration and rectification processes are alright, but I cannot find the cause for such results.
I appreciate any help, this situation is maddening I feel like I’m close to good results. Here is the full code:
Calibration
type her
import numpy as np
import cv2
import glob
import time
import matplotlib.pyplot as plt
import random
#- Function to get only x% of calib pictures
def sample_matching_elements(list1, list2, sample_size):
indices = random.sample(range(len(list1)), sample_size)
sampled_elements_list1 = [list1[i] for i in indices]
sampled_elements_list2 = [list2[i] for i in indices]
return sampled_elements_list1, sampled_elements_list2
# Critères de terminaison pour la précision des coins du damier
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# Préparer les points d'objet (0,0,0), (1,0,0), (2,0,0), ..., (6,5,0)
chessboardSize = (8,6)
frameSize = (800,600)
# termination criteria
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((chessboardSize[0] * chessboardSize[1], 3), np.float32)
objp[:,:2] = np.mgrid[0:chessboardSize[0],0:chessboardSize[1]].T.reshape(-1,2)
size_of_chessboard_squares_mm = 30
objp = objp * size_of_chessboard_squares_mm
# Tableaux pour stocker les points d'objet et les points d'image des deux caméras
objpoints = [] # points 3D dans l'espace réel
imgpoints_left = [] # points 2D dans l'image de la caméra gauche
imgpoints_right = [] # points 2D dans l'image de la caméra droite
# Charger les images de calibration
images_left = sorted(glob.glob('left/*.jpg'))
images_right = sorted(glob.glob('right/*.jpg'))
import os
#images_left,images_right = sample_matching_elements(images_left, images_right, 30)
for img_left, img_right in zip(images_left, images_right):
imgL = cv2.imread(img_left)
imgR = cv2.imread(img_right)
grayL = cv2.cvtColor(imgL, cv2.COLOR_BGR2GRAY)
grayR = cv2.cvtColor(imgR, cv2.COLOR_BGR2GRAY)
retL, cornersL = cv2.findChessboardCorners(grayL, chessboardSize, None)
retR, cornersR = cv2.findChessboardCorners(grayR, chessboardSize, None)
#print(retL,retR)
if retL and retR:
objpoints.append(objp)
corners2L = cv2.cornerSubPix(grayL, cornersL, (11,11), (-1,-1), criteria)
imgpoints_left.append(corners2L)
corners2R = cv2.cornerSubPix(grayR, cornersR, (11,11), (-1,-1), criteria)
imgpoints_right.append(corners2R)
cv2.drawChessboardCorners(imgL, (8,6), corners2L, retL)
cv2.drawChessboardCorners(imgR, (8,6), corners2R, retR)
cv2.imshow('imgL', imgL)
cv2.imshow('imgR', imgR)
res = cv2.waitKey(100000) & 0xFF
if res == ord('s'):
print('Save')
elif res == ord('r'):
print(img_left,img_right)
os.remove(img_left)
os.remove(img_right)
else:
print(img_left,img_right)
os.remove(img_left)
os.remove(img_right)
#
cv2.destroyAllWindows()
# Calibration de chaque caméra
retL, mtxL, distL, rvecsL, tvecsL = cv2.calibrateCamera(objpoints, imgpoints_left, grayL.shape[::-1], None, None)
h, w = imgL.shape[:2]
newCameraMatrixL, roi_L = cv2.getOptimalNewCameraMatrix(mtxL, distL, (w,h), 0, (w,h))
retR, mtxR, distR, rvecsR, tvecsR = cv2.calibrateCamera(objpoints, imgpoints_right, grayR.shape[::-1], None, None)
h, w = imgR.shape[:2]
newCameraMatrixR, roi_R = cv2.getOptimalNewCameraMatrix(mtxR, distR, (w,h), 0, (w,h))
########## Stereo Vision Calibration #############################################
flags = 0
#flags |= cv2.CALIB_USE_INTRINSIC_GUESS
#flags |= cv2.CALIB_USE_EXTRINSIC_GUESS
criteria_stereo= (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# This step is performed to transformation between the two cameras and calculate Essential and Fundamenatl matrix
retStereo, stereo_mtxL, stereo_distL, stereo_mtxR, stereo_distR, stereo_rot, stereo_trans, essentialMatrix, fundamentalMatrix = cv2.stereoCalibrate(objpoints, imgpoints_left, imgpoints_right, newCameraMatrixL, distL, newCameraMatrixR, distR, grayL.shape[::-1], criteria_stereo, flags)
########## Stereo Rectification #################################################
rectifyScale= 0
rectL, rectR, projMatrixL, projMatrixR, Q, roi_L, roi_R= cv2.stereoRectify(stereo_mtxL, stereo_distL, stereo_mtxR, stereo_distR, grayL.shape[::-1], stereo_rot, stereo_trans)
stereoMapL = cv2.initUndistortRectifyMap(stereo_mtxL, stereo_distL, rectL, projMatrixL, grayL.shape[::-1], cv2.CV_16SC2)
stereoMapR = cv2.initUndistortRectifyMap(stereo_mtxR, stereo_distR, rectR, projMatrixR, grayL.shape[::-1], cv2.CV_16SC2)
stereoMapL_x = stereoMapL[0]
stereoMapL_y = stereoMapL[1]
stereoMapR_x = stereoMapR[0]
stereoMapR_y = stereoMapR[1]
np.savez('calibration/stereo_calib.npz', mtxL=mtxL, distL=distL, mtxR=mtxR, distR=distR, R1=projMatrixL, R2=projMatrixR, P1=projMatrixL, P2=projMatrixR, Q=Q,R=stereo_rot)
cv_file = cv2.FileStorage('calibration/stereoMap.xml', cv2.FILE_STORAGE_WRITE)
cv_file.write('stereoMapL_x',stereoMapL[0])
cv_file.write('stereoMapL_y',stereoMapL[1])
cv_file.write('stereoMapR_x',stereoMapR[0])
cv_file.write('stereoMapR_y',stereoMapR[1])
cv_file.release()
Image rectification and depth map
import numpy as np
import cv2
import matplotlib.pyplot as plt
import os
def overlay_images(image1, image2, alpha=0.5):
# Vérification des dimensions des images
if image1.shape != image2.shape:
raise ValueError("Les dimensions des images ne correspondent pas.")
# Superposition des images avec un facteur d'opacité alpha
overlaid_image = cv2.addWeighted(image1, alpha, image2, 1 - alpha, 0)
return overlaid_image
def combine_images(image1, image2):
# Vérifiez si les images ont la même hauteur
if image1.shape[0] != image2.shape[0]:
# Redimensionnez les images pour avoir la même hauteur
height = min(image1.shape[0], image2.shape[0])
image1 = cv2.resize(image1, (int(image1.shape[1] * height / image1.shape[0]), height))
image2 = cv2.resize(image2, (int(image2.shape[1] * height / image2.shape[0]), height))
# Concaténer les images horizontalement
combined_image = np.hstack((image1, image2))
return combined_image
with np.load('calibration/stereo_calib.npz') as data:
mtxL = data['mtxL']
distL = data['distL']
mtxR = data['mtxR']
distR = data['distR']
# R = data['R']
# T = data['T']
R1 = data['R1']
R2 = data['R2']
projMatrixL = data['P1']
projMatrixR = data['P2']
Q = data['Q']
# Chargement des cartes de rectification
cv_file = cv2.FileStorage('calibration/stereoMap.xml', cv2.FILE_STORAGE_READ)
stereoMapL_x = cv_file.getNode('stereoMapL_x').mat()
stereoMapL_y = cv_file.getNode('stereoMapL_y').mat()
stereoMapR_x = cv_file.getNode('stereoMapR_x').mat()
stereoMapR_y = cv_file.getNode('stereoMapR_y').mat()
cv_file.release()
##### WHEN USING THE CAMERA #####
#image_size = (800, 600) # Taille de l'image
#cap_left = cv2.VideoCapture(0)
#cap_right = cv2.VideoCapture(4)
#for cap in [cap_left,cap_right]:
# cap.set(3,800)
# cap.set(4,600)
#
#ret_left, frame_left = cap_left.read()
#ret_right, frame_right = cap_right.read()
#base_image = combine_images(frame_left,frame_right)
#
#if not ret_left or not ret_right:
# print("Erreur: Impossible de lire les images des caméras.")
## break
## else:
#cap_left.release()
#cap_right.release()
#####################################
frame_left = cv2.imread('result/left_image.jpg')
frame_right= cv2.imread('result/right_image.jpg')
img_undistorted_left = cv2.undistort(frame_left, mtxL, distL, None, newCameraMatrix=mtxL)
img_undistorted_right = cv2.undistort(frame_right, mtxR, distR, None, newCameraMatrix=mtxR)
undistorted = combine_images(img_undistorted_left,img_undistorted_right)
rectified_left = cv2.remap(frame_left, stereoMapL_x, stereoMapL_y, cv2.INTER_LINEAR,cv2.BORDER_CONSTANT,0)
rectified_right = cv2.remap(frame_right, stereoMapR_x, stereoMapR_y, cv2.INTER_LINEAR,cv2.BORDER_CONSTANT,0)
rectified_pic = combine_images(rectified_left,rectified_right)
gray_left = cv2.cvtColor(rectified_left,cv2.COLOR_BGR2GRAY)
gray_right = cv2.cvtColor(rectified_right,cv2.COLOR_BGR2GRAY)
overlayed_base = overlay_images(frame_left,frame_right)
overlayed_rectified = overlay_images(rectified_left,rectified_right)
overlayed_combined = combine_images(overlayed_base,overlayed_rectified)
stereo = cv2.StereoSGBM_create(numDisparities=272, blockSize=5)
disparity = stereo.compute(gray_left,gray_right)
depth = cv2.reprojectImageTo3D(disparity,Q)
while True:
cv2.imshow("Only cam calib", undistorted)
cv2.imshow("Rectified", rectified_pic)
cv2.imshow("Disparity Map", disparity)
cv2.imshow("Depth Map", depth)
cv2.imshow('Overlayed',overlayed_combined)
# Attendre l'appui de la touche 'q' pour quitter la boucle
if cv2.waitKey(1) & 0xFF == ord('q'):
break
#for [image,savename] in [[frame_left,"left_image"],
# [frame_right,"right_image"],
# [rectified_left,"rectified_left"],
# [rectified_right,"rectified_right"],
# [overlayed_combined,"overlayed_combined"],
# [disparity,"disparity"],
# [depth,"depth_map"]]:
# cv2.imwrite("result/"+savename+".jpg",image)
cv2.destroyAllWindows()
New contributor
Clément is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.