from __future__ import print_function, division import argparse import logging import numpy as np import torch from tqdm import tqdm import time import os import cv2 import sys from core.defom_stereo import DEFOMStereo, autocast import core.stereo_datasets as datasets from core.utils.utils import InputPadder from core.utils.frame_utils import writePFM def makedirs(path): if not os.path.exists(path): os.makedirs(path) def StrToBytes(text): if sys.version_info[0] == 2: return text else: return bytes(text, 'UTF-8') def count_parameters(model): return sum(p.numel() for p in model.parameters() if p.requires_grad) @torch.no_grad() def test_eth3d(model, save_path, iters=32, scale_iters=3, mixed_prec=False): """ Peform validation using the ETH3D (train) split """ model.eval() aug_params = {} test_dataset = datasets.ETH3D(aug_params, split='testing', is_test=True) training_dataset = datasets.ETH3D(aug_params, split='training', is_test=True) dataset = test_dataset + training_dataset torch.backends.cudnn.benchmark = True for test_id in tqdm(range(len(dataset))): img1, img2, imageL_file = dataset[test_id] image1 = img1[None].cuda() image2 = img2[None].cuda() padder = InputPadder(image1.shape, divis_by=32) image1, image2 = padder.pad(image1, image2) with autocast(enabled=mixed_prec): start = time.time() disp_pr = model(image1, image2, iters=iters, scale_iters=scale_iters, test_mode=True) end = time.time() runtime = end - start disp = padder.unpad(disp_pr).cpu().squeeze().numpy() disp[disp < 0] = 0 disp[disp > 64] = 64 names = imageL_file.split("/") save_sub_path = os.path.join(save_path, "low_res_"+names[-3]) makedirs(save_sub_path) disp_path = os.path.join(save_sub_path, names[-2] + '.pfm') writePFM(disp_path, disp) txt_path = os.path.join(save_sub_path, names[-2] + '.txt') with open(txt_path, 'wb') as time_file: time_file.write(StrToBytes('runtime ' + str(runtime))) @torch.no_grad() def test_kitti(model, save_path, iters=32, scale_iters=3, split='15', mixed_prec=False): """ Peform testing on the KITTI-2015 (test) split """ model.eval() aug_params = {} save_path = os.path.join(save_path, "disp_0") makedirs(save_path) test_dataset = datasets.KITTI(aug_params, split=split, image_set='testing', is_test=True) runtime_sum = 0.0 runtime_count = 0 for test_id in tqdm(range(len(test_dataset))): img1, img2, imageL_file = test_dataset[test_id] image1 = img1[None].cuda() image2 = img2[None].cuda() padder = InputPadder(image1.shape, divis_by=32) image1, image2 = padder.pad(image1, image2) with autocast(enabled=mixed_prec): start = time.time() disp_pr = model(image1, image2, iters=iters, scale_iters=scale_iters, test_mode=True) end = time.time() runtime = end - start runtime_sum += runtime runtime_count += 1 disp = padder.unpad(disp_pr).cpu().squeeze().numpy() disp[disp < 0] = 0 disp[disp > 240] = 240 disp = np.uint16(disp*256) name = imageL_file.split('/')[-1] path = os.path.join(save_path, name) cv2.imwrite(path, disp, [cv2.IMWRITE_PNG_COMPRESSION, 9]) print('The average runtime on Kitti test images is (you will need this for the submission): ' + str(runtime_sum / runtime_count) + " seconds") @torch.no_grad() def test_middlebury(model, save_path, iters=32, scale_iters=8, split='F', mixed_prec=False, method_name="DEFOM-Stereo"): """ Peform validation using the Middlebury-V3 dataset """ model.eval() aug_params = {} test_dataset = datasets.Middlebury(aug_params, split=split, image_set='test', is_test=True) training_dataset = datasets.Middlebury(aug_params, split=split, image_set='training', is_test=True) dataset = test_dataset + training_dataset torch.backends.cudnn.benchmark = True for test_id in tqdm(range(len(dataset))): img1, img2, imageL_file = dataset[test_id] image1 = img1[None].cuda() image2 = img2[None].cuda() padder = InputPadder(image1.shape, divis_by=32) image1, image2 = padder.pad(image1, image2) with autocast(enabled=mixed_prec): start = time.time() disp_pr = model(image1, image2, iters=iters, scale_iters=scale_iters, test_mode=True) end = time.time() runtime = end - start disp = padder.unpad(disp_pr).cpu().squeeze().numpy() disp[disp < 0] = 0 disp[disp > 800] = 800 names = imageL_file.split("/") save_sub_path = os.path.join(save_path, names[-3], names[-2]) makedirs(save_sub_path) disp_path = os.path.join(save_sub_path, 'disp0' + method_name + '.pfm') writePFM(disp_path, disp) txt_path = os.path.join(save_sub_path, 'time' + method_name + '.txt') with open(txt_path, 'wb') as time_file: time_file.write(StrToBytes(str(runtime))) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--restore_ckpt', help="restore checkpoint", default=None) parser.add_argument('--datasets', nargs='+', type=str, help="dataset for evaluation", default=["kitti12", "kitti15"], choices=["eth3d", "kitti12", "kitti15"] + [f"middlebury_{s}" for s in 'FHQ']) parser.add_argument('--mixed_precision', action='store_true', help='use mixed precision') parser.add_argument('--valid_iters', type=int, default=32, help='number of disparity field updates during forward pass') parser.add_argument('--scale_iters', type=int, default=8, help="number of scaling updates to the disparity field in each forward pass.") parser.add_argument('--method_name', default="DEFOM-Stereo", help="the method to test") # Architecure choices parser.add_argument('--dinov2_encoder', type=str, default='vits', choices=['vits', 'vitb', 'vitl', 'vitg']) parser.add_argument('--idepth_scale', type=float, default=0.5, help="the scale of inverse depth to initialize disparity") parser.add_argument('--hidden_dims', nargs='+', type=int, default=[128] * 3, help="hidden state and context dimensions") parser.add_argument('--corr_implementation', choices=["reg", "alt", "reg_cuda", "alt_cuda"], default="reg", help="correlation volume implementation") parser.add_argument('--shared_backbone', action='store_true', help="use a single backbone for the context and feature encoders") parser.add_argument('--corr_levels', type=int, default=2, help="number of levels in the correlation pyramid") parser.add_argument('--corr_radius', type=int, default=4, help="width of the correlation pyramid") parser.add_argument('--scale_list', type=float, nargs='+', default=[0.125, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 2.0], help='the list of scaling factors of disparity') parser.add_argument('--scale_corr_radius', type=int, default=2, help="width of the correlation pyramid for scaled disparity") parser.add_argument('--n_downsample', type=int, default=2, choices=[2, 3], help="resolution of the disparity field (1/2^K)") parser.add_argument('--context_norm', type=str, default="batch", choices=['group', 'batch', 'instance', 'none'], help="normalization of context encoder") parser.add_argument('--n_gru_layers', type=int, default=3, help="number of hidden GRU levels") args = parser.parse_args() model = DEFOMStereo(args) logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s') if args.restore_ckpt is not None: assert args.restore_ckpt.endswith(".pth") logging.info("Loading checkpoint...") checkpoint = torch.load(args.restore_ckpt, map_location='cuda') model.load_state_dict(checkpoint, strict=True) logging.info(f"Done loading checkpoint") model.cuda() model.eval() print(f"The model has {format(count_parameters(model)/1e6, '.2f')}M learnable parameters.") # The CUDA implementations of the correlation volume prevent half-precision # rounding errors in the correlation lookup. This allows us to use mixed precision # in the entire forward pass, not just in the GRUs & feature extractors. use_mixed_precision = args.corr_implementation.endswith("_cuda") if 'eth3d' in args.datasets: save_path = os.path.abspath(args.restore_ckpt).split('.')[0] + '_' + "eth3d" makedirs(save_path) test_eth3d(model, save_path, iters=args.valid_iters, scale_iters=args.scale_iters, mixed_prec=use_mixed_precision) if 'kitti12' in args.datasets: save_path = os.path.abspath(args.restore_ckpt).split('.')[0] + '_' + "kitti12" makedirs(save_path) test_kitti(model, save_path, iters=args.valid_iters, scale_iters=args.scale_iters, mixed_prec=use_mixed_precision, split='12') if 'kitti15' in args.datasets: save_path = os.path.abspath(args.restore_ckpt).split('.')[0] + '_' + "kitti15" makedirs(save_path) test_kitti(model, save_path, iters=args.valid_iters, scale_iters=args.scale_iters, mixed_prec=use_mixed_precision, split='15') for s in 'FHQ': if f"middlebury_{s}" in args.datasets: save_path = os.path.abspath(args.restore_ckpt).split('.')[0] + '_' + f"middlebury_{s}" makedirs(save_path) test_middlebury(model, save_path, iters=args.valid_iters, scale_iters=args.scale_iters, split=s, method_name=args.method_name, mixed_prec=use_mixed_precision)