#!/usr/bin/env python3 """Run the FFSGWCVolume-plugin TensorRT engine from Python. This is the Python equivalent of cpp/src/ffs_single_depth_inference.cpp plus cpp/app/main.cpp for the single-engine plugin path: - engine inputs: left, right - engine output: disp - input tensors are RGB, CHW, float32, raw 0-255 values - resize is aspect-ratio-preserving with right/bottom replicate padding - disparity is cropped, nearest-neighbor upsampled, and scaled back to the original input-image pixel units """ import argparse import logging import os import sys from pathlib import Path import cv2 import imageio.v2 as imageio import numpy as np import torch import yaml code_dir = Path(__file__).resolve().parent repo_dir = code_dir.parent sys.path.append(str(repo_dir)) from Utils import set_logging_format, set_seed, vis_disparity from build_plugin_trt import ( PLUGIN_NAME, find_default_plugin_library, find_plugin_creator, load_plugin_library, ) def parse_args(): parser = argparse.ArgumentParser( description="Run Fast-FoundationStereo single TensorRT plugin engine." ) parser.add_argument( "--model_dir", type=Path, default=repo_dir / "engine1_plug", help="Directory containing fast_foundationstereo.engine and one YAML config.", ) parser.add_argument( "--engine_file", "--model_file", dest="engine_file", type=Path, default=None, help="Explicit TensorRT engine path. Defaults to /fast_foundationstereo.engine.", ) parser.add_argument( "--config_file", type=Path, default=None, help="Explicit YAML config path. Defaults to the single YAML in the engine directory.", ) parser.add_argument( "--plugin_lib", type=Path, default=None, help="Path to libffs_gwc_plugin.so. Defaults to cpp/build/libffs_gwc_plugin.so.", ) parser.add_argument("--left_file", type=Path, default=repo_dir / "demo_data" / "left.png") parser.add_argument("--right_file", type=Path, default=repo_dir / "demo_data" / "right.png") parser.add_argument( "--intrinsic_file", type=Path, default=repo_dir / "demo_data" / "K.txt", help="Text file with 3x3 K on line 1 and baseline in meters on line 2.", ) parser.add_argument("--out_dir", type=Path, default=repo_dir / "output_plugin_trt") return parser.parse_args() def resolve_engine_path(model_dir: Path, engine_file: Path | None) -> Path: path = engine_file if engine_file is not None else model_dir / "fast_foundationstereo.engine" if not path.exists(): raise FileNotFoundError(f"TensorRT engine does not exist: {path}") return path def resolve_config_path(engine_path: Path, model_dir: Path, config_file: Path | None) -> Path: if config_file is not None: if not config_file.exists(): raise FileNotFoundError(f"Config file does not exist: {config_file}") return config_file search_dir = model_dir if model_dir.exists() else engine_path.parent yaml_files = sorted( p for p in search_dir.iterdir() if p.is_file() and p.suffix.lower() in (".yaml", ".yml") ) if not yaml_files: raise FileNotFoundError(f"No YAML config found in: {search_dir}") if len(yaml_files) > 1: names = " ".join(str(p) for p in yaml_files) raise RuntimeError( f"Expected exactly one YAML config in {search_dir}, found {len(yaml_files)}: {names}" ) return yaml_files[0] def load_intrinsics(path: Path) -> tuple[np.ndarray, float]: with path.open("r") as f: lines = f.readlines() if len(lines) < 2: raise RuntimeError("intrinsic file must contain K on line 1 and baseline on line 2") k = np.array(list(map(float, lines[0].strip().split())), dtype=np.float32).reshape(3, 3) baseline = float(lines[1]) if not (k[0, 0] > 0.0 and baseline > 0.0): raise RuntimeError("invalid focal length or baseline in intrinsic file") return k, baseline def load_rgb(path: Path) -> np.ndarray: img = imageio.imread(path) if img.ndim == 2: img = np.tile(img[..., None], (1, 1, 3)) img = img[..., :3] if img.dtype != np.uint8: img = np.clip(img, 0, 255).astype(np.uint8) return np.ascontiguousarray(img) def resize_uniform_and_pad_rgb(img: np.ndarray, target_h: int, target_w: int): src_h, src_w = img.shape[:2] if src_h == target_h and src_w == target_w: return img.copy(), target_h, target_w scale = min(float(target_w) / src_w, float(target_h) / src_h) scaled_w = max(1, int(round(src_w * scale))) scaled_h = max(1, int(round(src_h * scale))) scaled = cv2.resize(img, (scaled_w, scaled_h), interpolation=cv2.INTER_LINEAR) padded = cv2.copyMakeBorder( scaled, 0, target_h - scaled_h, 0, target_w - scaled_w, cv2.BORDER_REPLICATE, ) return np.ascontiguousarray(padded), scaled_h, scaled_w def tensor_from_rgb_255(img: np.ndarray) -> torch.Tensor: arr = img.astype(np.float32, copy=False) return torch.as_tensor(arr, device="cuda").permute(2, 0, 1).unsqueeze(0).contiguous() def cpp_nearest_resize(src: np.ndarray, dst_h: int, dst_w: int) -> np.ndarray: src_h, src_w = src.shape xs = np.rint((np.arange(dst_w, dtype=np.float32) + 0.5) * src_w / dst_w - 0.5) ys = np.rint((np.arange(dst_h, dtype=np.float32) + 0.5) * src_h / dst_h - 0.5) xs = np.clip(xs.astype(np.int64), 0, src_w - 1) ys = np.clip(ys.astype(np.int64), 0, src_h - 1) return src[ys[:, None], xs[None, :]] def postprocess_disparity( raw_disp: np.ndarray, input_h: int, input_w: int, model_h: int, model_w: int, scaled_h: int, scaled_w: int, ) -> np.ndarray: disp = raw_disp.reshape(model_h, model_w).astype(np.float32, copy=False) disp = np.maximum(disp, 0.0) if input_h == model_h and input_w == model_w: return np.ascontiguousarray(disp) cropped = disp[:scaled_h, :scaled_w] upsampled = cpp_nearest_resize(cropped, input_h, input_w) return np.ascontiguousarray(upsampled * (float(input_w) / scaled_w), dtype=np.float32) def disparity_to_depth_cpp(disp: np.ndarray, fx: float, baseline_m: float) -> np.ndarray: d = disp.astype(np.float32, copy=True) xs = np.arange(d.shape[1], dtype=np.float32)[None, :] d[(xs - d) < 0.0] = np.inf with np.errstate(divide="ignore", invalid="ignore"): depth = np.float32(fx * baseline_m) / d return np.ascontiguousarray(depth, dtype=np.float32) def save_float_matrix_bin(path: Path, values: np.ndarray) -> None: arr = np.ascontiguousarray(values, dtype=np.float32) dims = np.array(arr.shape[:2], dtype=np.int32) with path.open("wb") as f: dims.tofile(f) arr.tofile(f) def colorize_depth(depth: np.ndarray) -> np.ndarray: valid = np.isfinite(depth) & (depth > 0.0) if not np.any(valid): return np.zeros((*depth.shape, 3), dtype=np.uint8) safe = np.zeros_like(depth, dtype=np.float32) safe[valid] = depth[valid] min_val = float(safe[valid].min()) max_val = float(safe[valid].max()) if max_val <= min_val: max_val = min_val + 1.0 depth_u8 = np.clip((safe - min_val) * (255.0 / (max_val - min_val)), 0, 255).astype(np.uint8) colored_bgr = cv2.applyColorMap(depth_u8, cv2.COLORMAP_TURBO) colored = colored_bgr[..., ::-1] colored[~valid] = 0 return colored class PluginTensorRTRunner: def __init__(self, engine_path: Path, plugin_lib: Path): import tensorrt as trt self.trt = trt self.logger = trt.Logger(trt.Logger.WARNING) trt.init_libnvinfer_plugins(self.logger, "") load_plugin_library(str(plugin_lib)) if not find_plugin_creator(trt): raise RuntimeError(f"{PLUGIN_NAME} plugin creator is not registered after loading {plugin_lib}") self.runtime = trt.Runtime(self.logger) self.engine = self.runtime.deserialize_cuda_engine(engine_path.read_bytes()) if self.engine is None: raise RuntimeError(f"Failed to deserialize TensorRT engine: {engine_path}") self.context = self.engine.create_execution_context() if self.context is None: raise RuntimeError(f"Failed to create execution context: {engine_path}") names = [self.engine.get_tensor_name(i) for i in range(self.engine.num_io_tensors)] missing = [name for name in ("left", "right", "disp") if name not in names] if missing: raise RuntimeError( f"Plugin engine must expose tensors named left, right, and disp. " f"Missing {missing}; found {names}" ) def _torch_dtype(self, trt_dtype): trt = self.trt mapping = { trt.DataType.FLOAT: torch.float32, trt.DataType.HALF: torch.float16, trt.DataType.INT32: torch.int32, trt.DataType.INT8: torch.int8, trt.DataType.BOOL: torch.bool, } if hasattr(trt.DataType, "BF16"): mapping[trt.DataType.BF16] = torch.bfloat16 if trt_dtype not in mapping: raise RuntimeError(f"Unsupported TensorRT dtype: {trt_dtype}") return mapping[trt_dtype] def infer(self, left: torch.Tensor, right: torch.Tensor) -> torch.Tensor: inputs = {"left": left, "right": right} for name, tensor in list(inputs.items()): expected = self._torch_dtype(self.engine.get_tensor_dtype(name)) if tensor.dtype != expected: tensor = tensor.to(expected) inputs[name] = tensor.contiguous() self.context.set_input_shape(name, tuple(inputs[name].shape)) out_shape = tuple(self.context.get_tensor_shape("disp")) out_dtype = self._torch_dtype(self.engine.get_tensor_dtype("disp")) output = torch.empty(out_shape, device="cuda", dtype=out_dtype) self.context.set_tensor_address("left", int(inputs["left"].data_ptr())) self.context.set_tensor_address("right", int(inputs["right"].data_ptr())) self.context.set_tensor_address("disp", int(output.data_ptr())) stream = torch.cuda.current_stream().cuda_stream if not self.context.execute_async_v3(stream): raise RuntimeError("TensorRT enqueue failed") return output def main() -> int: args = parse_args() set_logging_format() set_seed(0) torch.autograd.set_grad_enabled(False) args.out_dir.mkdir(parents=True, exist_ok=True) engine_path = resolve_engine_path(args.model_dir, args.engine_file) config_path = resolve_config_path(engine_path, args.model_dir, args.config_file) plugin_lib = args.plugin_lib or find_default_plugin_library() if plugin_lib is None or not plugin_lib.exists(): raise FileNotFoundError( "Could not find libffs_gwc_plugin.so. Build cpp first or pass " "--plugin_lib /path/to/libffs_gwc_plugin.so" ) with config_path.open("r") as f: cfg = yaml.safe_load(f) model_h, model_w = [int(v) for v in cfg["image_size"]] left = load_rgb(args.left_file) right = load_rgb(args.right_file) if left.shape != right.shape: raise RuntimeError(f"left/right image size mismatch: {left.shape} vs {right.shape}") input_h, input_w = left.shape[:2] logging.info(f"Engine: {engine_path}") logging.info(f"Plugin: {plugin_lib}") logging.info(f"Config: {config_path}") logging.info(f"Input images: {input_w}x{input_h}") logging.info(f"Model target resolution: {model_w}x{model_h}") left_model, scaled_h, scaled_w = resize_uniform_and_pad_rgb(left, model_h, model_w) right_model, right_scaled_h, right_scaled_w = resize_uniform_and_pad_rgb(right, model_h, model_w) if (scaled_h, scaled_w) != (right_scaled_h, right_scaled_w): raise RuntimeError("left/right images produced different resize scales") if (input_h, input_w) != (model_h, model_w): logging.info( f"Uniform resize+pad: {input_w}x{input_h} -> " f"{scaled_w}x{scaled_h} inside {model_w}x{model_h}" ) runner = PluginTensorRTRunner(engine_path, plugin_lib) t_left = tensor_from_rgb_255(left_model) t_right = tensor_from_rgb_255(right_model) logging.info("Running TensorRT inference") disp_raw_t = runner.infer(t_left, t_right) torch.cuda.current_stream().synchronize() logging.info("Inference done") disp_raw = disp_raw_t.float().detach().cpu().numpy() disp = postprocess_disparity( disp_raw, input_h=input_h, input_w=input_w, model_h=model_h, model_w=model_w, scaled_h=scaled_h, scaled_w=scaled_w, ) if not np.isfinite(disp).any(): raise RuntimeError("Model produced no finite disparity values") k, baseline = load_intrinsics(args.intrinsic_file) depth = disparity_to_depth_cpp(disp, float(k[0, 0]), baseline) disparity_path = args.out_dir / "disparity.bin" depth_bin_path = args.out_dir / "depth_meter.bin" depth_npy_path = args.out_dir / "depth_meter.npy" disp_vis_path = args.out_dir / "disp_vis.png" depth_vis_path = args.out_dir / "depth_vis.png" save_float_matrix_bin(disparity_path, disp) save_float_matrix_bin(depth_bin_path, depth) np.save(depth_npy_path, depth) disp_color = vis_disparity(disp, color_map=cv2.COLORMAP_TURBO) disp_vis = np.concatenate([left, right, disp_color], axis=1) imageio.imwrite(disp_vis_path, disp_vis) imageio.imwrite(depth_vis_path, colorize_depth(depth)) logging.info(f"Saved: {disparity_path}") logging.info(f"Saved: {depth_bin_path}") logging.info(f"Saved: {depth_npy_path}") logging.info(f"Saved: {disp_vis_path}") logging.info(f"Saved: {depth_vis_path}") return 0 if __name__ == "__main__": raise SystemExit(main())