isaac/cv/defom-stereo/core/defom_stereo.py

import torch
import torch.nn as nn
import torch.nn.functional as F
from core.update import BasicMultiUpdateBlock, ScaleBasicMultiUpdateBlock
from core.extractor import BasicEncoder, MultiBasicEncoder, ResidualBlock, DefomEncoder
from core.corr import CorrBlock1D, PytorchAlternateCorrBlock1D, CorrBlockFast1D, AlternateCorrBlock
from core.utils.utils import coords_grid, upflow, get_danv2_io_size


try:
    autocast = torch.cuda.amp.autocast
except:
    # dummy autocast for PyTorch < 1.6
    class autocast:
        def __init__(self, enabled):
            pass
        def __enter__(self):
            pass
        def __exit__(self, *args):
            pass


class DEFOMStereo(nn.Module):
    def __init__(self, args):
        super(DEFOMStereo, self).__init__()
        self.args = args

        self.register_buffer('mean', torch.tensor([[0.485, 0.456, 0.406]])[..., None, None] * 255)
        self.register_buffer('std', torch.tensor([[0.229, 0.224, 0.225]])[..., None, None] * 255)

        self.defomencoder = DefomEncoder(args.dinov2_encoder, idepth_scale=args.idepth_scale)

        context_dims = args.hidden_dims

        self.fnet = BasicEncoder(self.defomencoder.out_dim, output_dim=256, norm_fn='instance', downsample=args.n_downsample)

        self.context_zqr_convs = nn.ModuleList([nn.Conv2d(context_dims[i], args.hidden_dims[i]*3, 3, padding=3//2) for i in range(self.args.n_gru_layers)])

        self.update_block = BasicMultiUpdateBlock(self.args, hidden_dims=args.hidden_dims)
        self.scale_update_block = ScaleBasicMultiUpdateBlock(self.args, hidden_dims=args.hidden_dims)
        
        self.cnet = MultiBasicEncoder(self.defomencoder.out_dim, output_dim=[args.hidden_dims, context_dims],
                                      norm_fn=args.context_norm, downsample=args.n_downsample)

    def freeze_bn(self):
        for m in self.modules():
            if isinstance(m, nn.BatchNorm2d):
                m.eval()

    def initialize_coords(self, img):
        """ Disparity is represented as difference between two vertical coordinate grids disp
            = coords0[:, :1] - coords1[:, :1] """
        N, _, H, W = img.shape

        coords = coords_grid(N, H, W)[:, :1].to(img.device)

        return coords

    def upsample_flow(self, flow, mask):
        """ Upsample disparity field [H/scale, W/scale, 1] -> [H, W, 1] using convex combination """
        N, D, H, W = flow.shape
        factor = 2 ** self.args.n_downsample
        mask = mask.view(N, 1, 9, factor, factor, H, W)
        mask = torch.softmax(mask, dim=2)

        up_flow = F.unfold(factor * flow, [3, 3], padding=1)
        up_flow = up_flow.view(N, D, 9, 1, 1, H, W)

        up_flow = torch.sum(mask * up_flow, dim=2)
        up_flow = up_flow.permute(0, 1, 4, 2, 5, 3)
        return up_flow.reshape(N, D, factor * H, factor * W)

    def forward(self, image1, image2, iters=12, scale_iters=3, test_mode=False):
        """ Estimate optical flow between pair of frames """

        image1 = ((image1 - self.mean)/self.std).contiguous()
        image2 = ((image2 - self.mean)/self.std).contiguous()

        bs, _, h, w = image1.shape
        danv2_io_sizes = get_danv2_io_size(h, w, self.args.n_downsample)

        # run the context network
        with autocast(enabled=self.args.mixed_precision):
            d_features, dfeat1, dfeat2, disp = self.defomencoder([image1, image2], danv2_io_sizes)

            cnet_list = self.cnet(image1, d_features)
            fmap1, fmap2 = self.fnet([image1, image2], [dfeat1, dfeat2])
            net_list = [torch.tanh(x[0]) for x in cnet_list]
            inp_list = [torch.relu(x[1]) for x in cnet_list]
            # Rather than running the GRU's conv layers on the context features multiple times, we do it once at the beginning
            inp_list = [list(conv(i).split(split_size=conv.out_channels//3, dim=1)) for i, conv in zip(inp_list, self.context_zqr_convs)]

        coords = self.initialize_coords(net_list[0])

        fmap1, fmap2 = fmap1.float(), fmap2.float()
        disp = disp.float()
        corr_fn = CorrBlock1D(fmap1, fmap2, coords, radius=self.args.corr_radius, num_levels=self.args.corr_levels,
                              scale_list=self.args.scale_list, scale_corr_radius=self.args.scale_corr_radius)

        disp_predictions = []
        for itr in range(iters):
            disp = disp.detach()

            if itr < scale_iters:
                corr = corr_fn(disp, scaling=True)  # index correlation volume
                with autocast(enabled=self.args.mixed_precision):
                    net_list, up_mask, scale_disp = self.scale_update_block(net_list, inp_list, corr, disp,
                                                                            iter32=self.args.n_gru_layers == 3,
                                                                            iter16=self.args.n_gru_layers >= 2)

                # F(t+1) = \Scale(t) x F(t)
                disp = scale_disp * disp
            else:
                corr = corr_fn(disp, scaling=False)  # index correlation volume
                with autocast(enabled=self.args.mixed_precision):
                    net_list, up_mask, delta_disp = self.update_block(net_list, inp_list, corr, disp,
                                                                      iter32=self.args.n_gru_layers == 3,
                                                                      iter16=self.args.n_gru_layers >= 2)

                    # To avoid unstability, we limit the disparity update within the searching range.
                    delta_disp = torch.clip(delta_disp, min=-2**(self.args.corr_levels-1)*self.args.corr_radius,
                                            max=2**(self.args.corr_levels-1)*self.args.corr_radius)

                # F(t+1) = F(t) + \Delta(t)
                disp = disp + delta_disp

            # We do not need to upsample or output intermediate results in test_mode
            if test_mode and itr < iters - 1:
                continue

            # upsample predictions
            if up_mask is None:
                disp_up = upflow(disp, factor=2 ** self.n_downsample)
            else:
                disp_up = self.upsample_flow(disp, up_mask)

            disp_predictions.append(disp_up)

        if test_mode:
            return disp_up

        return disp_predictions