import logging import sys import torch import torch.optim as optim from torch.utils.tensorboard import SummaryWriter import numpy as np import random def seed_everything(seed): torch.manual_seed(seed) # Current CPU torch.cuda.manual_seed(seed) # Current GPU np.random.seed(seed) # Numpy module random.seed(seed) # Python random module torch.backends.cudnn.benchmark = False # Close optimization torch.backends.cudnn.deterministic = True # Close optimization torch.cuda.manual_seed_all(seed) # All GPU (Optional) def sequence_loss(flow_preds, flow_gt, valid, loss_gamma=0.9, max_flow=700): """ Loss function defined over sequence of flow predictions """ n_predictions = len(flow_preds) assert n_predictions >= 1 flow_loss = 0.0 # exlude invalid pixels and extremely large diplacements mag = torch.sum(flow_gt ** 2, dim=1, keepdim=True).sqrt() # exclude extremly large displacements valid = ((valid >= 0.5) & (mag < max_flow)) assert valid.shape == flow_gt.shape, [valid.shape, flow_gt.shape] assert not torch.isinf(flow_gt[valid.bool()]).any() for i in range(n_predictions): assert not torch.isnan(flow_preds[i]).any() and not torch.isinf(flow_preds[i]).any() # We adjust the loss_gamma so it is consistent for any number of RAFT-Stereo iterations adjusted_loss_gamma = loss_gamma ** (15 / (n_predictions)) i_weight = adjusted_loss_gamma ** (n_predictions - i) i_loss = (flow_preds[i] - flow_gt).abs() assert i_loss.shape == valid.shape, [i_loss.shape, valid.shape, flow_gt.shape, flow_preds[i].shape] flow_loss += i_weight * i_loss[valid.bool()].mean() epe = torch.sum((flow_preds[-1] - flow_gt) ** 2, dim=1).sqrt() epe = epe.view(-1)[valid.view(-1)] metrics = { 'epe': epe.mean().item(), '1px': (epe < 1).float().mean().item(), '3px': (epe < 3).float().mean().item(), '5px': (epe < 5).float().mean().item(), } return flow_loss, metrics def fetch_optimizer(args, model, last_epoch=-1, checkpoint=None): """ Create the optimizer and learning rate scheduler """ trainable_params = filter(lambda p: p.requires_grad, model.parameters()) optimizer = optim.AdamW(trainable_params, lr=args.lr, weight_decay=args.wdecay, eps=1e-8) if checkpoint is not None: optimizer.load_state_dict(checkpoint['optimizer']) scheduler = optim.lr_scheduler.OneCycleLR(optimizer, args.lr, args.num_steps + 100, pct_start=0.01, cycle_momentum=False, anneal_strategy='linear', last_epoch=last_epoch) return optimizer, scheduler class Logger: SUM_FREQ = 100 def __init__(self, model, scheduler, name): self.model = model self.scheduler = scheduler self.total_steps = 0 self.running_loss = {} self.log_dir = 'runs/' + name self.writer = SummaryWriter(log_dir=self.log_dir) def _print_training_status(self): metrics_data = [self.running_loss[k] / Logger.SUM_FREQ for k in sorted(self.running_loss.keys())] training_str = "[{:6d}, {:10.7f}] ".format(self.total_steps + 1, self.scheduler.get_last_lr()[0]) metrics_str = ("{:10.4f}, " * len(metrics_data)).format(*metrics_data) # print the training status logging.info(f"Training Metrics ({self.total_steps}): {training_str + metrics_str}") if self.writer is None: self.writer = SummaryWriter(log_dir=self.log_dir) for k in self.running_loss: self.writer.add_scalar("train/" + k, self.running_loss[k] / Logger.SUM_FREQ, self.total_steps) self.running_loss[k] = 0.0 def push(self, metrics): self.total_steps += 1 for key in metrics: if key not in self.running_loss: self.running_loss[key] = 0.0 self.running_loss[key] += metrics[key] if self.total_steps % Logger.SUM_FREQ == Logger.SUM_FREQ - 1: self._print_training_status() self.running_loss = {} def write_dict(self, results): if self.writer is None: self.writer = SummaryWriter(log_dir=self.log_dir) for key in results: self.writer.add_scalar("valid/" + key, results[key], self.total_steps) def close(self): self.writer.close()