''' VNET Model Adapted from MedicalZooPytorch: https://github.com/black0017/MedicalZooPytorch Implementation of this model is borrowed and modified (to support multi-channels and latest pytorch version) from here: https://github.com/Dawn90/V-Net.pytorch ''' from torchsummary import summary import os from abc import ABC, abstractmethod import torch from monai.networks.nets import SwinUNETR import torch.nn as nn import torch.nn.functional as F class BaseModel(nn.Module, ABC): r""" BaseModel with basic functionalities for checkpointing and restoration. """ # def __init__(self): # super().__init__() # self.best_loss = 1000000 @abstractmethod def forward(self, x): pass @abstractmethod def test(self): """ To be implemented by the subclass so that models can perform a forward propagation :return: """ pass @property def device(self): return next(self.parameters()).device def restore_checkpoint(self, ckpt_file, optimizer=None): r""" Restores checkpoint from a pth file and restores optimizer state. Args: ckpt_file (str): A PyTorch pth file containing model weights. optimizer (Optimizer): A vanilla optimizer to have its state restored from. Returns: int: Global step variable where the model was last checkpointed. """ if not ckpt_file: raise ValueError("No checkpoint file to be restored.") try: ckpt_dict = torch.load(ckpt_file) except RuntimeError: ckpt_dict = torch.load(ckpt_file, map_location=lambda storage, loc: storage) # Restore model weights self.load_state_dict(ckpt_dict['model_state_dict']) # Restore optimizer status if existing. Evaluation doesn't need this # TODO return optimizer????? if optimizer: optimizer.load_state_dict(ckpt_dict['optimizer_state_dict']) # Return global step return ckpt_dict['epoch'] def save_checkpoint(self, directory, epoch, loss, optimizer=None, name=None): r""" Saves checkpoint at a certain global step during training. Optimizer state is also saved together. Args: directory (str): Path to save checkpoint to. epoch (int): The training. epoch optimizer (Optimizer): Optimizer state to be saved concurrently. name (str): The name to save the checkpoint file as. Returns: None """ # Create directory to save to if not os.path.exists(directory): os.makedirs(directory) # Build checkpoint dict to save. ckpt_dict = { 'model_state_dict': self.state_dict(), 'optimizer_state_dict': optimizer.state_dict() if optimizer is not None else None, 'epoch': epoch } # Save the file with specific name if name is None: name = "{}_{}_epoch.pth".format( os.path.basename(directory), # netD or netG 'last') torch.save(ckpt_dict, os.path.join(directory, name)) if self.best_loss > loss: self.best_loss = loss name = "{}_BEST.pth".format( os.path.basename(directory)) torch.save(ckpt_dict, os.path.join(directory, name)) return name def count_params(self): r""" Computes the number of parameters in this model. Args: None Returns: int: Total number of weight parameters for this model. int: Total number of trainable parameters for this model. """ num_total_params = sum(p.numel() for p in self.parameters()) num_trainable_params = sum(p.numel() for p in self.parameters() if p.requires_grad) return num_total_params, num_trainable_params def inference(self, input_tensor): self.eval() with torch.no_grad(): output = self.forward(input_tensor) if isinstance(output, tuple): output = output[0] return output.cpu().detach() def passthrough(x, **kwargs): return x def ELUCons(elu, nchan): if elu: return nn.ELU(inplace=True) else: return nn.PReLU(nchan) class Swinunetr(BaseModel,SwinUNETR): def __init__(self,img_size=(128, 128, 128), in_channels=1, classes=2): # super(Swinunetr, self).__init__() super(Swinunetr, self).__init__( img_size=img_size, in_channels=in_channels, out_channels=classes, ) self.classes = classes self.in_channels = in_channels # self.swinunetr = SwinUNETR( # img_size=img_size, # in_channels=in_channels, # out_channels=classes, # feature_size=48, # spatial_dims=3, # ) def forward(self, x): return super(Swinunetr, self).forward(x) def test(self,device='cpu'): input_tensor = torch.rand(1, self.in_channels, 32, 32, 32) ideal_out = torch.rand(1, self.classes, 32, 32, 32) out = self.forward(input_tensor) assert ideal_out.shape == out.shape summary(self.to(torch.device(device)), (self.in_channels, 32, 32, 32),device=device) # import torchsummaryX # torchsummaryX.summary(self, input_tensor.to(device)) print("Vnet test is complete") class LUConv(nn.Module): def __init__(self, nchan, elu): super(LUConv, self).__init__() self.relu1 = ELUCons(elu, nchan) self.conv1 = nn.Conv3d(nchan, nchan, kernel_size=5, padding=2) self.bn1 = torch.nn.BatchNorm3d(nchan) def forward(self, x): out = self.relu1(self.bn1(self.conv1(x))) return out def _make_nConv(nchan, depth, elu): layers = [] for _ in range(depth): layers.append(LUConv(nchan, elu)) return nn.Sequential(*layers) class InputTransition(nn.Module): def __init__(self, in_channels, elu): super(InputTransition, self).__init__() self.num_features = 16 self.in_channels = in_channels self.conv1 = nn.Conv3d(self.in_channels, self.num_features, kernel_size=5, padding=2) self.bn1 = torch.nn.BatchNorm3d(self.num_features) self.relu1 = ELUCons(elu, self.num_features) def forward(self, x): out = self.conv1(x) repeat_rate = int(self.num_features / self.in_channels) out = self.bn1(out) x16 = x.repeat(1, repeat_rate, 1, 1, 1) return self.relu1(torch.add(out, x16)) class DownTransition(nn.Module): def __init__(self, inChans, nConvs, elu, dropout=False): super(DownTransition, self).__init__() outChans = 2 * inChans self.down_conv = nn.Conv3d(inChans, outChans, kernel_size=2, stride=2) self.bn1 = torch.nn.BatchNorm3d(outChans) self.do1 = passthrough self.relu1 = ELUCons(elu, outChans) self.relu2 = ELUCons(elu, outChans) if dropout: self.do1 = nn.Dropout3d() self.ops = _make_nConv(outChans, nConvs, elu) def forward(self, x): down = self.relu1(self.bn1(self.down_conv(x))) out = self.do1(down) out = self.ops(out) out = self.relu2(torch.add(out, down)) return out class UpTransition(nn.Module): def __init__(self, inChans, outChans, nConvs, elu, dropout=False): super(UpTransition, self).__init__() self.up_conv = nn.ConvTranspose3d(inChans, outChans // 2, kernel_size=2, stride=2) self.bn1 = torch.nn.BatchNorm3d(outChans // 2) self.do1 = passthrough self.do2 = nn.Dropout3d() self.relu1 = ELUCons(elu, outChans // 2) self.relu2 = ELUCons(elu, outChans) if dropout: self.do1 = nn.Dropout3d() self.ops = _make_nConv(outChans, nConvs, elu) def forward(self, x, skipx): out = self.do1(x) skipxdo = self.do2(skipx) out = self.relu1(self.bn1(self.up_conv(out))) xcat = torch.cat((out, skipxdo), 1) out = self.ops(xcat) out = self.relu2(torch.add(out, xcat)) return out class OutputTransition(nn.Module): def __init__(self, in_channels, classes, elu): super(OutputTransition, self).__init__() self.classes = classes self.conv1 = nn.Conv3d(in_channels, classes, kernel_size=5, padding=2) self.bn1 = torch.nn.BatchNorm3d(classes) self.conv2 = nn.Conv3d(classes, classes, kernel_size=1) self.relu1 = ELUCons(elu, classes) def forward(self, x): # convolve 32 down to channels as the desired classes out = self.relu1(self.bn1(self.conv1(x))) out = self.conv2(out) return out class AttentionGate3D(nn.Module): """ 3D Attention Gate as in "Attention U-Net", extended to volumetric data. Gating signal g: from decoder (coarser features) Skip connection x: from encoder (finer features) """ def __init__(self, F_g, F_l, F_int): super(AttentionGate3D, self).__init__() # 1x1x1 conv for gating signal self.W_g = nn.Sequential( nn.Conv3d(F_g, F_int, kernel_size=1, stride=1, padding=0, bias=True), nn.BatchNorm3d(F_int) ) # 1x1x1 conv for skip connection self.W_x = nn.Sequential( nn.Conv3d(F_l, F_int, kernel_size=1, stride=1, padding=0, bias=True), nn.BatchNorm3d(F_int) ) # Psi: combines features and outputs attention coefficients self.psi = nn.Sequential( nn.Conv3d(F_int, 1, kernel_size=1, stride=1, padding=0, bias=True), nn.BatchNorm3d(1), nn.Sigmoid() ) self.relu = nn.ReLU(inplace=True) def forward(self, g, x): # apply convs g1 = self.W_g(g) x1 = self.W_x(x) # upsample gating signal to match skip connection spatial dims if g1.shape[2:] != x1.shape[2:]: g1 = F.interpolate(g1, size=x1.shape[2:], mode='trilinear', align_corners=True) # combine and activate psi = self.relu(g1 + x1) psi = self.psi(psi) # multiply attention map return x * psi class VNetAttention(BaseModel): """ VNet with Attention Gates inserted before each upsampling block. """ def __init__(self, elu=True, in_channels=1, classes=2): super(VNetAttention, self).__init__() self.classes = classes self.in_channels = in_channels # Encoder self.in_tr = InputTransition(in_channels, elu=elu) self.down_tr32 = DownTransition(16, 1, elu) self.down_tr64 = DownTransition(32, 2, elu) self.down_tr128 = DownTransition(64, 3, elu, dropout=True) self.down_tr256 = DownTransition(128, 2, elu, dropout=True) # Attention gates for skip connections # skip from out128, gating out256 self.att256 = AttentionGate3D(F_g=256, F_l=128, F_int=128) # skip from out64, gating from up_tr256 output channels=256 self.att128 = AttentionGate3D(F_g=256, F_l=64, F_int=64) # skip from out32, gating from up_tr128 output channels=128 self.att64 = AttentionGate3D(F_g=128, F_l=32, F_int=32) # skip from out16, gating from up_tr64 output channels=64 self.att32 = AttentionGate3D(F_g=64, F_l=16, F_int=16) # Decoder self.up_tr256 = UpTransition(256, 256, 2, elu, dropout=True) self.up_tr128 = UpTransition(256, 128, 2, elu, dropout=True) self.up_tr64 = UpTransition(128, 64, 1, elu) self.up_tr32 = UpTransition(64, 32, 1, elu) self.out_tr = OutputTransition(32, classes, elu) def forward(self, x): # Encoder pass out16 = self.in_tr(x) out32 = self.down_tr32(out16) out64 = self.down_tr64(out32) out128 = self.down_tr128(out64) out256 = self.down_tr256(out128) # Attention gating + Decoder skip128 = self.att256(g=out256, x=out128) out = self.up_tr256(out256, skip128) skip64 = self.att128(g=out, x=out64) out = self.up_tr128(out, skip64) skip32 = self.att64(g=out, x=out32) out = self.up_tr64(out, skip32) skip16 = self.att32(g=out, x=out16) out = self.up_tr32(out, skip16) out = self.out_tr(out) return out def test(self, device='cpu'): input_tensor = torch.rand(1, self.in_channels, 32, 32, 32) out = self.forward(input_tensor) assert out.shape == (1, self.classes, 32, 32, 32), \ f"Expected shape (1,{self.classes},32,32,32), got {out.shape}" print("VNet with Attention test is complete. Output shape:", out.shape) class VNet(BaseModel): """ Implementations based on the Vnet paper: https://arxiv.org/abs/1606.04797 """ def __init__(self, elu=True, in_channels=1, classes=1): super(VNet, self).__init__() self.classes = classes self.in_channels = in_channels self.in_tr = InputTransition(in_channels, elu=elu) self.down_tr32 = DownTransition(16, 1, elu) self.down_tr64 = DownTransition(32, 2, elu) self.down_tr128 = DownTransition(64, 3, elu, dropout=True) self.down_tr256 = DownTransition(128, 2, elu, dropout=True) self.up_tr256 = UpTransition(256, 256, 2, elu, dropout=True) self.up_tr128 = UpTransition(256, 128, 2, elu, dropout=True) self.up_tr64 = UpTransition(128, 64, 1, elu) self.up_tr32 = UpTransition(64, 32, 1, elu) self.out_tr = OutputTransition(32, classes, elu) def forward(self, x): out16 = self.in_tr(x) out32 = self.down_tr32(out16) out64 = self.down_tr64(out32) out128 = self.down_tr128(out64) out256 = self.down_tr256(out128) out = self.up_tr256(out256, out128) out = self.up_tr128(out, out64) out = self.up_tr64(out, out32) out = self.up_tr32(out, out16) out = self.out_tr(out) return out def test(self,device='cpu'): input_tensor = torch.rand(1, self.in_channels, 32, 32, 32) ideal_out = torch.rand(1, self.classes, 32, 32, 32) out = self.forward(input_tensor) assert ideal_out.shape == out.shape summary(self.to(torch.device(device)), (self.in_channels, 32, 32, 32),device=device) # import torchsummaryX # torchsummaryX.summary(self, input_tensor.to(device)) print("Vnet test is complete") class VNetLight(BaseModel): """ A lighter version of Vnet that skips down_tr256 and up_tr256 in oreder to reduce time and space complexity """ def __init__(self, elu=True, in_channels=1, classes=2): super(VNetLight, self).__init__() self.classes = classes self.in_channels = in_channels self.in_tr = InputTransition(in_channels, elu) self.down_tr32 = DownTransition(16, 1, elu) self.down_tr64 = DownTransition(32, 2, elu) self.down_tr128 = DownTransition(64, 3, elu, dropout=True) self.up_tr128 = UpTransition(128, 128, 2, elu, dropout=True) self.up_tr64 = UpTransition(128, 64, 1, elu) self.up_tr32 = UpTransition(64, 32, 1, elu) self.out_tr = OutputTransition(32, classes, elu) def forward(self, x): out16 = self.in_tr(x) out32 = self.down_tr32(out16) out64 = self.down_tr64(out32) out128 = self.down_tr128(out64) out = self.up_tr128(out128, out64) out = self.up_tr64(out, out32) out = self.up_tr32(out, out16) out = self.out_tr(out) return out def test(self,device='cpu'): input_tensor = torch.rand(1, self.in_channels, 32, 32, 32) ideal_out = torch.rand(1, self.classes, 32, 32, 32) out = self.forward(input_tensor) assert ideal_out.shape == out.shape summary(self.to(torch.device(device)), (self.in_channels, 32, 32, 32),device=device) # import torchsummaryX # torchsummaryX.summary(self, input_tensor.to(device)) print("Vnet light test is complete") #m = VNet(in_channels=1,num_classes=2) #m.test()