Added runtime configuration and improved model

image-inpainting/.gitignore

@@ -2,4 +2,5 @@ data/*
 *.zip
 *.jpg
 *.pt
-__pycache__/
+__pycache__/
+runtime_predictions.npz

image-inpainting/results/runtime_config.json

@@ -0,0 +1,13 @@
+{
+  "n_updates": 35000,
+  "plot_at": 500,
+  "early_stopping_patience": 20,
+  "print_stats_at": 200,
+  "print_train_stats_at": 50,
+  "validate_at": 400,
+  "commands": {
+    "save_checkpoint": false,
+    "run_test_validation": false,
+    "generate_predictions": false
+  }
+}

image-inpainting/src/architecture.py

@@ -86,6 +86,34 @@ class EfficientAttention(nn.Module):
         return x
 
 
+class SelfAttention(nn.Module):
+    """Self-attention module for long-range dependencies"""
+    def __init__(self, in_channels, reduction=8):
+        super().__init__()
+        self.query = nn.Conv2d(in_channels, in_channels // reduction, 1)
+        self.key = nn.Conv2d(in_channels, in_channels // reduction, 1)
+        self.value = nn.Conv2d(in_channels, in_channels, 1)
+        self.gamma = nn.Parameter(torch.zeros(1))
+        self.softmax = nn.Softmax(dim=-1)
+    
+    def forward(self, x):
+        batch_size, C, H, W = x.size()
+        
+        # Generate query, key, value
+        query = self.query(x).view(batch_size, -1, H * W).permute(0, 2, 1)
+        key = self.key(x).view(batch_size, -1, H * W)
+        value = self.value(x).view(batch_size, -1, H * W)
+        
+        # Attention map
+        attention = self.softmax(torch.bmm(query, key))
+        out = torch.bmm(value, attention.permute(0, 2, 1))
+        out = out.view(batch_size, C, H, W)
+        
+        # Residual connection with learnable weight
+        out = self.gamma * out + x
+        return out
+
+
 class ConvBlock(nn.Module):
     """Convolutional block with Conv2d -> BatchNorm -> LeakyReLU"""
     def __init__(self, in_channels, out_channels, kernel_size=3, padding=1, dilation=1, dropout=0.0, separable=False):
@@ -150,7 +178,7 @@ class ResidualConvBlock(nn.Module):
 
 class DownBlock(nn.Module):
     """Enhanced downsampling block with dense and residual connections"""
-    def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False):
+    def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False, use_self_attention=False):
         super().__init__()
         self.conv1 = ConvBlock(in_channels, out_channels, dropout=dropout, separable=True)
         self.conv2 = ConvBlock(out_channels, out_channels, dropout=dropout)
@@ -159,18 +187,20 @@ class DownBlock(nn.Module):
         else:
             self.dense = ResidualConvBlock(out_channels, dropout=dropout)
         self.attention = EfficientAttention(out_channels) if use_attention else nn.Identity()
+        self.self_attention = SelfAttention(out_channels) if use_self_attention else nn.Identity()
         self.pool = nn.MaxPool2d(2)
     
     def forward(self, x):
         x = self.conv1(x)
         x = self.conv2(x)
         x = self.dense(x)
-        skip = self.attention(x)
+        x = self.attention(x)
+        skip = self.self_attention(x)
         return self.pool(skip), skip
 
 class UpBlock(nn.Module):
     """Enhanced upsampling block with gated skip connections"""
-    def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False):
+    def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False, use_self_attention=False):
         super().__init__()
         self.up = nn.ConvTranspose2d(in_channels, out_channels, kernel_size=2, stride=2)
         # Skip connection has in_channels, upsampled has out_channels
@@ -183,6 +213,7 @@ class UpBlock(nn.Module):
         else:
             self.dense = ResidualConvBlock(out_channels, dropout=dropout)
         self.attention = EfficientAttention(out_channels) if use_attention else nn.Identity()
+        self.self_attention = SelfAttention(out_channels) if use_self_attention else nn.Identity()
     
     def forward(self, x, skip):
         x = self.up(x)
@@ -194,6 +225,7 @@ class UpBlock(nn.Module):
         x = self.conv2(x)
         x = self.dense(x)
         x = self.attention(x)
+        x = self.self_attention(x)
         return x
 
 class MyModel(nn.Module):
@@ -225,19 +257,20 @@ class MyModel(nn.Module):
         # Encoder with progressive feature extraction
         self.down1 = DownBlock(base_channels, base_channels * 2, dropout=dropout, use_attention=False, use_dense=False)
         self.down2 = DownBlock(base_channels * 2, base_channels * 4, dropout=dropout, use_attention=True, use_dense=True)
-        self.down3 = DownBlock(base_channels * 4, base_channels * 8, dropout=dropout, use_attention=True, use_dense=True)
+        self.down3 = DownBlock(base_channels * 4, base_channels * 8, dropout=dropout, use_attention=True, use_dense=True, use_self_attention=True)
         
-        # Enhanced bottleneck with multi-scale features and dense connections
+        # Enhanced bottleneck with multi-scale features, dense connections, and self-attention
         self.bottleneck = nn.Sequential(
             ConvBlock(base_channels * 8, base_channels * 8, dropout=dropout),
-            DenseBlock(base_channels * 8, growth_rate=10, num_layers=3, dropout=dropout),
+            DenseBlock(base_channels * 8, growth_rate=12, num_layers=3, dropout=dropout),
+            SelfAttention(base_channels * 8, reduction=4),
             ConvBlock(base_channels * 8, base_channels * 8, dilation=2, padding=2, dropout=dropout),
             ResidualConvBlock(base_channels * 8, dropout=dropout),
             EfficientAttention(base_channels * 8)
         )
         
         # Decoder with progressive reconstruction
-        self.up1 = UpBlock(base_channels * 8, base_channels * 4, dropout=dropout, use_attention=True, use_dense=True)
+        self.up1 = UpBlock(base_channels * 8, base_channels * 4, dropout=dropout, use_attention=True, use_dense=True, use_self_attention=True)
         self.up2 = UpBlock(base_channels * 4, base_channels * 2, dropout=dropout, use_attention=True, use_dense=True)
         self.up3 = UpBlock(base_channels * 2, base_channels, dropout=dropout, use_attention=False, use_dense=False)
         

image-inpainting/src/datasets.py

@@ -10,7 +10,8 @@ import numpy as np
 import random
 import glob
 import os
-from PIL import Image, ImageEnhance
+from PIL import Image, ImageEnhance, ImageFilter
+from scipy.ndimage import gaussian_filter, map_coordinates
 
 IMAGE_DIMENSION = 100
 
@@ -37,7 +38,43 @@ def preprocess(input_array: np.ndarray):
     input_array = np.asarray(input_array, dtype=np.float32) / 255.0
     return input_array
 
-def augment_image(img: Image, strength: float = 0.7) -> Image:
+def elastic_transform(image: np.ndarray, alpha: float = 20, sigma: float = 4) -> np.ndarray:
+    """Apply elastic deformation to image array"""
+    shape = image.shape[:2]
+    dx = gaussian_filter((np.random.rand(*shape) * 2 - 1), sigma) * alpha
+    dy = gaussian_filter((np.random.rand(*shape) * 2 - 1), sigma) * alpha
+    
+    x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]))
+    indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1))
+    
+    # Apply to each channel
+    transformed = np.zeros_like(image)
+    for i in range(image.shape[2]):
+        transformed[:, :, i] = map_coordinates(image[:, :, i], indices, order=1, mode='reflect').reshape(shape)
+    
+    return transformed
+
+def add_noise(img_array: np.ndarray, noise_type: str = 'gaussian', strength: float = 0.02) -> np.ndarray:
+    """Add various types of noise to image"""
+    if noise_type == 'gaussian':
+        noise = np.random.normal(0, strength, img_array.shape)
+        noisy = img_array + noise
+    elif noise_type == 'salt_pepper':
+        noisy = img_array.copy()
+        # Salt
+        num_salt = int(strength * img_array.size * 0.5)
+        coords = [np.random.randint(0, i, num_salt) for i in img_array.shape]
+        noisy[coords[0], coords[1], :] = 1
+        # Pepper
+        num_pepper = int(strength * img_array.size * 0.5)
+        coords = [np.random.randint(0, i, num_pepper) for i in img_array.shape]
+        noisy[coords[0], coords[1], :] = 0
+    else:
+        noisy = img_array
+    
+    return np.clip(noisy, 0, 1)
+
+def augment_image(img: Image, strength: float = 0.8) -> Image:
     """Apply comprehensive data augmentation for better generalization"""
     # Random horizontal flip
     if random.random() > 0.5:
@@ -47,26 +84,62 @@ def augment_image(img: Image, strength: float = 0.7) -> Image:
     if random.random() > 0.5:
         img = img.transpose(Image.FLIP_TOP_BOTTOM)
     
-    # Random rotation (90, 180, 270 degrees)
+    # Random rotation (90, 180, 270 degrees, or small angles)
     if random.random() > 0.5:
-        angle = random.choice([90, 180, 270])
-        img = img.rotate(angle)
+        if random.random() > 0.7:
+            # Large rotation
+            angle = random.choice([90, 180, 270])
+            img = img.rotate(angle)
+        else:
+            # Small rotation for more variation
+            angle = random.uniform(-15, 15)
+            img = img.rotate(angle, fillcolor=(128, 128, 128))
     
-    # Color augmentation - more aggressive for long training
-    rand = random.random()
-    if rand > 0.75:
+    # More aggressive color augmentation
+    if random.random() > 0.3:
         # Brightness
-        factor = 1.0 + random.uniform(-0.2, 0.2) * strength
+        factor = 1.0 + random.uniform(-0.3, 0.3) * strength
         img = ImageEnhance.Brightness(img).enhance(factor)
-    elif rand > 0.5:
+    
+    if random.random() > 0.3:
         # Contrast
-        factor = 1.0 + random.uniform(-0.2, 0.2) * strength
+        factor = 1.0 + random.uniform(-0.3, 0.3) * strength
         img = ImageEnhance.Contrast(img).enhance(factor)
-    elif rand > 0.25:
+    
+    if random.random() > 0.3:
         # Saturation
-        factor = 1.0 + random.uniform(-0.15, 0.15) * strength
+        factor = 1.0 + random.uniform(-0.25, 0.25) * strength
         img = ImageEnhance.Color(img).enhance(factor)
     
+    if random.random() > 0.7:
+        # Sharpness
+        factor = 1.0 + random.uniform(-0.3, 0.5) * strength
+        img = ImageEnhance.Sharpness(img).enhance(factor)
+    
+    # Gaussian blur for robustness
+    if random.random() > 0.8:
+        radius = random.uniform(0.5, 1.5) * strength
+        img = img.filter(ImageFilter.GaussianBlur(radius=radius))
+    
+    # Convert to array for elastic transform and noise
+    img_array = np.array(img).astype(np.float32) / 255.0
+    
+    # Elastic deformation
+    if random.random() > 0.7:
+        alpha = random.uniform(15, 30) * strength
+        sigma = random.uniform(3, 5)
+        img_array = elastic_transform(img_array, alpha=alpha, sigma=sigma)
+    
+    # Add noise
+    if random.random() > 0.6:
+        noise_type = random.choice(['gaussian', 'salt_pepper'])
+        noise_strength = random.uniform(0.01, 0.03) * strength
+        img_array = add_noise(img_array, noise_type=noise_type, strength=noise_strength)
+    
+    # Convert back to PIL Image
+    img_array = np.clip(img_array * 255, 0, 255).astype(np.uint8)
+    img = Image.fromarray(img_array)
+    
     return img
 
 class ImageDataset(torch.utils.data.Dataset):
@@ -74,7 +147,7 @@ class ImageDataset(torch.utils.data.Dataset):
     Dataset class for loading images from a folder with augmentation support
     """
 
-    def __init__(self, datafolder: str, augment: bool = True, augment_strength: float = 0.7):
+    def __init__(self, datafolder: str, augment: bool = True, augment_strength: float = 0.8):
         self.imagefiles = sorted(glob.glob(os.path.join(datafolder,"**","*.jpg"),recursive=True))
         self.augment = augment
         self.augment_strength = augment_strength

image-inpainting/src/main.py

@@ -24,39 +24,67 @@ if __name__ == '__main__':
     config_dict['results_path'] = os.path.join(project_root, "results")
     config_dict['data_path'] = os.path.join(project_root, "data", "dataset")
     config_dict['device'] = None
-    config_dict['learningrate'] = 8e-4  # Optimized for long training
-    config_dict['weight_decay'] = 1e-4  # Better regularization for long training
-    config_dict['n_updates'] = 30000  # Full day of training (~24 hours)
-    config_dict['batchsize'] = 64  # Balanced for memory and quality
-    config_dict['early_stopping_patience'] = 15  # More patience for better convergence
+    config_dict['learningrate'] = 5e-4  # Lower initial LR with warmup
+    config_dict['weight_decay'] = 5e-5  # Reduced for more capacity
+    config_dict['n_updates'] = 35000  # Extended training for better convergence
+    config_dict['batchsize'] = 48  # Reduced for larger model and mixed precision
+    config_dict['early_stopping_patience'] = 20  # More patience for complex model
     config_dict['use_wandb'] = False
 
     config_dict['print_train_stats_at'] = 50
     config_dict['print_stats_at'] = 200
     config_dict['plot_at'] = 500
-    config_dict['validate_at'] = 500  # Regular validation
+    config_dict['validate_at'] = 400  # More frequent validation
 
     network_config = {
         'n_in_channels': 4,
-        'base_channels': 44,  # Optimal capacity for 16GB VRAM
-        'dropout': 0.12  # Higher dropout for longer training
+        'base_channels': 52,  # Increased capacity for better feature extraction
+        'dropout': 0.15  # Slightly higher dropout for regularization
     }
     
     config_dict['network_config'] = network_config
+    
+    # Prepare paths for runtime predictions
+    testset_path = os.path.join(project_root, "data", "challenge_testset.npz")
+    save_path = os.path.join(config_dict['results_path'], "runtime_predictions")
+    plot_path_predictions = os.path.join(config_dict['results_path'], "runtime_predictions", "plots")
+    
+    config_dict['testset_path'] = testset_path
+    config_dict['save_path'] = save_path
+    config_dict['plot_path_predictions'] = plot_path_predictions
+    
+    print("="*60)
+    print("RUNTIME CONFIGURATION ENABLED")
+    print("="*60)
+    print("During training, you can modify these parameters by editing:")
+    print(f"{os.path.join(config_dict['results_path'], 'runtime_config.json')}")
+    print("\nModifiable parameters:")
+    print("  - n_updates: Maximum training steps")
+    print("  - plot_at: How often to save plots")
+    print("  - early_stopping_patience: Patience for early stopping")
+    print("  - print_stats_at: How often to print detailed stats")
+    print("  - print_train_stats_at: How often to print training loss")
+    print("  - validate_at: How often to run validation")
+    print("\nRuntime commands (set to true to execute):")
+    print("  - save_checkpoint: Save model at current step")
+    print("  - run_test_validation: Run validation on final test set")
+    print("  - generate_predictions: Generate predictions on challenge testset")
+    print("\nChanges will be applied within 100 steps.")
+    print("="*60)
+    print()
 
     rmse_value = train(**config_dict)
     
-    testset_path = os.path.join(project_root, "data", "challenge_testset.npz")
     state_dict_path = os.path.join(config_dict['results_path'], "best_model.pt")
-    save_path = os.path.join(config_dict['results_path'], "testset", "tikaiz")
-    plot_path = os.path.join(config_dict['results_path'], "testset", "plots")
-    os.makedirs(plot_path, exist_ok=True)
-    for name in os.listdir(plot_path):
-        p = os.path.join(plot_path, name)
+    final_save_path = os.path.join(config_dict['results_path'], "testset", "tikaiz")
+    final_plot_path = os.path.join(config_dict['results_path'], "testset", "plots")
+    os.makedirs(final_plot_path, exist_ok=True)
+    for name in os.listdir(final_plot_path):
+        p = os.path.join(final_plot_path, name)
         if os.path.isfile(p) or os.path.islink(p):
             os.unlink(p)
         elif os.path.isdir(p):
             shutil.rmtree(p)
 
     # Comment out, if predictions are required
-    create_predictions(config_dict['network_config'], state_dict_path, testset_path, None, save_path, plot_path, plot_at=20, rmse_value=rmse_value)
+    create_predictions(config_dict['network_config'], state_dict_path, testset_path, None, final_save_path, final_plot_path, plot_at=20, rmse_value=rmse_value)

image-inpainting/src/train.py

@@ -12,6 +12,9 @@ import torch
 import torch.nn as nn
 import numpy as np
 import os
+import json
+from torchvision import models
+import torch.nn.functional as F
 
 from torch.utils.data import DataLoader
 from torch.utils.data import Subset
@@ -19,6 +22,53 @@ from torch.utils.data import Subset
 import wandb
 
 
+def load_runtime_config(config_path, current_params):
+    """Load runtime configuration from JSON file and update parameters"""
+    try:
+        if os.path.exists(config_path):
+            with open(config_path, 'r') as f:
+                new_config = json.load(f)
+            
+            # Update modifiable parameters
+            updated = False
+            modifiable_keys = ['n_updates', 'plot_at', 'early_stopping_patience', 
+                             'print_stats_at', 'print_train_stats_at', 'validate_at']
+            
+            for key in modifiable_keys:
+                if key in new_config and new_config[key] != current_params.get(key):
+                    old_val = current_params.get(key)
+                    current_params[key] = new_config[key]
+                    print(f"\n[CONFIG UPDATE] {key}: {old_val} -> {new_config[key]}")
+                    updated = True
+            
+            # Check for command flags
+            commands = new_config.get('commands', {})
+            current_params['commands'] = commands
+            
+            if updated:
+                print("[CONFIG UPDATE] Runtime configuration updated successfully!\n")
+    except Exception as e:
+        print(f"Warning: Could not load runtime config: {e}")
+    
+    return current_params
+
+
+def clear_command_flag(config_path, command_name):
+    """Clear a specific command flag after execution"""
+    try:
+        if os.path.exists(config_path):
+            with open(config_path, 'r') as f:
+                config = json.load(f)
+            
+            if 'commands' in config and command_name in config['commands']:
+                config['commands'][command_name] = False
+                
+                with open(config_path, 'w') as f:
+                    json.dump(config, f, indent=2)
+    except Exception as e:
+        print(f"Warning: Could not clear command flag: {e}")
+
+
 class RMSELoss(nn.Module):
     """RMSE loss for direct optimization of evaluation metric"""
     def __init__(self):
@@ -27,13 +77,104 @@ class RMSELoss(nn.Module):
     
     def forward(self, pred, target):
         mse = self.mse(pred, target)
-        rmse = torch.sqrt(mse + 1e-8)  # Add epsilon for numerical stability
+        # Larger epsilon for numerical stability
+        rmse = torch.sqrt(mse + 1e-6)
         return rmse
 
 
+class PerceptualLoss(nn.Module):
+    """Perceptual loss using VGG16 features for better texture and detail preservation"""
+    def __init__(self, device):
+        super().__init__()
+        # Load pre-trained VGG16 and use specific layers
+        vgg = models.vgg16(pretrained=True).features.to(device).eval()
+        # Freeze VGG parameters
+        for param in vgg.parameters():
+            param.requires_grad = False
+        
+        # Use early and middle layers for perceptual loss
+        self.slice1 = nn.Sequential(*list(vgg.children())[:4])   # relu1_2
+        self.slice2 = nn.Sequential(*list(vgg.children())[4:9])  # relu2_2
+        self.slice3 = nn.Sequential(*list(vgg.children())[9:16]) # relu3_3
+        
+        # Normalization for VGG (ImageNet stats)
+        self.register_buffer('mean', torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1))
+        self.register_buffer('std', torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1))
+    
+    def normalize(self, x):
+        """Normalize images for VGG with clamping for stability"""
+        # Clamp input to valid range
+        x = torch.clamp(x, 0.0, 1.0)
+        return (x - self.mean) / (self.std + 1e-8)
+    
+    def forward(self, pred, target):
+        # Clamp inputs to prevent extreme values
+        pred = torch.clamp(pred, 0.0, 1.0)
+        target = torch.clamp(target, 0.0, 1.0)
+        
+        # Normalize inputs
+        pred = self.normalize(pred)
+        target = self.normalize(target)
+        
+        # Extract features from multiple layers
+        pred_f1 = self.slice1(pred)
+        pred_f2 = self.slice2(pred_f1)
+        pred_f3 = self.slice3(pred_f2)
+        
+        target_f1 = self.slice1(target)
+        target_f2 = self.slice2(target_f1)
+        target_f3 = self.slice3(target_f2)
+        
+        # Compute losses at multiple scales
+        loss = F.l1_loss(pred_f1, target_f1) + \
+               F.l1_loss(pred_f2, target_f2) + \
+               F.l1_loss(pred_f3, target_f3)
+        
+        return loss
+
+
+class CombinedLoss(nn.Module):
+    """Combined loss optimized for RMSE evaluation with optional perceptual component"""
+    def __init__(self, device, use_perceptual=True, perceptual_weight=0.05):
+        super().__init__()
+        self.use_perceptual = use_perceptual
+        if use_perceptual:
+            self.perceptual_loss = PerceptualLoss(device)
+        # Use MSE instead of RMSE for training (more stable gradients)
+        self.mse_loss = nn.MSELoss()
+        self.rmse_loss = RMSELoss()  # For logging only
+        
+        self.perceptual_weight = perceptual_weight
+        self.mse_weight = 1.0 - perceptual_weight
+    
+    def forward(self, pred, target):
+        # Clamp predictions to valid range
+        pred = torch.clamp(pred, 0.0, 1.0)
+        
+        # Primary loss: MSE (equivalent to RMSE but more stable)
+        mse = self.mse_loss(pred, target)
+        rmse = self.rmse_loss(pred, target)  # For logging
+        
+        if self.use_perceptual:
+            # Optional small perceptual component for texture quality
+            perceptual = self.perceptual_loss(pred, target)
+            total_loss = self.mse_weight * mse + self.perceptual_weight * perceptual
+        else:
+            # Pure MSE optimization
+            perceptual = torch.tensor(0.0, device=pred.device)
+            total_loss = mse
+        
+        # Validate loss is not NaN or Inf
+        if not torch.isfinite(total_loss):
+            # Return MSE only as fallback
+            total_loss = mse
+        
+        return total_loss, perceptual, mse, rmse
+
+
 def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_stopping_patience, device, learningrate,
           weight_decay, n_updates, use_wandb, print_train_stats_at, print_stats_at, plot_at, validate_at, batchsize,
-          network_config: dict):
+          network_config: dict, testset_path=None, save_path=None, plot_path_predictions=None):
     np.random.seed(seed=seed)
     torch.manual_seed(seed=seed)
 
@@ -46,7 +187,10 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st
     
     # Enable mixed precision training for memory efficiency
     use_amp = torch.cuda.is_available()
-    scaler = torch.amp.GradScaler('cuda') if use_amp else None
+    if use_amp:
+        scaler = torch.amp.GradScaler('cuda', init_scale=2048.0, growth_interval=100)
+    else:
+        scaler = None
 
     if use_wandb:
         wandb.login()
@@ -91,17 +235,27 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st
     network.to(device)
     network.train()
 
-    # defining the loss - RMSE for direct optimization of evaluation metric
-    rmse_loss = RMSELoss().to(device)
+    # defining the loss - Optimized for RMSE evaluation
+    # Set use_perceptual=False for pure MSE training, or keep True with 5% weight for texture quality
+    combined_loss = CombinedLoss(device, use_perceptual=True, perceptual_weight=0.05).to(device)
     mse_loss = torch.nn.MSELoss()  # Keep for evaluation
 
     # defining the optimizer with AdamW for better weight decay handling
     optimizer = torch.optim.AdamW(network.parameters(), lr=learningrate, weight_decay=weight_decay, betas=(0.9, 0.999))
     
+    # Learning rate warmup
+    warmup_steps = min(1000, n_updates // 10)
+    
     # Cosine annealing with warm restarts for long training
-    scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
+    scheduler_main = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(
         optimizer, T_0=n_updates//4, T_mult=1, eta_min=learningrate/100
     )
+    
+    # Warmup scheduler
+    def get_lr_scale(step):
+        if step < warmup_steps:
+            return step / warmup_steps
+        return 1.0
 
     if use_wandb:
         wandb.watch(network, mse_loss, log="all", log_freq=10)
@@ -112,14 +266,94 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st
     loss_list = []
 
     saved_model_path = os.path.join(results_path, "best_model.pt")
-
+    
+    # Save runtime configuration to JSON file for dynamic updates
+    config_json_path = os.path.join(results_path, "runtime_config.json")
+    runtime_params = {
+        'n_updates': n_updates,
+        'plot_at': plot_at,
+        'early_stopping_patience': early_stopping_patience,
+        'print_stats_at': print_stats_at,
+        'print_train_stats_at': print_train_stats_at,
+        'validate_at': validate_at,
+        'commands': {
+            'save_checkpoint': False,
+            'run_test_validation': False,
+            'generate_predictions': False
+        }
+    }
+    
+    with open(config_json_path, 'w') as f:
+        json.dump(runtime_params, f, indent=2)
+    
     print(f"Started training on device {device}")
+    print(f"Runtime config saved to: {config_json_path}")
+    print(f"You can modify this file during training to change parameters dynamically!")
+    print(f"Set command flags to true to trigger actions (save_checkpoint, run_test_validation, generate_predictions)\n")
 
     while i < n_updates:
 
         for input, target in dataloader_train:
 
             input, target = input.to(device), target.to(device)
+            
+            # Check for runtime config updates every 50 steps
+            if i % 50 == 0 and i > 0:
+                runtime_params = load_runtime_config(config_json_path, runtime_params)
+                n_updates = runtime_params['n_updates']
+                plot_at = runtime_params['plot_at']
+                early_stopping_patience = runtime_params['early_stopping_patience']
+                print_stats_at = runtime_params['print_stats_at']
+                print_train_stats_at = runtime_params['print_train_stats_at']
+                validate_at = runtime_params['validate_at']
+                
+                # Execute runtime commands
+                commands = runtime_params.get('commands', {})
+                
+                # Command: Save checkpoint
+                if commands.get('save_checkpoint', False):
+                    checkpoint_path = os.path.join(results_path, f"checkpoint_step_{i}.pt")
+                    torch.save(network.state_dict(), checkpoint_path)
+                    print(f"\n[COMMAND] Checkpoint saved to: {checkpoint_path}\n")
+                    clear_command_flag(config_json_path, 'save_checkpoint')
+                
+                # Command: Generate predictions
+                if commands.get('generate_predictions', False) and testset_path is not None:
+                    print(f"\n[COMMAND] Generating predictions at step {i}...")
+                    try:
+                        from utils import create_predictions
+                        pred_save_path = save_path or os.path.join(results_path, "runtime_predictions", f"step_{i}")
+                        pred_plot_path = plot_path_predictions or os.path.join(results_path, "runtime_predictions", "plots", f"step_{i}")
+                        os.makedirs(pred_plot_path, exist_ok=True)
+                        
+                        # Save current state temporarily
+                        temp_state_path = os.path.join(results_path, f"temp_state_step_{i}.pt")
+                        torch.save(network.state_dict(), temp_state_path)
+                        
+                        # Generate predictions
+                        create_predictions(network_config, temp_state_path, testset_path, None, 
+                                         pred_save_path, pred_plot_path, plot_at=20, rmse_value=None)
+                        
+                        print(f"[COMMAND] Predictions saved to: {pred_save_path}")
+                        print(f"[COMMAND] Plots saved to: {pred_plot_path}\n")
+                        
+                        # Clean up temp file
+                        if os.path.exists(temp_state_path):
+                            os.remove(temp_state_path)
+                    except Exception as e:
+                        print(f"[COMMAND] Error generating predictions: {e}\n")
+                    
+                    network.train()
+                    clear_command_flag(config_json_path, 'generate_predictions')
+                
+                # Command: Run test validation
+                if commands.get('run_test_validation', False):
+                    print(f"\n[COMMAND] Running test set validation at step {i}...")
+                    network.eval()
+                    test_loss, test_rmse = evaluate_model(network, dataloader_test, mse_loss, device)
+                    print(f"[COMMAND] Test Loss: {test_loss:.6f}, Test RMSE: {test_rmse:.6f}\n")
+                    network.train()
+                    clear_command_flag(config_json_path, 'run_test_validation')
 
             if (i + 1) % print_train_stats_at == 0:
                 print(f'Update Step {i + 1} of {n_updates}: Current loss: {loss_list[-1]}')
@@ -130,33 +364,96 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st
             if use_amp:
                 with torch.amp.autocast('cuda'):
                     output = network(input)
-                    loss = rmse_loss(output, target)
+                    total_loss, perceptual, mse, rmse = combined_loss(output, target)
                 
-                scaler.scale(loss).backward()
+                # Check for NaN before backward
+                if not torch.isfinite(total_loss):
+                    continue
                 
-                # Gradient clipping for training stability
+                scaler.scale(total_loss).backward()
+                
+                # Unscale and check gradients
                 scaler.unscale_(optimizer)
-                torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=1.0)
+                
+                # Check for NaN in gradients
+                has_nan = False
+                for name, param in network.named_parameters():
+                    if param.grad is not None:
+                        if not torch.isfinite(param.grad).all():
+                            print(f"NaN gradient detected in {name}")
+                            has_nan = True
+                            break
+                
+                if has_nan:
+                    print(f"Skipping step {i+1}: NaN gradients detected")
+                    optimizer.zero_grad()
+                    scaler.update()
+                    continue
+                
+                # More aggressive gradient clipping for stability
+                grad_norm = torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=0.5)
+                
+                # Skip update if gradient norm is too large
+                if grad_norm > 100.0:
+                    print(f"Skipping step {i+1}: Gradient norm too large: {grad_norm:.2f}")
+                    optimizer.zero_grad()
+                    scaler.update()
+                    continue
                 
                 scaler.step(optimizer)
                 scaler.update()
             else:
                 output = network(input)
-                loss = rmse_loss(output, target)
-                loss.backward()
+                total_loss, perceptual, mse, rmse = combined_loss(output, target)
                 
-                # Gradient clipping for training stability
-                torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=1.0)
+                # Check for NaN before backward
+                if not torch.isfinite(total_loss):
+                    print(f"Skipping step {i+1}: NaN or Inf loss detected")
+                    continue
+                
+                total_loss.backward()
+                
+                # Check for NaN in gradients
+                has_nan = False
+                for param in network.parameters():
+                    if param.grad is not None and not torch.isfinite(param.grad).all():
+                        has_nan = True
+                        break
+                
+                if has_nan:
+                    print(f"Skipping step {i+1}: NaN gradients detected")
+                    optimizer.zero_grad()
+                    continue
+                
+                # More aggressive gradient clipping
+                grad_norm = torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=0.5)
+                
+                if grad_norm > 100.0:
+                    print(f"Skipping step {i+1}: Gradient norm too large: {grad_norm:.2f}")
+                    optimizer.zero_grad()
+                    continue
                 
                 optimizer.step()
             
-            scheduler.step()
+            # Apply learning rate scheduling with warmup
+            lr_scale = get_lr_scale(i)
+            for param_group in optimizer.param_groups:
+                param_group['lr'] = learningrate * lr_scale
+            
+            if i >= warmup_steps:
+                scheduler_main.step()
 
-            loss_list.append(loss.item())
+            loss_list.append(total_loss.item())
 
             # writing the stats to wandb
             if use_wandb and (i+1) % print_stats_at == 0:
-                wandb.log({"training/loss_per_batch": loss.item()}, step=i)
+                wandb.log({
+                    "training/loss_total": total_loss.item(),
+                    "training/loss_mse": mse.item(),
+                    "training/loss_rmse": rmse.item(),
+                    "training/loss_perceptual": perceptual.item() if isinstance(perceptual, torch.Tensor) else perceptual,
+                    "training/learning_rate": optimizer.param_groups[0]['lr']
+                }, step=i)
 
             # plotting
             if (i + 1) % plot_at == 0:

image-inpainting/src/utils.py

@@ -122,6 +122,13 @@ def create_predictions(model_config, state_dict_path, testset_path, device, save
 
     predictions = np.stack(predictions, axis=0)
 
+    # Handle NaN and inf values before conversion
+    nan_mask = ~np.isfinite(predictions)
+    if nan_mask.any():
+        nan_count = nan_mask.sum()
+        print(f"Warning: Found {nan_count} NaN/Inf values in predictions. Replacing with 0.")
+        predictions = np.nan_to_num(predictions, nan=0.0, posinf=1.0, neginf=0.0)
+    
     predictions = (np.clip(predictions, 0, 1) * 255.0).astype(np.uint8)
 
     data = {