diff --git a/image-inpainting/.gitignore b/image-inpainting/.gitignore index 1d5d60b..0e60ef1 100644 --- a/image-inpainting/.gitignore +++ b/image-inpainting/.gitignore @@ -2,4 +2,5 @@ data/* *.zip *.jpg *.pt -__pycache__/ \ No newline at end of file +__pycache__/ +runtime_predictions.npz \ No newline at end of file diff --git a/image-inpainting/results/runtime_config.json b/image-inpainting/results/runtime_config.json new file mode 100644 index 0000000..3eccfcf --- /dev/null +++ b/image-inpainting/results/runtime_config.json @@ -0,0 +1,13 @@ +{ + "n_updates": 35000, + "plot_at": 500, + "early_stopping_patience": 20, + "print_stats_at": 200, + "print_train_stats_at": 50, + "validate_at": 400, + "commands": { + "save_checkpoint": false, + "run_test_validation": false, + "generate_predictions": false + } +} \ No newline at end of file diff --git a/image-inpainting/src/architecture.py b/image-inpainting/src/architecture.py index e37c5e6..6b52929 100644 --- a/image-inpainting/src/architecture.py +++ b/image-inpainting/src/architecture.py @@ -86,6 +86,34 @@ class EfficientAttention(nn.Module): return x +class SelfAttention(nn.Module): + """Self-attention module for long-range dependencies""" + def __init__(self, in_channels, reduction=8): + super().__init__() + self.query = nn.Conv2d(in_channels, in_channels // reduction, 1) + self.key = nn.Conv2d(in_channels, in_channels // reduction, 1) + self.value = nn.Conv2d(in_channels, in_channels, 1) + self.gamma = nn.Parameter(torch.zeros(1)) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x): + batch_size, C, H, W = x.size() + + # Generate query, key, value + query = self.query(x).view(batch_size, -1, H * W).permute(0, 2, 1) + key = self.key(x).view(batch_size, -1, H * W) + value = self.value(x).view(batch_size, -1, H * W) + + # Attention map + attention = self.softmax(torch.bmm(query, key)) + out = torch.bmm(value, attention.permute(0, 2, 1)) + out = out.view(batch_size, C, H, W) + + # Residual connection with learnable weight + out = self.gamma * out + x + return out + + class ConvBlock(nn.Module): """Convolutional block with Conv2d -> BatchNorm -> LeakyReLU""" def __init__(self, in_channels, out_channels, kernel_size=3, padding=1, dilation=1, dropout=0.0, separable=False): @@ -150,7 +178,7 @@ class ResidualConvBlock(nn.Module): class DownBlock(nn.Module): """Enhanced downsampling block with dense and residual connections""" - def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False): + def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False, use_self_attention=False): super().__init__() self.conv1 = ConvBlock(in_channels, out_channels, dropout=dropout, separable=True) self.conv2 = ConvBlock(out_channels, out_channels, dropout=dropout) @@ -159,18 +187,20 @@ class DownBlock(nn.Module): else: self.dense = ResidualConvBlock(out_channels, dropout=dropout) self.attention = EfficientAttention(out_channels) if use_attention else nn.Identity() + self.self_attention = SelfAttention(out_channels) if use_self_attention else nn.Identity() self.pool = nn.MaxPool2d(2) def forward(self, x): x = self.conv1(x) x = self.conv2(x) x = self.dense(x) - skip = self.attention(x) + x = self.attention(x) + skip = self.self_attention(x) return self.pool(skip), skip class UpBlock(nn.Module): """Enhanced upsampling block with gated skip connections""" - def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False): + def __init__(self, in_channels, out_channels, dropout=0.1, use_attention=True, use_dense=False, use_self_attention=False): super().__init__() self.up = nn.ConvTranspose2d(in_channels, out_channels, kernel_size=2, stride=2) # Skip connection has in_channels, upsampled has out_channels @@ -183,6 +213,7 @@ class UpBlock(nn.Module): else: self.dense = ResidualConvBlock(out_channels, dropout=dropout) self.attention = EfficientAttention(out_channels) if use_attention else nn.Identity() + self.self_attention = SelfAttention(out_channels) if use_self_attention else nn.Identity() def forward(self, x, skip): x = self.up(x) @@ -194,6 +225,7 @@ class UpBlock(nn.Module): x = self.conv2(x) x = self.dense(x) x = self.attention(x) + x = self.self_attention(x) return x class MyModel(nn.Module): @@ -225,19 +257,20 @@ class MyModel(nn.Module): # Encoder with progressive feature extraction self.down1 = DownBlock(base_channels, base_channels * 2, dropout=dropout, use_attention=False, use_dense=False) self.down2 = DownBlock(base_channels * 2, base_channels * 4, dropout=dropout, use_attention=True, use_dense=True) - self.down3 = DownBlock(base_channels * 4, base_channels * 8, dropout=dropout, use_attention=True, use_dense=True) + self.down3 = DownBlock(base_channels * 4, base_channels * 8, dropout=dropout, use_attention=True, use_dense=True, use_self_attention=True) - # Enhanced bottleneck with multi-scale features and dense connections + # Enhanced bottleneck with multi-scale features, dense connections, and self-attention self.bottleneck = nn.Sequential( ConvBlock(base_channels * 8, base_channels * 8, dropout=dropout), - DenseBlock(base_channels * 8, growth_rate=10, num_layers=3, dropout=dropout), + DenseBlock(base_channels * 8, growth_rate=12, num_layers=3, dropout=dropout), + SelfAttention(base_channels * 8, reduction=4), ConvBlock(base_channels * 8, base_channels * 8, dilation=2, padding=2, dropout=dropout), ResidualConvBlock(base_channels * 8, dropout=dropout), EfficientAttention(base_channels * 8) ) # Decoder with progressive reconstruction - self.up1 = UpBlock(base_channels * 8, base_channels * 4, dropout=dropout, use_attention=True, use_dense=True) + self.up1 = UpBlock(base_channels * 8, base_channels * 4, dropout=dropout, use_attention=True, use_dense=True, use_self_attention=True) self.up2 = UpBlock(base_channels * 4, base_channels * 2, dropout=dropout, use_attention=True, use_dense=True) self.up3 = UpBlock(base_channels * 2, base_channels, dropout=dropout, use_attention=False, use_dense=False) diff --git a/image-inpainting/src/datasets.py b/image-inpainting/src/datasets.py index d7341a3..a454d2f 100644 --- a/image-inpainting/src/datasets.py +++ b/image-inpainting/src/datasets.py @@ -10,7 +10,8 @@ import numpy as np import random import glob import os -from PIL import Image, ImageEnhance +from PIL import Image, ImageEnhance, ImageFilter +from scipy.ndimage import gaussian_filter, map_coordinates IMAGE_DIMENSION = 100 @@ -37,7 +38,43 @@ def preprocess(input_array: np.ndarray): input_array = np.asarray(input_array, dtype=np.float32) / 255.0 return input_array -def augment_image(img: Image, strength: float = 0.7) -> Image: +def elastic_transform(image: np.ndarray, alpha: float = 20, sigma: float = 4) -> np.ndarray: + """Apply elastic deformation to image array""" + shape = image.shape[:2] + dx = gaussian_filter((np.random.rand(*shape) * 2 - 1), sigma) * alpha + dy = gaussian_filter((np.random.rand(*shape) * 2 - 1), sigma) * alpha + + x, y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0])) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)) + + # Apply to each channel + transformed = np.zeros_like(image) + for i in range(image.shape[2]): + transformed[:, :, i] = map_coordinates(image[:, :, i], indices, order=1, mode='reflect').reshape(shape) + + return transformed + +def add_noise(img_array: np.ndarray, noise_type: str = 'gaussian', strength: float = 0.02) -> np.ndarray: + """Add various types of noise to image""" + if noise_type == 'gaussian': + noise = np.random.normal(0, strength, img_array.shape) + noisy = img_array + noise + elif noise_type == 'salt_pepper': + noisy = img_array.copy() + # Salt + num_salt = int(strength * img_array.size * 0.5) + coords = [np.random.randint(0, i, num_salt) for i in img_array.shape] + noisy[coords[0], coords[1], :] = 1 + # Pepper + num_pepper = int(strength * img_array.size * 0.5) + coords = [np.random.randint(0, i, num_pepper) for i in img_array.shape] + noisy[coords[0], coords[1], :] = 0 + else: + noisy = img_array + + return np.clip(noisy, 0, 1) + +def augment_image(img: Image, strength: float = 0.8) -> Image: """Apply comprehensive data augmentation for better generalization""" # Random horizontal flip if random.random() > 0.5: @@ -47,26 +84,62 @@ def augment_image(img: Image, strength: float = 0.7) -> Image: if random.random() > 0.5: img = img.transpose(Image.FLIP_TOP_BOTTOM) - # Random rotation (90, 180, 270 degrees) + # Random rotation (90, 180, 270 degrees, or small angles) if random.random() > 0.5: - angle = random.choice([90, 180, 270]) - img = img.rotate(angle) + if random.random() > 0.7: + # Large rotation + angle = random.choice([90, 180, 270]) + img = img.rotate(angle) + else: + # Small rotation for more variation + angle = random.uniform(-15, 15) + img = img.rotate(angle, fillcolor=(128, 128, 128)) - # Color augmentation - more aggressive for long training - rand = random.random() - if rand > 0.75: + # More aggressive color augmentation + if random.random() > 0.3: # Brightness - factor = 1.0 + random.uniform(-0.2, 0.2) * strength + factor = 1.0 + random.uniform(-0.3, 0.3) * strength img = ImageEnhance.Brightness(img).enhance(factor) - elif rand > 0.5: + + if random.random() > 0.3: # Contrast - factor = 1.0 + random.uniform(-0.2, 0.2) * strength + factor = 1.0 + random.uniform(-0.3, 0.3) * strength img = ImageEnhance.Contrast(img).enhance(factor) - elif rand > 0.25: + + if random.random() > 0.3: # Saturation - factor = 1.0 + random.uniform(-0.15, 0.15) * strength + factor = 1.0 + random.uniform(-0.25, 0.25) * strength img = ImageEnhance.Color(img).enhance(factor) + if random.random() > 0.7: + # Sharpness + factor = 1.0 + random.uniform(-0.3, 0.5) * strength + img = ImageEnhance.Sharpness(img).enhance(factor) + + # Gaussian blur for robustness + if random.random() > 0.8: + radius = random.uniform(0.5, 1.5) * strength + img = img.filter(ImageFilter.GaussianBlur(radius=radius)) + + # Convert to array for elastic transform and noise + img_array = np.array(img).astype(np.float32) / 255.0 + + # Elastic deformation + if random.random() > 0.7: + alpha = random.uniform(15, 30) * strength + sigma = random.uniform(3, 5) + img_array = elastic_transform(img_array, alpha=alpha, sigma=sigma) + + # Add noise + if random.random() > 0.6: + noise_type = random.choice(['gaussian', 'salt_pepper']) + noise_strength = random.uniform(0.01, 0.03) * strength + img_array = add_noise(img_array, noise_type=noise_type, strength=noise_strength) + + # Convert back to PIL Image + img_array = np.clip(img_array * 255, 0, 255).astype(np.uint8) + img = Image.fromarray(img_array) + return img class ImageDataset(torch.utils.data.Dataset): @@ -74,7 +147,7 @@ class ImageDataset(torch.utils.data.Dataset): Dataset class for loading images from a folder with augmentation support """ - def __init__(self, datafolder: str, augment: bool = True, augment_strength: float = 0.7): + def __init__(self, datafolder: str, augment: bool = True, augment_strength: float = 0.8): self.imagefiles = sorted(glob.glob(os.path.join(datafolder,"**","*.jpg"),recursive=True)) self.augment = augment self.augment_strength = augment_strength diff --git a/image-inpainting/src/main.py b/image-inpainting/src/main.py index 7bccb7b..670de26 100644 --- a/image-inpainting/src/main.py +++ b/image-inpainting/src/main.py @@ -24,39 +24,67 @@ if __name__ == '__main__': config_dict['results_path'] = os.path.join(project_root, "results") config_dict['data_path'] = os.path.join(project_root, "data", "dataset") config_dict['device'] = None - config_dict['learningrate'] = 8e-4 # Optimized for long training - config_dict['weight_decay'] = 1e-4 # Better regularization for long training - config_dict['n_updates'] = 30000 # Full day of training (~24 hours) - config_dict['batchsize'] = 64 # Balanced for memory and quality - config_dict['early_stopping_patience'] = 15 # More patience for better convergence + config_dict['learningrate'] = 5e-4 # Lower initial LR with warmup + config_dict['weight_decay'] = 5e-5 # Reduced for more capacity + config_dict['n_updates'] = 35000 # Extended training for better convergence + config_dict['batchsize'] = 48 # Reduced for larger model and mixed precision + config_dict['early_stopping_patience'] = 20 # More patience for complex model config_dict['use_wandb'] = False config_dict['print_train_stats_at'] = 50 config_dict['print_stats_at'] = 200 config_dict['plot_at'] = 500 - config_dict['validate_at'] = 500 # Regular validation + config_dict['validate_at'] = 400 # More frequent validation network_config = { 'n_in_channels': 4, - 'base_channels': 44, # Optimal capacity for 16GB VRAM - 'dropout': 0.12 # Higher dropout for longer training + 'base_channels': 52, # Increased capacity for better feature extraction + 'dropout': 0.15 # Slightly higher dropout for regularization } config_dict['network_config'] = network_config + + # Prepare paths for runtime predictions + testset_path = os.path.join(project_root, "data", "challenge_testset.npz") + save_path = os.path.join(config_dict['results_path'], "runtime_predictions") + plot_path_predictions = os.path.join(config_dict['results_path'], "runtime_predictions", "plots") + + config_dict['testset_path'] = testset_path + config_dict['save_path'] = save_path + config_dict['plot_path_predictions'] = plot_path_predictions + + print("="*60) + print("RUNTIME CONFIGURATION ENABLED") + print("="*60) + print("During training, you can modify these parameters by editing:") + print(f"{os.path.join(config_dict['results_path'], 'runtime_config.json')}") + print("\nModifiable parameters:") + print(" - n_updates: Maximum training steps") + print(" - plot_at: How often to save plots") + print(" - early_stopping_patience: Patience for early stopping") + print(" - print_stats_at: How often to print detailed stats") + print(" - print_train_stats_at: How often to print training loss") + print(" - validate_at: How often to run validation") + print("\nRuntime commands (set to true to execute):") + print(" - save_checkpoint: Save model at current step") + print(" - run_test_validation: Run validation on final test set") + print(" - generate_predictions: Generate predictions on challenge testset") + print("\nChanges will be applied within 100 steps.") + print("="*60) + print() rmse_value = train(**config_dict) - testset_path = os.path.join(project_root, "data", "challenge_testset.npz") state_dict_path = os.path.join(config_dict['results_path'], "best_model.pt") - save_path = os.path.join(config_dict['results_path'], "testset", "tikaiz") - plot_path = os.path.join(config_dict['results_path'], "testset", "plots") - os.makedirs(plot_path, exist_ok=True) - for name in os.listdir(plot_path): - p = os.path.join(plot_path, name) + final_save_path = os.path.join(config_dict['results_path'], "testset", "tikaiz") + final_plot_path = os.path.join(config_dict['results_path'], "testset", "plots") + os.makedirs(final_plot_path, exist_ok=True) + for name in os.listdir(final_plot_path): + p = os.path.join(final_plot_path, name) if os.path.isfile(p) or os.path.islink(p): os.unlink(p) elif os.path.isdir(p): shutil.rmtree(p) # Comment out, if predictions are required - create_predictions(config_dict['network_config'], state_dict_path, testset_path, None, save_path, plot_path, plot_at=20, rmse_value=rmse_value) + create_predictions(config_dict['network_config'], state_dict_path, testset_path, None, final_save_path, final_plot_path, plot_at=20, rmse_value=rmse_value) diff --git a/image-inpainting/src/train.py b/image-inpainting/src/train.py index f0c9840..2098787 100644 --- a/image-inpainting/src/train.py +++ b/image-inpainting/src/train.py @@ -12,6 +12,9 @@ import torch import torch.nn as nn import numpy as np import os +import json +from torchvision import models +import torch.nn.functional as F from torch.utils.data import DataLoader from torch.utils.data import Subset @@ -19,6 +22,53 @@ from torch.utils.data import Subset import wandb +def load_runtime_config(config_path, current_params): + """Load runtime configuration from JSON file and update parameters""" + try: + if os.path.exists(config_path): + with open(config_path, 'r') as f: + new_config = json.load(f) + + # Update modifiable parameters + updated = False + modifiable_keys = ['n_updates', 'plot_at', 'early_stopping_patience', + 'print_stats_at', 'print_train_stats_at', 'validate_at'] + + for key in modifiable_keys: + if key in new_config and new_config[key] != current_params.get(key): + old_val = current_params.get(key) + current_params[key] = new_config[key] + print(f"\n[CONFIG UPDATE] {key}: {old_val} -> {new_config[key]}") + updated = True + + # Check for command flags + commands = new_config.get('commands', {}) + current_params['commands'] = commands + + if updated: + print("[CONFIG UPDATE] Runtime configuration updated successfully!\n") + except Exception as e: + print(f"Warning: Could not load runtime config: {e}") + + return current_params + + +def clear_command_flag(config_path, command_name): + """Clear a specific command flag after execution""" + try: + if os.path.exists(config_path): + with open(config_path, 'r') as f: + config = json.load(f) + + if 'commands' in config and command_name in config['commands']: + config['commands'][command_name] = False + + with open(config_path, 'w') as f: + json.dump(config, f, indent=2) + except Exception as e: + print(f"Warning: Could not clear command flag: {e}") + + class RMSELoss(nn.Module): """RMSE loss for direct optimization of evaluation metric""" def __init__(self): @@ -27,13 +77,104 @@ class RMSELoss(nn.Module): def forward(self, pred, target): mse = self.mse(pred, target) - rmse = torch.sqrt(mse + 1e-8) # Add epsilon for numerical stability + # Larger epsilon for numerical stability + rmse = torch.sqrt(mse + 1e-6) return rmse +class PerceptualLoss(nn.Module): + """Perceptual loss using VGG16 features for better texture and detail preservation""" + def __init__(self, device): + super().__init__() + # Load pre-trained VGG16 and use specific layers + vgg = models.vgg16(pretrained=True).features.to(device).eval() + # Freeze VGG parameters + for param in vgg.parameters(): + param.requires_grad = False + + # Use early and middle layers for perceptual loss + self.slice1 = nn.Sequential(*list(vgg.children())[:4]) # relu1_2 + self.slice2 = nn.Sequential(*list(vgg.children())[4:9]) # relu2_2 + self.slice3 = nn.Sequential(*list(vgg.children())[9:16]) # relu3_3 + + # Normalization for VGG (ImageNet stats) + self.register_buffer('mean', torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)) + self.register_buffer('std', torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)) + + def normalize(self, x): + """Normalize images for VGG with clamping for stability""" + # Clamp input to valid range + x = torch.clamp(x, 0.0, 1.0) + return (x - self.mean) / (self.std + 1e-8) + + def forward(self, pred, target): + # Clamp inputs to prevent extreme values + pred = torch.clamp(pred, 0.0, 1.0) + target = torch.clamp(target, 0.0, 1.0) + + # Normalize inputs + pred = self.normalize(pred) + target = self.normalize(target) + + # Extract features from multiple layers + pred_f1 = self.slice1(pred) + pred_f2 = self.slice2(pred_f1) + pred_f3 = self.slice3(pred_f2) + + target_f1 = self.slice1(target) + target_f2 = self.slice2(target_f1) + target_f3 = self.slice3(target_f2) + + # Compute losses at multiple scales + loss = F.l1_loss(pred_f1, target_f1) + \ + F.l1_loss(pred_f2, target_f2) + \ + F.l1_loss(pred_f3, target_f3) + + return loss + + +class CombinedLoss(nn.Module): + """Combined loss optimized for RMSE evaluation with optional perceptual component""" + def __init__(self, device, use_perceptual=True, perceptual_weight=0.05): + super().__init__() + self.use_perceptual = use_perceptual + if use_perceptual: + self.perceptual_loss = PerceptualLoss(device) + # Use MSE instead of RMSE for training (more stable gradients) + self.mse_loss = nn.MSELoss() + self.rmse_loss = RMSELoss() # For logging only + + self.perceptual_weight = perceptual_weight + self.mse_weight = 1.0 - perceptual_weight + + def forward(self, pred, target): + # Clamp predictions to valid range + pred = torch.clamp(pred, 0.0, 1.0) + + # Primary loss: MSE (equivalent to RMSE but more stable) + mse = self.mse_loss(pred, target) + rmse = self.rmse_loss(pred, target) # For logging + + if self.use_perceptual: + # Optional small perceptual component for texture quality + perceptual = self.perceptual_loss(pred, target) + total_loss = self.mse_weight * mse + self.perceptual_weight * perceptual + else: + # Pure MSE optimization + perceptual = torch.tensor(0.0, device=pred.device) + total_loss = mse + + # Validate loss is not NaN or Inf + if not torch.isfinite(total_loss): + # Return MSE only as fallback + total_loss = mse + + return total_loss, perceptual, mse, rmse + + def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_stopping_patience, device, learningrate, weight_decay, n_updates, use_wandb, print_train_stats_at, print_stats_at, plot_at, validate_at, batchsize, - network_config: dict): + network_config: dict, testset_path=None, save_path=None, plot_path_predictions=None): np.random.seed(seed=seed) torch.manual_seed(seed=seed) @@ -46,7 +187,10 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st # Enable mixed precision training for memory efficiency use_amp = torch.cuda.is_available() - scaler = torch.amp.GradScaler('cuda') if use_amp else None + if use_amp: + scaler = torch.amp.GradScaler('cuda', init_scale=2048.0, growth_interval=100) + else: + scaler = None if use_wandb: wandb.login() @@ -91,17 +235,27 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st network.to(device) network.train() - # defining the loss - RMSE for direct optimization of evaluation metric - rmse_loss = RMSELoss().to(device) + # defining the loss - Optimized for RMSE evaluation + # Set use_perceptual=False for pure MSE training, or keep True with 5% weight for texture quality + combined_loss = CombinedLoss(device, use_perceptual=True, perceptual_weight=0.05).to(device) mse_loss = torch.nn.MSELoss() # Keep for evaluation # defining the optimizer with AdamW for better weight decay handling optimizer = torch.optim.AdamW(network.parameters(), lr=learningrate, weight_decay=weight_decay, betas=(0.9, 0.999)) + # Learning rate warmup + warmup_steps = min(1000, n_updates // 10) + # Cosine annealing with warm restarts for long training - scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( + scheduler_main = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( optimizer, T_0=n_updates//4, T_mult=1, eta_min=learningrate/100 ) + + # Warmup scheduler + def get_lr_scale(step): + if step < warmup_steps: + return step / warmup_steps + return 1.0 if use_wandb: wandb.watch(network, mse_loss, log="all", log_freq=10) @@ -112,14 +266,94 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st loss_list = [] saved_model_path = os.path.join(results_path, "best_model.pt") - + + # Save runtime configuration to JSON file for dynamic updates + config_json_path = os.path.join(results_path, "runtime_config.json") + runtime_params = { + 'n_updates': n_updates, + 'plot_at': plot_at, + 'early_stopping_patience': early_stopping_patience, + 'print_stats_at': print_stats_at, + 'print_train_stats_at': print_train_stats_at, + 'validate_at': validate_at, + 'commands': { + 'save_checkpoint': False, + 'run_test_validation': False, + 'generate_predictions': False + } + } + + with open(config_json_path, 'w') as f: + json.dump(runtime_params, f, indent=2) + print(f"Started training on device {device}") + print(f"Runtime config saved to: {config_json_path}") + print(f"You can modify this file during training to change parameters dynamically!") + print(f"Set command flags to true to trigger actions (save_checkpoint, run_test_validation, generate_predictions)\n") while i < n_updates: for input, target in dataloader_train: input, target = input.to(device), target.to(device) + + # Check for runtime config updates every 50 steps + if i % 50 == 0 and i > 0: + runtime_params = load_runtime_config(config_json_path, runtime_params) + n_updates = runtime_params['n_updates'] + plot_at = runtime_params['plot_at'] + early_stopping_patience = runtime_params['early_stopping_patience'] + print_stats_at = runtime_params['print_stats_at'] + print_train_stats_at = runtime_params['print_train_stats_at'] + validate_at = runtime_params['validate_at'] + + # Execute runtime commands + commands = runtime_params.get('commands', {}) + + # Command: Save checkpoint + if commands.get('save_checkpoint', False): + checkpoint_path = os.path.join(results_path, f"checkpoint_step_{i}.pt") + torch.save(network.state_dict(), checkpoint_path) + print(f"\n[COMMAND] Checkpoint saved to: {checkpoint_path}\n") + clear_command_flag(config_json_path, 'save_checkpoint') + + # Command: Generate predictions + if commands.get('generate_predictions', False) and testset_path is not None: + print(f"\n[COMMAND] Generating predictions at step {i}...") + try: + from utils import create_predictions + pred_save_path = save_path or os.path.join(results_path, "runtime_predictions", f"step_{i}") + pred_plot_path = plot_path_predictions or os.path.join(results_path, "runtime_predictions", "plots", f"step_{i}") + os.makedirs(pred_plot_path, exist_ok=True) + + # Save current state temporarily + temp_state_path = os.path.join(results_path, f"temp_state_step_{i}.pt") + torch.save(network.state_dict(), temp_state_path) + + # Generate predictions + create_predictions(network_config, temp_state_path, testset_path, None, + pred_save_path, pred_plot_path, plot_at=20, rmse_value=None) + + print(f"[COMMAND] Predictions saved to: {pred_save_path}") + print(f"[COMMAND] Plots saved to: {pred_plot_path}\n") + + # Clean up temp file + if os.path.exists(temp_state_path): + os.remove(temp_state_path) + except Exception as e: + print(f"[COMMAND] Error generating predictions: {e}\n") + + network.train() + clear_command_flag(config_json_path, 'generate_predictions') + + # Command: Run test validation + if commands.get('run_test_validation', False): + print(f"\n[COMMAND] Running test set validation at step {i}...") + network.eval() + test_loss, test_rmse = evaluate_model(network, dataloader_test, mse_loss, device) + print(f"[COMMAND] Test Loss: {test_loss:.6f}, Test RMSE: {test_rmse:.6f}\n") + network.train() + clear_command_flag(config_json_path, 'run_test_validation') if (i + 1) % print_train_stats_at == 0: print(f'Update Step {i + 1} of {n_updates}: Current loss: {loss_list[-1]}') @@ -130,33 +364,96 @@ def train(seed, testset_ratio, validset_ratio, data_path, results_path, early_st if use_amp: with torch.amp.autocast('cuda'): output = network(input) - loss = rmse_loss(output, target) + total_loss, perceptual, mse, rmse = combined_loss(output, target) - scaler.scale(loss).backward() + # Check for NaN before backward + if not torch.isfinite(total_loss): + continue - # Gradient clipping for training stability + scaler.scale(total_loss).backward() + + # Unscale and check gradients scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=1.0) + + # Check for NaN in gradients + has_nan = False + for name, param in network.named_parameters(): + if param.grad is not None: + if not torch.isfinite(param.grad).all(): + print(f"NaN gradient detected in {name}") + has_nan = True + break + + if has_nan: + print(f"Skipping step {i+1}: NaN gradients detected") + optimizer.zero_grad() + scaler.update() + continue + + # More aggressive gradient clipping for stability + grad_norm = torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=0.5) + + # Skip update if gradient norm is too large + if grad_norm > 100.0: + print(f"Skipping step {i+1}: Gradient norm too large: {grad_norm:.2f}") + optimizer.zero_grad() + scaler.update() + continue scaler.step(optimizer) scaler.update() else: output = network(input) - loss = rmse_loss(output, target) - loss.backward() + total_loss, perceptual, mse, rmse = combined_loss(output, target) - # Gradient clipping for training stability - torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=1.0) + # Check for NaN before backward + if not torch.isfinite(total_loss): + print(f"Skipping step {i+1}: NaN or Inf loss detected") + continue + + total_loss.backward() + + # Check for NaN in gradients + has_nan = False + for param in network.parameters(): + if param.grad is not None and not torch.isfinite(param.grad).all(): + has_nan = True + break + + if has_nan: + print(f"Skipping step {i+1}: NaN gradients detected") + optimizer.zero_grad() + continue + + # More aggressive gradient clipping + grad_norm = torch.nn.utils.clip_grad_norm_(network.parameters(), max_norm=0.5) + + if grad_norm > 100.0: + print(f"Skipping step {i+1}: Gradient norm too large: {grad_norm:.2f}") + optimizer.zero_grad() + continue optimizer.step() - scheduler.step() + # Apply learning rate scheduling with warmup + lr_scale = get_lr_scale(i) + for param_group in optimizer.param_groups: + param_group['lr'] = learningrate * lr_scale + + if i >= warmup_steps: + scheduler_main.step() - loss_list.append(loss.item()) + loss_list.append(total_loss.item()) # writing the stats to wandb if use_wandb and (i+1) % print_stats_at == 0: - wandb.log({"training/loss_per_batch": loss.item()}, step=i) + wandb.log({ + "training/loss_total": total_loss.item(), + "training/loss_mse": mse.item(), + "training/loss_rmse": rmse.item(), + "training/loss_perceptual": perceptual.item() if isinstance(perceptual, torch.Tensor) else perceptual, + "training/learning_rate": optimizer.param_groups[0]['lr'] + }, step=i) # plotting if (i + 1) % plot_at == 0: diff --git a/image-inpainting/src/utils.py b/image-inpainting/src/utils.py index b12d2cf..dc065fd 100644 --- a/image-inpainting/src/utils.py +++ b/image-inpainting/src/utils.py @@ -122,6 +122,13 @@ def create_predictions(model_config, state_dict_path, testset_path, device, save predictions = np.stack(predictions, axis=0) + # Handle NaN and inf values before conversion + nan_mask = ~np.isfinite(predictions) + if nan_mask.any(): + nan_count = nan_mask.sum() + print(f"Warning: Found {nan_count} NaN/Inf values in predictions. Replacing with 0.") + predictions = np.nan_to_num(predictions, nan=0.0, posinf=1.0, neginf=0.0) + predictions = (np.clip(predictions, 0, 1) * 255.0).astype(np.uint8) data = {