Deep Learning with PyTorch

x → [W₁, b₁] → ReLU → [W₂, b₂] → ReLU → [W₃, b₃] → sigmoid → ŷ → L

z₁ = W₁x + b₁
a₁ = ReLU(z₁)
z₂ = W₂a₁ + b₂
a₂ = ReLU(z₂)
z₃ = W₃a₂ + b₃
ŷ  = sigmoid(z₃)
L  = BCE(ŷ, y)

∂L/∂z₃  = ŷ - y                    (BCE + sigmoid derivative cancel cleanly)
∂L/∂W₃  = (∂L/∂z₃) · a₂ᵀ / m       (m = batch size)
∂L/∂b₃  = mean(∂L/∂z₃)
∂L/∂a₂  = W₃ᵀ · (∂L/∂z₃)
∂L/∂z₂  = (∂L/∂a₂) · ReLU'(z₂)     (ReLU' = 1 if z>0 else 0)
∂L/∂W₂  = (∂L/∂z₂) · a₁ᵀ / m
...and so on back to W₁

import torch
import torch.nn as nn
import torch.nn.functional as F

# ── Tensor creation ───────────────────────────────────────────────────────────
x = torch.tensor([1.0, 2.0, 3.0])            # From Python list
X = torch.randn(32, 784)                       # Random normal, shape (32, 784)
W = torch.zeros(784, 256, requires_grad=True)  # Requires gradient tracking
I = torch.eye(5)                               # Identity matrix
Z = torch.zeros_like(W)                        # Same shape, all zeros

# ── GPU support ───────────────────────────────────────────────────────────────
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
X = X.to(device)
W = W.to(device)

# ── Autograd: automatic differentiation ───────────────────────────────────────
# Every tensor with requires_grad=True builds a computation graph
W = torch.randn(784, 256, requires_grad=True)
b = torch.zeros(256, requires_grad=True)

# Forward pass — builds the computation graph
out = X @ W + b              # Linear layer
out = F.relu(out)            # ReLU activation
loss = out.sum()

# Backward pass — computes all gradients via backprop
loss.backward()

print(W.grad.shape)          # (784, 256) — same shape as W
print(b.grad.shape)          # (256,)

# Zero gradients before next step! Gradients accumulate by default
W.grad.zero_()
b.grad.zero_()

# ── Context managers ──────────────────────────────────────────────────────────
with torch.no_grad():        # Disable gradient tracking (inference, val)
    predictions = model(X_val)

with torch.inference_mode(): # Faster than no_grad — stronger guarantees
    predictions = model(X_test)

class MLP(nn.Module):
    """Multi-Layer Perceptron with BatchNorm and Dropout."""

    def __init__(self, input_size: int, hidden: list[int], output_size: int,
                 dropout: float = 0.3):
        super().__init__()

        layers: list[nn.Module] = []
        sizes = [input_size] + hidden

        for i in range(len(sizes) - 1):
            layers += [
                nn.Linear(sizes[i], sizes[i+1]),
                nn.BatchNorm1d(sizes[i+1]),   # Normalize layer inputs
                nn.ReLU(),
                nn.Dropout(dropout),          # Random zeroing
            ]

        layers.append(nn.Linear(hidden[-1], output_size))
        self.net = nn.Sequential(*layers)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.net(x)

model = MLP(input_size=784, hidden=[512, 256, 128], output_size=10)
print(model)
print(f"Total parameters: {sum(p.numel() for p in model.parameters()):,}")

from torch.utils.data import DataLoader, TensorDataset
import torch.optim as optim

def train(model, train_loader, val_loader, epochs=30):
    optimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)
    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)
    criterion = nn.CrossEntropyLoss()

    best_val_acc = 0.0
    patience_counter = 0

    for epoch in range(epochs):
        # ── Training phase ────────────────────────────────────────────────────
        model.train()  # Enables Dropout and BatchNorm training mode
        train_loss = 0.0

        for X_batch, y_batch in train_loader:
            X_batch, y_batch = X_batch.to(device), y_batch.to(device)

            optimizer.zero_grad()              # Step 1: clear old gradients
            logits = model(X_batch)            # Step 2: forward pass
            loss = criterion(logits, y_batch)  # Step 3: compute loss
            loss.backward()                    # Step 4: backprop
            nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)  # Clip
            optimizer.step()                   # Step 5: update weights

            train_loss += loss.item() * len(X_batch)

        # ── Validation phase ──────────────────────────────────────────────────
        model.eval()  # Disables Dropout; BatchNorm uses running stats
        correct = total = 0

        with torch.no_grad():
            for X_batch, y_batch in val_loader:
                X_batch, y_batch = X_batch.to(device), y_batch.to(device)
                logits = model(X_batch)
                preds  = logits.argmax(dim=1)
                correct += (preds == y_batch).sum().item()
                total   += len(y_batch)

        val_acc = correct / total
        scheduler.step()   # Adjust learning rate

        # Early stopping
        if val_acc > best_val_acc:
            best_val_acc = val_acc
            torch.save(model.state_dict(), "best_model.pt")
            patience_counter = 0
        else:
            patience_counter += 1
            if patience_counter >= 7:
                print(f"Early stopping at epoch {epoch}")
                break

        print(f"Epoch {epoch+1:2d}: loss={train_loss/len(train_loader.dataset):.4f}"
              f"  val_acc={val_acc:.4f}  lr={scheduler.get_last_lr()[0]:.6f}")

    # Load best weights
    model.load_state_dict(torch.load("best_model.pt"))
    return model

import torch
import torch.nn as nn
import torchvision

class ConvBlock(nn.Module):
    """Conv → BatchNorm → ReLU → optional MaxPool"""
    def __init__(self, in_channels, out_channels, pool=False):
        super().__init__()
        layers = [
            nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
        ]
        if pool:
            layers.append(nn.MaxPool2d(2))
        self.block = nn.Sequential(*layers)

    def forward(self, x):
        return self.block(x)

class CNN(nn.Module):
    """CNN for CIFAR-10 (32x32 RGB images, 10 classes)."""

    def __init__(self, n_classes=10):
        super().__init__()
        self.features = nn.Sequential(
            ConvBlock(3,  64,  pool=False),  # (3,32,32) → (64,32,32)
            ConvBlock(64, 64,  pool=True),   # → (64,16,16)
            ConvBlock(64, 128, pool=False),  # → (128,16,16)
            ConvBlock(128,128, pool=True),   # → (128,8,8)
            ConvBlock(128,256, pool=False),  # → (256,8,8)
            ConvBlock(256,256, pool=True),   # → (256,4,4)
        )
        self.classifier = nn.Sequential(
            nn.AdaptiveAvgPool2d(1),         # → (256,1,1) — handles any input size
            nn.Flatten(),                    # → (256,)
            nn.Linear(256, 128),
            nn.ReLU(),
            nn.Dropout(0.5),
            nn.Linear(128, n_classes),
        )

    def forward(self, x):
        return self.classifier(self.features(x))

# Transfer learning: fine-tune ResNet-18
def make_finetune_model(n_classes=10, freeze_backbone=True):
    model = torchvision.models.resnet18(weights="DEFAULT")

    if freeze_backbone:
        for param in model.parameters():
            param.requires_grad = False

    # Replace the final fully connected layer
    in_features = model.fc.in_features    # 512 for ResNet-18
    model.fc = nn.Sequential(
        nn.Linear(in_features, 256),
        nn.ReLU(),
        nn.Dropout(0.4),
        nn.Linear(256, n_classes),
    )
    return model

# Only train the new head initially, then unfreeze backbone for fine-tuning
model = make_finetune_model(n_classes=10, freeze_backbone=True)
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
total     = sum(p.numel() for p in model.parameters())
print(f"Trainable: {trainable:,} / {total:,} ({trainable/total:.1%})")