Deep Learning Study Guide

# pip install torch torchvision
import torch

# Create tensors
t1 = torch.tensor([1.0, 2.0, 3.0])
t2 = torch.zeros(3, 4)
t3 = torch.ones(2, 3)
t4 = torch.rand(3, 3)
t5 = torch.arange(0, 10, step=2, dtype=torch.float32)

print('t1:', t1)
print('t2 shape:', t2.shape)
print('t4:', t4)

# Operations
a = torch.tensor([[1., 2.], [3., 4.]])
b = torch.tensor([[5., 6.], [7., 8.]])
print('Add:', a + b)
print('Matmul:', a @ b)
print('Mean:', a.mean())

# Reshape
x = torch.arange(12, dtype=torch.float32)
print('Reshape:', x.reshape(3, 4))

import torch
import numpy as np

t = torch.rand(4, 5)

# Indexing (same as NumPy)
print('Row 0:', t[0])
print('Col 1:', t[:, 1])
print('Slice:', t[1:3, 2:4])

# Boolean mask
print('> 0.5:', t[t > 0.5][:5])

# NumPy <-> Tensor (shared memory on CPU)
arr = np.array([1.0, 2.0, 3.0])
tensor_from_np = torch.from_numpy(arr)
np_from_tensor = t.numpy()  # only works on CPU tensors
print('From numpy:', tensor_from_np)

# Device
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('Using device:', device)
t_device = t.to(device)
print('Tensor device:', t_device.device)

import torch

torch.manual_seed(0)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('Device:', device)

# Broadcasting: (3,1) + (1,4) -> (3,4)
a = torch.randn(3, 1, device=device)
b = torch.randn(1, 4, device=device)
c = a + b
print('Broadcast shape:', c.shape)  # (3, 4)

# torch.einsum: batch matrix multiply
# 'bij,bjk->bik' = batched matmul
X = torch.randn(8, 3, 4, device=device)  # batch of 8, (3x4)
Y = torch.randn(8, 4, 5, device=device)  # batch of 8, (4x5)
Z = torch.einsum('bij,bjk->bik', X, Y)
print('einsum batch matmul:', Z.shape)  # (8, 3, 5)

# L2 norm per row
M = torch.randn(3, 4, device=device)
row_norms = torch.norm(M, dim=1)
print('Row norms:', row_norms)

# Move to CPU and convert to numpy
M_np = M.cpu().numpy()
print('As numpy shape:', M_np.shape)

import torch

torch.manual_seed(0)

# --- Sparse Tensors ---
# Define a sparse COO tensor: indices (2, nnz) and values (nnz,)
indices = torch.tensor([[0, 1, 2], [1, 0, 3]])  # row, col positions
values  = torch.tensor([3.0, 5.0, 7.0])
sparse = torch.sparse_coo_tensor(indices, values, size=(4, 5))
print('Sparse tensor:')
print(sparse.to_dense())
print('nnz (non-zero):', sparse._nnz())

# --- Boolean Mask Indexing ---
x = torch.randn(4, 5)
print('\nOriginal tensor:\n', x.round(decimals=2))

# Select elements where value > 0
mask = x > 0
print('Positive values:', x[mask][:6].round(decimals=3))

# Zero-out negatives in-place
x_clipped = x.clone()
x_clipped[x_clipped < 0] = 0.0
print('ReLU via mask (min):', x_clipped.min().item())

# Row-wise mask: keep rows where row mean > 0
row_mask = x.mean(dim=1) > 0
print('Rows with positive mean:', x[row_mask].shape)

import torch

# requires_grad=True tracks operations
x = torch.tensor(3.0, requires_grad=True)
y = x ** 2 + 2 * x + 1   # y = x^2 + 2x + 1
print('y:', y.item())

# Compute dy/dx
y.backward()
print('dy/dx at x=3:', x.grad.item())  # 2x + 2 = 8

# Computation graph example
a = torch.tensor(2.0, requires_grad=True)
b = torch.tensor(3.0, requires_grad=True)
c = a * b + b ** 2    # c = ab + b^2
c.backward()
print('dc/da:', a.grad.item())   # b = 3
print('dc/db:', b.grad.item())   # a + 2b = 8

# Stop tracking
with torch.no_grad():
    z = a * b
print('z requires_grad:', z.requires_grad)

import torch

# Fit y = 2x + 1 using gradient descent
torch.manual_seed(42)
X = torch.linspace(0, 1, 100).unsqueeze(1)
y_true = 2 * X + 1 + 0.1 * torch.randn_like(X)

# Parameters to learn
w = torch.randn(1, requires_grad=True)
b = torch.randn(1, requires_grad=True)

lr = 0.1
for epoch in range(200):
    y_pred = X * w + b
    loss = ((y_pred - y_true) ** 2).mean()
    loss.backward()

    with torch.no_grad():
        w -= lr * w.grad
        b -= lr * b.grad
    w.grad.zero_()
    b.grad.zero_()

    if (epoch + 1) % 50 == 0:
        print(f'Epoch {epoch+1}: loss={loss.item():.4f} w={w.item():.3f} b={b.item():.3f}')

import torch
from torch.autograd.functional import jacobian, hessian

# Scalar function: f(x) = x1^2 + x1*x2 + x2^2
def f(x):
    return (x[0]**2 + x[0]*x[1] + x[1]**2).unsqueeze(0)

x0 = torch.tensor([1.0, 2.0])

# Jacobian: df/dx — shape (1, 2)
J = jacobian(f, x0)
print('Jacobian:', J)  # [2x0+x1, x0+2x1] = [4, 5]

# Hessian: d^2f/dx^2 — shape (2, 2)
def f_scalar(x):
    return x[0]**2 + x[0]*x[1] + x[1]**2

H = hessian(f_scalar, x0)
print('Hessian:')
print(H)  # [[2, 1], [1, 2]]

# Verify: condition number of Hessian
eigvals = torch.linalg.eigvalsh(H)
print('Eigenvalues:', eigvals)
print('Condition number:', (eigvals.max() / eigvals.min()).item())

import torch
import torch.nn as nn

torch.manual_seed(42)

# --- torch.no_grad: disable gradient tracking ---
x = torch.randn(4, 8, requires_grad=True)
with torch.no_grad():
    y = x * 2  # no graph built
print('y requires_grad:', y.requires_grad)  # False

# --- Gradient Clipping ---
model = nn.Linear(8, 4)
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)

# Simulate large gradients
out = model(x)
loss = (out * 1000).mean()  # artificially huge loss
loss.backward()

before = max(p.grad.abs().max().item() for p in model.parameters())
torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
after  = max(p.grad.abs().max().item() for p in model.parameters())
print(f'Max grad before clip: {before:.2f} | after clip: {after:.4f}')

# --- register_hook: inspect gradients during backprop ---
grad_log = []
h = torch.randn(3, requires_grad=True)
h.register_hook(lambda g: grad_log.append(g.clone()))

loss2 = (h ** 2).sum()
loss2.backward()
print('Gradient via hook:', grad_log[0])  # should be 2*h
print('Gradient via .grad:', h.grad)

import torch
import torch.nn as nn

class MLP(nn.Module):
    def __init__(self, in_features, hidden, out_features):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_features, hidden),
            nn.ReLU(),
            nn.Linear(hidden, hidden),
            nn.ReLU(),
            nn.Linear(hidden, out_features)
        )

    def forward(self, x):
        return self.net(x)

model = MLP(in_features=10, hidden=64, out_features=1)
print(model)

# Count parameters
total = sum(p.numel() for p in model.parameters())
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f'Total params: {total:,} | Trainable: {trainable:,}')

# Test forward pass
x = torch.randn(32, 10)  # batch of 32
out = model(x)
print('Output shape:', out.shape)

import torch
import torch.nn as nn

# Quick model with Sequential
model = nn.Sequential(
    nn.Linear(20, 128),
    nn.BatchNorm1d(128),
    nn.ReLU(),
    nn.Dropout(0.3),
    nn.Linear(128, 64),
    nn.ReLU(),
    nn.Linear(64, 3)  # 3-class output
)

x = torch.randn(16, 20)
logits = model(x)
print('Logits shape:', logits.shape)

# Apply softmax for probabilities
probs = torch.softmax(logits, dim=1)
print('Probs sum per sample:', probs.sum(dim=1)[:3].round(decimals=4))

# Loss functions
targets = torch.randint(0, 3, (16,))
ce_loss = nn.CrossEntropyLoss()(logits, targets)
print('CrossEntropy loss:', ce_loss.item().__round__(4))

import torch
import torch.nn as nn

# nn.ModuleList: dynamic list of layers
class FlexMLP(nn.Module):
    def __init__(self, layer_sizes):
        super().__init__()
        self.layers = nn.ModuleList([
            nn.Linear(layer_sizes[i], layer_sizes[i+1])
            for i in range(len(layer_sizes) - 1)
        ])
        self.acts = nn.ModuleList([
            nn.ReLU() for _ in range(len(layer_sizes) - 2)
        ])

    def forward(self, x):
        for i, layer in enumerate(self.layers[:-1]):
            x = self.acts[i](layer(x))
        return self.layers[-1](x)

model = FlexMLP([10, 64, 32, 1])
print(model)

# nn.ModuleDict: named modules
heads = nn.ModuleDict({
    'classifier': nn.Linear(64, 5),
    'regressor':  nn.Linear(64, 1),
})
feat = torch.randn(8, 64)
print('Classifier out:', heads['classifier'](feat).shape)
print('Regressor out: ', heads['regressor'](feat).shape)

x = torch.randn(16, 10)
print('FlexMLP output:', model(x).shape)

import torch
import torch.nn as nn

torch.manual_seed(42)

# --- Parameter Sharing: two layers share the SAME weight tensor ---
class TiedAutoEncoder(nn.Module):
    '''Encoder and decoder share the same weight matrix (tied weights).'''
    def __init__(self, in_dim, hid_dim):
        super().__init__()
        self.encoder = nn.Linear(in_dim, hid_dim, bias=True)
        # Decoder reuses encoder weight (transposed), has its own bias
        self.dec_bias = nn.Parameter(torch.zeros(in_dim))

    def encode(self, x):
        return torch.relu(self.encoder(x))

    def decode(self, h):
        # W^T h + bias  (tied weights)
        return h @ self.encoder.weight + self.dec_bias

    def forward(self, x):
        return self.decode(self.encode(x))

model = TiedAutoEncoder(in_dim=16, hid_dim=8)

# Verify encoder & decoder share the same weight object
enc_weight = model.encoder.weight
print('Encoder weight shape:', enc_weight.shape)  # (8, 16)

x = torch.randn(4, 16)
recon = model(x)
print('Reconstruction shape:', recon.shape)  # (4, 16)

# Count parameters (shared weight counted once)
total = sum(p.numel() for p in model.parameters())
print(f'Total params: {total}')  # 8*16 + 8 (enc bias) + 16 (dec bias)

# Gradient flows through the shared weight from both encoder & decoder
loss = nn.MSELoss()(recon, x)
loss.backward()
print('Encoder weight grad shape:', model.encoder.weight.grad.shape)

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# Data
X, y = make_classification(n_samples=2000, n_features=10, random_state=42)
X_tr, X_val, y_tr, y_val = train_test_split(X, y, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_tr = scaler.fit_transform(X_tr)
X_val = scaler.transform(X_val)

to_tensor = lambda a, t: torch.tensor(a, dtype=t)
train_ds = TensorDataset(to_tensor(X_tr, torch.float32), to_tensor(y_tr, torch.long))
val_ds   = TensorDataset(to_tensor(X_val, torch.float32), to_tensor(y_val, torch.long))
train_dl = DataLoader(train_ds, batch_size=64, shuffle=True)
val_dl   = DataLoader(val_ds, batch_size=64)

model = nn.Sequential(nn.Linear(10,64), nn.ReLU(), nn.Linear(64,32), nn.ReLU(), nn.Linear(32,2))
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
criterion = nn.CrossEntropyLoss()

for epoch in range(1, 6):
    model.train()
    train_loss = 0
    for xb, yb in train_dl:
        optimizer.zero_grad()
        loss = criterion(model(xb), yb)
        loss.backward()
        optimizer.step()
        train_loss += loss.item()

    model.eval()
    correct = 0
    with torch.no_grad():
        for xb, yb in val_dl:
            preds = model(xb).argmax(dim=1)
            correct += (preds == yb).sum().item()
    acc = correct / len(val_ds)
    print(f'Epoch {epoch}: train_loss={train_loss/len(train_dl):.4f} val_acc={acc:.4f}')

import torch
import torch.nn as nn

model = nn.Sequential(nn.Linear(10, 64), nn.ReLU(), nn.Linear(64, 1))
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)

# ReduceLROnPlateau — halve lr when val loss plateaus
scheduler_plateau = torch.optim.lr_scheduler.ReduceLROnPlateau(
    optimizer, mode='min', factor=0.5, patience=5
)

# CosineAnnealingLR
optimizer2 = torch.optim.SGD(model.parameters(), lr=0.1)
scheduler_cos = torch.optim.lr_scheduler.CosineAnnealingLR(
    optimizer2, T_max=50, eta_min=1e-4
)

# Simulate training
for epoch in range(1, 11):
    # Fake val loss
    val_loss = 1.0 / (epoch + 1) + 0.01
    scheduler_plateau.step(val_loss)
    scheduler_cos.step()
    lr1 = optimizer.param_groups[0]['lr']
    lr2 = optimizer2.param_groups[0]['lr']
    print(f'Epoch {epoch:2d}: plateau_lr={lr1:.6f}  cosine_lr={lr2:.5f}')

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

torch.manual_seed(42)
X = torch.randn(500, 10)
y = torch.randint(0, 2, (500,))
dl = DataLoader(TensorDataset(X, y), batch_size=64, shuffle=True)

model = nn.Sequential(
    nn.Linear(10, 64), nn.ReLU(),
    nn.Linear(64, 32), nn.ReLU(),
    nn.Linear(32, 2)
)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
    optimizer, T_max=20, eta_min=1e-5
)
criterion = nn.CrossEntropyLoss()

best_loss, patience, wait = float('inf'), 5, 0

for epoch in range(1, 21):
    model.train()
    epoch_loss = 0
    for xb, yb in dl:
        optimizer.zero_grad()
        loss = criterion(model(xb), yb)
        loss.backward()
        # Gradient clipping: prevent exploding gradients
        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
        optimizer.step()
        epoch_loss += loss.item()
    scheduler.step()
    avg = epoch_loss / len(dl)
    lr  = optimizer.param_groups[0]['lr']
    if avg < best_loss:
        best_loss, wait = avg, 0
    else:
        wait += 1
        if wait >= patience:
            print(f'Early stop at epoch {epoch}')
            break
    if epoch <= 3 or epoch % 5 == 0:
        print(f'Epoch {epoch:2d}: loss={avg:.4f} lr={lr:.6f} wait={wait}')

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

torch.manual_seed(42)
X = torch.randn(600, 10)
y = torch.randint(0, 2, (600,))
dl = DataLoader(TensorDataset(X, y), batch_size=64, shuffle=True)

model = nn.Sequential(
    nn.Linear(10, 32), nn.ReLU(), nn.Linear(32, 2)
)
base_lr = 1e-3
warmup_epochs = 5
total_epochs  = 25
optimizer = torch.optim.Adam(model.parameters(), lr=base_lr)
criterion = nn.CrossEntropyLoss()

# Linear warmup: lr ramps from 0 -> base_lr over warmup_epochs
def warmup_lambda(epoch):
    if epoch < warmup_epochs:
        return (epoch + 1) / warmup_epochs  # ramp up
    # Cosine decay after warmup
    progress = (epoch - warmup_epochs) / max(total_epochs - warmup_epochs, 1)
    return 0.5 * (1 + torch.cos(torch.tensor(3.14159 * progress)).item())

scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=warmup_lambda)

# Early stopping state
best_loss, patience, wait = float('inf'), 6, 0

for epoch in range(1, total_epochs + 1):
    model.train()
    epoch_loss = 0
    for xb, yb in dl:
        optimizer.zero_grad()
        loss = criterion(model(xb), yb)
        loss.backward()
        optimizer.step()
        epoch_loss += loss.item()
    scheduler.step()
    avg = epoch_loss / len(dl)
    lr  = optimizer.param_groups[0]['lr']
    # Early stopping check
    if avg < best_loss - 1e-4:
        best_loss, wait = avg, 0
    else:
        wait += 1
        if wait >= patience:
            print(f'Early stop at epoch {epoch}')
            break
    if epoch <= warmup_epochs or epoch % 5 == 0:
        print(f'Epoch {epoch:2d}: loss={avg:.4f} lr={lr:.6f} wait={wait}')

import torch
import torch.nn as nn

class SimpleCNN(nn.Module):
    def __init__(self, num_classes=10):
        super().__init__()
        self.features = nn.Sequential(
            nn.Conv2d(1, 32, kernel_size=3, padding=1),  # grayscale in
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(2, 2),          # 14x14
            nn.Conv2d(32, 64, kernel_size=3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2, 2)           # 7x7
        )
        self.classifier = nn.Sequential(
            nn.Flatten(),
            nn.Linear(64 * 7 * 7, 256),
            nn.ReLU(),
            nn.Dropout(0.5),
            nn.Linear(256, num_classes)
        )

    def forward(self, x):
        return self.classifier(self.features(x))

model = SimpleCNN(num_classes=10)
x = torch.randn(8, 1, 28, 28)  # batch of 8 MNIST-like images
print('Output:', model(x).shape)
total = sum(p.numel() for p in model.parameters())
print(f'Parameters: {total:,}')

import torch
import torch.nn as nn

# Simple edge-detection filters (like early CNN layers learn)
conv = nn.Conv2d(1, 4, kernel_size=3, padding=1, bias=False)

# Manual: set Sobel-like weights
with torch.no_grad():
    # Horizontal edge filter
    conv.weight[0, 0] = torch.tensor([[-1.,-2.,-1.],[0.,0.,0.],[1.,2.,1.]])
    # Vertical edge filter
    conv.weight[1, 0] = torch.tensor([[-1.,0.,1.],[-2.,0.,2.],[-1.,0.,1.]])
    # Blur
    conv.weight[2, 0] = torch.ones(3, 3) / 9
    # Sharpen
    conv.weight[3, 0] = torch.tensor([[0.,-1.,0.],[-1.,5.,-1.],[0.,-1.,0.]])

# Apply to a synthetic gradient image
img = torch.linspace(0, 1, 28*28).reshape(1, 1, 28, 28)
fmaps = conv(img)
print('Input:', img.shape)
print('Feature maps:', fmaps.shape)  # 4 channels
for i, name in enumerate(['Horiz','Vert','Blur','Sharpen']):
    print(f'  {name}: min={fmaps[0,i].min():.3f} max={fmaps[0,i].max():.3f}')

import torch
import torch.nn as nn

class ResBlock(nn.Module):
    '''Basic residual block: F(x) + x with optional projection.'''
    def __init__(self, in_ch, out_ch, stride=1):
        super().__init__()
        self.conv1 = nn.Conv2d(in_ch, out_ch, 3, stride=stride, padding=1, bias=False)
        self.bn1   = nn.BatchNorm2d(out_ch)
        self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1, bias=False)
        self.bn2   = nn.BatchNorm2d(out_ch)
        self.relu  = nn.ReLU(inplace=True)
        # Skip connection: project if dims change
        self.skip = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, 1, stride=stride, bias=False),
            nn.BatchNorm2d(out_ch)
        ) if (in_ch != out_ch or stride != 1) else nn.Identity()

    def forward(self, x):
        out = self.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        return self.relu(out + self.skip(x))  # residual

# Stack residual blocks into a tiny CNN
model = nn.Sequential(
    nn.Conv2d(1, 16, 3, padding=1), nn.BatchNorm2d(16), nn.ReLU(),
    ResBlock(16, 16),
    ResBlock(16, 32, stride=2),  # downsample
    ResBlock(32, 32),
    nn.AdaptiveAvgPool2d(1),
    nn.Flatten(),
    nn.Linear(32, 10)
)

x = torch.randn(4, 1, 28, 28)
print('Output:', model(x).shape)
print('Params:', sum(p.numel() for p in model.parameters()))

import torch
import torch.nn as nn

torch.manual_seed(42)

# --- Depthwise Separable Convolution ---
# Standard Conv2d: C_in * C_out * K * K params
# DW-Sep: C_in * K * K (depthwise) + C_in * C_out (pointwise) — much fewer params
class DepthwiseSeparableConv(nn.Module):
    def __init__(self, in_ch, out_ch, kernel_size=3, padding=1):
        super().__init__()
        # Depthwise: one filter per channel (groups=in_ch)
        self.dw = nn.Conv2d(in_ch, in_ch, kernel_size,
                            padding=padding, groups=in_ch, bias=False)
        # Pointwise: 1x1 conv to mix channels
        self.pw = nn.Conv2d(in_ch, out_ch, 1, bias=False)
        self.bn = nn.BatchNorm2d(out_ch)
        self.relu = nn.ReLU()

    def forward(self, x):
        return self.relu(self.bn(self.pw(self.dw(x))))

# Compare parameter counts
standard = nn.Conv2d(32, 64, 3, padding=1, bias=False)
dw_sep   = DepthwiseSeparableConv(32, 64)
std_params = sum(p.numel() for p in standard.parameters())
dws_params = sum(p.numel() for p in dw_sep.parameters())
print(f'Standard Conv  params: {std_params:,}')
print(f'DW-Sep Conv    params: {dws_params:,}  ({dws_params/std_params:.1%} of standard)')

# --- Global Average Pooling vs Flatten ---
x = torch.randn(4, 64, 7, 7)  # feature maps after conv layers
gap = nn.AdaptiveAvgPool2d(1)  # collapses spatial dims to 1x1
out_gap = gap(x).squeeze(-1).squeeze(-1)  # (4, 64)
out_flat = x.flatten(1)                   # (4, 64*7*7 = 3136)
print(f'GAP output shape:     {out_gap.shape}   (no spatial params!)')
print(f'Flatten output shape: {out_flat.shape}')

# Full MobileNet-style block: DWS + GAP + Linear
backbone = nn.Sequential(
    DepthwiseSeparableConv(3, 32),
    DepthwiseSeparableConv(32, 64),
    nn.AdaptiveAvgPool2d(1),
    nn.Flatten(),
    nn.Linear(64, 5)
)
x_img = torch.randn(8, 3, 28, 28)
print('Backbone output:', backbone(x_img).shape)  # (8, 5)
print('Backbone params:', sum(p.numel() for p in backbone.parameters()))

import torch
import torch.nn as nn
import torchvision.models as models

# Load pretrained ResNet18
model = models.resnet18(weights='IMAGENET1K_V1')

# Freeze all layers
for param in model.parameters():
    param.requires_grad = False

# Replace the final FC layer for our task (e.g., 5 classes)
num_features = model.fc.in_features
model.fc = nn.Sequential(
    nn.Linear(num_features, 256),
    nn.ReLU(),
    nn.Dropout(0.4),
    nn.Linear(256, 5)  # 5 output classes
)

# Only the new head is trainable
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
total = sum(p.numel() for p in model.parameters())
print(f'Trainable: {trainable:,} / {total:,} ({trainable/total:.1%})')

# Test forward pass
x = torch.randn(4, 3, 224, 224)
out = model(x)
print('Output shape:', out.shape)

import torch
import torch.nn as nn
import torchvision.models as models

model = models.resnet18(weights='IMAGENET1K_V1')

# Replace head for 3-class task
model.fc = nn.Linear(model.fc.in_features, 3)

# Phase 1: only train head
for param in model.parameters():
    param.requires_grad = False
for param in model.fc.parameters():
    param.requires_grad = True

print('Phase 1 trainable params:',
      sum(p.numel() for p in model.parameters() if p.requires_grad))

# Phase 2: unfreeze layer4 + head
for param in model.layer4.parameters():
    param.requires_grad = True

print('Phase 2 trainable params:',
      sum(p.numel() for p in model.parameters() if p.requires_grad))

# Phase 3: unfreeze all
for param in model.parameters():
    param.requires_grad = True

print('Phase 3 (all) trainable params:',
      sum(p.numel() for p in model.parameters() if p.requires_grad))

import torch
import torch.nn as nn
import torchvision.models as models

def count_params(m):
    total = sum(p.numel() for p in m.parameters())
    trainable = sum(p.numel() for p in m.parameters() if p.requires_grad)
    return total, trainable

# --- Mode 1: Feature Extraction (freeze all, new head only) ---
model_fe = models.resnet18(weights='IMAGENET1K_V1')
for p in model_fe.parameters():
    p.requires_grad = False
model_fe.fc = nn.Linear(model_fe.fc.in_features, 4)
t, tr = count_params(model_fe)
print(f'Feature Extraction: {tr:,}/{t:,} trainable ({tr/t:.2%})')

# --- Mode 2: Fine-tune last N=2 blocks (layer3, layer4, fc) ---
model_ft = models.resnet18(weights='IMAGENET1K_V1')
for p in model_ft.parameters():
    p.requires_grad = False
for p in model_ft.layer3.parameters():
    p.requires_grad = True
for p in model_ft.layer4.parameters():
    p.requires_grad = True
model_ft.fc = nn.Linear(model_ft.fc.in_features, 4)
t, tr = count_params(model_ft)
print(f'Fine-tune last 2 blocks: {tr:,}/{t:,} trainable ({tr/t:.2%})')

# Both accept same input
x = torch.randn(2, 3, 224, 224)
print('FE output:', model_fe(x).shape)
print('FT output:', model_ft(x).shape)

import torch
import torch.nn as nn
import torchvision.models as models

torch.manual_seed(42)

# Load pretrained ResNet18
model = models.resnet18(weights='IMAGENET1K_V1')
model.fc = nn.Linear(model.fc.in_features, 5)

# --- Strategy: differential learning rates ---
# Early layers: very low lr (already well-trained)
# Later layers: medium lr
# New head:     full lr
param_groups = [
    {'params': list(model.layer1.parameters()) +
               list(model.layer2.parameters()), 'lr': 1e-5},
    {'params': list(model.layer3.parameters()) +
               list(model.layer4.parameters()), 'lr': 1e-4},
    {'params': model.fc.parameters(),            'lr': 1e-3},
]
optimizer = torch.optim.Adam(param_groups)

print('Param groups:')
for g in optimizer.param_groups:
    n = sum(p.numel() for p in g['params'])
    print(f'  lr={g["lr"]}  params={n:,}')

# --- Freeze then unfreeze on schedule ---
def set_requires_grad(module, value):
    for p in module.parameters():
        p.requires_grad = value

# Phase 1: freeze backbone, train only head
set_requires_grad(model, False)
set_requires_grad(model.fc, True)
phase1_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f'Phase 1 trainable: {phase1_trainable:,}')

# Phase 2: unfreeze layer4 for fine-tuning
set_requires_grad(model.layer4, True)
phase2_trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f'Phase 2 trainable: {phase2_trainable:,}')

# Verify forward pass works in both phases
x = torch.randn(2, 3, 224, 224)
print('Output shape:', model(x).shape)

import torch
from torch.utils.data import Dataset, DataLoader
import numpy as np

class TabularDataset(Dataset):
    def __init__(self, X, y):
        self.X = torch.tensor(X, dtype=torch.float32)
        self.y = torch.tensor(y, dtype=torch.long)

    def __len__(self):
        return len(self.y)

    def __getitem__(self, idx):
        return self.X[idx], self.y[idx]

# Create dataset
np.random.seed(42)
X = np.random.randn(1000, 8)
y = (X[:, 0] + X[:, 1] > 0).astype(int)

dataset = TabularDataset(X, y)
print(f'Dataset size: {len(dataset)}')
print(f'Sample: {dataset[0]}')

# DataLoader: batching + shuffling
loader = DataLoader(dataset, batch_size=32, shuffle=True, num_workers=0)
xb, yb = next(iter(loader))
print(f'Batch X: {xb.shape}, y: {yb.shape}')

import torch
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as T

# Standard ImageNet transforms
train_transforms = T.Compose([
    T.ToPILImage(),
    T.Resize((224, 224)),
    T.RandomHorizontalFlip(p=0.5),
    T.RandomRotation(15),
    T.ColorJitter(brightness=0.2, contrast=0.2),
    T.ToTensor(),
    T.Normalize(mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225])
])
val_transforms = T.Compose([
    T.ToPILImage(),
    T.Resize((224, 224)),
    T.ToTensor(),
    T.Normalize(mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225])
])

# Apply to a random image
import numpy as np
img_np = np.random.randint(0, 255, (100, 100, 3), dtype=np.uint8)
tensor_img = train_transforms(img_np)
print('Augmented shape:', tensor_img.shape)
print('Value range: [{:.3f}, {:.3f}]'.format(tensor_img.min().item(), tensor_img.max().item()))

import torch
from torch.utils.data import IterableDataset, DataLoader
import numpy as np

# IterableDataset: useful for streaming / large on-disk data
class StreamDataset(IterableDataset):
    def __init__(self, n_samples, n_features, seed=0):
        self.n = n_samples
        self.d = n_features
        self.seed = seed

    def __iter__(self):
        rng = np.random.default_rng(self.seed)
        for _ in range(self.n):
            x = rng.standard_normal(self.d).astype(np.float32)
            y = int(x[0] > 0)
            yield torch.tensor(x), torch.tensor(y)

stream_ds = StreamDataset(200, 8)
dl = DataLoader(stream_ds, batch_size=16)
xb, yb = next(iter(dl))
print('Streamed batch:', xb.shape, yb.shape)

# collate_fn: handle variable-length sequences (pad to max)
def pad_collate(batch):
    seqs, labels = zip(*batch)
    lengths = [s.size(0) for s in seqs]
    padded = torch.zeros(len(seqs), max(lengths))
    for i, s in enumerate(seqs):
        padded[i, :lengths[i]] = s
    return padded, torch.tensor(labels), torch.tensor(lengths)

var_data = [(torch.randn(np.random.randint(3, 10)), torch.tensor(i % 2))
            for i in range(8)]
padded, lbls, lens = pad_collate(var_data)
print('Padded shape:', padded.shape)
print('Lengths:', lens.tolist())

import torch
from torch.utils.data import Dataset, DataLoader, WeightedRandomSampler
import numpy as np

# Simulate imbalanced dataset: 90% class 0, 10% class 1
np.random.seed(42)
n = 1000
X = torch.randn(n, 8)
y = torch.tensor(np.random.choice([0, 1], n, p=[0.9, 0.1]), dtype=torch.long)
print(f'Class distribution: 0={( y==0).sum()}, 1={(y==1).sum()}')

class TabularDataset(Dataset):
    def __init__(self, X, y): self.X, self.y = X, y
    def __len__(self): return len(self.y)
    def __getitem__(self, i): return self.X[i], self.y[i]

dataset = TabularDataset(X, y)

# WeightedRandomSampler: oversample minority class
class_counts   = torch.bincount(y)
class_weights  = 1.0 / class_counts.float()
sample_weights = class_weights[y]
sampler = WeightedRandomSampler(sample_weights, num_samples=500, replacement=True)

loader = DataLoader(dataset, batch_size=32, sampler=sampler)
# Verify balance in sampled batches
all_labels = []
for _, labels in loader:
    all_labels.extend(labels.tolist())
sampled_0 = all_labels.count(0)
sampled_1 = all_labels.count(1)
print(f'Sampled class distribution: 0={sampled_0}, 1={sampled_1}')
print(f'Balance ratio: {sampled_1/max(sampled_0,1):.2f}')

import torch
import torch.nn as nn

class RegularizedNet(nn.Module):
    def __init__(self, dropout_rate=0.3):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(20, 256),
            nn.BatchNorm1d(256),   # normalize activations
            nn.ReLU(),
            nn.Dropout(dropout_rate),  # randomly zero activations
            nn.Linear(256, 128),
            nn.BatchNorm1d(128),
            nn.ReLU(),
            nn.Dropout(dropout_rate),
            nn.Linear(128, 1)
        )

    def forward(self, x):
        return self.net(x)

model = RegularizedNet(dropout_rate=0.4)

x = torch.randn(16, 20)
model.train()  # Dropout ACTIVE
out_train = model(x)
model.eval()   # Dropout INACTIVE
with torch.no_grad():
    out_eval = model(x)
print('Train std:', out_train.std().item().__round__(4))
print('Eval  std:', out_eval.std().item().__round__(4))

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

torch.manual_seed(42)
X = torch.randn(500, 10)
y = (X[:, 0] > 0).float()
dl = DataLoader(TensorDataset(X, y), batch_size=32, shuffle=True)

model = nn.Sequential(nn.Linear(10,64), nn.ReLU(), nn.Linear(64,1))

# weight_decay = L2 regularization
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-4)
criterion = nn.BCEWithLogitsLoss()

# Early stopping
best_loss, patience, wait = float('inf'), 5, 0

for epoch in range(1, 30):
    total = 0
    for xb, yb in dl:
        optimizer.zero_grad()
        loss = criterion(model(xb).squeeze(), yb)
        loss.backward()
        optimizer.step()
        total += loss.item()
    avg = total / len(dl)
    if avg < best_loss:
        best_loss, wait = avg, 0
    else:
        wait += 1
    if wait >= patience:
        print(f'Early stop at epoch {epoch}')
        break
    if epoch <= 3 or epoch % 5 == 0:
        print(f'Epoch {epoch:2d}: loss={avg:.4f} (wait={wait})')

import torch
import torch.nn as nn
import numpy as np

torch.manual_seed(42)

# --- Mixup: blend two samples and their labels ---
def mixup_batch(x, y, alpha=0.4, num_classes=3):
    lam = np.random.beta(alpha, alpha)
    idx = torch.randperm(x.size(0))
    x_mix = lam * x + (1 - lam) * x[idx]
    # One-hot then mix
    y_oh  = torch.zeros(x.size(0), num_classes).scatter_(1, y.unsqueeze(1), 1)
    y_mix = lam * y_oh + (1 - lam) * y_oh[idx]
    return x_mix, y_mix

# --- Label Smoothing: soften hard targets ---
class LabelSmoothingLoss(nn.Module):
    def __init__(self, num_classes, smoothing=0.1):
        super().__init__()
        self.eps = smoothing
        self.K   = num_classes

    def forward(self, logits, targets):
        log_probs = torch.log_softmax(logits, dim=1)
        # Smooth targets: (1-eps)*one_hot + eps/K
        with torch.no_grad():
            smooth = torch.zeros_like(log_probs).fill_(self.eps / self.K)
            smooth.scatter_(1, targets.unsqueeze(1), 1 - self.eps + self.eps / self.K)
        return -(smooth * log_probs).sum(dim=1).mean()

x = torch.randn(16, 10)
y = torch.randint(0, 3, (16,))
x_mix, y_mix = mixup_batch(x, y, num_classes=3)
print('Mixup x shape:', x_mix.shape)
print('Mixup y (soft):', y_mix[:2].round(decimals=3))

model = nn.Linear(10, 3)
ls_loss = LabelSmoothingLoss(3, smoothing=0.1)
ce_loss = nn.CrossEntropyLoss()
logits = model(x)
print('Label Smoothing loss:', ls_loss(logits, y).item().__round__(4))
print('Plain CE loss:       ', ce_loss(logits, y).item().__round__(4))

import torch
import torch.nn as nn
import torch.nn.functional as F

# Focal Loss — down-weights easy examples, focuses on hard ones
class FocalLoss(nn.Module):
    def __init__(self, alpha=1.0, gamma=2.0):
        super().__init__()
        self.alpha, self.gamma = alpha, gamma

    def forward(self, logits, targets):
        ce   = F.cross_entropy(logits, targets, reduction='none')
        pt   = torch.exp(-ce)
        loss = self.alpha * (1 - pt) ** self.gamma * ce
        return loss.mean()

# Label smoothing — prevents overconfident predictions
class LabelSmoothingLoss(nn.Module):
    def __init__(self, num_classes, smoothing=0.1):
        super().__init__()
        self.smoothing = smoothing
        self.cls = num_classes

    def forward(self, logits, targets):
        log_probs = F.log_softmax(logits, dim=-1)
        smooth    = self.smoothing / (self.cls - 1)
        one_hot   = torch.full_like(log_probs, smooth)
        one_hot.scatter_(1, targets.unsqueeze(1), 1 - self.smoothing)
        return -(one_hot * log_probs).sum(dim=-1).mean()

# Compare losses on synthetic predictions
torch.manual_seed(42)
logits  = torch.randn(8, 3)   # 8 samples, 3 classes
targets = torch.randint(0, 3, (8,))

ce_loss = nn.CrossEntropyLoss()(logits, targets)
fl_loss = FocalLoss(gamma=2.0)(logits, targets)
ls_loss = LabelSmoothingLoss(3, 0.1)(logits, targets)
print(f'CrossEntropy:    {ce_loss.item():.4f}')
print(f'FocalLoss:       {fl_loss.item():.4f}')
print(f'LabelSmoothing:  {ls_loss.item():.4f}')

# Weighted CrossEntropy for class imbalance
weights = torch.tensor([1.0, 5.0, 3.0])  # class 1 is rare, upweighted
wce = nn.CrossEntropyLoss(weight=weights)(logits, targets)
print(f'WeightedCE:      {wce.item():.4f}')

import torch
import torch.nn as nn
import os, tempfile

# Build and 'train' a model
model = nn.Sequential(
    nn.Linear(10, 64), nn.ReLU(),
    nn.Linear(64, 32), nn.ReLU(),
    nn.Linear(32, 3)
)

tmp = tempfile.gettempdir()
path = os.path.join(tmp, 'model.pth')

# Save weights only (recommended)
torch.save(model.state_dict(), path)
size_kb = os.path.getsize(path) / 1024
print(f'Saved model: {size_kb:.1f} KB')

# Load into same architecture
loaded = nn.Sequential(
    nn.Linear(10, 64), nn.ReLU(),
    nn.Linear(64, 32), nn.ReLU(),
    nn.Linear(32, 3)
)
loaded.load_state_dict(torch.load(path, weights_only=True))
loaded.eval()

x = torch.randn(4, 10)
print('Output:', loaded(x).shape)
print('Weights match:', torch.allclose(
    list(model.parameters())[0],
    list(loaded.parameters())[0]
))

import torch
import torch.nn as nn
import os, tempfile

model = nn.Linear(10, 1)
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)
epoch = 5
best_val_loss = 0.042

tmp = tempfile.gettempdir()
ckpt_path = os.path.join(tmp, 'checkpoint.pth')

# Save full checkpoint
torch.save({
    'epoch': epoch,
    'model_state_dict': model.state_dict(),
    'optimizer_state_dict': optimizer.state_dict(),
    'best_val_loss': best_val_loss
}, ckpt_path)
print(f'Checkpoint saved ({os.path.getsize(ckpt_path)/1024:.1f} KB)')

# Resume training
checkpoint = torch.load(ckpt_path, weights_only=False)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
start_epoch = checkpoint['epoch'] + 1
print(f'Resuming from epoch {start_epoch}')
print(f'Best val loss was: {checkpoint["best_val_loss"]}')

import torch
import torch.nn as nn
import os, tempfile

torch.manual_seed(42)

class SmallNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(8, 16)
        self.fc2 = nn.Linear(16, 1)
    def forward(self, x):
        return self.fc2(torch.relu(self.fc1(x)))

model = SmallNet()
tmp = tempfile.gettempdir()

# Method 1: state_dict (recommended, portable)
sd_path = os.path.join(tmp, 'net_sd.pth')
torch.save(model.state_dict(), sd_path)
m2 = SmallNet()
m2.load_state_dict(torch.load(sd_path, weights_only=True))
print('state_dict load OK')

# Method 2: full model save (pickle-based, less portable)
full_path = os.path.join(tmp, 'net_full.pth')
torch.save(model, full_path)
m3 = torch.load(full_path, weights_only=False)
print('Full model load OK')

# Method 3: TorchScript (portable, no Python needed at inference)
scripted = torch.jit.script(model)
ts_path = os.path.join(tmp, 'net_scripted.pt')
scripted.save(ts_path)
loaded_ts = torch.jit.load(ts_path)
x = torch.randn(4, 8)
print('TorchScript output:', loaded_ts(x).shape)
print('Outputs match:', torch.allclose(model(x), loaded_ts(x), atol=1e-5))

import torch
import torch.nn as nn
import io

class MLP(nn.Module):
    def __init__(self, in_f, hidden, out_f):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_f, hidden), nn.ReLU(),
            nn.Linear(hidden, out_f)
        )
    def forward(self, x): return self.net(x)

model = MLP(16, 64, 4)

# Save/load state dict to buffer (no file system needed)
buf = io.BytesIO()
torch.save(model.state_dict(), buf)
buf.seek(0)
size_kb = buf.getbuffer().nbytes / 1024
print(f'State dict size: {size_kb:.2f} KB')

# Load into new model
model2 = MLP(16, 64, 4)
model2.load_state_dict(torch.load(buf, weights_only=True))
model2.eval()

# Verify identical outputs
torch.manual_seed(0)
x = torch.randn(4, 16)
with torch.no_grad():
    out1 = model(x)
    out2 = model2(x)
print(f'Outputs identical: {torch.allclose(out1, out2)}')

# Parameter count per layer
print('\nModel parameter summary:')
total = 0
for name, p in model.named_parameters():
    n = p.numel()
    total += n
    print(f'  {name:25s}: {list(p.shape)} = {n:,} params')
print(f'  Total: {total:,} parameters')

import torch
import torch.nn as nn

class LSTMClassifier(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, num_classes):
        super().__init__()
        self.lstm = nn.LSTM(
            input_size, hidden_size,
            num_layers=num_layers,
            batch_first=True,
            dropout=0.2 if num_layers > 1 else 0
        )
        self.fc = nn.Linear(hidden_size, num_classes)

    def forward(self, x):
        # x: [batch, seq_len, input_size]
        out, (h_n, c_n) = self.lstm(x)
        return self.fc(out[:, -1, :])  # last timestep

model = LSTMClassifier(
    input_size=10, hidden_size=64,
    num_layers=2, num_classes=3
)
print(model)

# Batch of 16 sequences, each 30 timesteps, 10 features
x = torch.randn(16, 30, 10)
out = model(x)
print('Output shape:', out.shape)  # [16, 3]
print('Params:', sum(p.numel() for p in model.parameters()))

import torch
import torch.nn as nn
import numpy as np

class GRUForecaster(nn.Module):
    def __init__(self, input_size=1, hidden=64, horizon=1):
        super().__init__()
        self.gru = nn.GRU(input_size, hidden, batch_first=True)
        self.fc = nn.Linear(hidden, horizon)

    def forward(self, x):
        out, _ = self.gru(x)
        return self.fc(out[:, -1, :])

model = GRUForecaster(input_size=1, hidden=64, horizon=1)

# Generate sine wave data
np.random.seed(42)
t = np.linspace(0, 20 * np.pi, 1000)
signal = np.sin(t) + 0.1 * np.random.randn(1000)

# Create sliding window sequences (window=30)
W = 30
X = np.array([signal[i:i+W] for i in range(len(signal)-W)])
y = signal[W:]

X_t = torch.tensor(X, dtype=torch.float32).unsqueeze(2)  # [n, 30, 1]
y_t = torch.tensor(y, dtype=torch.float32).unsqueeze(1)

out = model(X_t[:8])
print('Forecast shape:', out.shape)  # [8, 1]
loss = nn.MSELoss()(out, y_t[:8])
print('Initial MSE:', loss.item().__round__(4))

import torch
import torch.nn as nn
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence

# Bidirectional LSTM: processes sequence forward AND backward
class BiLSTM(nn.Module):
    def __init__(self, input_size, hidden, num_classes):
        super().__init__()
        self.lstm = nn.LSTM(
            input_size, hidden,
            batch_first=True,
            bidirectional=True  # 2x hidden size in output
        )
        self.fc = nn.Linear(hidden * 2, num_classes)

    def forward(self, x, lengths):
        # Pack variable-length sequences for efficiency
        packed = pack_padded_sequence(x, lengths, batch_first=True,
                                      enforce_sorted=False)
        out_packed, (h_n, _) = self.lstm(packed)
        out, _ = pad_packed_sequence(out_packed, batch_first=True)
        # Concatenate last forward and first backward hidden states
        last = torch.cat([h_n[0], h_n[1]], dim=1)
        return self.fc(last)

model = BiLSTM(input_size=8, hidden=32, num_classes=4)

# Variable-length batch (padded)
batch_size = 6
max_len = 15
lengths = torch.tensor([15, 12, 10, 8, 7, 5])
x = torch.zeros(batch_size, max_len, 8)
for i, l in enumerate(lengths):
    x[i, :l] = torch.randn(l, 8)

out = model(x, lengths)
print('BiLSTM output:', out.shape)  # [6, 4]
print('Params:', sum(p.numel() for p in model.parameters()))

import torch
import torch.nn as nn
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence, pad_sequence

torch.manual_seed(42)

# Bidirectional LSTM
bi_lstm = nn.LSTM(input_size=16, hidden_size=32, num_layers=2,
                  batch_first=True, bidirectional=True, dropout=0.2)

x = torch.randn(8, 20, 16)  # batch=8, seq_len=20, features=16
out, (h_n, c_n) = bi_lstm(x)
print('Bidirectional LSTM:')
print(f'  Input:  {list(x.shape)}')
print(f'  Output: {list(out.shape)}  (hidden*2={32*2} for bidirectional)')
print(f'  h_n:    {list(h_n.shape)}  (layers*2, batch, hidden)')

# Packed sequences — handle variable-length inputs efficiently
sequences = [torch.randn(length, 8) for length in [10, 7, 5, 3]]
lengths   = torch.tensor([10, 7, 5, 3])
padded    = pad_sequence(sequences, batch_first=True)  # (4, 10, 8)

lstm = nn.LSTM(input_size=8, hidden_size=16, batch_first=True)
packed = pack_padded_sequence(padded, lengths, batch_first=True, enforce_sorted=True)
out_packed, _ = lstm(packed)
out_padded, out_lengths = pad_packed_sequence(out_packed, batch_first=True)

print('\nPacked sequences:')
print(f'  Padded input:  {list(padded.shape)}')
print(f'  Output:        {list(out_padded.shape)}')
print(f'  Out lengths:   {out_lengths.tolist()}')

import numpy as np

def softmax(x, axis=-1):
    x = x - x.max(axis=axis, keepdims=True)
    e = np.exp(x)
    return e / e.sum(axis=axis, keepdims=True)

def scaled_dot_product_attention(Q, K, V, mask=None):
    d_k = Q.shape[-1]
    scores = Q @ K.swapaxes(-2, -1) / np.sqrt(d_k)  # (batch, heads, seq, seq)
    if mask is not None:
        scores = np.where(mask, scores, -1e9)
    weights = softmax(scores, axis=-1)
    return weights @ V, weights

np.random.seed(42)
batch, seq_len, d_model = 2, 5, 8
Q = np.random.randn(batch, seq_len, d_model)
K = np.random.randn(batch, seq_len, d_model)
V = np.random.randn(batch, seq_len, d_model)

output, weights = scaled_dot_product_attention(Q, K, V)
print('Q shape:', Q.shape)
print('Attention output shape:', output.shape)
print('Attention weights shape:', weights.shape)
print('Attention weights (sample 0, row 0):', weights[0, 0].round(3))
print('Row sum (should be 1.0):', weights[0, 0].sum(axis=-1).round(4))

import numpy as np

def softmax(x, axis=-1):
    x -= x.max(axis=axis, keepdims=True)
    return np.exp(x) / np.exp(x).sum(axis=axis, keepdims=True)

class MultiHeadAttention:
    def __init__(self, d_model=64, n_heads=4, seed=42):
        self.h = n_heads
        self.d_k = d_model // n_heads
        rng = np.random.default_rng(seed)
        self.W_Q = rng.standard_normal((d_model, d_model)) * 0.1
        self.W_K = rng.standard_normal((d_model, d_model)) * 0.1
        self.W_V = rng.standard_normal((d_model, d_model)) * 0.1
        self.W_O = rng.standard_normal((d_model, d_model)) * 0.1

    def split_heads(self, x):
        B, L, D = x.shape
        return x.reshape(B, L, self.h, self.d_k).transpose(0, 2, 1, 3)

    def forward(self, x):
        Q = self.split_heads(x @ self.W_Q)
        K = self.split_heads(x @ self.W_K)
        V = self.split_heads(x @ self.W_V)
        scores = Q @ K.swapaxes(-2,-1) / self.d_k**0.5
        attn   = softmax(scores) @ V
        concat = attn.transpose(0,2,1,3).reshape(x.shape[0], x.shape[1], -1)
        return concat @ self.W_O

np.random.seed(0)
B, L, D = 2, 10, 64
x = np.random.randn(B, L, D)
mha = MultiHeadAttention(d_model=D, n_heads=4)
out = mha.forward(x)
print(f'Input:  {x.shape}')
print(f'Output: {out.shape}')
print(f'Mean abs output: {np.abs(out).mean():.4f}')

import numpy as np

def softmax(x, axis=-1):
    x = x - x.max(axis=axis, keepdims=True)
    return np.exp(x) / np.exp(x).sum(axis=axis, keepdims=True)

def causal_attention(Q, K, V):
    d_k = Q.shape[-1]
    seq = Q.shape[-2]
    # Causal mask: upper triangle = -inf
    mask = np.tril(np.ones((seq, seq))).astype(bool)
    scores = Q @ K.swapaxes(-2,-1) / np.sqrt(d_k)
    scores = np.where(mask, scores, -1e9)
    weights = softmax(scores)
    return weights @ V, weights

np.random.seed(42)
L, D = 6, 16
Q = np.random.randn(1, L, D)
K = np.random.randn(1, L, D)
V = np.random.randn(1, L, D)

out, attn = causal_attention(Q, K, V)
print('Causal attention weights (6x6):')
print(attn[0].round(3))
print('\nRow sums:', attn[0].sum(axis=-1).round(4))
print('Upper triangle is zero (causal mask verified):', (attn[0] * np.triu(np.ones((L,L)),1) < 1e-6).all())

try:
    import torch
    import torch.nn as nn

    torch.manual_seed(42)
    d_model, n_heads = 64, 4
    mha = nn.MultiheadAttention(embed_dim=d_model, num_heads=n_heads, batch_first=True)

    B, L = 2, 10
    x = torch.randn(B, L, d_model)

    # Self-attention (Q=K=V=x)
    out, weights = mha(x, x, x)
    print(f'Input:   {tuple(x.shape)}')
    print(f'Output:  {tuple(out.shape)}')
    print(f'Weights: {tuple(weights.shape)}')

    # Causal (autoregressive) mask
    mask = torch.triu(torch.ones(L, L), diagonal=1).bool()
    out_causal, w_causal = mha(x, x, x, attn_mask=mask)
    print(f'Causal output: {tuple(out_causal.shape)}')
    print(f'Params: {sum(p.numel() for p in mha.parameters()):,}')
except ImportError:
    print('pip install torch')
    print('nn.MultiheadAttention: built-in PyTorch multi-head attention')
    print('batch_first=True: input shape (batch, seq, features)')

import numpy as np

np.random.seed(42)

def relu(x): return np.maximum(0, x)
def sigmoid(x): return 1 / (1 + np.exp(-np.clip(x, -100, 100)))

class Autoencoder:
    def __init__(self, in_dim=28, latent=4, lr=0.01):
        self.lr = lr
        # Encoder
        self.W_enc = np.random.randn(in_dim, 8) * 0.1
        self.W_lat = np.random.randn(8, latent) * 0.1
        # Decoder
        self.W_dec = np.random.randn(latent, 8) * 0.1
        self.W_out = np.random.randn(8, in_dim) * 0.1

    def encode(self, x):
        self.h1 = relu(x @ self.W_enc)
        return self.h1 @ self.W_lat

    def decode(self, z):
        self.h2 = relu(z @ self.W_dec)
        return sigmoid(self.h2 @ self.W_out)

    def forward(self, x):
        self.z   = self.encode(x)
        self.rec = self.decode(self.z)
        return self.rec

    def loss(self, x, rec):
        return -np.mean(x * np.log(rec + 1e-8) + (1-x) * np.log(1-rec + 1e-8))

# Train on random binary data
X = (np.random.rand(500, 28) > 0.5).astype(float)
ae = Autoencoder(in_dim=28, latent=4)

for epoch in range(50):
    rec = ae.forward(X)
    if epoch % 10 == 0:
        print(f'Epoch {epoch}: loss={ae.loss(X, rec):.4f}')

import numpy as np

np.random.seed(42)

# Demonstrate the reparameterization trick
# Instead of z ~ N(mu, sigma^2), sample:
# epsilon ~ N(0, 1), then z = mu + sigma * epsilon

def sample_vae(mu, log_var):
    """Reparameterization: z = mu + exp(0.5*log_var) * eps"""
    eps = np.random.standard_normal(mu.shape)
    sigma = np.exp(0.5 * log_var)
    return mu + sigma * eps

def kl_divergence(mu, log_var):
    """KL(N(mu,sigma) || N(0,1)) in closed form."""
    return -0.5 * np.mean(1 + log_var - mu**2 - np.exp(log_var))

def vae_loss(x, x_rec, mu, log_var):
    recon = np.mean((x - x_rec)**2)  # MSE reconstruction
    kl    = kl_divergence(mu, log_var)
    return recon + kl, recon, kl

# Simulate encoder outputs
batch, latent = 32, 8
mu      = np.random.randn(batch, latent) * 0.5
log_var = np.random.randn(batch, latent) * 0.5 - 1  # log_var < 0 -> small sigma

z = sample_vae(mu, log_var)
x = np.random.randn(batch, 16)
x_rec = x + np.random.randn(*x.shape) * 0.3  # simulated reconstruction

total, recon, kl = vae_loss(x, x_rec, mu, log_var)
print(f'Total VAE loss: {total:.4f}')
print(f'Reconstruction: {recon:.4f}')
print(f'KL divergence:  {kl:.4f}')
print(f'Latent z shape: {z.shape}')

import numpy as np

np.random.seed(42)

def sigmoid(x): return 1 / (1 + np.exp(-np.clip(x, -10, 10)))
def bce(y_true, y_pred): return -np.mean(y_true*np.log(y_pred+1e-8) + (1-y_true)*np.log(1-y_pred+1e-8))

# Minimal GAN training loop concept
# Real distribution: N(4, 0.5)
real_data = np.random.normal(4, 0.5, (1000, 1))

# Generator: noise -> fake samples
class Generator:
    def __init__(self):
        self.W1 = np.random.randn(1, 8) * 0.1
        self.W2 = np.random.randn(8, 1) * 0.1
    def generate(self, z):
        h = np.tanh(z @ self.W1)
        return h @ self.W2  # fake samples

# Discriminator: sample -> P(real)
class Discriminator:
    def __init__(self):
        self.W1 = np.random.randn(1, 8) * 0.1
        self.W2 = np.random.randn(8, 1) * 0.1
    def discriminate(self, x):
        h = np.maximum(0, x @ self.W1)
        return sigmoid(h @ self.W2)

G = Generator(); D = Discriminator()
batch = 32
print('GAN Training Loop (concepts):')
for step in range(5):
    # Discriminator: maximize log D(real) + log(1-D(fake))
    real = real_data[np.random.randint(0, len(real_data), batch)]
    z    = np.random.randn(batch, 1)
    fake = G.generate(z)
    d_real = D.discriminate(real)
    d_fake = D.discriminate(fake)
    d_loss = bce(np.ones((batch,1)), d_real) + bce(np.zeros((batch,1)), d_fake)
    # Generator: maximize log D(fake) = minimize -log D(fake)
    g_loss = bce(np.ones((batch,1)), d_fake)
    print(f'Step {step+1}: D_loss={d_loss:.4f} G_loss={g_loss:.4f} | '
          f'fake_mean={fake.mean():.2f} (target~4.0)')

try:
    import torch
    import torch.nn as nn
    import torch.nn.functional as F

    class VAE(nn.Module):
        def __init__(self, input_dim=784, latent_dim=20, hidden=256):
            super().__init__()
            self.encoder = nn.Sequential(nn.Linear(input_dim, hidden), nn.ReLU())
            self.mu_layer     = nn.Linear(hidden, latent_dim)
            self.logvar_layer = nn.Linear(hidden, latent_dim)
            self.decoder = nn.Sequential(
                nn.Linear(latent_dim, hidden), nn.ReLU(),
                nn.Linear(hidden, input_dim), nn.Sigmoid()
            )

        def reparameterize(self, mu, logvar):
            std = torch.exp(0.5 * logvar)
            return mu + std * torch.randn_like(std)

        def forward(self, x):
            h   = self.encoder(x)
            mu  = self.mu_layer(h)
            lv  = self.logvar_layer(h)
            z   = self.reparameterize(mu, lv)
            rec = self.decoder(z)
            kl  = -0.5 * (1 + lv - mu**2 - lv.exp()).mean()
            return rec, kl

    torch.manual_seed(42)
    vae = VAE(input_dim=784, latent_dim=16)
    x = torch.rand(32, 784)
    rec, kl = vae(x)
    recon_loss = F.binary_cross_entropy(rec, x, reduction='mean')
    loss = recon_loss + kl
    print(f'VAE total loss: {loss.item():.4f} (recon={recon_loss.item():.4f}, kl={kl.item():.4f})')
    print(f'Params: {sum(p.numel() for p in vae.parameters()):,}')
except ImportError:
    print('pip install torch')

try:
    import shap
    import numpy as np
    from sklearn.ensemble import GradientBoostingClassifier
    from sklearn.datasets import make_classification

    np.random.seed(42)
    X, y = make_classification(n_samples=1000, n_features=8, n_informative=5, random_state=42)
    feat_names = [f'feature_{i}' for i in range(8)]

    model = GradientBoostingClassifier(n_estimators=50, random_state=0).fit(X, y)

    explainer   = shap.TreeExplainer(model)
    shap_values = explainer.shap_values(X[:100])

    print('SHAP values shape:', shap_values.shape)
    print('\nMean |SHAP| per feature (global importance):')
    importances = np.abs(shap_values).mean(axis=0)
    for name, imp in sorted(zip(feat_names, importances), key=lambda x: -x[1]):
        bar = '|' * int(imp * 20)
        print(f'{name}: {imp:.4f} {bar}')
    print('\nSHAP values for sample 0:')
    for name, sv in zip(feat_names, shap_values[0]):
        print(f'  {name}: {sv:+.4f}')
except ImportError:
    print('pip install shap')

import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.inspection import permutation_importance
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

np.random.seed(42)
X, y = make_classification(n_samples=2000, n_features=10, n_informative=5, random_state=42)
feat_names = [f'f{i}' for i in range(10)]
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.3, random_state=42)

model = GradientBoostingClassifier(n_estimators=50, random_state=0).fit(X_tr, y_tr)

result = permutation_importance(
    model, X_te, y_te, n_repeats=10, random_state=42, scoring='roc_auc'
)

print('Permutation Feature Importance (ROC-AUC drop):')
print(f'{"Feature":<12} {"Mean Drop":>12} {"Std":>8}')
print('-' * 35)
order = np.argsort(-result.importances_mean)
for i in order:
    print(f'{feat_names[i]:<12} {result.importances_mean[i]:>12.4f} {result.importances_std[i]:>8.4f}')

import numpy as np
from sklearn.linear_model import Ridge
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

np.random.seed(42)
X, y = make_classification(n_samples=1000, n_features=8, n_informative=5, random_state=42)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.2, random_state=42)
model = GradientBoostingClassifier(n_estimators=50, random_state=0).fit(X_tr, y_tr)

def lime_explain(model, x_instance, X_train, n_samples=500, sigma=0.1):
    """Approximate local explanation around x_instance."""
    rng = np.random.default_rng(42)
    # Sample neighborhood around x_instance
    X_perturbed = x_instance + rng.normal(0, sigma, (n_samples, len(x_instance)))
    # Get black-box predictions
    y_hat = model.predict_proba(X_perturbed)[:, 1]
    # Weight by distance
    distances = np.linalg.norm(X_perturbed - x_instance, axis=1)
    weights = np.exp(-distances / distances.mean())
    # Fit local linear model
    local_model = Ridge(alpha=0.1)
    local_model.fit(X_perturbed, y_hat, sample_weight=weights)
    return local_model.coef_

x0 = X_te[0]
coef = lime_explain(model, x0, X_tr)
feat_names = [f'f{i}' for i in range(8)]
print(f'Sample 0 prediction: {model.predict_proba(x0.reshape(1,-1))[0,1]:.3f}')
print('LIME explanation (local coefficients):')
for name, c in sorted(zip(feat_names, coef), key=lambda x: -abs(x[1])):
    print(f'  {name}: {c:+.4f}')

import numpy as np

np.random.seed(42)

# Simulate a simple neural network with gradient computation
def relu(x): return np.maximum(0, x)
def relu_grad(x): return (x > 0).astype(float)
def sigmoid(x): return 1 / (1 + np.exp(-np.clip(x, -10, 10)))

class SimpleNet:
    def __init__(self, d=8, h=16):
        self.W1 = np.random.randn(d, h) * 0.1
        self.W2 = np.random.randn(h, 1) * 0.1

    def forward(self, x):
        self.x  = x
        self.h  = relu(x @ self.W1)
        return sigmoid(self.h @ self.W2)

    def saliency(self, x):
        out = self.forward(x)
        # Backprop to input
        d_out = out * (1 - out)  # sigmoid grad
        d_h   = d_out @ self.W2.T
        d_h   *= relu_grad(x @ self.W1)
        d_x   = d_h @ self.W1.T
        return np.abs(d_x)

    def integrated_gradients(self, x, n_steps=50):
        baseline = np.zeros_like(x)
        alphas   = np.linspace(0, 1, n_steps)
        grads    = [self.saliency(baseline + a * x) for a in alphas]
        ig = (x - baseline) * np.mean(grads, axis=0)
        return ig

net = SimpleNet(d=8)
x   = np.random.randn(1, 8)
sal = net.saliency(x)[0]
ig  = net.integrated_gradients(x)[0]
print('Saliency scores:', sal.round(4))
print('Integrated Gradients:', ig.round(4))
print('Top feature (saliency):', np.argmax(sal))
print('Top feature (IG):', np.argmax(np.abs(ig)))

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset
import numpy as np

np.random.seed(42); torch.manual_seed(42)
# Simulate image-like data
X = torch.randn(1000, 1, 28, 28)
y = torch.randint(0, 10, (1000,))

class SimpleCNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, padding=1)
        self.pool  = nn.MaxPool2d(2, 2)
        self.relu  = nn.ReLU()
        self.fc1   = nn.Linear(32 * 7 * 7, 128)
        self.fc2   = nn.Linear(128, 10)
        self.dropout = nn.Dropout(0.3)

    def forward(self, x):
        x = self.pool(self.relu(self.conv1(x)))  # -> (16, 14, 14)
        x = self.pool(self.relu(self.conv2(x)))  # -> (32, 7, 7)
        x = x.view(x.size(0), -1)
        x = self.relu(self.fc1(x))
        x = self.dropout(x)
        return self.fc2(x)

model = SimpleCNN()
print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")
loader = DataLoader(TensorDataset(X, y), batch_size=32)
x_batch, _ = next(iter(loader))
out = model(x_batch)
print(f"Output shape: {out.shape}")

import torch
import torch.nn as nn
import numpy as np

class ResidualBlock(nn.Module):
    def __init__(self, channels):
        super().__init__()
        self.conv1 = nn.Conv2d(channels, channels, 3, padding=1)
        self.bn1   = nn.BatchNorm2d(channels)
        self.conv2 = nn.Conv2d(channels, channels, 3, padding=1)
        self.bn2   = nn.BatchNorm2d(channels)
        self.relu  = nn.ReLU()

    def forward(self, x):
        identity = x
        out = self.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        return self.relu(out + identity)  # skip connection

class MiniResNet(nn.Module):
    def __init__(self, n_classes=10):
        super().__init__()
        self.stem   = nn.Conv2d(1, 32, 3, padding=1)
        self.res1   = ResidualBlock(32)
        self.pool   = nn.AdaptiveAvgPool2d(4)
        self.fc     = nn.Linear(32 * 4 * 4, n_classes)

    def forward(self, x):
        x = torch.relu(self.stem(x))
        x = self.res1(x)
        x = self.pool(x)
        return self.fc(x.flatten(1))

model = MiniResNet()
x = torch.randn(8, 1, 28, 28)
print(f"Output shape: {model(x).shape}")
print(f"Params: {sum(p.numel() for p in model.parameters()):,}")

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset
import numpy as np

torch.manual_seed(0); np.random.seed(0)
# Multi-class classification
X = torch.randn(2000, 3, 32, 32)
y = torch.randint(0, 5, (2000,))

class CNN3(nn.Module):
    def __init__(self, n_classes=5):
        super().__init__()
        self.net = nn.Sequential(
            nn.Conv2d(3, 32, 3, padding=1), nn.BatchNorm2d(32), nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(32, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1), nn.ReLU(),
            nn.AdaptiveAvgPool2d(2),
            nn.Flatten(),
            nn.Linear(128*4, 256), nn.ReLU(), nn.Dropout(0.4),
            nn.Linear(256, n_classes)
        )
    def forward(self, x): return self.net(x)

model = CNN3()
loader = DataLoader(TensorDataset(X[:200], y[:200]), batch_size=32, shuffle=True)
opt  = optim.Adam(model.parameters(), lr=1e-3)
loss_fn = nn.CrossEntropyLoss()
for epoch in range(3):
    total_loss = 0
    for xb, yb in loader:
        opt.zero_grad()
        loss = loss_fn(model(xb), yb)
        loss.backward(); opt.step()
        total_loss += loss.item()
    print(f"Epoch {epoch+1}: loss={total_loss/len(loader):.4f}")

import torch
import torch.nn as nn
import numpy as np

# Transfer learning simulation with pretrained-like frozen backbone
class FrozenBackbone(nn.Module):
    def __init__(self):
        super().__init__()
        # Simulated frozen backbone (fixed weights)
        self.backbone = nn.Sequential(
            nn.Conv2d(3, 64, 3, padding=1), nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Conv2d(64, 128, 3, padding=1), nn.ReLU(),
            nn.AdaptiveAvgPool2d(4),
        )
        # Freeze backbone
        for p in self.backbone.parameters():
            p.requires_grad = False
        # Trainable head
        self.head = nn.Sequential(
            nn.Flatten(),
            nn.Linear(128*16, 256), nn.ReLU(), nn.Dropout(0.3),
            nn.Linear(256, 3)   # 3 classes
        )
    def forward(self, x):
        with torch.no_grad():
            features = self.backbone(x)
        return self.head(features)

model = FrozenBackbone()
frozen = sum(p.numel() for p in model.parameters() if not p.requires_grad)
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f"Frozen params:    {frozen:,}")
print(f"Trainable params: {trainable:,}")
x = torch.randn(4, 3, 64, 64)
print(f"Output: {model(x).shape}")

import torch
import torch.nn as nn
import numpy as np

class TransformerBlock(nn.Module):
    def __init__(self, d_model, n_heads, d_ff, dropout=0.1):
        super().__init__()
        self.attn  = nn.MultiheadAttention(d_model, n_heads, batch_first=True, dropout=dropout)
        self.ff    = nn.Sequential(
            nn.Linear(d_model, d_ff), nn.GELU(), nn.Linear(d_ff, d_model)
        )
        self.ln1 = nn.LayerNorm(d_model)
        self.ln2 = nn.LayerNorm(d_model)
        self.drop = nn.Dropout(dropout)

    def forward(self, x, mask=None):
        # Pre-norm formulation
        attn_out, weights = self.attn(self.ln1(x), self.ln1(x), self.ln1(x))
        x = x + self.drop(attn_out)
        x = x + self.drop(self.ff(self.ln2(x)))
        return x, weights

d_model, n_heads, d_ff = 64, 4, 256
block = TransformerBlock(d_model, n_heads, d_ff)
x = torch.randn(8, 20, d_model)  # batch=8, seq_len=20
out, weights = block(x)
print(f"Output shape:  {out.shape}")
print(f"Attn weights:  {weights.shape}")

import torch
import torch.nn as nn
import numpy as np

class PositionalEncoding(nn.Module):
    def __init__(self, d_model, max_len=512, dropout=0.1):
        super().__init__()
        self.dropout = nn.Dropout(dropout)
        pe = torch.zeros(max_len, d_model)
        pos = torch.arange(max_len).unsqueeze(1)
        div = torch.exp(torch.arange(0, d_model, 2) * (-torch.log(torch.tensor(10000.0)) / d_model))
        pe[:, 0::2] = torch.sin(pos * div)
        pe[:, 1::2] = torch.cos(pos * div)
        self.register_buffer("pe", pe.unsqueeze(0))

    def forward(self, x):
        return self.dropout(x + self.pe[:, :x.size(1)])

class TextTransformer(nn.Module):
    def __init__(self, vocab_size=1000, d_model=64, n_heads=4, n_layers=2, n_classes=3):
        super().__init__()
        self.embed   = nn.Embedding(vocab_size, d_model, padding_idx=0)
        self.pos_enc = PositionalEncoding(d_model)
        encoder_layer = nn.TransformerEncoderLayer(d_model, n_heads, dim_feedforward=256,
                                                    batch_first=True, norm_first=True)
        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=n_layers)
        self.classifier = nn.Linear(d_model, n_classes)

    def forward(self, x, src_key_padding_mask=None):
        x = self.pos_enc(self.embed(x))
        x = self.encoder(x, src_key_padding_mask=src_key_padding_mask)
        return self.classifier(x.mean(dim=1))  # mean pooling

model = TextTransformer()
tokens = torch.randint(1, 1000, (4, 30))  # batch=4, seq_len=30
out = model(tokens)
print(f"Output: {out.shape}, params: {sum(p.numel() for p in model.parameters()):,}")

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np

torch.manual_seed(0)
# Text classification with Transformer
vocab_size, d_model, n_classes = 500, 32, 4
class MiniTransformer(nn.Module):
    def __init__(self):
        super().__init__()
        self.embed   = nn.Embedding(vocab_size, d_model, padding_idx=0)
        layer = nn.TransformerEncoderLayer(d_model, 4, 128, batch_first=True, norm_first=True)
        self.encoder = nn.TransformerEncoder(layer, num_layers=2)
        self.head    = nn.Linear(d_model, n_classes)
    def forward(self, x):
        mask = (x == 0)
        return self.head(self.encoder(self.embed(x), src_key_padding_mask=mask).mean(1))

model = MiniTransformer()
X = torch.randint(0, vocab_size, (200, 20))
X[:, 15:] = 0  # padding
y = torch.randint(0, n_classes, (200,))
loader = torch.utils.data.DataLoader(
    torch.utils.data.TensorDataset(X, y), batch_size=32, shuffle=True)
opt = optim.Adam(model.parameters(), lr=3e-4)
for epoch in range(5):
    total = 0
    for xb, yb in loader:
        opt.zero_grad(); loss = nn.CrossEntropyLoss()(model(xb), yb)
        loss.backward(); opt.step(); total += loss.item()
    print(f"Epoch {epoch+1}: loss={total/len(loader):.4f}")

import torch
import torch.nn as nn
import numpy as np

# Variational Autoencoder (VAE)
class VAE(nn.Module):
    def __init__(self, input_dim=784, hidden=256, latent=16):
        super().__init__()
        self.encoder = nn.Sequential(nn.Linear(input_dim, hidden), nn.ReLU())
        self.mu_layer  = nn.Linear(hidden, latent)
        self.log_var_layer = nn.Linear(hidden, latent)
        self.decoder = nn.Sequential(
            nn.Linear(latent, hidden), nn.ReLU(),
            nn.Linear(hidden, input_dim), nn.Sigmoid()
        )
    def encode(self, x):
        h = self.encoder(x)
        return self.mu_layer(h), self.log_var_layer(h)
    def reparameterize(self, mu, log_var):
        std = torch.exp(0.5 * log_var)
        eps = torch.randn_like(std)
        return mu + eps * std
    def forward(self, x):
        mu, log_var = self.encode(x)
        z = self.reparameterize(mu, log_var)
        return self.decoder(z), mu, log_var

vae = VAE()
x = torch.randn(32, 784).clamp(0, 1)
recon, mu, log_var = vae(x)
recon_loss = nn.functional.binary_cross_entropy(recon, x, reduction="sum")
kl_loss = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())
loss = recon_loss + kl_loss
print(f"Recon loss: {recon_loss.item():.2f}, KL loss: {kl_loss.item():.2f}")
print(f"Latent shape: {mu.shape}")

import torch
import torch.nn as nn
import numpy as np

# Simple GAN for 1D distribution
class Generator(nn.Module):
    def __init__(self, z_dim=8, out_dim=1):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(z_dim, 32), nn.LeakyReLU(0.2),
            nn.Linear(32, 64), nn.LeakyReLU(0.2),
            nn.Linear(64, out_dim)
        )
    def forward(self, z): return self.net(z)

class Discriminator(nn.Module):
    def __init__(self, in_dim=1):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_dim, 64), nn.LeakyReLU(0.2),
            nn.Linear(64, 32), nn.LeakyReLU(0.2),
            nn.Linear(32, 1), nn.Sigmoid()
        )
    def forward(self, x): return self.net(x)

torch.manual_seed(0)
G, D = Generator(), Discriminator()
G_opt = torch.optim.Adam(G.parameters(), lr=2e-4, betas=(0.5, 0.999))
D_opt = torch.optim.Adam(D.parameters(), lr=2e-4, betas=(0.5, 0.999))
bce = nn.BCELoss()
# Target: N(3, 0.5) distribution
for step in range(300):
    real = torch.randn(64, 1) * 0.5 + 3.0
    z = torch.randn(64, 8)
    fake = G(z)
    d_loss = bce(D(real), torch.ones(64,1)) + bce(D(fake.detach()), torch.zeros(64,1))
    D_opt.zero_grad(); d_loss.backward(); D_opt.step()
    g_loss = bce(D(G(torch.randn(64,8))), torch.ones(64,1))
    G_opt.zero_grad(); g_loss.backward(); G_opt.step()
with torch.no_grad():
    samples = G(torch.randn(1000, 8)).squeeze()
print(f"Generated: mean={samples.mean():.3f}, std={samples.std():.3f}")
print(f"Target:    mean=3.000, std=0.500")

import torch
import torch.nn as nn
import numpy as np

# Autoencoder for anomaly detection
class Autoencoder(nn.Module):
    def __init__(self, input_dim=20, bottleneck=4):
        super().__init__()
        self.encoder = nn.Sequential(
            nn.Linear(input_dim, 12), nn.ReLU(),
            nn.Linear(12, bottleneck), nn.ReLU()
        )
        self.decoder = nn.Sequential(
            nn.Linear(bottleneck, 12), nn.ReLU(),
            nn.Linear(12, input_dim)
        )
    def forward(self, x):
        return self.decoder(self.encoder(x))

torch.manual_seed(5)
ae = Autoencoder()
# Train on normal data
X_normal = torch.randn(500, 20)
opt = torch.optim.Adam(ae.parameters(), lr=1e-3)
loader = torch.utils.data.DataLoader(X_normal, batch_size=32, shuffle=True)
for epoch in range(20):
    for xb in loader:
        recon = ae(xb)
        loss = nn.MSELoss()(recon, xb)
        opt.zero_grad(); loss.backward(); opt.step()
# Anomaly detection
X_test_normal = torch.randn(50, 20)
X_test_anomaly = torch.randn(10, 20) * 3  # out-of-distribution
X_test = torch.cat([X_test_normal, X_test_anomaly])
labels = torch.cat([torch.zeros(50), torch.ones(10)])
with torch.no_grad():
    errors = ((ae(X_test) - X_test)**2).mean(dim=1)
threshold = errors[:50].mean() + 2*errors[:50].std()
preds = (errors > threshold).float()
accuracy = (preds == labels).float().mean()
print(f"Anomaly threshold: {threshold:.4f}")
print(f"Detection accuracy: {accuracy:.4f}")

import torch
import torch.nn as nn
import numpy as np

torch.manual_seed(42); np.random.seed(42)

t = np.linspace(0, 8*np.pi, 500)
signal = np.sin(t) + 0.1*np.random.randn(len(t))

def make_sequences(data, seq_len=20):
    X, y = [], []
    for i in range(len(data) - seq_len):
        X.append(data[i:i+seq_len])
        y.append(data[i+seq_len])
    return np.array(X, dtype=np.float32), np.array(y, dtype=np.float32)

X, y = make_sequences(signal)
split = int(len(X)*0.8)
X_tr = torch.tensor(X[:split]).unsqueeze(-1)
X_te = torch.tensor(X[split:]).unsqueeze(-1)
y_tr = torch.tensor(y[:split]).unsqueeze(-1)
y_te = torch.tensor(y[split:]).unsqueeze(-1)

class LSTMForecaster(nn.Module):
    def __init__(self):
        super().__init__()
        self.lstm = nn.LSTM(1, 32, 2, batch_first=True, dropout=0.2)
        self.fc = nn.Linear(32, 1)
    def forward(self, x):
        out, _ = self.lstm(x)
        return self.fc(out[:, -1, :])

model = LSTMForecaster()
opt = torch.optim.Adam(model.parameters(), lr=1e-3)
crit = nn.MSELoss()
for epoch in range(50):
    model.train()
    loss = crit(model(X_tr), y_tr)
    opt.zero_grad(); loss.backward(); opt.step()
    if (epoch+1) % 10 == 0:
        model.eval()
        with torch.no_grad():
            vl = crit(model(X_te), y_te).item()
        print(f"Epoch {epoch+1}: train={loss.item():.4f}, val={vl:.4f}")

import torch
import torch.nn as nn

torch.manual_seed(42)
VOCAB = list("abcdefghijklmnopqrstuvwxyz ")
char2idx = {c: i+1 for i, c in enumerate(VOCAB)}
MAX = 40

def encode(text):
    enc = [char2idx.get(c, 0) for c in text.lower()[:MAX]]
    return enc + [0]*(MAX-len(enc))

texts = ["great product love it", "terrible quality broke",
         "amazing fast shipping", "waste of money poor",
         "highly recommend excellent", "disappointed does not work"]
labels = [1, 0, 1, 0, 1, 0]

X = torch.tensor([encode(t) for t in texts], dtype=torch.long)
y = torch.tensor(labels, dtype=torch.float)

class GRUClassifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.embed = nn.Embedding(len(VOCAB)+2, 16, padding_idx=0)
        self.gru = nn.GRU(16, 32, batch_first=True, bidirectional=True)
        self.fc = nn.Linear(64, 1)
    def forward(self, x):
        x = self.embed(x)
        _, h = self.gru(x)
        h = torch.cat([h[-2], h[-1]], dim=-1)
        return torch.sigmoid(self.fc(h)).squeeze()

model = GRUClassifier()
opt = torch.optim.Adam(model.parameters(), lr=1e-3)
crit = nn.BCELoss()
for _ in range(100):
    loss = crit(model(X), y)
    opt.zero_grad(); loss.backward(); opt.step()

model.eval()
with torch.no_grad():
    preds = (model(X) > 0.5).int().tolist()
acc = sum(p==l for p,l in zip(preds, labels))/len(labels)
print(f"GRU Accuracy: {acc:.2f}")
for t, p, l in zip(texts, preds, labels):
    print(f"  [{'OK' if p==l else 'X'}] {t}: pred={p}")

import torch
import torch.nn as nn
import time
import numpy as np

torch.manual_seed(42)

# Compare LSTM vs GRU on same task
X = torch.randn(64, 50, 10)  # batch=64, seq=50, features=10
y = torch.randint(0, 2, (64,)).float()

class RNNModel(nn.Module):
    def __init__(self, cell='lstm', hidden=64):
        super().__init__()
        if cell == 'lstm':
            self.rnn = nn.LSTM(10, hidden, 2, batch_first=True)
        else:
            self.rnn = nn.GRU(10, hidden, 2, batch_first=True)
        self.fc = nn.Linear(hidden, 1)
    def forward(self, x):
        out, _ = self.rnn(x)
        return torch.sigmoid(self.fc(out[:, -1])).squeeze()

results = {}
for cell in ['lstm', 'gru']:
    model = RNNModel(cell=cell)
    opt = torch.optim.Adam(model.parameters(), lr=1e-3)
    crit = nn.BCELoss()
    n_params = sum(p.numel() for p in model.parameters())
    start = time.time()
    for _ in range(100):
        loss = crit(model(X), y)
        opt.zero_grad(); loss.backward(); opt.step()
    elapsed = time.time() - start
    acc = ((model(X) > 0.5).float() == y).float().mean().item()
    results[cell] = {'params': n_params, 'time': elapsed, 'acc': acc, 'loss': loss.item()}

print("LSTM vs GRU Comparison:")
for cell, r in results.items():
    print(f"  {cell.upper()}: params={r['params']:,}, time={r['time']:.2f}s, "
          f"acc={r['acc']:.4f}, loss={r['loss']:.4f}")
print("\nConclusion: GRU has fewer params and is faster; LSTM often better for long deps")

import torch
import torch.nn as nn
from torch.utils.data import TensorDataset, DataLoader

torch.manual_seed(42)

class PretrainedBackbone(nn.Module):
    def __init__(self):
        super().__init__()
        self.features = nn.Sequential(
            nn.Linear(128, 64), nn.ReLU(),
            nn.Linear(64, 32), nn.ReLU(),
        )
        for p in self.parameters():
            p.requires_grad = False  # Freeze
    def forward(self, x): return self.features(x)

class TransferModel(nn.Module):
    def __init__(self, n_classes=4):
        super().__init__()
        self.backbone = PretrainedBackbone()
        self.head = nn.Sequential(
            nn.Linear(32, 16), nn.ReLU(), nn.Dropout(0.3),
            nn.Linear(16, n_classes)
        )
    def forward(self, x): return self.head(self.backbone(x))

model = TransferModel()
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
total = sum(p.numel() for p in model.parameters())
print(f"Trainable: {trainable}/{total} ({trainable/total*100:.1f}%)")

X = torch.randn(200, 128); y = torch.randint(0, 4, (200,))
loader = DataLoader(TensorDataset(X, y), batch_size=32, shuffle=True)
opt = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)
crit = nn.CrossEntropyLoss()

for epoch in range(15):
    for xb, yb in loader:
        loss = crit(model(xb), yb)
        opt.zero_grad(); loss.backward(); opt.step()
    if (epoch+1) % 5 == 0:
        model.eval()
        with torch.no_grad():
            acc = (model(X).argmax(1)==y).float().mean().item()
        model.train()
        print(f"Epoch {epoch+1}: loss={loss.item():.4f}, acc={acc:.4f}")

import torch
import torch.nn as nn

torch.manual_seed(42)

class PretrainedModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.layer1 = nn.Linear(64, 32)
        self.layer2 = nn.Linear(32, 16)
        self.head = nn.Linear(16, 3)
        self.relu = nn.ReLU()
        for layer in [self.layer1, self.layer2]:
            for p in layer.parameters():
                p.requires_grad = False

    def forward(self, x):
        return self.head(self.relu(self.layer2(self.relu(self.layer1(x)))))

X = torch.randn(200, 64); y = torch.randint(0, 3, (200,))
crit = nn.CrossEntropyLoss()
model = PretrainedModel()

# Phase 1: head only
opt = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)
for _ in range(20):
    loss = crit(model(X), y); opt.zero_grad(); loss.backward(); opt.step()
print(f"Phase 1 (head only): {loss.item():.4f}")

# Phase 2: unfreeze layer2 with smaller LR
for p in model.layer2.parameters(): p.requires_grad = True
opt = torch.optim.Adam([
    {"params": model.layer2.parameters(), "lr": 1e-4},
    {"params": model.head.parameters(), "lr": 1e-3},
])
for _ in range(20):
    loss = crit(model(X), y); opt.zero_grad(); loss.backward(); opt.step()
print(f"Phase 2 (+layer2): {loss.item():.4f}")

# Phase 3: unfreeze all
for p in model.layer1.parameters(): p.requires_grad = True
opt = torch.optim.Adam([
    {"params": model.layer1.parameters(), "lr": 1e-5},
    {"params": model.layer2.parameters(), "lr": 1e-4},
    {"params": model.head.parameters(), "lr": 1e-3},
])
for _ in range(30):
    loss = crit(model(X), y); opt.zero_grad(); loss.backward(); opt.step()
acc = (model(X).argmax(1)==y).float().mean().item()
print(f"Phase 3 (all): loss={loss.item():.4f}, acc={acc:.4f}")

import torch, torch.nn as nn, numpy as np

torch.manual_seed(42)

class Encoder(nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(50, 32)
        self.fc2 = nn.Linear(32, 16)
        self.relu = nn.ReLU()
    def forward(self, x): return self.relu(self.fc2(self.relu(self.fc1(x))))

class TargetModel(nn.Module):
    def __init__(self, freeze=True):
        super().__init__()
        self.encoder = Encoder()
        self.head = nn.Linear(16, 2)
        if freeze:
            for p in self.encoder.parameters(): p.requires_grad = False
    def forward(self, x): return self.head(self.encoder(x))

X_src = torch.randn(500, 50); y_src = torch.randint(0, 2, (500,))
X_tgt = torch.randn(50, 50); y_tgt = torch.randint(0, 2, (50,))  # small target

crit = nn.CrossEntropyLoss()
for mode in ["frozen", "full_finetune"]:
    model = TargetModel(freeze=(mode=="frozen"))
    opt = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-3)
    for _ in range(50):
        loss = crit(model(X_tgt), y_tgt); opt.zero_grad(); loss.backward(); opt.step()
    model.eval()
    with torch.no_grad():
        src_acc = (model(X_src).argmax(1)==y_src).float().mean().item()
        tgt_acc = (model(X_tgt).argmax(1)==y_tgt).float().mean().item()
    print(f"{mode:>16}: target_acc={tgt_acc:.4f}, source_acc={src_acc:.4f}")

print("\nTip: Frozen backbone works best with limited target data (< 500 samples)")
print("Tip: Full fine-tuning risks catastrophic forgetting on small datasets")

import torch
import torch.nn as nn

torch.manual_seed(42)

def sdp_attention(Q, K, V, mask=None):
    d_k = Q.size(-1)
    scores = torch.matmul(Q, K.transpose(-2, -1)) / (d_k**0.5)
    if mask is not None:
        scores = scores.masked_fill(mask==0, float('-inf'))
    weights = torch.softmax(scores, dim=-1)
    return torch.matmul(weights, V), weights

class MultiHeadAttention(nn.Module):
    def __init__(self, d_model=64, n_heads=4):
        super().__init__()
        self.d_k = d_model // n_heads
        self.n_heads = n_heads
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)

    def split(self, x):
        B, T, D = x.size()
        return x.view(B, T, self.n_heads, self.d_k).transpose(1, 2)

    def forward(self, Q, K, V):
        B = Q.size(0)
        Q, K, V = self.split(self.W_q(Q)), self.split(self.W_k(K)), self.split(self.W_v(V))
        out, attn = sdp_attention(Q, K, V)
        out = out.transpose(1,2).contiguous().view(B, -1, self.n_heads*self.d_k)
        return self.W_o(out), attn

B, T, D = 2, 10, 64
x = torch.randn(B, T, D)
mha = MultiHeadAttention(d_model=64, n_heads=4)
out, attn = mha(x, x, x)
print(f"Input: {x.shape} -> Output: {out.shape}")
print(f"Attention weights: {attn.shape}")
print(f"Attention sums to 1: {attn[0,0,0].sum().item():.4f}")

import torch, torch.nn as nn, math

class PositionalEncoding(nn.Module):
    def __init__(self, d_model, max_len=1000):
        super().__init__()
        pe = torch.zeros(max_len, d_model)
        pos = torch.arange(0, max_len).unsqueeze(1).float()
        div = torch.exp(torch.arange(0, d_model, 2).float()*(-math.log(10000.0)/d_model))
        pe[:, 0::2] = torch.sin(pos*div)
        pe[:, 1::2] = torch.cos(pos*div)
        self.register_buffer('pe', pe.unsqueeze(0))
    def forward(self, x): return x + self.pe[:, :x.size(1)]

class TransformerBlock(nn.Module):
    def __init__(self, d=64, n_heads=4, ff=256, drop=0.1):
        super().__init__()
        self.attn = nn.MultiheadAttention(d, n_heads, dropout=drop, batch_first=True)
        self.ff = nn.Sequential(nn.Linear(d, ff), nn.GELU(), nn.Dropout(drop), nn.Linear(ff, d))
        self.n1, self.n2 = nn.LayerNorm(d), nn.LayerNorm(d)
        self.drop = nn.Dropout(drop)
    def forward(self, x):
        a, _ = self.attn(x, x, x)
        x = self.n1(x + self.drop(a))
        return self.n2(x + self.drop(self.ff(x)))

class TransformerClassifier(nn.Module):
    def __init__(self, vocab=100, d=64, n_heads=4, n_layers=2, n_classes=3):
        super().__init__()
        self.embed = nn.Embedding(vocab, d, padding_idx=0)
        self.pos = PositionalEncoding(d)
        self.layers = nn.Sequential(*[TransformerBlock(d, n_heads) for _ in range(n_layers)])
        self.head = nn.Linear(d, n_classes)
    def forward(self, x): return self.head(self.layers(self.pos(self.embed(x))).mean(1))

torch.manual_seed(42)
model = TransformerClassifier()
x = torch.randint(1, 100, (4, 20)); y = torch.randint(0, 3, (4,))
opt = torch.optim.Adam(model.parameters(), lr=1e-3)
for i in range(30):
    loss = nn.CrossEntropyLoss()(model(x), y)
    opt.zero_grad(); loss.backward(); opt.step()
print(f"Transformer: {sum(p.numel() for p in model.parameters()):,} params")
print(f"After 30 steps: loss={loss.item():.4f}, preds={model(x).argmax(1).tolist()}")

import torch, torch.nn as nn, numpy as np

torch.manual_seed(42)

def sdp_attention(Q, K, V):
    d_k = Q.size(-1)
    scores = torch.matmul(Q, K.transpose(-2,-1)) / (d_k**0.5)
    weights = torch.softmax(scores, dim=-1)
    return torch.matmul(weights, V), weights

# Simple sequence: "the cat sat on the mat"
tokens = ["the", "cat", "sat", "on", "the", "mat"]
d_model = 8
x = torch.randn(1, len(tokens), d_model)

# Single-head attention
W_q = nn.Linear(d_model, d_model, bias=False)
W_k = nn.Linear(d_model, d_model, bias=False)
W_v = nn.Linear(d_model, d_model, bias=False)

Q, K, V = W_q(x), W_k(x), W_v(x)
out, attn = sdp_attention(Q, K, V)
attn_matrix = attn[0].detach().numpy()

print("Attention weights matrix (rows=query, cols=key):")
print("Tokens:", tokens)
header = "      " + "".join(f"{t:>6}" for t in tokens)
print(header)
for i, row in enumerate(attn_matrix):
    row_str = f"{tokens[i]:>6}" + "".join(f"{v:>6.3f}" for v in row)
    print(row_str)

# Check: each row sums to 1
print(f"\nRow sums: {attn_matrix.sum(axis=1).round(4)}")
print(f"Diagonal (self-attention strength): {attn_matrix.diagonal().round(4)}")

import torch, torch.nn as nn, numpy as np

torch.manual_seed(42); np.random.seed(42)

class PlainNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(40, 128), nn.ReLU(),
                                  nn.Linear(128, 64), nn.ReLU(), nn.Linear(64, 1))
    def forward(self, x): return self.net(x).squeeze()

class RegNet(nn.Module):
    def __init__(self, dropout=0.3):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(40, 128), nn.BatchNorm1d(128), nn.ReLU(), nn.Dropout(dropout),
            nn.Linear(128, 64), nn.BatchNorm1d(64), nn.ReLU(), nn.Dropout(dropout),
            nn.Linear(64, 1))
    def forward(self, x): return self.net(x).squeeze()

X_tr = torch.randn(100, 40); y_tr = (X_tr[:, 0] > 0).float()
X_va = torch.randn(500, 40); y_va = (X_va[:, 0] > 0).float()
crit = nn.BCEWithLogitsLoss()

def train(model, n=100, wd=0.0):
    opt = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=wd)
    for _ in range(n):
        model.train()
        loss = crit(model(X_tr), y_tr); opt.zero_grad(); loss.backward(); opt.step()
    model.eval()
    with torch.no_grad():
        tr_acc = ((model(X_tr)>0).float()==y_tr).float().mean().item()
        va_acc = ((model(X_va)>0).float()==y_va).float().mean().item()
    return tr_acc, va_acc

for name, model, wd in [
    ("No regularization", PlainNet(), 0),
    ("BN + Dropout(0.3)", RegNet(0.3), 0),
    ("BN + Drop + L2", RegNet(0.3), 1e-4),
]:
    tr, va = train(model, n=100, wd=wd)
    print(f"{name:<22}: train={tr:.4f}, val={va:.4f}, overfit={tr-va:.4f}")

import torch, torch.nn as nn

torch.manual_seed(42)
B, T, D = 4, 10, 32
x = torch.randn(B, T, D)

# BatchNorm (normalizes over N,L per channel D)
bn = nn.BatchNorm1d(D)
x2d = x.view(-1, D)
out_bn = bn(x2d).view(B, T, D)

# LayerNorm (normalizes over D per token position)
ln = nn.LayerNorm(D)
out_ln = ln(x)

print(f"Input:     mean={x.mean():.4f}, std={x.std():.4f}")
print(f"BatchNorm: mean={out_bn.mean():.6f}, std={out_bn.std():.4f}")
print(f"LayerNorm: mean={out_ln.mean():.6f}, std={out_ln.std():.4f}")

# Verify LayerNorm normalizes per token
for pos in [0, 3, 7]:
    m = out_ln[0, pos].mean().item()
    s = out_ln[0, pos].std().item()
    print(f"  LayerNorm pos {pos}: mean={m:.6f}, std={s:.4f}")

guide = {
    "BatchNorm":  "CNNs, fixed-length, large batches (N >= 16)",
    "LayerNorm":  "Transformers, NLP, variable-length sequences",
    "GroupNorm":  "Small batches (N < 8), object detection",
    "InstanceNorm": "Style transfer, per-sample normalization",
}
print("\nNormalization Guide:")
for k, v in guide.items():
    print(f"  {k:<14}: {v}")

import torch, torch.nn as nn, numpy as np

torch.manual_seed(42)

# Critical: model.eval() disables dropout and uses running BN stats
class NetWithDropout(nn.Module):
    def __init__(self, dropout=0.5):
        super().__init__()
        self.fc1 = nn.Linear(20, 64)
        self.bn = nn.BatchNorm1d(64)
        self.drop = nn.Dropout(dropout)
        self.fc2 = nn.Linear(64, 1)
    def forward(self, x):
        return self.fc2(self.drop(torch.relu(self.bn(self.fc1(x))))).squeeze()

model = NetWithDropout(dropout=0.5)
x = torch.randn(10, 20)

# Demonstrate train vs eval mode difference
model.train()
out_train1 = model(x).detach()
out_train2 = model(x).detach()

model.eval()
out_eval1 = model(x).detach()
out_eval2 = model(x).detach()

print("Train mode outputs (different each call due to dropout):")
print(f"  Call 1: {out_train1[:5].numpy().round(3)}")
print(f"  Call 2: {out_train2[:5].numpy().round(3)}")
print(f"  Same:   {torch.allclose(out_train1, out_train2)}")

print("\nEval mode outputs (deterministic):")
print(f"  Call 1: {out_eval1[:5].numpy().round(3)}")
print(f"  Call 2: {out_eval2[:5].numpy().round(3)}")
print(f"  Same:   {torch.allclose(out_eval1, out_eval2)}")

print("\nKey rule: ALWAYS call model.eval() before inference!")

import torch, torch.nn as nn

torch.manual_seed(42)

def run_scheduler(sched_name, n_epochs=60, base_lr=0.1):
    model = nn.Linear(10, 1)
    opt = torch.optim.SGD(model.parameters(), lr=base_lr)
    if sched_name == "StepLR":
        sched = torch.optim.lr_scheduler.StepLR(opt, step_size=15, gamma=0.5)
    elif sched_name == "CosineAnnealing":
        sched = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=n_epochs)
    elif sched_name == "ExponentialLR":
        sched = torch.optim.lr_scheduler.ExponentialLR(opt, gamma=0.95)
    else:
        sched = torch.optim.lr_scheduler.OneCycleLR(
            opt, max_lr=base_lr, total_steps=n_epochs)

    lrs = [opt.param_groups[0]['lr']]
    for _ in range(n_epochs):
        sched.step()
        lrs.append(opt.param_groups[0]['lr'])
    return lrs

schedules = ["StepLR", "CosineAnnealing", "ExponentialLR", "OneCycleLR"]
checkpoints = [0, 15, 30, 45, 60]

for name in schedules:
    lrs = run_scheduler(name)
    vals = [f"{lrs[i]:.5f}" for i in checkpoints]
    print(f"{name:<20}: {' -> '.join(vals)}")

import torch, torch.nn as nn, math

class WarmupCosine:
    def __init__(self, opt, warmup, total, min_lr=1e-6):
        self.opt = opt
        self.warmup = warmup
        self.total = total
        self.min_lr = min_lr
        self.base_lr = opt.param_groups[0]['lr']
        self.step_n = 0

    def step(self):
        self.step_n += 1
        if self.step_n <= self.warmup:
            lr = self.base_lr * self.step_n / self.warmup
        else:
            p = (self.step_n - self.warmup) / (self.total - self.warmup)
            lr = self.min_lr + 0.5*(self.base_lr-self.min_lr)*(1+math.cos(math.pi*p))
        for g in self.opt.param_groups: g['lr'] = lr
        return lr

torch.manual_seed(42)
model = nn.Linear(10, 1)
opt = torch.optim.AdamW(model.parameters(), lr=1e-3)
sched = WarmupCosine(opt, warmup=10, total=100)

lrs = [sched.step() for _ in range(100)]
print("Warmup + Cosine LR at key steps:")
for s in [1, 5, 10, 25, 50, 75, 100]:
    print(f"  Step {s:>3}: {lrs[s-1]:.6f}")

# Simulate training
X = torch.randn(100, 10); y = torch.randn(100, 1)
opt2 = torch.optim.AdamW(model.parameters(), lr=1e-3)
sched2 = WarmupCosine(opt2, warmup=10, total=100)
for step in range(100):
    loss = nn.MSELoss()(model(X), y)
    opt2.zero_grad(); loss.backward(); opt2.step()
    sched2.step()
    if (step+1) % 25 == 0:
        print(f"  Step {step+1}: loss={loss.item():.4f}, lr={opt2.param_groups[0]['lr']:.6f}")

import torch, torch.nn as nn, numpy as np

torch.manual_seed(42)

# ReduceLROnPlateau: auto-reduces LR when metric stagnates
model = nn.Sequential(nn.Linear(20, 64), nn.ReLU(), nn.Linear(64, 1))
opt = torch.optim.Adam(model.parameters(), lr=1e-2)
plateau_sched = torch.optim.lr_scheduler.ReduceLROnPlateau(
    opt, mode='min', factor=0.5, patience=5, min_lr=1e-6, verbose=False)

X = torch.randn(200, 20); y = X[:, :1]
X_val = torch.randn(50, 20); y_val = X_val[:, :1]
crit = nn.MSELoss()

lr_history = []
for epoch in range(60):
    model.train()
    loss = crit(model(X), y)
    opt.zero_grad(); loss.backward(); opt.step()
    model.eval()
    with torch.no_grad():
        val_loss = crit(model(X_val), y_val).item()
    plateau_sched.step(val_loss)
    lr_history.append(opt.param_groups[0]['lr'])
    if (epoch+1) % 10 == 0:
        print(f"Epoch {epoch+1}: val_loss={val_loss:.4f}, lr={opt.param_groups[0]['lr']:.6f}")

# Simple LR range test (find optimal LR)
print("\nLR Range Test (exponential sweep):")
model2 = nn.Linear(20, 1)
opt2 = torch.optim.SGD(model2.parameters(), lr=1e-6)
min_lr, max_lr, n_steps = 1e-6, 1e-1, 20
lr_mult = (max_lr/min_lr)**(1/n_steps)
for step in range(n_steps):
    lr = min_lr * (lr_mult**step)
    for g in opt2.param_groups: g['lr'] = lr
    loss = crit(model2(X[:32]), y[:32])
    opt2.zero_grad(); loss.backward(); opt2.step()
    if step % 5 == 0:
        print(f"  lr={lr:.2e}: loss={loss.item():.4f}")

import torch, torch.nn as nn, os

torch.manual_seed(42)

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(20,64), nn.ReLU(), nn.Linear(64,1))
    def forward(self, x): return self.net(x).squeeze()

def save_ckpt(model, opt, epoch, val_loss, path):
    torch.save({'epoch': epoch, 'model': model.state_dict(),
                'optimizer': opt.state_dict(), 'val_loss': val_loss}, path)
    print(f"  Saved: epoch={epoch}, val_loss={val_loss:.4f}")

def load_ckpt(model, opt, path):
    if not os.path.exists(path): return 0, float('inf')
    ckpt = torch.load(path, weights_only=True)
    model.load_state_dict(ckpt['model'])
    opt.load_state_dict(ckpt['optimizer'])
    print(f"  Resumed from epoch {ckpt['epoch']}")
    return ckpt['epoch'], ckpt['val_loss']

model = Net()
opt = torch.optim.Adam(model.parameters(), lr=1e-3)
crit = nn.MSELoss()
X_tr = torch.randn(200,20); y_tr = X_tr[:,0]
X_va = torch.randn(50,20); y_va = X_va[:,0]
best_val, ckpt_path = float('inf'), "/tmp/best.pt"

for epoch in range(30):
    model.train()
    loss = crit(model(X_tr), y_tr); opt.zero_grad(); loss.backward(); opt.step()
    model.eval()
    with torch.no_grad():
        vl = crit(model(X_va), y_va).item()
    if vl < best_val:
        best_val = vl; save_ckpt(model, opt, epoch+1, vl, ckpt_path)

# Reload best
model2 = Net(); opt2 = torch.optim.Adam(model2.parameters())
load_ckpt(model2, opt2, ckpt_path)
model2.eval()
with torch.no_grad():
    print(f"Loaded model val_loss: {crit(model2(X_va), y_va).item():.4f}")

import torch, torch.nn as nn, copy

class EarlyStopping:
    def __init__(self, patience=10, min_delta=1e-4):
        self.patience = patience
        self.min_delta = min_delta
        self.best_loss = float('inf')
        self.best_weights = None
        self.counter = 0

    def __call__(self, model, val_loss):
        if val_loss < self.best_loss - self.min_delta:
            self.best_loss = val_loss
            self.best_weights = copy.deepcopy(model.state_dict())
            self.counter = 0
        else:
            self.counter += 1
        if self.counter >= self.patience:
            model.load_state_dict(self.best_weights)
            return True
        return False

torch.manual_seed(42)
model = nn.Sequential(nn.Linear(10,32), nn.ReLU(), nn.Linear(32,1))
opt = torch.optim.Adam(model.parameters(), lr=1e-2)
crit = nn.MSELoss()
es = EarlyStopping(patience=8, min_delta=1e-3)

X = torch.randn(100,10); y = X[:,0:1]
Xv = torch.randn(50,10); yv = Xv[:,0:1]

for epoch in range(200):
    model.train()
    loss = crit(model(X), y); opt.zero_grad(); loss.backward(); opt.step()
    model.eval()
    with torch.no_grad():
        vl = crit(model(Xv), yv).item()
    if (epoch+1)%20==0:
        print(f"Epoch {epoch+1}: val={vl:.4f}, patience_ctr={es.counter}")
    if es(model, vl):
        print(f"Early stop at epoch {epoch+1}, best_val={es.best_loss:.4f}")
        break

import torch, torch.nn as nn, copy, os

class TrainingManager:
    def __init__(self, model, optimizer, patience=10, ckpt_every=5, save_dir="/tmp"):
        self.model = model; self.opt = optimizer
        self.patience = patience; self.ckpt_every = ckpt_every
        self.save_dir = save_dir; self.best_loss = float('inf')
        self.best_state = None; self.counter = 0; self.history = []

    def update(self, epoch, train_loss, val_loss):
        self.history.append({'epoch': epoch, 'train': train_loss, 'val': val_loss})
        if (epoch+1) % self.ckpt_every == 0:
            path = os.path.join(self.save_dir, f"ckpt_ep{epoch+1}.pt")
            torch.save({'epoch': epoch+1, 'model': self.model.state_dict(), 'val': val_loss}, path)
            print(f"  [ckpt] ep{epoch+1} saved")
        if val_loss < self.best_loss - 1e-4:
            self.best_loss = val_loss
            self.best_state = copy.deepcopy(self.model.state_dict())
            self.counter = 0
        else:
            self.counter += 1
        if self.counter >= self.patience:
            self.model.load_state_dict(self.best_state)
            print(f"  Early stop ep {epoch+1}, best={self.best_loss:.4f}")
            return True
        return False

torch.manual_seed(42)
model = nn.Sequential(nn.Linear(20,64), nn.ReLU(), nn.Linear(64,1))
opt = torch.optim.Adam(model.parameters(), lr=5e-3)
mgr = TrainingManager(model, opt, patience=10, ckpt_every=5)
X = torch.randn(200,20); y = X[:,:1]; Xv = torch.randn(50,20); yv = Xv[:,:1]
crit = nn.MSELoss()

for epoch in range(80):
    model.train()
    l = crit(model(X), y); opt.zero_grad(); l.backward(); opt.step()
    model.eval()
    with torch.no_grad():
        vl = crit(model(Xv), yv).item()
    if (epoch+1)%20==0:
        print(f"Epoch {epoch+1}: train={l.item():.4f}, val={vl:.4f}")
    if mgr.update(epoch, l.item(), vl): break

import torch, torch.nn as nn, numpy as np

torch.manual_seed(42)

class DeepRNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.rnn = nn.RNN(10, 64, 4, batch_first=True)
        self.fc = nn.Linear(64, 1)
    def forward(self, x):
        out, _ = self.rnn(x)
        return self.fc(out[:, -1]).squeeze()

def grad_norm(model):
    return sum(p.grad.data.norm(2).item()**2 for p in model.parameters()
               if p.grad is not None)**0.5

X = torch.randn(16, 50, 10); y = torch.randn(16)
model = DeepRNN()
opt = torch.optim.SGD(model.parameters(), lr=0.1)
crit = nn.MSELoss()

print("Gradient norms (before/after clip):")
for step in range(10):
    loss = crit(model(X), y)
    opt.zero_grad(); loss.backward()
    gn_before = grad_norm(model)
    torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
    gn_after = grad_norm(model)
    opt.step()
    if step < 5:
        print(f"  Step {step+1}: before={gn_before:.2f}, after={gn_after:.2f} (clipped={gn_before>1.0})")

import torch, torch.nn as nn, time

torch.manual_seed(42)

class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(128, 256), nn.ReLU(), nn.Linear(256, 128), nn.ReLU(), nn.Linear(128, 10))
    def forward(self, x): return self.net(x)

X = torch.randn(256, 128); y = torch.randint(0, 10, (256,))
crit = nn.CrossEntropyLoss()

# Standard FP32
model1 = Model()
opt1 = torch.optim.AdamW(model1.parameters(), lr=1e-3)
t0 = time.time()
for _ in range(200):
    l = crit(model1(X), y); opt1.zero_grad(); l.backward(); opt1.step()
fp32_t = time.time()-t0

# AMP pattern (works on CPU too, GPU gets real speedup)
model2 = Model()
opt2 = torch.optim.AdamW(model2.parameters(), lr=1e-3)
scaler = torch.amp.GradScaler('cpu', enabled=False)
t0 = time.time()
for _ in range(200):
    with torch.amp.autocast('cpu', dtype=torch.float32):
        l = crit(model2(X), y)
    scaler.scale(l).backward()
    scaler.unscale_(opt2)
    torch.nn.utils.clip_grad_norm_(model2.parameters(), 1.0)
    scaler.step(opt2); scaler.update(); opt2.zero_grad()
amp_t = time.time()-t0

print(f"FP32 time: {fp32_t:.2f}s, AMP pattern time: {amp_t:.2f}s")
print(f"FP32 loss: {crit(model1(X),y).item():.4f}")
print(f"AMP  loss: {crit(model2(X),y).item():.4f}")
print("\nAMP Best Practices:")
for tip in ["Use autocast for forward pass only",
            "Use GradScaler to prevent FP16 underflow",
            "Clip gradients AFTER scaler.unscale_()",
            "BF16 more stable than FP16 (Ampere+ GPUs only)"]:
    print(f"  - {tip}")

import torch, torch.nn as nn, numpy as np

torch.manual_seed(42)

class MultiLayerNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.layers = nn.ModuleList([
            nn.Linear(20, 64), nn.Linear(64, 32), nn.Linear(32, 16), nn.Linear(16, 1)])
        self.relu = nn.ReLU()
    def forward(self, x):
        for layer in self.layers[:-1]:
            x = self.relu(layer(x))
        return self.layers[-1](x).squeeze()

model = MultiLayerNet()
opt = torch.optim.Adam(model.parameters(), lr=1e-3)
crit = nn.MSELoss()
X = torch.randn(64, 20); y = torch.randn(64)

print("Gradient statistics per layer:")
for epoch in [1, 10, 50]:
    for _ in range(epoch if epoch==1 else 9):
        loss = crit(model(X), y); opt.zero_grad(); loss.backward(); opt.step()
    print(f"\nEpoch {epoch}:")
    for name, p in model.named_parameters():
        if p.grad is not None:
            gn = p.grad.norm().item()
            wn = p.data.norm().item()
            print(f"  {name:<25}: grad_norm={gn:.4f}, weight_norm={wn:.4f}, ratio={gn/wn:.4f}")

# Detect vanishing/exploding gradients
print("\nGradient health check:")
for name, p in model.named_parameters():
    if p.grad is not None:
        gn = p.grad.norm().item()
        status = "EXPLODING" if gn > 10 else ("VANISHING" if gn < 1e-5 else "OK")
        print(f"  {name:<25}: {status} (norm={gn:.6f})")

import torch, torch.nn as nn, os

torch.manual_seed(42)

class Classifier(nn.Module):
    def __init__(self, n_in=20, n_classes=3):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(n_in, 64), nn.ReLU(), nn.Dropout(0.2),
            nn.Linear(64, 32), nn.ReLU(), nn.Linear(32, n_classes))
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return torch.softmax(self.net(x), dim=-1)

model = Classifier(); model.eval()
x = torch.randn(5, 20)
original_out = model(x)

# trace: for fixed control flow
traced = torch.jit.trace(model, x)

# script: for dynamic control flow
scripted = torch.jit.script(model)
scripted.save("/tmp/classifier.pt")

loaded = torch.jit.load("/tmp/classifier.pt"); loaded.eval()
loaded_out = loaded(x)

print(f"TorchScript size: {os.path.getsize('/tmp/classifier.pt')/1024:.1f} KB")
print(f"Outputs match: {torch.allclose(original_out, loaded_out, atol=1e-5)}")
print(f"Batch preds: {loaded(torch.randn(8,20)).argmax(1).tolist()}")

import torch, torch.nn as nn, os

torch.manual_seed(42)

class Regressor(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(10,32), nn.ReLU(), nn.Linear(32,1))
    def forward(self, x): return self.net(x)

model = Regressor(); model.eval()
dummy = torch.randn(1, 10)

torch.onnx.export(
    model, dummy, "/tmp/regressor.onnx",
    input_names=["features"], output_names=["prediction"],
    dynamic_axes={"features": {0: "batch"}, "prediction": {0: "batch"}},
    opset_version=17)

print(f"ONNX file: {os.path.getsize('/tmp/regressor.onnx')/1024:.1f} KB")

try:
    import onnx
    m = onnx.load("/tmp/regressor.onnx")
    onnx.checker.check_model(m)
    print("ONNX check: PASSED")
    print(f"  Inputs:  {[i.name for i in m.graph.input]}")
    print(f"  Outputs: {[o.name for o in m.graph.output]}")
except ImportError:
    print("Install onnx: pip install onnx")

# Deployment comparison
print("\nDeployment Format Comparison:")
for fmt, use_case in [
    ("TorchScript", "C++ microservice, mobile (TorchMobile)"),
    ("ONNX",        "Cross-framework, ONNX Runtime, mobile"),
    ("Pickle",      "Python-only, sklearn, quick prototyping"),
    ("TF SavedModel","TensorFlow Serving, TFLite mobile"),
]:
    print(f"  {fmt:<15}: {use_case}")

import torch, torch.nn as nn, pickle, json, time, os

torch.manual_seed(42)

# Full deployment pipeline: train -> validate -> export -> health check
class ProductionModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(10,32), nn.ReLU(), nn.Linear(32,3))
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return torch.softmax(self.net(x), dim=-1)

# Train
model = ProductionModel()
X = torch.randn(200, 10); y = torch.randint(0, 3, (200,))
opt = torch.optim.Adam(model.parameters(), lr=1e-3)
for _ in range(100):
    l = nn.CrossEntropyLoss()(model(X), y); opt.zero_grad(); l.backward(); opt.step()

model.eval()
test_acc = (model(X).argmax(1)==y).float().mean().item()
print(f"Model accuracy: {test_acc:.4f}")

# Export
scripted = torch.jit.script(model)
scripted.save("/tmp/prod_model.pt")
scripted_size = os.path.getsize("/tmp/prod_model.pt")/1024

# Health check function
def health_check(model_path, test_input_shape=(1, 10)):
    loaded = torch.jit.load(model_path); loaded.eval()
    x_test = torch.randn(*test_input_shape)
    t0 = time.time()
    with torch.no_grad():
        out = loaded(x_test)
    latency_ms = (time.time()-t0)*1000
    return {
        "status": "healthy",
        "output_shape": list(out.shape),
        "output_sum_to_1": bool(abs(out.sum().item()-1) < 1e-4),
        "latency_ms": round(latency_ms, 3),
        "model_size_kb": round(scripted_size, 1),
    }

health = health_check("/tmp/prod_model.pt")
print("\nHealth Check:", json.dumps(health, indent=2))

# Deployment manifest
manifest = {
    "model_path": "/tmp/prod_model.pt",
    "format": "TorchScript",
    "input": {"name": "features", "shape": [-1, 10], "dtype": "float32"},
    "output": {"name": "probabilities", "shape": [-1, 3]},
    "accuracy": round(test_acc, 4),
}
print("\nDeployment Manifest:", json.dumps(manifest, indent=2))