Neural Networks

Let's build neural networks in PyTorch! 🧠

Simple Neural Network

1import torch
2import torch.nn as nn
3import torch.optim as optim
4
5# Define architecture
6class SafariClassifier(nn.Module):
7    def __init__(self, input_size, hidden_size, num_classes):
8        super(SafariClassifier, self).__init__()
9
10        self.layer1 = nn.Linear(input_size, hidden_size)
11        self.relu = nn.ReLU()
12        self.layer2 = nn.Linear(hidden_size, hidden_size)
13        self.layer3 = nn.Linear(hidden_size, num_classes)
14        self.dropout = nn.Dropout(0.3)
15
16    def forward(self, x):
17        x = self.layer1(x)
18        x = self.relu(x)
19        x = self.dropout(x)
20        x = self.layer2(x)
21        x = self.relu(x)
22        x = self.layer3(x)
23        return x
24
25# Create model
26model = SafariClassifier(input_size=10, hidden_size=64, num_classes=3)
27print(model)

Sequential API

1# Simpler way to define a model
2model = nn.Sequential(
3    nn.Linear(10, 64),
4    nn.ReLU(),
5    nn.Dropout(0.3),
6    nn.Linear(64, 32),
7    nn.ReLU(),
8    nn.Linear(32, 3)
9)

Training the Model

1# Preparation
2device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
3model = SafariClassifier(10, 64, 3).to(device)
4
5# Loss function and optimizer
6criterion = nn.CrossEntropyLoss()
7optimizer = optim.Adam(model.parameters(), lr=0.001)
8
9# Training loop
10num_epochs = 100
11
12for epoch in range(num_epochs):
13    model.train()
14    total_loss = 0
15
16    for batch_X, batch_y in train_loader:
17        batch_X, batch_y = batch_X.to(device), batch_y.to(device)
18
19        # Forward pass
20        outputs = model(batch_X)
21        loss = criterion(outputs, batch_y)
22
23        # Backward pass
24        optimizer.zero_grad()
25        loss.backward()
26        optimizer.step()
27
28        total_loss += loss.item()
29
30    if (epoch + 1) % 10 == 0:
31        print(f"Epoch [{epoch+1}/{num_epochs}], Loss: {total_loss:.4f}")

Evaluation

1def evaluate(model, test_loader, device):
2    model.eval()
3    correct = 0
4    total = 0
5
6    with torch.no_grad():  # Disable gradients
7        for batch_X, batch_y in test_loader:
8            batch_X, batch_y = batch_X.to(device), batch_y.to(device)
9
10            outputs = model(batch_X)
11            _, predicted = torch.max(outputs, 1)
12
13            total += batch_y.size(0)
14            correct += (predicted == batch_y).sum().item()
15
16    accuracy = correct / total
17    return accuracy
18
19accuracy = evaluate(model, test_loader, device)
20print(f"Test Accuracy: {accuracy:.2%}")

Activation Functions

1# ReLU - most popular
2relu = nn.ReLU()
3
4# Sigmoid - for binary output
5sigmoid = nn.Sigmoid()
6
7# Softmax - for multi-class classification
8softmax = nn.Softmax(dim=1)
9
10# Tanh
11tanh = nn.Tanh()
12
13# LeakyReLU - prevents "dying" neurons
14leaky_relu = nn.LeakyReLU(0.01)

Regularization

1# Dropout - randomly disables neurons
2dropout = nn.Dropout(p=0.5)
3
4# Batch Normalization - normalizes layer outputs
5batch_norm = nn.BatchNorm1d(num_features=64)
6
7# L2 regularization (weight decay)
8optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)

Saving and Loading a Model

1# Save model
2torch.save(model.state_dict(), 'safari_model.pth')
3
4# Load model
5model = SafariClassifier(10, 64, 3)
6model.load_state_dict(torch.load('safari_model.pth'))
7model.eval()
8
9# Save entire model (less flexible)
10torch.save(model, 'full_model.pth')
11model = torch.load('full_model.pth')