Loss functions measure how wrong your model's predictions are. Choosing the right loss function is crucial—it defines what "good" means and guides the optimization process.

Regression Loss Functions

Mean Squared Error (MSE)

import numpy as np

def mse(y_true, y_pred):
    return np.mean((y_true - y_pred)**2)

# Most common for regression
# Penalizes large errors more (squared term)
# Sensitive to outliers

# Example
y_true = np.array([3, -0.5, 2, 7])
y_pred = np.array([2.5, 0.0, 2, 8])
print(f"MSE: {mse(y_true, y_pred):.3f}")

Mean Absolute Error (MAE)

def mae(y_true, y_pred):
    return np.mean(np.abs(y_true - y_pred))

# Less sensitive to outliers than MSE
# All errors weighted equally
# Use when outliers shouldn't dominate

print(f"MAE: {mae(y_true, y_pred):.3f}")

Huber Loss (Hybrid)

def huber_loss(y_true, y_pred, delta=1.0):
    error = y_true - y_pred
    is_small = np.abs(error) <= delta
    squared_loss = 0.5 * error**2
    linear_loss = delta * (np.abs(error) - 0.5 * delta)
    return np.mean(np.where(is_small, squared_loss, linear_loss))

# MSE for small errors, MAE for large errors
# Robust to outliers while maintaining smoothness

Classification Loss Functions

Binary Cross-Entropy

def binary_crossentropy(y_true, y_pred):
    # Clip predictions to avoid log(0)
    y_pred = np.clip(y_pred, 1e-7, 1 - 1e-7)
    return -np.mean(y_true * np.log(y_pred) +
                    (1 - y_true) * np.log(1 - y_pred))

# For binary classification (0 or 1)
# Penalizes confident wrong predictions heavily
# Most common for logistic regression

y_true = np.array([1, 0, 1, 0])
y_pred = np.array([0.9, 0.1, 0.8, 0.3])
print(f"Binary Cross-Entropy: {binary_crossentropy(y_true, y_pred):.3f}")

Categorical Cross-Entropy

def categorical_crossentropy(y_true, y_pred):
    # y_true: one-hot encoded
    # y_pred: predicted probabilities
    y_pred = np.clip(y_pred, 1e-7, 1)
    return -np.sum(y_true * np.log(y_pred)) / len(y_true)

# For multi-class classification
# Use with softmax output layer

# Example: 3 classes
y_true = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
y_pred = np.array([[0.7, 0.2, 0.1], [0.1, 0.8, 0.1], [0.2, 0.2, 0.6]])
print(f"Categorical Cross-Entropy: {categorical_crossentropy(y_true, y_pred):.3f}")

Choosing the Right Loss Function

Task	Loss Function
Linear Regression	MSE
Regression with outliers	MAE or Huber
Binary Classification	Binary Cross-Entropy
Multi-class Classification	Categorical Cross-Entropy
Imbalanced Classification	Weighted Cross-Entropy

Using Loss Functions in sklearn

from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.metrics import mean_squared_error, log_loss

# Regression
model = LinearRegression()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
mse = mean_squared_error(y_test, y_pred)

# Classification
model = LogisticRegression()
model.fit(X_train, y_train)
y_pred_proba = model.predict_proba(X_test)
cross_entropy = log_loss(y_test, y_pred_proba)

Custom Loss Functions

# Example: Asymmetric loss (penalize under-prediction more)
def asymmetric_mse(y_true, y_pred, weight_over=1.0, weight_under=2.0):
    error = y_true - y_pred
    loss = np.where(error > 0,
                   weight_under * error**2,  # Under-predicted
                   weight_over * error**2)    # Over-predicted
    return np.mean(loss)

# Useful when one type of error is costlier
# Example: Inventory (under-stock worse than over-stock)

Monitoring Loss During Training

import matplotlib.pyplot as plt

# Track training and validation loss
history = {
    'train_loss': [],
    'val_loss': []
}

for epoch in range(100):
    # Train
    train_loss = calculate_loss(train_data)
    history['train_loss'].append(train_loss)

    # Validate
    val_loss = calculate_loss(val_data)
    history['val_loss'].append(val_loss)

# Plot
plt.plot(history['train_loss'], label='Training Loss')
plt.plot(history['val_loss'], label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()

# Validation loss increasing = overfitting!