Gradient Boosting - XGBoost, LightGBM, CatBoost

Gradient Boosting is the most powerful technique for tabular data! Algorithms like XGBoost win most ML competitions.

XGBoost

1pip install xgboost

1import xgboost as xgb
2from sklearn.model_selection import train_test_split
3
4# Safari data
5X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
6
7# XGBoost model
8model = xgb.XGBClassifier(
9    n_estimators=100,           # number of trees
10    max_depth=6,                # tree depth
11    learning_rate=0.1,          # learning rate
12    subsample=0.8,              # % of samples per tree
13    colsample_bytree=0.8,       # % of features per tree
14    early_stopping_rounds=10,   # early stopping (XGBoost 2.0+: in the constructor)
15    random_state=42
16)
17
18# Training with early stopping - eval_set monitors the validation set
19model.fit(
20    X_train, y_train,
21    eval_set=[(X_test, y_test)]
22)
23
24# Predictions
25predictions = model.predict(X_test)
26probabilities = model.predict_proba(X_test)
27
28# Feature importance
29import matplotlib.pyplot as plt
30xgb.plot_importance(model)
31plt.show()

LightGBM

1pip install lightgbm

1import lightgbm as lgb
2
3# LightGBM model - faster than XGBoost for large datasets
4model = lgb.LGBMClassifier(
5    n_estimators=100,
6    max_depth=6,
7    learning_rate=0.1,
8    num_leaves=31,         # maximum number of leaves
9    min_child_samples=20,  # min samples in a leaf
10    random_state=42
11)
12
13model.fit(
14    X_train, y_train,
15    eval_set=[(X_test, y_test)]
16)
17
18# Feature importance
19lgb.plot_importance(model, max_num_features=10)
20plt.show()

CatBoost

1pip install catboost

1from catboost import CatBoostClassifier
2
3# CatBoost - best for categorical data
4model = CatBoostClassifier(
5    iterations=100,
6    depth=6,
7    learning_rate=0.1,
8    cat_features=['habitat', 'diet'],  # categorical columns
9    verbose=False
10)
11
12model.fit(X_train, y_train, eval_set=(X_test, y_test))
13
14# Feature importance
15feature_importance = model.get_feature_importance()

Hyperparameter Tuning

1from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
2import xgboost as xgb
3
4# Grid Search - searches all combinations
5param_grid = {
6    'max_depth': [3, 5, 7],
7    'learning_rate': [0.01, 0.1, 0.2],
8    'n_estimators': [50, 100, 200],
9    'subsample': [0.7, 0.8, 0.9]
10}
11
12model = xgb.XGBClassifier()
13grid_search = GridSearchCV(
14    model, param_grid,
15    cv=5,
16    scoring='accuracy',
17    n_jobs=-1
18)
19grid_search.fit(X_train, y_train)
20
21print(f"Best params: {grid_search.best_params_}")
22print(f"Best score: {grid_search.best_score_:.4f}")
23best_model = grid_search.best_estimator_
24
25# Random Search - random combinations (faster)
26from scipy.stats import uniform, randint
27
28param_dist = {
29    'max_depth': randint(3, 10),
30    'learning_rate': uniform(0.01, 0.2),
31    'n_estimators': randint(50, 300)
32}
33
34random_search = RandomizedSearchCV(
35    model, param_dist,
36    n_iter=50,     # number of random combinations
37    cv=5,
38    random_state=42
39)
40random_search.fit(X_train, y_train)

Optuna - Advanced Tuning

1pip install optuna

1import optuna
2
3def objective(trial):
4    params = {
5        'max_depth': trial.suggest_int('max_depth', 3, 10),
6        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),
7        'n_estimators': trial.suggest_int('n_estimators', 50, 300),
8        'subsample': trial.suggest_float('subsample', 0.6, 1.0)
9    }
10
11    model = xgb.XGBClassifier(**params)
12    model.fit(X_train, y_train)
13    accuracy = model.score(X_val, y_val)
14    return accuracy
15
16# Optimization
17study = optuna.create_study(direction='maximize')
18study.optimize(objective, n_trials=100)
19
20print(f"Best params: {study.best_params}")
21print(f"Best accuracy: {study.best_value:.4f}")