Gradient Boosting - XGBoost, LightGBM, CatBoost

Gradient Boosting to najpotężniejsza technika dla danych tabelarycznych! Algorytmy takie jak XGBoost wygrywają większość konkursów ML.

XGBoost

1pip install xgboost

1import xgboost as xgb
2from sklearn.model_selection import train_test_split
3
4# Dane Safari
5X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
6
7# Model XGBoost
8model = xgb.XGBClassifier(
9    n_estimators=100,           # liczba drzew
10    max_depth=6,                # głębokość drzewa
11    learning_rate=0.1,          # szybkość uczenia
12    subsample=0.8,              # % próbek na drzewo
13    colsample_bytree=0.8,       # % cech na drzewo
14    early_stopping_rounds=10,   # early stopping (XGBoost 2.0+: w konstruktorze)
15    random_state=42
16)
17
18# Trenowanie z early stopping - eval_set monitoruje zbiór walidacyjny
19model.fit(
20    X_train, y_train,
21    eval_set=[(X_test, y_test)]
22)
23
24# Przewidywanie
25predictions = model.predict(X_test)
26probabilities = model.predict_proba(X_test)
27
28# Feature importance
29import matplotlib.pyplot as plt
30xgb.plot_importance(model)
31plt.show()

LightGBM

1pip install lightgbm

1import lightgbm as lgb
2
3# Model LightGBM - szybszy niż XGBoost dla dużych danych
4model = lgb.LGBMClassifier(
5    n_estimators=100,
6    max_depth=6,
7    learning_rate=0.1,
8    num_leaves=31,         # maksymalna liczba liści
9    min_child_samples=20,  # min próbek w liściu
10    random_state=42
11)
12
13model.fit(
14    X_train, y_train,
15    eval_set=[(X_test, y_test)]
16)
17
18# Feature importance
19lgb.plot_importance(model, max_num_features=10)
20plt.show()

CatBoost

1pip install catboost

1from catboost import CatBoostClassifier
2
3# CatBoost - najlepszy dla danych kategorycznych
4model = CatBoostClassifier(
5    iterations=100,
6    depth=6,
7    learning_rate=0.1,
8    cat_features=['habitat', 'diet'],  # kolumny kategoryczne
9    verbose=False
10)
11
12model.fit(X_train, y_train, eval_set=(X_test, y_test))
13
14# Feature importance
15feature_importance = model.get_feature_importance()

Hyperparameter Tuning

1from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
2import xgboost as xgb
3
4# Grid Search - przeszukuje wszystkie kombinacje
5param_grid = {
6    'max_depth': [3, 5, 7],
7    'learning_rate': [0.01, 0.1, 0.2],
8    'n_estimators': [50, 100, 200],
9    'subsample': [0.7, 0.8, 0.9]
10}
11
12model = xgb.XGBClassifier()
13grid_search = GridSearchCV(
14    model, param_grid,
15    cv=5,
16    scoring='accuracy',
17    n_jobs=-1
18)
19grid_search.fit(X_train, y_train)
20
21print(f"Best params: {grid_search.best_params_}")
22print(f"Best score: {grid_search.best_score_:.4f}")
23best_model = grid_search.best_estimator_
24
25# Random Search - losowe kombinacje (szybszy)
26from scipy.stats import uniform, randint
27
28param_dist = {
29    'max_depth': randint(3, 10),
30    'learning_rate': uniform(0.01, 0.2),
31    'n_estimators': randint(50, 300)
32}
33
34random_search = RandomizedSearchCV(
35    model, param_dist,
36    n_iter=50,     # liczba losowych kombinacji
37    cv=5,
38    random_state=42
39)
40random_search.fit(X_train, y_train)

Optuna - zaawansowane strojenie

1pip install optuna

1import optuna
2
3def objective(trial):
4    params = {
5        'max_depth': trial.suggest_int('max_depth', 3, 10),
6        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),
7        'n_estimators': trial.suggest_int('n_estimators', 50, 300),
8        'subsample': trial.suggest_float('subsample', 0.6, 1.0)
9    }
10
11    model = xgb.XGBClassifier(**params)
12    model.fit(X_train, y_train)
13    accuracy = model.score(X_val, y_val)
14    return accuracy
15
16# Optymalizacja
17study = optuna.create_study(direction='maximize')
18study.optimize(objective, n_trials=100)
19
20print(f"Best params: {study.best_params}")
21print(f"Best accuracy: {study.best_value:.4f}")