Scikit-Learn Model Selection Patterns¶

In [1]:
import pandas as pd
from sklearn.datasets import load_iris

X, y = load_iris(return_X_y=True)
In [2]:
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV, StratifiedKFold
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import StandardScaler

# Make the model
pipeline = Pipeline([("scale", None), ("predict", None)])

# Set up some parameters
params = [
    {
        "scale": [None, StandardScaler()],
        "predict": [LogisticRegression(max_iter=1000)],
        "predict__C": [0.1, 1, 10, 100, 1000],
    },
    {
        "scale": [None, StandardScaler()],
        "predict": [MLPClassifier(max_iter=1000)],
    },
]

gs = GridSearchCV(
    pipeline,
    params,
    scoring=["accuracy", "f1_macro"],
    refit="accuracy",
    cv=StratifiedKFold(),
)
gs.fit(X, y)

res = pd.DataFrame(gs.cv_results_)
pd.options.display.max_colwidth = 100

print(gs.best_estimator_)
remove = (
    lambda x: x.startswith("split")
    or x.endswith("time")
    or x.startswith("std")
    or x == "params"
    or x.startswith("rank")
)
cols = [c for c in res.columns if not remove(c)]
res[cols]

Pipeline(steps=[('scale', None),
                ('predict', LogisticRegression(C=100, max_iter=1000))])
Out[2]:
param_predict param_predict__C param_scale mean_test_accuracy mean_test_f1_macro
0 LogisticRegression(C=100, max_iter=1000) 0.1 None 0.946667 0.946229
1 LogisticRegression(C=100, max_iter=1000) 0.1 StandardScaler() 0.926667 0.925698
2 LogisticRegression(C=100, max_iter=1000) 1 None 0.973333 0.973165
3 LogisticRegression(C=100, max_iter=1000) 1 StandardScaler() 0.960000 0.959798
4 LogisticRegression(C=100, max_iter=1000) 10 None 0.973333 0.973300
5 LogisticRegression(C=100, max_iter=1000) 10 StandardScaler() 0.973333 0.973199
6 LogisticRegression(C=100, max_iter=1000) 100 None 0.980000 0.979983
7 LogisticRegression(C=100, max_iter=1000) 100 StandardScaler() 0.973333 0.973199
8 LogisticRegression(C=100, max_iter=1000) 1000 None 0.973333 0.973199
9 LogisticRegression(C=100, max_iter=1000) 1000 StandardScaler() 0.973333 0.973199
10 MLPClassifier(max_iter=1000) NaN None 0.973333 0.973199
11 MLPClassifier(max_iter=1000) NaN StandardScaler() 0.960000 0.959832