I have a simple binary classification experiment I am trying to perform. Here is the code
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC
from sklearn.model_selection import cross_val_score
import xgboost as xgb
models = [
RandomForestClassifier(n_estimators=100, max_depth=3, random_state=0),
LinearSVC(),
MultinomialNB(),
LogisticRegression(random_state=0),
xgb.XGBClassifier(max_depth=3,
objective='binary:logistic',
n_estimators=100,
num_classes=2,
n_jobs = -1)
]
CV = 5
cv_df = pd.DataFrame(index=range(CV * len(models)))
entries = []
for model in models:
model_name = model.__class__.__name__
f1_score = cross_val_score(model, X_prep, labels, scoring = make_scorer(f1_score, average='weighted', labels=[2]), cv=CV)
for fold_idx, f1score in enumerate(f1_score):
entries.append((model_name, fold_idx, f1score))
cv_df = pd.DataFrame(entries, columns=['model_name', 'fold_idx', 'f1score'])
I am getting the following error message:
---------------------------------------------------------------------------
InvalidParameterError Traceback (most recent call last)
Cell In[34], line 22
20 for model in models:
21 model_name = model.__class__.__name__
---> 22 f1_score = cross_val_score(model, X_prep, labels, scoring = make_scorer(f1_score, average='weighted', labels=[2]), cv=CV)
23 for fold_idx, f1score in enumerate(f1_score):
24 entries.append((model_name, fold_idx, f1score))
File ~/anaconda3/envs/gpt-ds/lib/python3.10/site-packages/sklearn/utils/_param_validation.py:203, in validate_params.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
200 to_ignore += ["self", "cls"]
201 params = {k: v for k, v in params.arguments.items() if k not in to_ignore}
--> 203 validate_parameter_constraints(
204 parameter_constraints, params, caller_name=func.__qualname__
205 )
207 try:
208 with config_context(
209 skip_parameter_validation=(
210 prefer_skip_nested_validation or global_skip_validation
211 )
212 ):
File ~/anaconda3/envs/gpt-ds/lib/python3.10/site-packages/sklearn/utils/_param_validation.py:95, in validate_parameter_constraints(parameter_constraints, params, caller_name)
89 else:
90 constraints_str = (
91 f"{', '.join([str(c) for c in constraints[:-1]])} or"
92 f" {constraints[-1]}"
93 )
---> 95 raise InvalidParameterError(
96 f"The {param_name!r} parameter of {caller_name} must be"
97 f" {constraints_str}. Got {param_val!r} instead."
98 )
InvalidParameterError: The 'score_func' parameter of make_scorer must be a callable. Got array([0., 0., 0., 0., 0.]) instead.
I am just trying to get the mac_avg f1-score for the cross_val_score. Any suggestions are appreciated, I tried some solutions that were relevant but this one in particular did not work for me.