Datasets
Models
Downloads: 1
Diff of
/03-Experiments/04-XGBoost_With_Optuna.ipynb
[000000]
..
[b4c0b6]
Switch to side-by-side view
--- a +++ b/03-Experiments/04-XGBoost_With_Optuna.ipynb @@ -0,0 +1,1091 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Global Experiment Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "<Experiment: artifact_location='/Users/arham/Downloads/Projects/03-Experiments/mlruns/2', creation_time=1713912394972, experiment_id='2', last_update_time=1713912394972, lifecycle_stage='active', name='XGBoost', tags={}>" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import mlflow\n", + "\n", + "\n", + "# Set the MLflow tracking URI to a new SQLite URI\n", + "mlflow.set_tracking_uri(\"sqlite:///new_mlflow.db\")\n", + "mlflow.set_experiment(\"XGBoost\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "import numpy as np\n", + "from sklearn.preprocessing import MinMaxScaler\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "import lightgbm as lgb\n", + "from sklearn.metrics import accuracy_score\n", + "import warnings\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "from sklearn.model_selection import cross_val_score\n", + "from sklearn.metrics import accuracy_score, precision_score, recall_score\n", + "import xgboost as xgb\n", + "from sklearn.metrics import accuracy_score, precision_score, recall_score\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "def load_data(path):\n", + " df = pd.read_csv(path)\n", + " train_df, test_df = train_test_split(df, test_size=0.35, random_state=42)\n", + " train_df, val_df, = train_test_split(train_df, test_size=0.20, random_state=42)\n", + " train_df = train_df.drop(['id'], axis=1).drop_duplicates().reset_index(drop=True)\n", + " test_df = test_df.drop(['id'], axis=1).drop_duplicates().reset_index(drop=True)\n", + " val_df = val_df.drop(['id'], axis=1).drop_duplicates().reset_index(drop=True)\n", + " return train_df, val_df, test_df\n", + "\n", + "def encode_target(train):\n", + " target_key = {'Insufficient_Weight': 0, 'Normal_Weight': 1, 'Overweight_Level_I': 2, 'Overweight_Level_II': 3, 'Obesity_Type_I': 4,'Obesity_Type_II' : 5, 'Obesity_Type_III': 6}\n", + " train['NObeyesdad'] = train['NObeyesdad'].map(target_key)\n", + " return train\n", + "\n", + "def make_gender_binary(train):\n", + " train['Gender'] = train['Gender'].map({'Male':0, 'Female':1})\n", + "\n", + "def datatypes(train):\n", + " train['Weight'] = train['Weight'].astype(float)\n", + " train['Age'] = train['Age'].astype(float)\n", + " train['Height'] = train['Height'].astype(float)\n", + " return train\n", + "\n", + "# def age_binning(train_df):\n", + "# # train_df['Age_Group'] = pd.cut(train_df['Age'], bins=[0, 20, 30, 40, 50, train_df['Age'].max()], labels=['0-20', '21-30', '31-40', '41-50', '50+'])\n", + "# train_df['Age_Group'] = pd.cut(train_df['Age'], bins=[0, 20, 30, 40, 50, train_df['Age'].max()], labels=[1, 2, 3, 4, 5])\n", + "# train_df['Age_Group'] = train_df['Age_Group'].astype(int)\n", + "# return train_df\n", + "\n", + "def age_binning(df):\n", + " age_groups = []\n", + " for age in df['Age']:\n", + " if age <= 20:\n", + " age_group = 1\n", + " elif age <= 30:\n", + " age_group = 2\n", + " elif age <= 40:\n", + " age_group = 3\n", + " elif age <= 50:\n", + " age_group = 4\n", + " else:\n", + " age_group = 5\n", + " age_groups.append(age_group)\n", + " df['Age_Group'] = age_groups\n", + " return df\n", + "\n", + "def age_scaling_log(train_df):\n", + " train_df['Age'] = train_df['Age'].astype(float)\n", + " train_df['Log_Age'] = np.log1p(train_df['Age'])\n", + " return train_df\n", + "\n", + "def age_scaling_minmax(train_df):\n", + " train_df['Age'] = train_df['Age'].astype(float)\n", + " scaler_age = MinMaxScaler()\n", + " train_df['Scaled_Age'] = scaler_age.fit_transform(train_df['Age'].values.reshape(-1, 1))\n", + " return train_df, scaler_age\n", + "\n", + "def weight_scaling_log(train_df):\n", + " train_df['Weight'] = train_df['Weight'].astype(float)\n", + " train_df['Log_Weight'] = np.log1p(train_df['Weight'])\n", + " return train_df\n", + "\n", + "def weight_scaling_minmax(train_df):\n", + " train_df['Weight'] = train_df['Weight'].astype(float)\n", + " scaler_weight = MinMaxScaler()\n", + " train_df['Scaled_Weight'] = scaler_weight.fit_transform(train_df['Weight'].values.reshape(-1, 1))\n", + " return train_df, scaler_weight\n", + "\n", + "def height_scaling_log(train_df):\n", + " train_df['Log_Height'] = np.log1p(train_df['Height'])\n", + " return train_df\n", + "\n", + "def height_scaling_minmax(train_df):\n", + " scaler_height = MinMaxScaler()\n", + " train_df['Scaled_Height'] = scaler_height.fit_transform(train_df['Height'].values.reshape(-1, 1))\n", + " return train_df, scaler_height\n", + "\n", + "def make_gender_binary(train):\n", + " train['Gender'] = train['Gender'].map({'Female':1, 'Male':0})\n", + " return train\n", + "\n", + "def fix_binary_columns(train):\n", + " Binary_Cols = ['family_history_with_overweight','FAVC', 'SCC','SMOKE']\n", + " # if yes then 1 else 0\n", + " for col in Binary_Cols:\n", + " train[col] = train[col].map({'yes': 1, 'no': 0})\n", + " # column datatype integer\n", + " train[col] = train[col].astype(int)\n", + " return train\n", + "\n", + "def freq_cat_cols(train):\n", + " # One hot encoding\n", + " cat_cols = ['CAEC', 'CALC']\n", + " for col in cat_cols:\n", + " train[col] = train[col].map({'no': 0, 'Sometimes': 1, 'Frequently': 2, 'Always': 3})\n", + " return train\n", + "\n", + "def Mtrans(train):\n", + " \"\"\"\n", + " Public_Transportation 8692\n", + " Automobile 1835\n", + " Walking 231\n", + " Motorbike 19\n", + " Bike 16\n", + " \"\"\"\n", + " # train['MTRANS'] = train['MTRANS'].map({'Public_Transportation': 3, 'Automobile': 5, 'Walking': 1, 'Motorbike': 4, 'Bike': 2})\n", + " # dummify column\n", + " train = pd.get_dummies(train, columns=['MTRANS'])\n", + " # convert these columns to integer\n", + " train['MTRANS_Automobile'] = train['MTRANS_Automobile'].astype(int)\n", + " train['MTRANS_Walking'] = train['MTRANS_Walking'].astype(int)\n", + " train['MTRANS_Motorbike'] = train['MTRANS_Motorbike'].astype(int)\n", + " train['MTRANS_Bike'] = train['MTRANS_Bike'].astype(int)\n", + " train['MTRANS_Public_Transportation'] = train['MTRANS_Public_Transportation'].astype(int)\n", + " return train\n", + "\n", + "\n", + "def other_features(train):\n", + " train['BMI'] = train['Weight'] / (train['Height'] ** 2)\n", + " # train['Age'*'Gender'] = train['Age'] * train['Gender']\n", + " polynomial_features = PolynomialFeatures(degree=2)\n", + " X_poly = polynomial_features.fit_transform(train[['Age', 'BMI']])\n", + " poly_features_df = pd.DataFrame(X_poly, columns=['Age^2', 'Age^3', 'BMI^2', 'Age * BMI', 'Age * BMI^2', 'Age^2 * BMI^2'])\n", + " train = pd.concat([train, poly_features_df], axis=1)\n", + " return train\n", + "\n", + "\n", + "def test_pipeline(test, scaler_age, scaler_weight, scaler_height):\n", + " test = datatypes(test)\n", + " test = encode_target(test)\n", + " test = age_binning(test)\n", + " test = age_scaling_log(test)\n", + " test['Scaled_Age'] = scaler_age.transform(test['Age'].values.reshape(-1, 1))\n", + " test = weight_scaling_log(test)\n", + " test['Scaled_Weight'] = scaler_weight.transform(test['Weight'].values.reshape(-1, 1))\n", + " test = height_scaling_log(test)\n", + " test['Scaled_Height'] = scaler_height.transform(test['Height'].values.reshape(-1, 1))\n", + " test = make_gender_binary(test)\n", + " test = fix_binary_columns(test)\n", + " test = freq_cat_cols(test)\n", + " test = Mtrans(test)\n", + " test = other_features(test)\n", + "\n", + " return test\n", + "\n", + "def train_model(params, X_train, y_train):\n", + " lgb_train = lgb.Dataset(X_train, y_train)\n", + " model = lgb.train(params, lgb_train, num_boost_round=1000)\n", + " return model\n", + "\n", + "def evaluate_model(model, X_val, y_val):\n", + " y_pred = model.predict(X_val)\n", + " y_pred = [np.argmax(y) for y in y_pred]\n", + " accuracy = accuracy_score(y_val, y_pred)\n", + " return accuracy\n", + "\n", + "def objective(trial, X_train, y_train):\n", + " params = {\n", + " 'objective': 'multiclass',\n", + " 'num_class': 7,\n", + " 'metric': 'multi_logloss',\n", + " 'boosting_type': 'gbdt',\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.005, 0.5),\n", + " 'num_leaves': trial.suggest_int('num_leaves', 10, 1000),\n", + " 'max_depth': trial.suggest_int('max_depth', -1, 20),\n", + " 'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.6, 0.95),\n", + " 'feature_fraction': trial.suggest_uniform('feature_fraction', 0.6, 0.95),\n", + " 'verbosity': -1\n", + " }\n", + "\n", + " n_splits = 5\n", + " kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n", + " scores = []\n", + "\n", + " for train_index, val_index in kf.split(X_train, y_train):\n", + " X_tr, X_val = X_train.iloc[train_index], X_train.iloc[val_index]\n", + " y_tr, y_val = y_train.iloc[train_index], y_train.iloc[val_index]\n", + "\n", + " model = train_model(params, X_tr, y_tr)\n", + " accuracy = evaluate_model(model, X_val, y_val)\n", + " scores.append(accuracy)\n", + "\n", + " return np.mean(scores)\n", + "\n", + "def optimize_hyperparameters(X_train, y_train, n_trials=2):\n", + " study = optuna.create_study(direction='maximize')\n", + " study.optimize(lambda trial: objective(trial, X_train, y_train), n_trials=n_trials)\n", + " return study.best_params\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### XGB With Optuna" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "path = '/Users/arham/Downloads/Projects/01-Dataset/01-Data-for-model-building/train.csv'\n", + "train_df, val_df, test_df = load_data(path)\n", + "\n", + "train_df = datatypes(train_df)\n", + "train_df = encode_target(train_df)\n", + "train_df = age_binning(train_df)\n", + "train_df, scaler_age = age_scaling_minmax(train_df)\n", + "train_df = age_scaling_log(train_df)\n", + "train_df, scaler_weight = weight_scaling_minmax(train_df)\n", + "train_df = weight_scaling_log(train_df)\n", + "train_df, scaler_height = height_scaling_minmax(train_df)\n", + "train_df = height_scaling_log(train_df)\n", + "train_df = make_gender_binary(train_df)\n", + "train_df = fix_binary_columns(train_df)\n", + "train_df = freq_cat_cols(train_df)\n", + "train_df = Mtrans(train_df)\n", + "train_df = other_features(train_df)\n", + "\n", + "val_df = test_pipeline(val_df, scaler_age, scaler_weight, scaler_height)\n", + "test_df = test_pipeline(test_df, scaler_age, scaler_weight, scaler_height)\n", + "\n", + "Target = 'NObeyesdad'\n", + "# features = train_df.columns.drop(Target)\n", + "features = ['Gender', 'Age', 'Height', 'Weight', 'family_history_with_overweight',\n", + " 'FAVC', 'FCVC', 'NCP', 'CAEC', 'SMOKE', 'CH2O', 'SCC', 'FAF', 'TUE',\n", + " 'CALC', 'Age_Group', \n", + " 'MTRANS_Automobile', 'MTRANS_Bike', 'MTRANS_Motorbike',\n", + " 'MTRANS_Public_Transportation', 'MTRANS_Walking', 'BMI', 'Age^2',\n", + " 'Age^3', 'BMI^2', 'Age * BMI', 'Age * BMI^2', 'Age^2 * BMI^2'] \n", + "\n", + " #'Scaled_Age', 'Log_Age', 'Scaled_Weight', 'Log_Weight', 'Scaled_Height', 'Log_Height',\n", + "\n", + "\n", + "X_train = train_df[features]\n", + "y_train = train_df[Target]\n", + "X_val = val_df[features]\n", + "y_val = val_df[Target]\n", + "X_test = test_df[features]\n", + "y_test = test_df[Target]\n", + "\n", + "# save X_train, y_train, X_val, X_test, y_test\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:07:02,465] A new study created in memory with name: no-name-64002971-707f-4151-be7c-8ed470ee8bc1\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.88003705 0.88235294 0.86984715 0.86376274 0.87673772]\n", + "Mean CV Accuracy (XGBoost): 0.8745475220438529\n", + "\n", + "Accuracy (XGBoost): 0.8684698036309745\n", + "Precision (XGBoost): 0.8684692480457514\n", + "Recall (XGBoost): 0.8684698036309745\n", + "F1 (XGBoost): 0.8684695258382741\n", + "Recall for class 0: 0.9113924050632911\n", + "Recall for class 1: 0.8770053475935828\n", + "Recall for class 2: 0.7432835820895523\n", + "Recall for class 3: 0.7355623100303952\n", + "Recall for class 4: 0.8218527315914489\n", + "Recall for class 5: 0.9330143540669856\n", + "Recall for class 6: 0.9901185770750988\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:07:19,496] Trial 0 finished with value: 0.8745475220438529 and parameters: {'max_depth': 5, 'learning_rate': 0.013394988506670473, 'n_estimators': 77, 'min_child_weight': 10, 'subsample': 1.0, 'colsample_bytree': 0.7, 'gamma': 6.905297068817435e-07, 'reg_alpha': 0.678891339417352, 'reg_lambda': 2.8230926058910324}. Best is trial 0 with value: 0.8745475220438529.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.90504863 0.90690134 0.90273275 0.8952734 0.90037071]\n", + "Mean CV Accuracy (XGBoost): 0.9020653676808635\n", + "\n", + "Accuracy (XGBoost): 0.9018154872174879\n", + "Precision (XGBoost): 0.9021161905127771\n", + "Recall (XGBoost): 0.9018154872174879\n", + "F1 (XGBoost): 0.9019658138025226\n", + "Recall for class 0: 0.9272151898734177\n", + "Recall for class 1: 0.9010695187165776\n", + "Recall for class 2: 0.7761194029850746\n", + "Recall for class 3: 0.8237082066869301\n", + "Recall for class 4: 0.8646080760095012\n", + "Recall for class 5: 0.9688995215311005\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:10:12,141] Trial 1 finished with value: 0.9020653676808635 and parameters: {'max_depth': 10, 'learning_rate': 0.09249377976464467, 'n_estimators': 721, 'min_child_weight': 7, 'subsample': 0.9, 'colsample_bytree': 0.8, 'gamma': 0.00012452373684121342, 'reg_alpha': 0.71016682383988, 'reg_lambda': 1.2412790195572406e-06}. Best is trial 1 with value: 0.9020653676808635.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.90504863 0.91014358 0.90736452 0.89944393 0.90222428]\n", + "Mean CV Accuracy (XGBoost): 0.9048449901075782\n", + "\n", + "Accuracy (XGBoost): 0.9040385327899222\n", + "Precision (XGBoost): 0.9045766402024644\n", + "Recall (XGBoost): 0.9040385327899222\n", + "F1 (XGBoost): 0.9043075064461108\n", + "Recall for class 0: 0.9335443037974683\n", + "Recall for class 1: 0.9037433155080213\n", + "Recall for class 2: 0.7611940298507462\n", + "Recall for class 3: 0.8328267477203647\n", + "Recall for class 4: 0.8741092636579573\n", + "Recall for class 5: 0.9712918660287081\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:10:55,359] Trial 2 finished with value: 0.9048449901075782 and parameters: {'max_depth': 8, 'learning_rate': 0.1480724813458801, 'n_estimators': 201, 'min_child_weight': 10, 'subsample': 0.7, 'colsample_bytree': 0.6, 'gamma': 4.241734906017588e-07, 'reg_alpha': 1.1757709861728217e-05, 'reg_lambda': 3.398219063416917}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.89717462 0.8994905 0.89161649 0.88878591 0.89712697]\n", + "Mean CV Accuracy (XGBoost): 0.8948388988311532\n", + "\n", + "Accuracy (XGBoost): 0.889218228973694\n", + "Precision (XGBoost): 0.8896005944786827\n", + "Recall (XGBoost): 0.889218228973694\n", + "F1 (XGBoost): 0.8894093706305529\n", + "Recall for class 0: 0.9208860759493671\n", + "Recall for class 1: 0.8823529411764706\n", + "Recall for class 2: 0.7522388059701492\n", + "Recall for class 3: 0.8206686930091185\n", + "Recall for class 4: 0.838479809976247\n", + "Recall for class 5: 0.9593301435406698\n", + "Recall for class 6: 0.9940711462450593\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:11:45,267] Trial 3 finished with value: 0.8948388988311532 and parameters: {'max_depth': 6, 'learning_rate': 0.0915177926130075, 'n_estimators': 339, 'min_child_weight': 5, 'subsample': 0.9, 'colsample_bytree': 0.7, 'gamma': 0.0813995583239973, 'reg_alpha': 20.619635618013948, 'reg_lambda': 3.974196714305311e-07}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.89995368 0.90736452 0.89810097 0.89712697 0.89759036]\n", + "Mean CV Accuracy (XGBoost): 0.9000273012812284\n", + "\n", + "Accuracy (XGBoost): 0.8984809188588366\n", + "Precision (XGBoost): 0.898791107190973\n", + "Recall (XGBoost): 0.8984809188588366\n", + "F1 (XGBoost): 0.8986359862574483\n", + "Recall for class 0: 0.9272151898734177\n", + "Recall for class 1: 0.8903743315508021\n", + "Recall for class 2: 0.7522388059701492\n", + "Recall for class 3: 0.8358662613981763\n", + "Recall for class 4: 0.8669833729216152\n", + "Recall for class 5: 0.9641148325358851\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:13:55,859] Trial 4 finished with value: 0.9000273012812284 and parameters: {'max_depth': 9, 'learning_rate': 0.030870513505594194, 'n_estimators': 288, 'min_child_weight': 1, 'subsample': 1.0, 'colsample_bytree': 1.0, 'gamma': 0.9549119955309026, 'reg_alpha': 0.007762320297440327, 'reg_lambda': 2.0978432882529218e-05}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.90829088 0.90736452 0.90504863 0.89759036 0.90315107]\n", + "Mean CV Accuracy (XGBoost): 0.9042890913781608\n", + "\n", + "Accuracy (XGBoost): 0.9058910707669507\n", + "Precision (XGBoost): 0.9059794693613187\n", + "Recall (XGBoost): 0.9058910707669507\n", + "F1 (XGBoost): 0.9059352679077136\n", + "Recall for class 0: 0.930379746835443\n", + "Recall for class 1: 0.9090909090909091\n", + "Recall for class 2: 0.7791044776119403\n", + "Recall for class 3: 0.8358662613981763\n", + "Recall for class 4: 0.8669833729216152\n", + "Recall for class 5: 0.9712918660287081\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:15:44,305] Trial 5 finished with value: 0.9042890913781608 and parameters: {'max_depth': 8, 'learning_rate': 0.04624235053164599, 'n_estimators': 385, 'min_child_weight': 3, 'subsample': 0.6, 'colsample_bytree': 0.6, 'gamma': 2.4010178478007346e-07, 'reg_alpha': 1.2972918999055175e-08, 'reg_lambda': 0.010144470159191883}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.89671144 0.90458546 0.89671144 0.89388323 0.89851715]\n", + "Mean CV Accuracy (XGBoost): 0.8980817415813537\n", + "\n", + "Accuracy (XGBoost): 0.8969988884772138\n", + "Precision (XGBoost): 0.897313793211423\n", + "Recall (XGBoost): 0.8969988884772138\n", + "F1 (XGBoost): 0.8971563132111772\n", + "Recall for class 0: 0.9208860759493671\n", + "Recall for class 1: 0.8983957219251337\n", + "Recall for class 2: 0.7731343283582089\n", + "Recall for class 3: 0.7993920972644377\n", + "Recall for class 4: 0.8622327790973872\n", + "Recall for class 5: 0.9688995215311005\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:18:36,520] Trial 6 finished with value: 0.8980817415813537 and parameters: {'max_depth': 6, 'learning_rate': 0.21361268583130158, 'n_estimators': 989, 'min_child_weight': 6, 'subsample': 0.6, 'colsample_bytree': 0.7, 'gamma': 0.15715908126940034, 'reg_alpha': 0.002876421704879931, 'reg_lambda': 0.00020071476895386069}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.89532191 0.90273275 0.8994905 0.8952734 0.90176089]\n", + "Mean CV Accuracy (XGBoost): 0.8989158901612793\n", + "\n", + "Accuracy (XGBoost): 0.9007039644312709\n", + "Precision (XGBoost): 0.9010058877096547\n", + "Recall (XGBoost): 0.9007039644312709\n", + "F1 (XGBoost): 0.9008549007729195\n", + "Recall for class 0: 0.9272151898734177\n", + "Recall for class 1: 0.9037433155080213\n", + "Recall for class 2: 0.7761194029850746\n", + "Recall for class 3: 0.817629179331307\n", + "Recall for class 4: 0.8598574821852731\n", + "Recall for class 5: 0.9688995215311005\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:19:40,518] Trial 7 finished with value: 0.8989158901612793 and parameters: {'max_depth': 7, 'learning_rate': 0.2701374472461074, 'n_estimators': 220, 'min_child_weight': 4, 'subsample': 0.7, 'colsample_bytree': 1.0, 'gamma': 6.952295076389228e-06, 'reg_alpha': 4.7746605073070416e-08, 'reg_lambda': 0.010461996036680242}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.88652154 0.89022696 0.87633164 0.87395737 0.88693234]\n", + "Mean CV Accuracy (XGBoost): 0.8827939684773225\n", + "\n", + "Accuracy (XGBoost): 0.8806965542793628\n", + "Precision (XGBoost): 0.8807374877985908\n", + "Recall (XGBoost): 0.8806965542793628\n", + "F1 (XGBoost): 0.880717020563355\n", + "Recall for class 0: 0.9335443037974683\n", + "Recall for class 1: 0.8663101604278075\n", + "Recall for class 2: 0.7432835820895523\n", + "Recall for class 3: 0.7993920972644377\n", + "Recall for class 4: 0.8218527315914489\n", + "Recall for class 5: 0.9545454545454546\n", + "Recall for class 6: 0.9901185770750988\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:20:20,776] Trial 8 finished with value: 0.8827939684773225 and parameters: {'max_depth': 10, 'learning_rate': 0.030824226564898152, 'n_estimators': 161, 'min_child_weight': 10, 'subsample': 1.0, 'colsample_bytree': 0.8, 'gamma': 0.0002761971902093278, 'reg_alpha': 34.12323171902436, 'reg_lambda': 1.1963423659959008e-06}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.90180639 0.90690134 0.90180639 0.89341983 0.90176089]\n", + "Mean CV Accuracy (XGBoost): 0.9011389699604345\n", + "\n", + "Accuracy (XGBoost): 0.9021859948128936\n", + "Precision (XGBoost): 0.9026219001901948\n", + "Recall (XGBoost): 0.9021859948128936\n", + "F1 (XGBoost): 0.9024038948606238\n", + "Recall for class 0: 0.9367088607594937\n", + "Recall for class 1: 0.8983957219251337\n", + "Recall for class 2: 0.7582089552238805\n", + "Recall for class 3: 0.8237082066869301\n", + "Recall for class 4: 0.8812351543942993\n", + "Recall for class 5: 0.9641148325358851\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:20:52,333] Trial 9 finished with value: 0.9011389699604345 and parameters: {'max_depth': 6, 'learning_rate': 0.30841942106618764, 'n_estimators': 132, 'min_child_weight': 7, 'subsample': 0.9, 'colsample_bytree': 0.7, 'gamma': 5.749686185921016e-07, 'reg_alpha': 1.953546914108059e-08, 'reg_lambda': 0.003027552093067664}. Best is trial 2 with value: 0.9048449901075782.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.91014358 0.91014358 0.90736452 0.90083411 0.90222428]\n", + "Mean CV Accuracy (XGBoost): 0.9061420155986472\n", + "\n", + "Accuracy (XGBoost): 0.9029270100037051\n", + "Precision (XGBoost): 0.9034148150281903\n", + "Recall (XGBoost): 0.9029270100037051\n", + "F1 (XGBoost): 0.9031708466497482\n", + "Recall for class 0: 0.930379746835443\n", + "Recall for class 1: 0.9117647058823529\n", + "Recall for class 2: 0.764179104477612\n", + "Recall for class 3: 0.8328267477203647\n", + "Recall for class 4: 0.8622327790973872\n", + "Recall for class 5: 0.9688995215311005\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:22:05,710] Trial 10 finished with value: 0.9061420155986472 and parameters: {'max_depth': 3, 'learning_rate': 0.14385928468253742, 'n_estimators': 574, 'min_child_weight': 8, 'subsample': 0.7, 'colsample_bytree': 0.6, 'gamma': 1.5922499564605758e-08, 'reg_alpha': 7.445790014646167e-06, 'reg_lambda': 95.07956676133519}. Best is trial 10 with value: 0.9061420155986472.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.91014358 0.91060676 0.90736452 0.90500463 0.90315107]\n", + "Mean CV Accuracy (XGBoost): 0.9072541135432812\n", + "\n", + "Accuracy (XGBoost): 0.9047795479807336\n", + "Precision (XGBoost): 0.9054304429990271\n", + "Recall (XGBoost): 0.9047795479807336\n", + "F1 (XGBoost): 0.9051048784691129\n", + "Recall for class 0: 0.9367088607594937\n", + "Recall for class 1: 0.9064171122994652\n", + "Recall for class 2: 0.7701492537313432\n", + "Recall for class 3: 0.8389057750759878\n", + "Recall for class 4: 0.8622327790973872\n", + "Recall for class 5: 0.9712918660287081\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:23:19,966] Trial 11 finished with value: 0.9072541135432812 and parameters: {'max_depth': 3, 'learning_rate': 0.14050588188423715, 'n_estimators': 601, 'min_child_weight': 9, 'subsample': 0.7, 'colsample_bytree': 0.6, 'gamma': 1.9866742273835196e-08, 'reg_alpha': 1.1945121332468573e-05, 'reg_lambda': 71.93048807006664}. Best is trial 11 with value: 0.9072541135432812.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.90968041 0.91060676 0.91060676 0.90454124 0.90083411]\n", + "Mean CV Accuracy (XGBoost): 0.9072538559840245\n", + "\n", + "Accuracy (XGBoost): 0.9044090403853279\n", + "Precision (XGBoost): 0.9053349503998932\n", + "Recall (XGBoost): 0.9044090403853279\n", + "F1 (XGBoost): 0.9048717585333201\n", + "Recall for class 0: 0.9367088607594937\n", + "Recall for class 1: 0.9117647058823529\n", + "Recall for class 2: 0.764179104477612\n", + "Recall for class 3: 0.8358662613981763\n", + "Recall for class 4: 0.8646080760095012\n", + "Recall for class 5: 0.9688995215311005\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:24:34,779] Trial 12 finished with value: 0.9072538559840245 and parameters: {'max_depth': 3, 'learning_rate': 0.13577039129548693, 'n_estimators': 604, 'min_child_weight': 8, 'subsample': 0.7, 'colsample_bytree': 0.6, 'gamma': 2.3371777592215328e-08, 'reg_alpha': 8.403097374567035e-06, 'reg_lambda': 41.38139839851449}. Best is trial 11 with value: 0.9072541135432812.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.8994905 0.90597499 0.90319592 0.89759036 0.89805375]\n", + "Mean CV Accuracy (XGBoost): 0.9008611064488117\n", + "\n", + "Accuracy (XGBoost): 0.9036680251945165\n", + "Precision (XGBoost): 0.9043871882647396\n", + "Recall (XGBoost): 0.9036680251945165\n", + "F1 (XGBoost): 0.9040274637042609\n", + "Recall for class 0: 0.9272151898734177\n", + "Recall for class 1: 0.9010695187165776\n", + "Recall for class 2: 0.7731343283582089\n", + "Recall for class 3: 0.8267477203647416\n", + "Recall for class 4: 0.8693586698337292\n", + "Recall for class 5: 0.9760765550239234\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:26:22,855] Trial 13 finished with value: 0.9008611064488117 and parameters: {'max_depth': 3, 'learning_rate': 0.45330520785367046, 'n_estimators': 709, 'min_child_weight': 8, 'subsample': 0.8, 'colsample_bytree': 0.9, 'gamma': 1.050198550207745e-08, 'reg_alpha': 1.4096829741801802e-05, 'reg_lambda': 54.59517406603631}. Best is trial 11 with value: 0.9072541135432812.\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:12: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:15: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:16: FutureWarning: suggest_discrete_uniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., step=...) instead.\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:17: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:18: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + "/var/folders/2t/c7s0z0zs4698zw0k9pj4f2r80000gn/T/ipykernel_20288/2035532747.py:19: FutureWarning: suggest_loguniform has been deprecated in v3.0.0. This feature will be removed in v6.0.0. See https://github.com/optuna/optuna/releases/tag/v3.0.0. Use suggest_float(..., log=True) instead.\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Target Drift For Each Class [0.004943133623686147, 0.011990707821925795, -0.0087675011457998, -0.001077949504617301, -0.017190035106736085, -0.00032756263090533144, 0.01042920694244659]\n", + "Cross-validation Scores (XGBoost): [0.90690134 0.91060676 0.90551181 0.90454124 0.90037071]\n", + "Mean CV Accuracy (XGBoost): 0.9055863744284867\n", + "\n", + "Accuracy (XGBoost): 0.9044090403853279\n", + "Precision (XGBoost): 0.9048073892271471\n", + "Recall (XGBoost): 0.9044090403853279\n", + "F1 (XGBoost): 0.9046081709525012\n", + "Recall for class 0: 0.9335443037974683\n", + "Recall for class 1: 0.9117647058823529\n", + "Recall for class 2: 0.7611940298507462\n", + "Recall for class 3: 0.8297872340425532\n", + "Recall for class 4: 0.8717339667458432\n", + "Recall for class 5: 0.9712918660287081\n", + "Recall for class 6: 0.9960474308300395\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[I 2024-04-25 15:27:41,681] Trial 14 finished with value: 0.9055863744284867 and parameters: {'max_depth': 4, 'learning_rate': 0.05892316102120539, 'n_estimators': 515, 'min_child_weight': 8, 'subsample': 0.8, 'colsample_bytree': 0.6, 'gamma': 0.002339216477344003, 'reg_alpha': 1.5648083825482848e-06, 'reg_lambda': 0.3956617258269376}. Best is trial 11 with value: 0.9072541135432812.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of finished trials: 15\n", + "Best trial:\n", + " Value: 0.90725\n", + " Params: \n", + " max_depth: 3\n", + " learning_rate: 0.14050588188423715\n", + " n_estimators: 601\n", + " min_child_weight: 9\n", + " subsample: 0.7\n", + " colsample_bytree: 0.6\n", + " gamma: 1.9866742273835196e-08\n", + " reg_alpha: 1.1945121332468573e-05\n", + " reg_lambda: 71.93048807006664\n" + ] + } + ], + "source": [ + "import optuna\n", + "from sklearn.model_selection import cross_val_score\n", + "from xgboost import XGBClassifier\n", + "from sklearn.metrics import accuracy_score, precision_score, recall_score, precision_recall_fscore_support\n", + "import mlflow\n", + "\n", + "mlflow.sklearn.autolog(disable=True)\n", + "\n", + "def objective(trial):\n", + " params = {\n", + " 'max_depth': trial.suggest_int('max_depth', 3, 10),\n", + " 'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.5),\n", + " 'n_estimators': trial.suggest_int('n_estimators', 50, 1000),\n", + " 'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n", + " 'subsample': trial.suggest_discrete_uniform('subsample', 0.6, 1, 0.1),\n", + " 'colsample_bytree': trial.suggest_discrete_uniform('colsample_bytree', 0.6, 1, 0.1),\n", + " 'gamma': trial.suggest_loguniform('gamma', 1e-8, 1.0),\n", + " 'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-8, 100.0),\n", + " 'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-8, 100.0),\n", + " 'random_state': 42,\n", + " 'objective': 'multi:softmax',\n", + " 'num_class': 7,\n", + " 'eval_metric': 'mlogloss'\n", + " }\n", + "\n", + " with mlflow.start_run(run_name=\"XGB_with_Feature_Engineering_optuna_\" + str(trial.number), nested=True):\n", + " # Log hyperparameters as a single dictionary\n", + " mlflow.log_params(params)\n", + "\n", + " # Calculate target drift and log\n", + " class_counts_train = [y_train[y_train == i].count() / y_train.count() for i in range(7)]\n", + " class_counts_val = [y_val[y_val == i].count() / y_val.count() for i in range(7)]\n", + " target_drift = [(train_count - val_count) for train_count, val_count in zip(class_counts_train, class_counts_val)]\n", + " mlflow.log_params({'Target_Drift_' + str(i): freq for i, freq in enumerate(target_drift)})\n", + " print(f\"Target Drift For Each Class {target_drift}\")\n", + "\n", + " # Train XGBoost model\n", + " model = XGBClassifier(**params)\n", + " cv_scores_xgb = cross_val_score(model, X_train, y_train, cv=5, scoring='accuracy')\n", + " print(\"Cross-validation Scores (XGBoost):\", cv_scores_xgb)\n", + " print(\"Mean CV Accuracy (XGBoost):\", cv_scores_xgb.mean())\n", + " \n", + "\n", + " # Log CV metrics\n", + " mlflow.log_metric('mean_cv_accuracy', cv_scores_xgb.mean())\n", + "\n", + " # Fit model on full training data\n", + " model.fit(X_train, y_train)\n", + " y_val_pred_xgb = model.predict(X_val)\n", + "\n", + " # Calculate evaluation metrics\n", + " accuracy_xgb = accuracy_score(y_val, y_val_pred_xgb)\n", + " precision_xgb = precision_score(y_val, y_val_pred_xgb, average='weighted')\n", + " recall_xgb = recall_score(y_val, y_val_pred_xgb, average='weighted')\n", + " f1_xgb = 2 * (precision_xgb * recall_xgb) / (precision_xgb + recall_xgb)\n", + " print(\"\\nAccuracy (XGBoost):\", accuracy_xgb)\n", + " print(\"Precision (XGBoost):\", precision_xgb)\n", + " print(\"Recall (XGBoost):\", recall_xgb)\n", + " print(\"F1 (XGBoost):\", f1_xgb)\n", + "\n", + " # Log evaluation metrics\n", + " mlflow.log_metric('accuracy', accuracy_xgb)\n", + " mlflow.log_metric('precision', precision_xgb)\n", + " mlflow.log_metric('recall', recall_xgb)\n", + " mlflow.log_metric('f1', f1_xgb)\n", + "\n", + " # Calculate and log precision, recall for each class\n", + " precision_per_class, recall_per_class, _, _ = precision_recall_fscore_support(y_val, y_val_pred_xgb, average=None)\n", + " for i in range(len(recall_per_class)):\n", + " print(f\"Recall for class {i}: {recall_per_class[i]}\")\n", + " mlflow.log_metric(f'recall_class_{i}', recall_per_class[i])\n", + "\n", + " # Log XGBoost model\n", + " mlflow.xgboost.log_model(model, 'model')\n", + "\n", + " # Set tags\n", + " mlflow.set_tag('experiments', 'Arham A.')\n", + " mlflow.set_tag('model_name', 'XGBoost')\n", + " mlflow.set_tag('preprocessing', 'Yes')\n", + "\n", + " return cv_scores_xgb.mean()\n", + "\n", + "study = optuna.create_study(direction='maximize')\n", + "study.optimize(objective, n_trials=15)\n", + "\n", + "print('Number of finished trials:', len(study.trials))\n", + "print('Best trial:')\n", + "trial = study.best_trial\n", + "print(' Value: {:.5f}'.format(trial.value))\n", + "print(' Params: ')\n", + "for key, value in trial.params.items():\n", + " print(' {}: {}'.format(key, value))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2024-04-26 12:25:15 -0400] [60478] [INFO] Starting gunicorn 21.2.0\n", + "[2024-04-26 12:25:15 -0400] [60478] [INFO] Listening at: http://127.0.0.1:5000 (60478)\n", + "[2024-04-26 12:25:15 -0400] [60478] [INFO] Using worker: sync\n", + "[2024-04-26 12:25:15 -0400] [60479] [INFO] Booting worker with pid: 60479\n", + "[2024-04-26 12:25:15 -0400] [60480] [INFO] Booting worker with pid: 60480\n", + "[2024-04-26 12:25:15 -0400] [60481] [INFO] Booting worker with pid: 60481\n", + "[2024-04-26 12:25:16 -0400] [60482] [INFO] Booting worker with pid: 60482\n", + "^C\n", + "[2024-04-26 14:01:01 -0400] [60478] [INFO] Handling signal: int\n", + "[2024-04-26 14:01:01 -0400] [60482] [INFO] Worker exiting (pid: 60482)\n", + "[2024-04-26 14:01:01 -0400] [60480] [INFO] Worker exiting (pid: 60480)\n", + "[2024-04-26 14:01:01 -0400] [60479] [INFO] Worker exiting (pid: 60479)\n", + "[2024-04-26 14:01:01 -0400] [60481] [INFO] Worker exiting (pid: 60481)\n" + ] + } + ], + "source": [ + "!mlflow ui --backend-store-uri \"sqlite:////Users/arham/Downloads/Projects/03-Experiments/new_mlflow.db\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "DataScience", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}