{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# How to work on Colab\n## Email Notification & Session Auto Shutdown\n\n\n\nThis was supposed to be my baseline to work on [Predict Health Outcomes of Horses (playground-series-s3e22)](https://www.kaggle.com/competitions/playground-series-s3e22) with optuna.
\nHowever, I gave too much of `n_trials`, and the Kaggle notebook crashed...
\nTo prevent that to happen again, I decided to move to Colab.
\n\nBelow is the code to enable email notification & Session Auto Shutdown.
\n
\n### Email Notification\nYou have to search the config for your own email provider.
\nFor gmail users, simply `ctrl` / `cmd` + `f` change `email` to `gmail`.
\nOnce you get your App Passwords from
`Security` - `2-step verification` setting, enter that password for email_pw,
it will work fine for you.
\n```\nimport smtplib\nfrom smtplib import SMTPException\nfrom getpass import getpass\n\nemail_user = 'your_@email.com'\nemail_pw = getpass('Enter your Password : ')\n\nmail_from = email_user\nmail_to = 'maybe_another_@email.com'\n\nmail_subject = 'TEST'\nmail_message_body = 'test_message_body'\n\nmail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\ntry:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\nexcept SMTPException:\n print(\"Error: unable to send email\")\n```\n
\n\n### Session Auto Shutdown:\nI believe this is essential for `Pro` & `Pro+` users.
\nYou can place the code below, into `try` & `except` phrase 😊\n
\n```\nfrom google.colab import runtime\nruntime.unassign()\n```\n
\n\nNow the baseline starts, you will be able to see the implementations of above functions when you download this and bring it to your colab.","metadata":{}},{"cell_type":"code","source":"## Use below in Colab\n# !pip install -q optuna tensorflow catboost","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.679849Z","iopub.execute_input":"2023-09-27T13:12:37.680126Z","iopub.status.idle":"2023-09-27T13:12:37.684872Z","shell.execute_reply.started":"2023-09-27T13:12:37.680095Z","shell.execute_reply":"2023-09-27T13:12:37.683689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nwarnings.simplefilter(\"ignore\")\nimport logging\nlogging.disable(logging.ERROR)\nimport os\nos.environ['TRANSFORMERS_NO_ADVISORY_WARNINGS'] = 'true'\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n\n## Use below in Colab\n# import shutil\n# from getpass import getpass\n\nimport os\nimport optuna\nimport pickle\nimport random\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tqdm.auto import tqdm\nfrom sklearn.svm import SVC\nfrom functools import partial\nfrom sklearn.metrics import auc\nfrom sklearn.impute import KNNImputer\nfrom tensorflow.keras import Sequential\nfrom xgboost.sklearn import XGBClassifier\nfrom lightgbm import LGBMClassifier as lgb\nfrom sklearn.ensemble import VotingClassifier\nfrom catboost import CatBoostClassifier as cat\nfrom tensorflow.keras.optimizers import SGD, Adam\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom tensorflow.keras.layers import Input, Dense, ReLU, Softmax, Dropout\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom sklearn.metrics import roc_auc_score, roc_curve, make_scorer, f1_score\nfrom sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.690483Z","iopub.execute_input":"2023-09-27T13:12:37.690749Z","iopub.status.idle":"2023-09-27T13:12:37.709485Z","shell.execute_reply.started":"2023-09-27T13:12:37.690718Z","shell.execute_reply":"2023-09-27T13:12:37.708402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Use below in Colab\n'''\nsecret = getpass('Enter the secret value: ')\nos.environ['KAGGLE_USERNAME'] = 'your_kaggle_username'\nos.environ['KAGGLE_KEY'] = secret\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.711411Z","iopub.execute_input":"2023-09-27T13:12:37.711686Z","iopub.status.idle":"2023-09-27T13:12:37.732303Z","shell.execute_reply.started":"2023-09-27T13:12:37.711655Z","shell.execute_reply":"2023-09-27T13:12:37.731217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Use below in Colab\n# !kaggle competitions download -c playground-series-s3e22","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.734013Z","iopub.execute_input":"2023-09-27T13:12:37.734592Z","iopub.status.idle":"2023-09-27T13:12:37.743830Z","shell.execute_reply.started":"2023-09-27T13:12:37.734551Z","shell.execute_reply":"2023-09-27T13:12:37.743035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Use below in Colab\n'''\nfrom zipfile import ZipFile\nt = '/content/playground-series-s3e22.zip'\n\nwith ZipFile(t, 'r') as Zobject:\n Zobject.extractall(path='/content/')\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.746580Z","iopub.execute_input":"2023-09-27T13:12:37.747258Z","iopub.status.idle":"2023-09-27T13:12:37.758645Z","shell.execute_reply.started":"2023-09-27T13:12:37.747215Z","shell.execute_reply":"2023-09-27T13:12:37.757561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set configs\nis_tuning = True\nis_scaling = True\nis_pca = False\nif is_tuning:\n n_trials=1\n\n# Keras model compile\nlearning_rate = 1e-3\nbatch_size = 64\nepochs = 20","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:12:37.760233Z","iopub.execute_input":"2023-09-27T13:12:37.760792Z","iopub.status.idle":"2023-09-27T13:12:37.770974Z","shell.execute_reply.started":"2023-09-27T13:12:37.760753Z","shell.execute_reply":"2023-09-27T13:12:37.769981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed: int = 42):\n random.seed(seed)\n np.random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n tf.random.set_seed(seed)\n\nseed_everything()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:12:37.772616Z","iopub.execute_input":"2023-09-27T13:12:37.772911Z","iopub.status.idle":"2023-09-27T13:12:37.791124Z","shell.execute_reply.started":"2023-09-27T13:12:37.772870Z","shell.execute_reply":"2023-09-27T13:12:37.790104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_f1(y_true, y_pred):\n f1 = f1_score(y_true, y_pred, average='micro')\n return tf.constant(f1, dtype=tf.float32)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:12:37.792566Z","iopub.execute_input":"2023-09-27T13:12:37.792878Z","iopub.status.idle":"2023-09-27T13:12:37.805064Z","shell.execute_reply.started":"2023-09-27T13:12:37.792836Z","shell.execute_reply":"2023-09-27T13:12:37.804275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s3e22/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s3e22/test.csv')\nsubmission = pd.read_csv('/kaggle/input/playground-series-s3e22/sample_submission.csv')\n\n## Use below in Colab\n'''\ntrain = pd.read_csv('/content/train.csv')\ntest = pd.read_csv('/content/test.csv')\nsubmission = pd.read_csv('/content/sample_submission.csv')\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.807898Z","iopub.execute_input":"2023-09-27T13:12:37.808695Z","iopub.status.idle":"2023-09-27T13:12:37.851114Z","shell.execute_reply.started":"2023-09-27T13:12:37.808652Z","shell.execute_reply":"2023-09-27T13:12:37.850359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.898130Z","iopub.execute_input":"2023-09-27T13:12:37.899228Z","iopub.status.idle":"2023-09-27T13:12:37.931328Z","shell.execute_reply.started":"2023-09-27T13:12:37.899182Z","shell.execute_reply":"2023-09-27T13:12:37.930393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.933783Z","iopub.execute_input":"2023-09-27T13:12:37.934099Z","iopub.status.idle":"2023-09-27T13:12:37.950718Z","shell.execute_reply.started":"2023-09-27T13:12:37.934058Z","shell.execute_reply":"2023-09-27T13:12:37.949696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping = {'yes' : 0, 'no' : 1}\ntrain.surgery = train.surgery.map(mapping)\ntest.surgery = test.surgery.map(mapping)\ntrain.surgical_lesion = train.surgical_lesion.map(mapping)\ntest.surgical_lesion = test.surgical_lesion.map(mapping)\ntrain.cp_data = train.cp_data.map(mapping)\ntest.cp_data = test.cp_data.map(mapping)\nmapping = {'adult' : 1, 'young' : 0}\ntrain.age = train.age.map(mapping)\ntest.age = test.age.map(mapping)\nmapping = {'cool' : 0, 'cold' : 1, 'normal' : 2, 'warm' : 3}\ntrain.temp_of_extremities = train.temp_of_extremities.map(mapping)\ntest.temp_of_extremities = test.temp_of_extremities.map(mapping)\nmapping = {'absent' : 0, 'reduced' : 1, 'normal' : 2, 'increased' : 3}\ntrain.peripheral_pulse = train.peripheral_pulse.map(mapping)\ntest.peripheral_pulse = test.peripheral_pulse.map(mapping)\nmapping = {'dark_cyanotic' : 0, 'pale_cyanotic' : 1, 'pale_pink' : 2, 'normal_pink' : 3, 'bright_pink' : 4, 'bright_red' : 5}\ntrain.mucous_membrane = train.mucous_membrane.map(mapping)\ntest.mucous_membrane = test.mucous_membrane.map(mapping)\nmapping = {'less_3_sec' : 0, 'more_3_sec' : 1, '3' : 3}\ntrain.capillary_refill_time = train.capillary_refill_time.map(mapping)\ntest.capillary_refill_time = test.capillary_refill_time.map(mapping)\nmapping = {'depressed' : 0, 'mild_pain' : 1, 'extreme_pain' : 2, 'alert': 3, 'severe_pain' : 4, 'slight' : 5}\ntrain.pain = train.pain.map(mapping)\ntest.pain = test.pain.map(mapping)\nmapping = {'absent' : 0, 'hypomotile' : 1, 'normal' : 2, 'hypermotile': 3, 'distend_small' : 4}\ntrain.peristalsis = train.peristalsis.map(mapping)\ntest.peristalsis = test.peristalsis.map(mapping)\nmapping = {'slight' : 0, 'moderate' : 1, 'none' : 2, 'severe': 3}\ntrain.abdominal_distention = train.abdominal_distention.map(mapping)\ntest.abdominal_distention = test.abdominal_distention.map(mapping)\nmapping = {'slight' : 0, 'significant' : 1, 'none' : 2}\ntrain.nasogastric_tube = train.nasogastric_tube.map(mapping)\ntest.nasogastric_tube = test.nasogastric_tube.map(mapping)\nmapping = {'less_1_liter' : 0, 'more_1_liter' : 1, 'none' : 2, 'slight' : 3}\ntrain.nasogastric_reflux = train.nasogastric_reflux.map(mapping)\ntest.nasogastric_reflux = test.nasogastric_reflux.map(mapping)\nmapping = {'decreased' : 0, 'absent' : 1, 'normal' : 2, 'increased' : 3, 'serosanguious' : 4}\ntrain.rectal_exam_feces = train.rectal_exam_feces.map(mapping)\ntest.rectal_exam_feces = test.rectal_exam_feces.map(mapping)\nmapping = {'distend_small' : 0, 'distend_large' : 1, 'normal' : 2, 'firm' : 3, 'other' : 4}\ntrain.abdomen = train.abdomen.map(mapping)\ntest.abdomen = test.abdomen.map(mapping)\nmapping = {'serosanguious' : 0, 'cloudy' : 1, 'clear' : 2}\ntrain.abdomo_appearance = train.abdomo_appearance.map(mapping)\ntest.abdomo_appearance = test.abdomo_appearance.map(mapping)\nmapping = {'lived' : 0, 'died' : 1, 'euthanized': 2}\ntrain.outcome = train.outcome.map(mapping)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:37.952371Z","iopub.execute_input":"2023-09-27T13:12:37.952990Z","iopub.status.idle":"2023-09-27T13:12:38.013341Z","shell.execute_reply.started":"2023-09-27T13:12:37.952949Z","shell.execute_reply":"2023-09-27T13:12:38.012512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.014642Z","iopub.execute_input":"2023-09-27T13:12:38.015556Z","iopub.status.idle":"2023-09-27T13:12:38.031202Z","shell.execute_reply.started":"2023-09-27T13:12:38.015512Z","shell.execute_reply":"2023-09-27T13:12:38.030130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.034770Z","iopub.execute_input":"2023-09-27T13:12:38.035000Z","iopub.status.idle":"2023-09-27T13:12:38.134078Z","shell.execute_reply.started":"2023-09-27T13:12:38.034972Z","shell.execute_reply":"2023-09-27T13:12:38.132998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imputer = KNNImputer(n_neighbors=3,)\ntr_columns = list(train.columns)\nte_columns = list(test.columns)\ntrain[tr_columns] = imputer.fit_transform(train[tr_columns])\ntest[te_columns] = imputer.fit_transform(test[te_columns])","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:14:02.835770Z","iopub.execute_input":"2023-09-27T13:14:02.836148Z","iopub.status.idle":"2023-09-27T13:14:03.014095Z","shell.execute_reply.started":"2023-09-27T13:14:02.836106Z","shell.execute_reply":"2023-09-27T13:14:03.012728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:14:05.452174Z","iopub.execute_input":"2023-09-27T13:14:05.452489Z","iopub.status.idle":"2023-09-27T13:14:05.470774Z","shell.execute_reply.started":"2023-09-27T13:14:05.452456Z","shell.execute_reply":"2023-09-27T13:14:05.469622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[variance_inflation_factor(train, i) for i in range (train.shape[1])]\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:14:27.404415Z","iopub.execute_input":"2023-09-27T13:14:27.404705Z","iopub.status.idle":"2023-09-27T13:14:27.619529Z","shell.execute_reply.started":"2023-09-27T13:14:27.404674Z","shell.execute_reply":"2023-09-27T13:14:27.618050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_vif(df):\n vifs = [variance_inflation_factor(df, i) for i in range(df.shape[1])]\n vif_df = pd.DataFrame({\"features\":df.columns, \"VIF\" : vifs})\n vif_df = vif_df.sort_values(by=\"VIF\", ascending=False)\n remove_col = vif_df.iloc[0, 0]\n top_vif = vif_df.iloc[0, 1]\n return vif_df, remove_col, top_vif","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:14:42.712360Z","iopub.execute_input":"2023-09-27T13:14:42.712677Z","iopub.status.idle":"2023-09-27T13:14:42.720225Z","shell.execute_reply.started":"2023-09-27T13:14:42.712635Z","shell.execute_reply":"2023-09-27T13:14:42.718916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_vif = 100\n\nwhile(top_vif > 5):\n vif_df, remove_col, top_vif = check_vif(train)\n print(remove_col, top_vif)\n if top_vif < 5:\n break\n train = train.drop(columns=remove_col)\n\ntrain","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:14:44.656154Z","iopub.execute_input":"2023-09-27T13:14:44.656444Z","iopub.status.idle":"2023-09-27T13:14:46.041023Z","shell.execute_reply.started":"2023-09-27T13:14:44.656412Z","shell.execute_reply":"2023-09-27T13:14:46.040010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop(columns=['outcome', 'id'])\ny = train['outcome']\n\nrf = RandomForestClassifier()\nrf.fit(X, y)\n# [(col, fi) for col, fi in zip(X.columns, rf.feature_importances_)]\nfi_df = pd.DataFrame({'feature': X.columns, 'importance' : rf.feature_importances_})\n\nfi_df.sort_values(by='importance', ascending=False).plot(kind='bar')","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:15:06.639583Z","iopub.execute_input":"2023-09-27T13:15:06.639886Z","iopub.status.idle":"2023-09-27T13:15:07.447671Z","shell.execute_reply.started":"2023-09-27T13:15:06.639854Z","shell.execute_reply":"2023-09-27T13:15:07.446548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_cols = fi_df.sort_values(by='importance', ascending=False)[:10]['feature'].values\nselected_cols","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:15:18.138254Z","iopub.execute_input":"2023-09-27T13:15:18.138520Z","iopub.status.idle":"2023-09-27T13:15:18.147090Z","shell.execute_reply.started":"2023-09-27T13:15:18.138491Z","shell.execute_reply":"2023-09-27T13:15:18.145955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class imbalance handling\n\n## 1. undersampling\nc0 = train[train.outcome==0]\nc1 = train[train.outcome==1]\nc2 = train[train.outcome==2]\n\n# print(c0.shape, c1.shape, c2.shape)\nc0 = c0.sample(n=c1.shape[0])\ntrain = pd.concat([c0, c1])\ntrain.shape","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:15:57.510811Z","iopub.execute_input":"2023-09-27T13:15:57.511085Z","iopub.status.idle":"2023-09-27T13:15:57.526122Z","shell.execute_reply.started":"2023-09-27T13:15:57.511056Z","shell.execute_reply":"2023-09-27T13:15:57.525191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.id","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:16:03.029787Z","iopub.execute_input":"2023-09-27T13:16:03.030105Z","iopub.status.idle":"2023-09-27T13:16:03.043819Z","shell.execute_reply.started":"2023-09-27T13:16:03.030072Z","shell.execute_reply":"2023-09-27T13:16:03.042653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\n\nX = train[selected_cols]\ny = train['outcome']\n\nX = pd.concat([train.id, X], axis=1)\nsmote = SMOTE(k_neighbors=5)\nX_resampled, y_resampled = smote.fit_resample(X, y)\nprint(X_resampled.shape, y_resampled.shape)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:16:08.822693Z","iopub.execute_input":"2023-09-27T13:16:08.822989Z","iopub.status.idle":"2023-09-27T13:16:08.840803Z","shell.execute_reply.started":"2023-09-27T13:16:08.822957Z","shell.execute_reply":"2023-09-27T13:16:08.839548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_resampled, y_resampled, test_size=0.15, stratify=y_resampled, random_state=42)\nprint(X_train.shape, X_val.shape, y_train.shape, y_val.shape)\nX_val.to_csv('test.csv', index=False)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:16:45.822837Z","iopub.execute_input":"2023-09-27T13:16:45.823124Z","iopub.status.idle":"2023-09-27T13:16:45.838592Z","shell.execute_reply.started":"2023-09-27T13:16:45.823091Z","shell.execute_reply":"2023-09-27T13:16:45.837557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nif is_scaling:\n scaler = StandardScaler()\n data_ = scaler.fit_transform(X_train.drop(columns='id'))\n X_train = pd.DataFrame(data=data_, columns=(X_train.drop(columns='id')).columns)\n data_ = scaler.transform(X_val.drop(columns='id'))\n X_val = pd.DataFrame(data=data_, columns=(X_val.drop(columns='id')).columns)\n display(X_train)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:17:19.130486Z","iopub.execute_input":"2023-09-27T13:17:19.130800Z","iopub.status.idle":"2023-09-27T13:17:19.167581Z","shell.execute_reply.started":"2023-09-27T13:17:19.130766Z","shell.execute_reply":"2023-09-27T13:17:19.166532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_pca:\n from sklearn.decomposition import PCA\n\n pca = PCA(n_components=0.90, random_state=42)\n data_ = pca.fit_transform(X_train)\n X_train = pd.DataFrame(data=data_, columns=[f\"PC{i}\" for i in range(1, data_.shape[1]+1)])\n data_ = pca.transform(X_val)\n X_val = pd.DataFrame(data=data_, columns=[f\"PC{i}\" for i in range(1, data_.shape[1]+1)])\n\n display(X_train)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:17:23.016413Z","iopub.execute_input":"2023-09-27T13:17:23.016748Z","iopub.status.idle":"2023-09-27T13:17:23.024712Z","shell.execute_reply.started":"2023-09-27T13:17:23.016712Z","shell.execute_reply":"2023-09-27T13:17:23.023563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"svm_params = {\"C\":93,\n# \"kernel\":'precomputed',\n \"class_weight\":'balanced',\n \"gamma\":'auto',\n 'degree':2,\n 'coef0':0.45366687207674355,\n 'decision_function_shape':'ovo',\n 'probability':True,\n 'cache_size':1000,\n 'random_state':42\n }\nlr_params = {'C':82,\n 'solver':'sag',\n 'max_iter':500,\n 'multi_class':'multinomial',\n 'class_weight':'balanced',\n 'penalty':'l2',\n 'tol':0.0004915719520170494,\n 'random_state':42,\n 'n_jobs':-1\n }\nxgb_params = {'booster':'gbtree',\n 'colsample_bytree':0.7471481597860571,\n 'learning_rate':0.0016662352927965512,\n 'max_depth':4,\n 'n_estimators':134,\n 'num_class':3,\n 'random_state':42,\n 'reg_lambda':1.0980941643506688\n }\nrf_params = {'max_depth':4,\n 'max_features':0.7558496313859349,\n 'criterion':'gini',\n 'n_estimators':67,\n 'class_weight':'balanced',\n 'random_state':42\n }\n\ncatb_params = {'iterations':53,\n 'depth':4,\n 'bootstrap_type':'Bayesian',\n 'l2_leaf_reg':99.29985063946806,\n 'learning_rate':0.002375762467646143,\n 'bagging_temperature':2.2182206203898582,\n 'random_state':42\n }\nlgbm_params = {'max_depth':34,\n 'objective':'multiclass',\n 'num_leaves':3,\n 'learning_rate':3.6229230616515386e-09,\n 'lambda_l1':3.5132581656827253,\n 'lambda_l2':0.013405538678100381,\n 'random_state':42\n }\n\n# nn = Sequential([\n# Input(shape=(X_train.shape[1],)),\n# Dense(20), ReLU(), Dropout(0.2),\n# Dense(20), ReLU(), Dropout(0.2),\n# Dense(10), ReLU(), Dropout(0.1),\n# Dense(20), ReLU(), Dropout(0.2),\n# Dense(20), ReLU(), Dropout(0.2),\n# Dense(3), Softmax()\n# ])\n# nn.summary()\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:17:58.274019Z","iopub.execute_input":"2023-09-27T13:17:58.274329Z","iopub.status.idle":"2023-09-27T13:17:58.288624Z","shell.execute_reply.started":"2023-09-27T13:17:58.274295Z","shell.execute_reply":"2023-09-27T13:17:58.287666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [\n (\"svm\", SVC(**svm_params)),\n (\"lr\", LogisticRegression(**lr_params)),\n (\"xgb\", XGBClassifier(**xgb_params)),\n (\"rf\", RandomForestClassifier(**rf_params)),\n (\"catboost\", cat(**catb_params)),\n (\"lgbm\", lgb(**lgbm_params)),\n]","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:17:59.649184Z","iopub.execute_input":"2023-09-27T13:17:59.649445Z","iopub.status.idle":"2023-09-27T13:17:59.658407Z","shell.execute_reply.started":"2023-09-27T13:17:59.649417Z","shell.execute_reply":"2023-09-27T13:17:59.657372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voting_clf = VotingClassifier(estimators=models,\n voting='soft',\n weights=[0.2, 0.2, 0.2, 0.2, 0.1, 0.1]\n )\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:18:01.487193Z","iopub.execute_input":"2023-09-27T13:18:01.487457Z","iopub.status.idle":"2023-09-27T13:18:01.493011Z","shell.execute_reply.started":"2023-09-27T13:18:01.487428Z","shell.execute_reply":"2023-09-27T13:18:01.491666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"voting_clf.fit(X_train, y_train)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:18:03.780141Z","iopub.execute_input":"2023-09-27T13:18:03.780410Z","iopub.status.idle":"2023-09-27T13:18:04.233984Z","shell.execute_reply.started":"2023-09-27T13:18:03.780372Z","shell.execute_reply":"2023-09-27T13:18:04.232119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val = pd.read_csv('/kaggle/working/test.csv')\ny_pred = voting_clf.predict(X_val)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:18:27.361161Z","iopub.execute_input":"2023-09-27T13:18:27.361436Z","iopub.status.idle":"2023-09-27T13:18:27.446669Z","shell.execute_reply.started":"2023-09-27T13:18:27.361407Z","shell.execute_reply":"2023-09-27T13:18:27.445210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val = pd.read_csv('/kaggle/working/test.csv')\nsubmission = pd.DataFrame({'id': X_val['id'], 'outcome': y_pred})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:18:30.788221Z","iopub.execute_input":"2023-09-27T13:18:30.788752Z","iopub.status.idle":"2023-09-27T13:18:30.945895Z","shell.execute_reply.started":"2023-09-27T13:18:30.788714Z","shell.execute_reply":"2023-09-27T13:18:30.944384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.371267Z","iopub.status.idle":"2023-09-27T13:12:38.371765Z","shell.execute_reply.started":"2023-09-27T13:12:38.371479Z","shell.execute_reply":"2023-09-27T13:12:38.371505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# optimizer = Adam(learning_rate=learning_rate)\n# loss_fn = tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.8) # [0.8, 0.2] <--> [0.9, 0] // [0, 0.9]\n# scheduler = ReduceLROnPlateau(monitor='val_loss',\n# factor=0.5,\n# patience=10,\n# min_lr=1e-6)\n# earlystopper = EarlyStopping(monitor='val_loss',\n# patience=20,\n# min_delta=1e-2)\n\n\n# nn.compile(optimizer=optimizer, loss=loss_fn, metrics=[calculate_f1])\n\n# nn_y_train = tf.keras.utils.to_categorical(y_train, num_classes=3)\n# nn_y_val = tf.keras.utils.to_categorical(y_val, num_classes=3)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:12:38.373162Z","iopub.status.idle":"2023-09-27T13:12:38.373650Z","shell.execute_reply.started":"2023-09-27T13:12:38.373366Z","shell.execute_reply":"2023-09-27T13:12:38.373391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"\\nFitting LogisticRegression...\")\n# lr.fit(X_train, y_train)\n# print(\"\\nFitting SVM...\")\n# svm.fit(X_train, y_train)\n# print(\"\\nFitting RandomForest...\")\n# rf.fit(X_train, y_train)\n# print(\"\\nFitting XGBoost...\")\n# xgb.fit(X_train, y_train)\n# print(\"\\nFitting CatBoost...\")\n# catb.fit(X_train, y_train)\n# print(\"\\nFitting Light GBM...\")\n# lgbm.fit(X_train, y_train)\n# # print(\"\\nFitting MLP...\")\n# # history = nn.fit(X_train, nn_y_train,\n# # batch_size=batch_size,\n# # epochs=epochs,\n# # validation_data=[X_val, nn_y_val],\n# # callbacks=[scheduler, earlystopper])\n","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:12:38.375532Z","iopub.status.idle":"2023-09-27T13:12:38.376424Z","shell.execute_reply.started":"2023-09-27T13:12:38.376143Z","shell.execute_reply":"2023-09-27T13:12:38.376170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"evaluation_metric = calculate_f1","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:01:44.810625Z","iopub.status.idle":"2023-09-27T13:01:44.810945Z","shell.execute_reply.started":"2023-09-27T13:01:44.810771Z","shell.execute_reply":"2023-09-27T13:01:44.810789Z"}}},{"cell_type":"markdown","source":"print(\"--- Prediction with LR ---\")\nlr_pred_train = lr.predict(X_train)\nlr_pred_val = lr.predict(X_val)\n\nlr_train_score = evaluation_metric(y_train, lr_pred_train)\nlr_val_score = evaluation_metric(y_val, lr_pred_val)\n\nprint(\"Train Score : %.4f\" % lr_train_score)\nprint(\"Test Score : %.4f\" % lr_val_score)\n\nprint(\"--- Prediction with SVM ---\")\nsvm_pred_train = svm.predict(X_train)\nsvm_pred_val = svm.predict(X_val)\n\nsvm_train_score = evaluation_metric(y_train, svm_pred_train)\nsvm_val_score = evaluation_metric(y_val, svm_pred_val)\n\nprint(\"Train Score : %.4f\" % svm_train_score)\nprint(\"Test Score : %.4f\" % svm_val_score)\n\nprint(\"--- Prediction with RF ---\")\nrf_pred_train = rf.predict(X_train)\nrf_pred_val = rf.predict(X_val)\n\nrf_train_score = evaluation_metric(y_train, rf_pred_train)\nrf_val_score = evaluation_metric(y_val, rf_pred_val)\n\nprint(\"Train Score : %.4f\" % rf_train_score)\nprint(\"Test Score : %.4f\" % rf_val_score)\n\nprint(\"--- Prediction with CAT ---\")\ncat_pred_train = catb.predict(X_train)\ncat_pred_val = catb.predict(X_val)\n\ncat_train_score = evaluation_metric(y_train, cat_pred_train)\ncat_val_score = evaluation_metric(y_val, cat_pred_val)\n\nprint(\"Train Score : %.4f\" % cat_train_score)\nprint(\"Test Score : %.4f\" % cat_val_score)\n\nprint(\"--- Prediction with LGBM ---\")\nlgb_pred_train = lgbm.predict(X_train)\nlgb_pred_val = lgbm.predict(X_val)\n\nlgb_train_score = evaluation_metric(y_train, lgb_pred_train)\nlgb_val_score = evaluation_metric(y_val, lgb_pred_val)\n\nprint(\"Train Score : %.4f\" % lgb_train_score)\nprint(\"Test Score : %.4f\" % lgb_val_score)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-27T13:01:44.812414Z","iopub.status.idle":"2023-09-27T13:01:44.812752Z","shell.execute_reply.started":"2023-09-27T13:01:44.812564Z","shell.execute_reply":"2023-09-27T13:01:44.812581Z"}}},{"cell_type":"code","source":"'''\nimport smtplib\nfrom smtplib import SMTPException\nfrom getpass import getpass\n\nemail_user = 'your_@email.com'\nemail_pw = getpass('Enter your Password :')\n\nmail_from = email_user\nmail_to = 'maybe_another_@email.com'\n\nmail_subject = 'TEST'\nmail_message_body = 'test_message_body'\n\nmail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\ntry:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\nexcept SMTPException:\n print(\"Error: unable to send email\")\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.378531Z","iopub.status.idle":"2023-09-27T13:12:38.379025Z","shell.execute_reply.started":"2023-09-27T13:12:38.378762Z","shell.execute_reply":"2023-09-27T13:12:38.378787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef lr_optimizer(trial, X, y, K):\n\n C = trial.suggest_int('C', 5, 100)\n solver = trial.suggest_categorical('solver', ['newton-cg', 'sag', 'lbfgs', 'saga'])\n\n\n model = LogisticRegression(C=C,\n solver=solver,\n max_iter=500,\n multi_class='multinomial',\n class_weight='balanced',\n random_state=42,\n n_jobs=-1)\n\n\n kf = StratifiedKFold(n_splits=K, shuffle=True, random_state=42)\n f1_scores = []\n\n for train_idx, val_idx in kf.split(X, y):\n X_train = X.iloc[train_idx, :]\n y_train = y.iloc[train_idx]\n\n X_val = X.iloc[val_idx, :]\n y_val = y.iloc[val_idx]\n\n model.fit(X_train, y_train)\n preds = model.predict(X_val)\n f1 = f1_score(y_val, preds, average='micro')\n f1_scores.append(f1)\n\n\n return -np.mean(f1_scores)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.380859Z","iopub.status.idle":"2023-09-27T13:12:38.381355Z","shell.execute_reply.started":"2023-09-27T13:12:38.381078Z","shell.execute_reply":"2023-09-27T13:12:38.381104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n%%time\ntry:\n K = 6 # set K of K-Fold\n opt_func = partial(lr_optimizer, X=X_train, y=y_train, K=K)\n\n if is_tuning:\n lr_study = optuna.create_study(direction=\"maximize\") # determine minimize or maximize sth\n lr_study.optimize(opt_func, n_trials=n_trials)\n mail_subject = 'Logistic Regression Finished'\n mail_message_body = 'Logistic Regression Finished.\\\n Starting Random Forest.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, msg=mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\nexcept:\n mail_subject = 'Logistic Regression Failed'\n mail_message_body = 'Logistic Regression Failed.\\\n Shutting Colab off.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\n\n from google.colab import runtime\n runtime.unassign()\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.382903Z","iopub.status.idle":"2023-09-27T13:12:38.383713Z","shell.execute_reply.started":"2023-09-27T13:12:38.383404Z","shell.execute_reply":"2023-09-27T13:12:38.383432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef rf_optimizer(trial, X, y, K):\n # define parameter to tune\n n_estimators = trial.suggest_int('n_estimators', 50, 500)\n max_depth = trial.suggest_int('max_depth', 4, 20)\n max_features = trial.suggest_float('max_features', 0.6, 0.8, log=True)\n\n model = RandomForestClassifier(n_estimators=n_estimators,\n max_depth=max_depth,\n max_features=max_features,\n criterion='gini',\n class_weight='balanced',\n random_state=42\n )\n kf = StratifiedKFold(n_splits=K, shuffle=True, random_state=42)\n f1_scores = []\n\n for train_idx, val_idx in kf.split(X, y):\n X_train, X_val = X.iloc[train_idx, :], X.iloc[val_idx, :]\n y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n model.fit(X_train, y_train)\n preds = model.predict(X_val)\n f1 = f1_score(y_val, preds, average='micro')\n f1_scores.append(f1)\n# print(f1)\n\n return -np.mean(f1_scores)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.385240Z","iopub.status.idle":"2023-09-27T13:12:38.386298Z","shell.execute_reply.started":"2023-09-27T13:12:38.386001Z","shell.execute_reply":"2023-09-27T13:12:38.386029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n%%time\n\nK = 6 # set K of K-Fold\nopt_func = partial(rf_optimizer, X=X_train, y=y_train, K=K)\ntry:\n if is_tuning:\n rf_study = optuna.create_study(direction=\"maximize\") # determine minimize or maximize sth\n rf_study.optimize(opt_func, n_trials=n_trials)\n mail_subject = 'Random Forest Finished'\n mail_message_body = 'Random Forest Finished.\\\n Starting Support Vector Machine.'\n \n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n \n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\nexcept:\n mail_subject = 'Random Forest Failed'\n mail_message_body = 'Random Forest Failed.\\\n Shutting Colab off.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\n\n from google.colab import runtime\n runtime.unassign()\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.388192Z","iopub.status.idle":"2023-09-27T13:12:38.388686Z","shell.execute_reply.started":"2023-09-27T13:12:38.388404Z","shell.execute_reply":"2023-09-27T13:12:38.388429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef svm_optimizer(trial, X, y, K):\n C = trial.suggest_int('C', 1, 100)\n\n model = SVC(C=C,\n kernel='rbf',\n class_weight='balanced', # if class imbalanced\n gamma='auto',\n decision_function_shape='ovo',\n probability=True,\n cache_size=1000,\n random_state=42\n )\n\n kf = StratifiedKFold(n_splits=K, shuffle=True, random_state=42)\n f1_scores = []\n\n for train_idx, val_idx in kf.split(X, y):\n X_train = X.iloc[train_idx, :]\n y_train = y.iloc[train_idx]\n\n X_val = X.iloc[val_idx, :]\n y_val = y.iloc[val_idx]\n\n model.fit(X_train, y_train)\n preds = model.predict(X_val)\n f1 = f1_score(y_val, preds, average='micro')\n f1_scores.append(f1)\n\n\n return -np.mean(f1_scores)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.390080Z","iopub.status.idle":"2023-09-27T13:12:38.390934Z","shell.execute_reply.started":"2023-09-27T13:12:38.390655Z","shell.execute_reply":"2023-09-27T13:12:38.390683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n%%time\n\nK = 6\nopt_func = partial(svm_optimizer, X=X_train, y=y_train, K=K)\n\ntry:\n if is_tuning:\n svm_study = optuna.create_study(direction=\"maximize\")\n svm_study.optimize(opt_func, n_trials=n_trials)\n mail_subject = 'Support Vector Machine Finished'\n mail_message_body = 'Support Vector Machine Finished.\\\n Starting XG Boost.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\nexcept:\n mail_subject = 'Support Vector Machine Failed'\n mail_message_body = 'Support Vector Machine Failed.\\\n Shutting Colab off.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\n\n from google.colab import runtime\n runtime.unassign()\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.392381Z","iopub.status.idle":"2023-09-27T13:12:38.392873Z","shell.execute_reply.started":"2023-09-27T13:12:38.392583Z","shell.execute_reply":"2023-09-27T13:12:38.392626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef xgb_optimizer(trial, X, y, K):\n n_estimators = trial.suggest_int('n_estimators', 50, 500)\n max_depth = trial.suggest_int('max_depth', 4, 20)\n colsample_bytree = trial.suggest_float('colsample_bytree', 0.5, 0.8, log=True)\n learning_rate = trial.suggest_float('learning_rate', 1e-3, 1e-2, log=True)\n num_class = trial.suggest_int('num_class', 2, 10)\n reg_lambda = trial.suggest_float('reg_lambda', 0.1, 2, log=True)\n booster = trial.suggest_categorical('booster', ['gbtree', 'dart'])\n\n\n model = XGBClassifier(objective='multi:softmax',\n n_estimators=n_estimators,\n max_depth=max_depth,\n colsample_bytree=colsample_bytree,\n num_class=num_class,\n eta=learning_rate,\n reg_lambda=reg_lambda,\n random_state=42\n )\n\n\n kf = StratifiedKFold(n_splits=K, shuffle=True, random_state=42)\n f1_scores = []\n\n for train_idx, val_idx in kf.split(X, y):\n X_train = X.iloc[train_idx, :]\n y_train = y.iloc[train_idx]\n\n X_val = X.iloc[val_idx, :]\n y_val = y.iloc[val_idx]\n\n model.fit(X_train, y_train)\n preds = model.predict(X_val)\n f1 = f1_score(y_val, preds, average='micro')\n f1_scores.append(f1)\n\n\n return -np.mean(f1_scores)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.394313Z","iopub.status.idle":"2023-09-27T13:12:38.394796Z","shell.execute_reply.started":"2023-09-27T13:12:38.394514Z","shell.execute_reply":"2023-09-27T13:12:38.394539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n%%time\n\nK = 6\nopt_func = partial(xgb_optimizer, X=X_train, y=y_train, K=K)\ntry:\n if is_tuning:\n xgb_study = optuna.create_study(direction=\"maximize\")\n xgb_study.optimize(opt_func, n_trials=n_trials)\n mail_subject = 'XG Boost Finished'\n mail_message_body = 'XG Boost Finished.\\\n Starting Cat Boost.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\nexcept:\n mail_subject = 'XG Boost Failed'\n mail_message_body = 'XG Boost Failed.\\\n Shutting Colab off.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Failed, Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\n\n from google.colab import runtime\n runtime.unassign()\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.397736Z","iopub.status.idle":"2023-09-27T13:12:38.398218Z","shell.execute_reply.started":"2023-09-27T13:12:38.397946Z","shell.execute_reply":"2023-09-27T13:12:38.397971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef cat_optimizer(trial, X, y, K):\n iterations = trial.suggest_int('iterations', 50, 200)\n depth = trial.suggest_int('depth', 4, 16)\n bootstrap_type = trial.suggest_categorical('bootstrap_type', ['Bayesian'])\n learning_rate = trial.suggest_float('learning_rate', 1e-3, 1e-2, log=True)\n l2_leaf_reg = trial.suggest_float('l2_leaf_reg', 1e-8, 100.0, log=True)\n bagging_temperature = trial.suggest_float('bagging_temperature', 0, 10, log=False)\n\n model = cat(iterations=iterations,\n depth=depth,\n bootstrap_type=bootstrap_type,\n l2_leaf_reg=l2_leaf_reg,\n learning_rate=learning_rate,\n bagging_temperature=bagging_temperature,\n random_state=42\n )\n\n\n kf = StratifiedKFold(n_splits=K, shuffle=True, random_state=42)\n f1_scores = []\n\n for train_idx, val_idx in kf.split(X, y):\n X_train = X.iloc[train_idx, :]\n y_train = y.iloc[train_idx]\n\n X_val = X.iloc[val_idx, :]\n y_val = y.iloc[val_idx]\n\n model.fit(X_train, y_train)\n preds = model.predict(X_val)\n f1 = f1_score(y_val, preds, average='micro')\n f1_scores.append(f1)\n\n\n return -np.mean(f1_scores)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.399839Z","iopub.status.idle":"2023-09-27T13:12:38.400829Z","shell.execute_reply.started":"2023-09-27T13:12:38.400535Z","shell.execute_reply":"2023-09-27T13:12:38.400561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n%%time\n\nK = 6\nopt_func = partial(cat_optimizer, X=X_train, y=y_train, K=K)\ntry:\n if is_tuning:\n cat_study = optuna.create_study(direction=\"maximize\")\n cat_study.optimize(opt_func, n_trials=n_trials)\n mail_subject = 'Cat Boost Finished'\n mail_message_body = 'Cat Boost Finished.\\\n Starting Light GBM.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\nexcept:\n mail_subject = 'Cat Boost Failed'\n mail_message_body = 'Cat Boost Failed.\\\n Shutting Colab off.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\n\n from google.colab import runtime\n runtime.unassign()\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.402073Z","iopub.status.idle":"2023-09-27T13:12:38.402927Z","shell.execute_reply.started":"2023-09-27T13:12:38.402646Z","shell.execute_reply":"2023-09-27T13:12:38.402674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef lgbm_optimizer(trial, X, y, K):\n num_leaves = trial.suggest_int('num_leaves', 100, 500)\n max_depth = trial.suggest_int('max_depth', 4, 20)\n boosting_type = trial.suggest_categorical('boosting_type', ['gbdt', 'dart', 'rf'])\n learning_rate = trial.suggest_float('learning_rate', 1e-3, 1e-2, log=True)\n\n\n model = lgb(max_depth=max_depth,\n objective='multiclass',\n metric='multi_logloss',\n num_leaves=num_leaves,\n boosting_type=boosting_type,\n learning_rate=learning_rate,\n random_state=42\n )\n\n\n kf = StratifiedKFold(n_splits=K, shuffle=True, random_state=42)\n f1_scores = []\n\n for train_idx, val_idx in kf.split(X, y):\n X_train = X.iloc[train_idx, :]\n y_train = y.iloc[train_idx]\n\n X_val = X.iloc[val_idx, :]\n y_val = y.iloc[val_idx]\n\n model.fit(X_train, y_train)\n preds = model.predict(X_val)\n f1 = f1_score(y_val, preds, average='micro')\n f1_scores.append(f1)\n\n\n return -np.mean(f1_scores)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.404444Z","iopub.status.idle":"2023-09-27T13:12:38.405150Z","shell.execute_reply.started":"2023-09-27T13:12:38.404853Z","shell.execute_reply":"2023-09-27T13:12:38.404880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n%%time\n\nK = 6\nopt_func = partial(lgbm_optimizer, X=X_train, y=y_train, K=K)\ntry:\n if is_tuning:\n lgbm_study = optuna.create_study(direction=\"maximize\")\n lgbm_study.optimize(opt_func, n_trials=n_trials)\n mail_subject = 'Light GBM Finished'\n mail_message_body = 'Light GBM Finished.\\\n Starting Training Done.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\nexcept:\n mail_subject = 'Light GBM Failed'\n mail_message_body = 'Light GBM Failed.\\\n Shutting Colab off.'\n\n mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n try:\n server = smtplib.SMTP_SSL('smtp.email.com', 465)\n server.login(email_user, email_pw)\n server.sendmail(mail_from, mail_to, mail_message)\n server.close()\n print(\"Successfully sent email\")\n except SMTPException:\n print(\"Error: unable to send email\")\n\n from google.colab import runtime\n runtime.unassign()\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.406736Z","iopub.status.idle":"2023-09-27T13:12:38.407633Z","shell.execute_reply.started":"2023-09-27T13:12:38.407325Z","shell.execute_reply":"2023-09-27T13:12:38.407353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# save all studies\nif is_tuning:\n with open(\"rf_study.pk\", 'wb') as f:\n pickle.dump(rf_study, f)\n with open(\"lr_study.pk\", 'wb') as f:\n pickle.dump(lr_study, f)\n with open(\"svm_study.pk\", 'wb') as f:\n pickle.dump(svm_study, f)\n with open(\"xgb_study.pk\", 'wb') as f:\n pickle.dump(xgb_study, f)\n with open(\"cat_study.pk\", 'wb') as f:\n pickle.dump(cat_study, f)\n with open(\"lgbm_study.pk\", 'wb') as f:\n pickle.dump(lgbm_study, f)\n\n# nn.save(\"./simple_nn_model.keras\")\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.409027Z","iopub.status.idle":"2023-09-27T13:12:38.409752Z","shell.execute_reply.started":"2023-09-27T13:12:38.409454Z","shell.execute_reply":"2023-09-27T13:12:38.409479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# visualize experiment logs\ndef display_experiment_log(study):\n display(study.trials_dataframe())\n print(\"Best Score: %.4f\" % study.best_value)\n print(\"Best params: \", study.best_trial.params)\n history = study.trials_dataframe()\n display(history[history.value == study.best_value])\n optuna.visualization.plot_optimization_history(study).show()\n optuna.visualization.plot_param_importances(study).show()\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.411215Z","iopub.status.idle":"2023-09-27T13:12:38.412004Z","shell.execute_reply.started":"2023-09-27T13:12:38.411702Z","shell.execute_reply":"2023-09-27T13:12:38.411730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nif is_tuning:\n display_experiment_log(rf_study)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.413532Z","iopub.status.idle":"2023-09-27T13:12:38.414168Z","shell.execute_reply.started":"2023-09-27T13:12:38.413883Z","shell.execute_reply":"2023-09-27T13:12:38.413909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nif is_tuning:\n display_experiment_log(lr_study)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.415820Z","iopub.status.idle":"2023-09-27T13:12:38.416507Z","shell.execute_reply.started":"2023-09-27T13:12:38.416228Z","shell.execute_reply":"2023-09-27T13:12:38.416254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nif is_tuning:\n display_experiment_log(svm_study)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.417961Z","iopub.status.idle":"2023-09-27T13:12:38.419069Z","shell.execute_reply.started":"2023-09-27T13:12:38.418783Z","shell.execute_reply":"2023-09-27T13:12:38.418810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nif is_tuning:\n display_experiment_log(xgb_study)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.420694Z","iopub.status.idle":"2023-09-27T13:12:38.421175Z","shell.execute_reply.started":"2023-09-27T13:12:38.420891Z","shell.execute_reply":"2023-09-27T13:12:38.420914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nif is_tuning:\n display_experiment_log(cat_study)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.422951Z","iopub.status.idle":"2023-09-27T13:12:38.423433Z","shell.execute_reply.started":"2023-09-27T13:12:38.423175Z","shell.execute_reply":"2023-09-27T13:12:38.423200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nif is_tuning:\n display_experiment_log(lgbm_study)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.424904Z","iopub.status.idle":"2023-09-27T13:12:38.425374Z","shell.execute_reply.started":"2023-09-27T13:12:38.425117Z","shell.execute_reply":"2023-09-27T13:12:38.425143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n## preprocessing in same way\nX_test = test[train.columns.drop(\"Class\")].fillna(test.mean())\nif is_scaling:\n X_test = scaler.transform(X_test)\n\nif is_pca:\n data_ = pca.transform(X_test)\n X_test = pd.DataFrame(data=data_, columns=[f\"PC{i}\" for i in range(1, data_.shape[1]+1)])\n\nX_test\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.427345Z","iopub.status.idle":"2023-09-27T13:12:38.427835Z","shell.execute_reply.started":"2023-09-27T13:12:38.427551Z","shell.execute_reply":"2023-09-27T13:12:38.427575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# # Finalize Models\n# if is_tuning:\n# rf_best_params = rf_study.best_params\n# lr_best_params = lr_study.best_params\n# xgb_best_params = xgb_study.best_params\n# svm_best_params = svm_study.best_params\n# lgbm_best_params = lgbm_study.best_params\n# cat_best_params = cat_study.best_params\n\n# best_rf = RandomForestClassifier(**rf_best_params)\n# best_lr = LogisticRegression(**lr_best_params)\n# best_xgb = XGBClassifier(**xgb_best_params)\n# best_svm = SVC(**svm_best_params, probability=True)\n# best_lgbm = lgb(**lgbm_best_params)\n# best_cat = cat(**cat_best_params)\n\n# try:\n# mail_subject = 'Best Parameters'\n\n# mail_message_body = f'''Random Forest : Best Params {rf_best_params}, Best_RF : {best_rf} \\n\\r\\\n# Logistic Regression : Best Params {lr_best_params}, Best_LR : {best_lr} \\n\\r\\\n# XGB : Best Params {xgb_best_params}, Best_xgb : {best_xgb} \\n\\r\\\n# SVM : Best Params {svm_best_params}, Best_svm : {best_svm} \\n\\r\\\n# LGBM : Best Params {lgbm_best_params}, Best_LGBM : {best_lgbm} \\n\\r\\\n# CatBoost : Best Params {cat_best_params}, Best_RF : {best_cat} \\n\\r\\\n# '''\n\n# mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n# try:\n# server = smtplib.SMTP_SSL('smtp.email.com', 465)\n# server.login(email_user, email_pw)\n# server.sendmail(mail_from, mail_to, mail_message)\n# server.close()\n# print(\"Successfully sent email\")\n# except SMTPException:\n# print(\"Error: unable to send email\")\n# except:\n# mail_subject = 'Best Parameters... Supposed to be...'\n# mail_message_body = f'Something Wrong...'\n\n# mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n# try:\n# server = smtplib.SMTP_SSL('smtp.email.com', 465)\n# server.login(email_user, email_pw)\n# server.sendmail(mail_from, mail_to, mail_message)\n# server.close()\n# print(\"Successfully sent email\")\n# except SMTPException:\n# print(\"Error: unable to send email\")\n# from google.colab import runtime\n# runtime.unassign()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.429677Z","iopub.status.idle":"2023-09-27T13:12:38.430159Z","shell.execute_reply.started":"2023-09-27T13:12:38.429886Z","shell.execute_reply":"2023-09-27T13:12:38.429911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nprint(rf_best_params)\n# print(rf_study.best_trial)\nprint(rf_study.best_value)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.431760Z","iopub.status.idle":"2023-09-27T13:12:38.432286Z","shell.execute_reply.started":"2023-09-27T13:12:38.432011Z","shell.execute_reply":"2023-09-27T13:12:38.432035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nprint(lr_best_params)\n# print(lr_study.best_trial)\nprint(lr_study.best_value)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.433550Z","iopub.status.idle":"2023-09-27T13:12:38.434026Z","shell.execute_reply.started":"2023-09-27T13:12:38.433766Z","shell.execute_reply":"2023-09-27T13:12:38.433791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nprint(xgb_best_params)\n# print(xgb_study.best_trial)\nprint(xgb_study.best_value)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.436013Z","iopub.status.idle":"2023-09-27T13:12:38.436844Z","shell.execute_reply.started":"2023-09-27T13:12:38.436638Z","shell.execute_reply":"2023-09-27T13:12:38.436663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nprint(svm_best_params)\n# print(svm_study.best_trial)\nprint(svm_study.best_value)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.437994Z","iopub.status.idle":"2023-09-27T13:12:38.438720Z","shell.execute_reply.started":"2023-09-27T13:12:38.438492Z","shell.execute_reply":"2023-09-27T13:12:38.438513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nprint(lgbm_best_params)\n# print(lgbm_study.best_trial)\nprint(lgbm_study.best_value)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.439927Z","iopub.status.idle":"2023-09-27T13:12:38.440560Z","shell.execute_reply.started":"2023-09-27T13:12:38.440375Z","shell.execute_reply":"2023-09-27T13:12:38.440394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nprint(cat_best_params)\n# print(cat_study.best_trial)\nprint(cat_study.best_value)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.441703Z","iopub.status.idle":"2023-09-27T13:12:38.442304Z","shell.execute_reply.started":"2023-09-27T13:12:38.442111Z","shell.execute_reply":"2023-09-27T13:12:38.442143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try:\n# # first ensebmle model, then check it.\n# best_rf.fit(X_train, y_train)\n# best_lr.fit(X_train, y_train)\n# best_xgb.fit(X_train, y_train)\n# best_svm.fit(X_train, y_train)\n# best_lgbm.fit(X_train, y_train)\n# best_cat.fit(X_train, y_train)\n\n# # OOF-prediction\n# v_rf = best_rf.predict(X_val)\n# v_lr = best_lr.predict(X_val)\n# v_xgb = best_xgb.predict(X_val)\n# v_svm = best_svm.predict(X_val)\n# v_lgbm = best_lgbm.predict(X_val)\n# v_cat = best_cat.predict(X_val)\n\n# print(v_rf.shape, v_lr.shape, v_xgb.shape, v_svm.shape, v_lgbm.shape, v_cat.shape)\n\n# preds_rf = best_rf.predict(X_test)\n# preds_lr = best_lr.predict(X_test)\n# preds_xgb = best_xgb.predict(X_test)\n# preds_svm = best_svm.predict(X_test)\n# preds_lgbm = best_lgbm.predict(X_test)\n# preds_cat = best_cat.predict(X_test)\n# print(preds_rf.shape, preds_lr.shape, preds_xgb.shape, preds_svm.shape, preds_lgbm.shape, preds_cat.shape)\n\n\n# mail_subject = 'Best Parameters'\n# mail_message_body = f'''Random Forest : Best Params {rf_best_params}, Best_RF : {best_rf} \\n\\\n# Logistic Regression : Best Params {lr_best_params}, Best_LR : {best_lr} \\n\\\n# XGB : Best Params {xgb_best_params}, Best_xgb : {best_xgb} \\n\\\n# SVM : Best Params {svm_best_params}, Best_svm : {best_svm} \\n\\\n# LGBM : Best Params {lgbm_best_params}, Best_LGBM : {best_lgbm} \\n\\\n# CatBoost : Best Params {cat_best_params}, Best_RF : {best_cat} \\n\\\n# '''\n\n# mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n# try:\n# server = smtplib.SMTP_SSL('smtp.email.com', 465)\n# server.login(email_user, email_pw)\n# server.sendmail(mail_from, mail_to, mail_message)\n# server.close()\n# print(\"Successfully sent email\")\n# except SMTPException:\n# print(\"Error: unable to send email\")\n# except:\n# mail_subject = 'Best Parameters... Supposed to be...'\n# mail_message_body = f'Something Wrong...'\n\n# mail_message = f'From: {mail_from}\\nTo: {mail_to}\\nSubject: {mail_subject}\\n\\n{mail_message_body}'\n\n# try:\n# server = smtplib.SMTP_SSL('smtp.email.com', 465)\n# server.login(email_user, email_pw)\n# server.sendmail(mail_from, mail_to, mail_message)\n# server.close()\n# print(\"Successfully sent email\")\n# except SMTPException:\n# print(\"Error: unable to send email\")\n# from google.colab import runtime\n# runtime.unassign()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.443403Z","iopub.status.idle":"2023-09-27T13:12:38.444021Z","shell.execute_reply.started":"2023-09-27T13:12:38.443834Z","shell.execute_reply":"2023-09-27T13:12:38.443854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # MLP predictions\n# v_nn = nn.predict(X_val)\n# preds_nn = nn.predict(X_test)\n# print(v_nn.shape, preds_nn.shape)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T13:12:38.445305Z","iopub.status.idle":"2023-09-27T13:12:38.445978Z","shell.execute_reply.started":"2023-09-27T13:12:38.445789Z","shell.execute_reply":"2023-09-27T13:12:38.445809Z"},"trusted":true},"execution_count":null,"outputs":[]}]}