{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Tabular Playground Series - Aug 2022","metadata":{}},{"cell_type":"markdown","source":"## NoteBook\n\n### LightGBM+XGBoost+ANN Ensemble Model\n* Data: [LightGBM+XGBoost+ANN | Ensemble](https://www.kaggle.com/code/juhjoo/0-585-tps-aug-lightgbm-xgboost-ann-ensemble)\n\n### TabNet Model\n* Data: [Tabnet BaseLine Model](https://www.kaggle.com/juhjoo/0-523-tps-aug-tabnet-baseline)\n","metadata":{}},{"cell_type":"markdown","source":"## My Strategy\n\n### Simple Data Analysis\n* overview of the dataset, Head, Describe, NaN Analysis\n\n### Data Pre-Processing\n* Fill missing values\n* Impute the missing values (KNNImputer) (22-08-09)\n* Encode Object Features\n\n### Feature Engineering\n\n### Model Development\n* Hyper Parameter Optimization by Bayesian\n* K-fold Cross validation loop based on Seed\n* Constructing of a Tabnet Model\n\n### Creating a Submission File","metadata":{}},{"cell_type":"markdown","source":"## Import Library","metadata":{}},{"cell_type":"code","source":"# Core\nimport pandas as pd\nimport numpy as np\nimport os\nimport cv2\nimport gc\nimport itertools\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom tqdm.notebook import tqdm\nfrom datetime import datetime\nimport json,itertools\nfrom typing import Optional\nfrom glob import glob\nimport warnings\nfrom IPython import display as ipd\nwarnings.filterwarnings(\"ignore\")\nimport matplotlib.gridspec as gridspec\nimport matplotlib.patches as mpatches\nimport matplotlib as mpl\nfrom matplotlib.patches import Rectangle\nimport seaborn as sns\nimport random\nfrom joblib import Parallel, delayed\nimport os, shutil\nimport datetime \n\nfrom sklearn.impute import KNNImputer\nfrom sklearn.preprocessing import LabelEncoder\n\nimport lightgbm as lgb\nfrom sklearn.metrics import r2_score\n\n\nfrom sklearn.model_selection import KFold, train_test_split, GroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nfrom sklearn.linear_model import LogisticRegression\n\nimport missingno as msno\nimport plotly.express as px\n\nimport xgboost as xgb\n\n# Keras\nfrom tensorflow import keras\nimport tensorflow as tf\nimport keras\nfrom keras import backend as K\nfrom keras.models import Model\nfrom tensorflow.keras.layers import LSTM, Flatten, TimeDistributed, Conv1D, Input, Dense, Multiply, Add, Activation, GRU, BatchNormalization\nfrom keras.layers.convolutional import Conv2D, Conv2DTranspose\nfrom keras.layers.pooling import MaxPooling2D\nfrom keras.layers.merge import concatenate\nfrom keras.losses import binary_crossentropy\nfrom keras.callbacks import Callback, ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom keras.models import load_model, save_model, Sequential\nfrom tensorflow.data import Dataset\nfrom tensorflow.keras.initializers import TruncatedNormal\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras import optimizers\nimport wandb\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Activation, Conv1D, Flatten,MaxPooling1D,BatchNormalization, Lambda, AveragePooling1D, Dropout, Input\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint,ReduceLROnPlateau\nimport tensorflow.keras as keras","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Install Tabnet Library","metadata":{}},{"cell_type":"code","source":"!pip install  \"git+https://github.com/dreamquark-ai/tabnet.git@develop#egg=pytorch_tabnet\" --upgrade","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import Tabnet","metadata":{}},{"cell_type":"code","source":"import torch\nfrom pytorch_tabnet.tab_model import TabNetClassifier\nfrom pytorch_tabnet.metrics import Metric","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Dataset","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest_df = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:31:09.766534Z","iopub.execute_input":"2022-08-09T12:31:09.768408Z","iopub.status.idle":"2022-08-09T12:31:10.044319Z","shell.execute_reply.started":"2022-08-09T12:31:09.768360Z","shell.execute_reply":"2022-08-09T12:31:10.043032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Pre-Processing","metadata":{}},{"cell_type":"code","source":"object_cols = [col for col in train_df.columns if train_df[col].dtypes == 'object']\nnull_value_cols = [col for col in train_df.columns if train_df[col].isnull().sum()!=0]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:31:10.045913Z","iopub.execute_input":"2022-08-09T12:31:10.047125Z","iopub.status.idle":"2022-08-09T12:31:10.076219Z","shell.execute_reply.started":"2022-08-09T12:31:10.047088Z","shell.execute_reply":"2022-08-09T12:31:10.074822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat([train_df,test_df],ignore_index=True)\ndf = df.drop(columns=['product_code'])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:31:10.079390Z","iopub.execute_input":"2022-08-09T12:31:10.080207Z","iopub.status.idle":"2022-08-09T12:31:10.171169Z","shell.execute_reply.started":"2022-08-09T12:31:10.080132Z","shell.execute_reply":"2022-08-09T12:31:10.169475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = train_df.columns.drop(['id', 'product_code', 'failure'])\ncols","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:31:10.172965Z","iopub.execute_input":"2022-08-09T12:31:10.173878Z","iopub.status.idle":"2022-08-09T12:31:10.184410Z","shell.execute_reply.started":"2022-08-09T12:31:10.173829Z","shell.execute_reply":"2022-08-09T12:31:10.183212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Encode Object Features","metadata":{}},{"cell_type":"code","source":"label = LabelEncoder()\n\ndf['attribute_0'] = label.fit_transform(df['attribute_0'])\ndf['attribute_1'] = label.fit_transform(df['attribute_1'])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:31:10.185962Z","iopub.execute_input":"2022-08-09T12:31:10.187081Z","iopub.status.idle":"2022-08-09T12:31:10.227016Z","shell.execute_reply.started":"2022-08-09T12:31:10.187027Z","shell.execute_reply":"2022-08-09T12:31:10.225755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Process Missing Data","metadata":{}},{"cell_type":"code","source":"Columns = ['loading',  'measurement_3', 'measurement_4', 'measurement_5', 'measurement_6', 'measurement_7',\n       'measurement_8', 'measurement_9', 'measurement_10', 'measurement_11',\n       'measurement_12', 'measurement_13', 'measurement_14', 'measurement_15',\n       'measurement_16', 'measurement_17']\n'''\nfor col in Columns:\n    if df[col].isnull().sum():\n        df[col].fillna(df[col].mean(), inplace=True)\n'''\nimputer = KNNImputer(n_neighbors=3)\nimputer.fit(train_df[Columns])\ndf[Columns] = imputer.transform(df[Columns])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:31:10.231497Z","iopub.execute_input":"2022-08-09T12:31:10.232769Z","iopub.status.idle":"2022-08-09T12:32:00.068595Z","shell.execute_reply.started":"2022-08-09T12:31:10.232715Z","shell.execute_reply":"2022-08-09T12:32:00.067229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.069783Z","iopub.execute_input":"2022-08-09T12:32:00.070192Z","iopub.status.idle":"2022-08-09T12:32:00.085333Z","shell.execute_reply.started":"2022-08-09T12:32:00.070144Z","shell.execute_reply":"2022-08-09T12:32:00.084033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Devide the Train and Test Dataset","metadata":{}},{"cell_type":"code","source":"train = df[:26570]\ntest = df[26570:]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.086870Z","iopub.execute_input":"2022-08-09T12:32:00.088024Z","iopub.status.idle":"2022-08-09T12:32:00.093437Z","shell.execute_reply.started":"2022-08-09T12:32:00.087928Z","shell.execute_reply":"2022-08-09T12:32:00.092344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add a Feature","metadata":{}},{"cell_type":"code","source":"train['attribute_2*3'] = train['attribute_2'] * train['attribute_3']\ntest['attribute_2*3'] = test['attribute_2'] * test['attribute_3']\n\nmeas_gr1_cols = [f\"measurement_{i:d}\" for i in list(range(3, 5)) + list(range(9, 17))]\ntrain['meas_gr1_avg'] = np.mean(train[meas_gr1_cols], axis=1)\ntrain['meas_gr1_std'] = np.std(train[meas_gr1_cols], axis=1)\n\ntest['meas_gr1_avg'] = np.mean(test[meas_gr1_cols], axis=1)\ntest['meas_gr1_std'] = np.std(test[meas_gr1_cols], axis=1) \n\nmeas_gr2_cols = [f\"measurement_{i:d}\" for i in list(range(5, 9))]\ntrain['meas_gr2_avg'] = np.mean(train[meas_gr2_cols], axis=1)\ntest['meas_gr2_avg'] = np.mean(test[meas_gr2_cols], axis=1)\n    \ntrain['meas17/meas_gr2_avg'] = train['measurement_17'] / train['meas_gr2_avg']\ntest['meas17/meas_gr2_avg'] = test['measurement_17'] / test['meas_gr2_avg']","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.097696Z","iopub.execute_input":"2022-08-09T12:32:00.098078Z","iopub.status.idle":"2022-08-09T12:32:00.146786Z","shell.execute_reply.started":"2022-08-09T12:32:00.098043Z","shell.execute_reply":"2022-08-09T12:32:00.145500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.reset_index(inplace=True)\ntest = test.drop(columns=['index','failure'])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.148415Z","iopub.execute_input":"2022-08-09T12:32:00.148790Z","iopub.status.idle":"2022-08-09T12:32:00.159819Z","shell.execute_reply.started":"2022-08-09T12:32:00.148756Z","shell.execute_reply":"2022-08-09T12:32:00.158498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = train.drop(columns=['failure'])\ntrain_Y = train['failure']","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.161463Z","iopub.execute_input":"2022-08-09T12:32:00.162083Z","iopub.status.idle":"2022-08-09T12:32:00.173831Z","shell.execute_reply.started":"2022-08-09T12:32:00.162046Z","shell.execute_reply":"2022-08-09T12:32:00.172573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.175601Z","iopub.execute_input":"2022-08-09T12:32:00.176428Z","iopub.status.idle":"2022-08-09T12:32:00.204754Z","shell.execute_reply.started":"2022-08-09T12:32:00.176390Z","shell.execute_reply":"2022-08-09T12:32:00.203724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.206171Z","iopub.execute_input":"2022-08-09T12:32:00.206854Z","iopub.status.idle":"2022-08-09T12:32:00.231491Z","shell.execute_reply.started":"2022-08-09T12:32:00.206810Z","shell.execute_reply":"2022-08-09T12:32:00.230444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_Y.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:00.233082Z","iopub.execute_input":"2022-08-09T12:32:00.233477Z","iopub.status.idle":"2022-08-09T12:32:00.245967Z","shell.execute_reply.started":"2022-08-09T12:32:00.233442Z","shell.execute_reply":"2022-08-09T12:32:00.245053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nunique = train_X.nunique()\ntypes = train_X.dtypes\n\ncategorical_columns = []\ncategorical_dims =  {}\n\nfor col in tqdm(train_X.columns):\n    if types[col] == 'object':\n        categorical_columns.append(col)\n        categorical_dims[col] = len(l_enc.classes_)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:43.244691Z","iopub.execute_input":"2022-08-09T12:32:43.245186Z","iopub.status.idle":"2022-08-09T12:32:43.320211Z","shell.execute_reply.started":"2022-08-09T12:32:43.245119Z","shell.execute_reply":"2022-08-09T12:32:43.318605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [ col for col in train_X.columns] \ncat_idxs = [ i for i, f in enumerate(features) if f in categorical_columns]\ncat_dims = [ categorical_dims[f] for i, f in enumerate(features) if f in categorical_columns]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:32:43.422612Z","iopub.execute_input":"2022-08-09T12:32:43.423514Z","iopub.status.idle":"2022-08-09T12:32:43.430858Z","shell.execute_reply.started":"2022-08-09T12:32:43.423456Z","shell.execute_reply":"2022-08-09T12:32:43.429578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tabnet Model Configration","metadata":{}},{"cell_type":"code","source":"class CFG:\n    DEBUG = False\n    model = 'tabnet_baseline'\n    N_folds = 5\n    seed = 42\n    batch_size = 512\n    max_epochs = 60","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:33:39.414129Z","iopub.execute_input":"2022-08-09T12:33:39.414612Z","iopub.status.idle":"2022-08-09T12:33:39.420962Z","shell.execute_reply.started":"2022-08-09T12:33:39.414572Z","shell.execute_reply":"2022-08-09T12:33:39.419628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Dataset for result","metadata":{}},{"cell_type":"code","source":"oof_predictions = np.zeros((train_X.shape[0]))\ntest_predictions = np.zeros(test.shape[0])\nfeature_importances = pd.DataFrame()\nfeature_importances[\"feature\"] = train_X.columns.tolist()\nstats = pd.DataFrame()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:33:40.035772Z","iopub.execute_input":"2022-08-09T12:33:40.037321Z","iopub.status.idle":"2022-08-09T12:33:40.048270Z","shell.execute_reply.started":"2022-08-09T12:33:40.037268Z","shell.execute_reply":"2022-08-09T12:33:40.046787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train and Infer Tabnet Model","metadata":{}},{"cell_type":"code","source":"kf = KFold(n_splits=5, shuffle=True, random_state=4885).split(train_X)\n\nfor fold, (train_idx, valid_idx) in enumerate(kf):\n\n    X_train, y_train = train_X.loc[train_idx], train_Y.loc[train_idx]\n    X_valid, y_valid = train_X.loc[valid_idx], train_Y.loc[valid_idx]        \n        \n    model = TabNetClassifier(n_d = 32,\n                             n_a = 32,\n                             n_steps = 3,\n                             gamma = 1.3,\n                             n_independent = 2,\n                             n_shared = 2,\n                             momentum = 0.02,\n                             clip_value = None,\n                             lambda_sparse = 1e-3,\n                             optimizer_fn = torch.optim.Adam,\n                             optimizer_params = dict(lr = 1e-3, weight_decay=1e-3),\n                             scheduler_fn = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts,\n                             scheduler_params = {'T_0':5,\n                                                 'eta_min':1e-4,\n                                                 'T_mult':1,\n                                                 'last_epoch':-1},\n                             mask_type = 'entmax',\n                             seed = CFG.seed)\n    \n    \n\n    ## train\n    model.fit(np.array(X_train),\n              np.array(y_train.values.ravel()),\n              eval_set = [(np.array(X_valid), np.array(y_valid.values.ravel()))],\n              max_epochs = CFG.max_epochs,\n              patience = 50,\n              batch_size = CFG.batch_size,\n              eval_metric = ['auc', 'accuracy']) # Last metric is used for early stopping\n    \n    # Saving best model\n    saving_path_name = f\"./fold{fold}\"\n    saved_filepath = model.save_model(saving_path_name)\n    \n    # Inference\n    oof_predictions[valid_idx] = model.predict_proba(X_valid.values)[:, 1]\n    \n    test_predictions += model.predict_proba(test.values)[:, 1]/5\n    feature_importances[f\"importance_fold{fold}+1\"] = model.feature_importances_\n    \n    # Loss , metric tracking\n    stats[f'fold{fold+1}_train_loss'] = model.history['loss']\n\n    del X_train, y_train\n    del X_valid, y_valid\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T12:56:09.808457Z","iopub.execute_input":"2022-08-09T12:56:09.809413Z","iopub.status.idle":"2022-08-09T13:06:44.146795Z","shell.execute_reply.started":"2022-08-09T12:56:09.809366Z","shell.execute_reply":"2022-08-09T13:06:44.145108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check Feature Importance","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfeature_importances['mean_importance']=feature_importances[['importance_fold0+1','importance_fold1+1']].mean(axis=1)\nfeature_importances.sort_values(by='mean_importance', ascending=False, inplace=True)\nsns.barplot(y=feature_importances['feature'][:50],x=feature_importances['mean_importance'][:50])\nplt.title('Mean Feature Importance')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:07:16.356176Z","iopub.execute_input":"2022-08-09T13:07:16.356595Z","iopub.status.idle":"2022-08-09T13:07:16.812099Z","shell.execute_reply.started":"2022-08-09T13:07:16.356560Z","shell.execute_reply":"2022-08-09T13:07:16.810826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save a Submission File","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/tabular-playground-series-aug-2022/sample_submission.csv\")\nsubmission['failure'] = test_predictions\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T13:07:29.174975Z","iopub.execute_input":"2022-08-09T13:07:29.175418Z","iopub.status.idle":"2022-08-09T13:07:29.250391Z","shell.execute_reply.started":"2022-08-09T13:07:29.175379Z","shell.execute_reply":"2022-08-09T13:07:29.249060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TabNet Baseline Model Public Score: 0.52372","metadata":{}}]}