{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Feature Engineering + CTGAN Synthetic dataset\n\nreference notebook : https://www.kaggle.com/code/takanashihumbert/tps-aug22-lb-0-59013","metadata":{}},{"cell_type":"code","source":"!pip install -qq ctgan\nfrom ctgan import CTGANSynthesizer\n\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\n\nfrom matplotlib.ticker import MaxNLocator\nimport seaborn as sns\nfrom cycler import cycler\nfrom IPython.display import display\nimport math\nimport os\nimport random\nimport gc\nimport sys\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport optuna\nfrom colorama import Fore, Back, Style\n\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import StratifiedGroupKFold, StratifiedKFold, train_test_split,GroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.calibration import CalibrationDisplay\nfrom sklearn.preprocessing import StandardScaler,RobustScaler,LabelEncoder\nfrom sklearn.impute import KNNImputer\nfrom sklearn import linear_model\nfrom sklearn.linear_model import HuberRegressor\nfrom sklearn.decomposition import PCA\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.calibration import CalibratedClassifierCV\n\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping\nfrom tensorflow.keras.layers import Input, Dense, Activation,  BatchNormalization, Dropout, Concatenate, Embedding,  Flatten, Conv1D\nfrom tensorflow.keras.models import Model","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-13T06:12:12.749502Z","iopub.execute_input":"2022-08-13T06:12:12.750040Z","iopub.status.idle":"2022-08-13T06:12:23.939453Z","shell.execute_reply.started":"2022-08-13T06:12:12.750001Z","shell.execute_reply":"2022-08-13T06:12:23.937801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\nsubmission = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')\ntarget = train['failure']\ntrain.drop('failure',axis=1, inplace = True)\ndata = pd.concat([train, test])\ntrain.shape,test.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:53:45.360039Z","iopub.execute_input":"2022-08-13T05:53:45.360626Z","iopub.status.idle":"2022-08-13T05:53:45.649575Z","shell.execute_reply.started":"2022-08-13T05:53:45.360592Z","shell.execute_reply":"2022-08-13T05:53:45.648578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<h1> Preprocessing","metadata":{}},{"cell_type":"markdown","source":"\nInitial preprocessing from :\n\nhttps://www.kaggle.com/code/desalegngeb/tps08-logisticregression-and-some-fe\n\nhttps://www.kaggle.com/code/alikayed/tps08-logisticregression-and-some-fe-c83a47","metadata":{}},{"cell_type":"code","source":"# library for coding string values :\n!pip install -qq feature_engine\nfrom feature_engine.encoding import WoEEncoder","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-13T05:53:45.651126Z","iopub.execute_input":"2022-08-13T05:53:45.651439Z","iopub.status.idle":"2022-08-13T05:53:56.982998Z","shell.execute_reply.started":"2022-08-13T05:53:45.651410Z","shell.execute_reply":"2022-08-13T05:53:56.981477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['m3_missing'] = data['measurement_3'].isnull().astype(np.int8)\ndata['m5_missing'] = data['measurement_5'].isnull().astype(np.int8)\ndata['area'] = data['attribute_2'] * data['attribute_3']\n\nfeature = [f for f in test.columns if f.startswith('measurement') or f=='loading']\n\n# dictionnary of dictionnaries (for the 11 best correlated measurement columns), \n# we will use the dictionnaries below to select the best correlated columns according to the product code)\n# Only for 'measurement_17' we make a 'manual' selection :\nfull_fill_dict ={}\nfull_fill_dict['measurement_17'] = {\n    'A': ['measurement_5','measurement_6','measurement_8'],\n    'B': ['measurement_4','measurement_5','measurement_7'],\n    'C': ['measurement_5','measurement_7','measurement_8','measurement_9'],\n    'D': ['measurement_5','measurement_6','measurement_7','measurement_8'],\n    'E': ['measurement_4','measurement_5','measurement_6','measurement_8'],\n    'F': ['measurement_4','measurement_5','measurement_6','measurement_7'],\n    'G': ['measurement_4','measurement_6','measurement_8','measurement_9'],\n    'H': ['measurement_4','measurement_5','measurement_7','measurement_8','measurement_9'],\n    'I': ['measurement_3','measurement_7','measurement_8']\n}\n\n# collect the name of the next 10 best measurement columns sorted by correlation (except 17 already done above):\ncol = [col for col in test.columns if 'measurement' not in col]+ ['loading','m3_missing','m5_missing']\na = []\nb =[]\nfor x in range(3,17):\n    corr = np.absolute(data.drop(col, axis=1).corr()[f'measurement_{x}']).sort_values(ascending=False)\n    a.append(np.round(np.sum(corr[1:4]),3)) # we add the 3 first lines of the correlation values to get the \"most correlated\"\n    b.append(f'measurement_{x}')\nc = pd.DataFrame()\nc['Selected columns'] = b\nc['correlation total'] = a\nc = c.sort_values(by = 'correlation total',ascending=False).reset_index(drop = True)\nprint(f'Columns selected by correlation sum of the 3 first rows : ')\ndisplay(c.head(10))\n\nfor i in range(10):\n    measurement_col = 'measurement_' + c.iloc[i,0][12:] # we select the next best correlated column \n    fill_dict ={}\n    for x in data.product_code.unique() : \n        corr = np.absolute(data[data.product_code == x].drop(col, axis=1).corr()[measurement_col]).sort_values(ascending=False)\n        measurement_col_dic = {}\n        measurement_col_dic[measurement_col] = corr[1:5].index.tolist()\n        fill_dict[x] = measurement_col_dic[measurement_col]\n    full_fill_dict[measurement_col] =fill_dict\n    \nfeature = [f for f in data.columns if f.startswith('measurement') or f=='loading']\nnullValue_cols = [col for col in train.columns if train[col].isnull().sum()!=0]\n    \nfor code in data.product_code.unique():\n    total_na_filled_by_linear_model = 0\n    print(f'\\n-------- Product code {code} ----------\\n')\n    print(f'filled by linear model :')\n    for measurement_col in list(full_fill_dict.keys()):\n        tmp = data[data.product_code==code]\n        column = full_fill_dict[measurement_col][code]\n        tmp_train = tmp[column+[measurement_col]].dropna(how='any')\n        tmp_test = tmp[(tmp[column].isnull().sum(axis=1)==0)&(tmp[measurement_col].isnull())]\n\n        model = HuberRegressor(epsilon=1.9)\n        model.fit(tmp_train[column], tmp_train[measurement_col])\n        data.loc[(data.product_code==code)&(data[column].isnull().sum(axis=1)==0)&(data[measurement_col].isnull()),measurement_col] = model.predict(tmp_test[column])\n        print(f'{measurement_col} : {len(tmp_test)}')\n        total_na_filled_by_linear_model += len(tmp_test)\n        \n    # others NA columns:\n    NA = data.loc[data[\"product_code\"] == code,nullValue_cols ].isnull().sum().sum()\n    model1 = KNNImputer(n_neighbors=3)\n    data.loc[data.product_code==code, feature] = model1.fit_transform(data.loc[data.product_code==code, feature])\n    print(f'\\n{total_na_filled_by_linear_model} filled by linear model ') \n    print(f'{NA} filled by KNN ')\n    \ndata['measurement_avg'] = data[[f'measurement_{i}' for i in range(3, 17)]].mean(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:53:56.986700Z","iopub.execute_input":"2022-08-13T05:53:56.987559Z","iopub.status.idle":"2022-08-13T05:54:19.295352Z","shell.execute_reply.started":"2022-08-13T05:53:56.987519Z","shell.execute_reply":"2022-08-13T05:54:19.294169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _scale(train_data, val_data, test_data, feats):\n    scaler = StandardScaler()\n    # scaler = PowerTransformer()\n    \n    scaled_train = scaler.fit_transform(train_data[feats])\n    scaled_val = scaler.transform(val_data[feats])\n    scaled_test = scaler.transform(test_data[feats])\n    \n    #back to dataframe\n    new_train = train_data.copy()\n    new_val = val_data.copy()\n    new_test = test_data.copy()\n    \n    new_train[feats] = scaled_train\n    new_val[feats] = scaled_val\n    new_test[feats] = scaled_test\n    \n    assert len(train_data) == len(new_train)\n    assert len(val_data) == len(new_val)\n    assert len(test_data) == len(new_test)\n    \n    return new_train, new_val, new_test","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:54:19.296771Z","iopub.execute_input":"2022-08-13T05:54:19.297102Z","iopub.status.idle":"2022-08-13T05:54:19.304844Z","shell.execute_reply.started":"2022-08-13T05:54:19.297073Z","shell.execute_reply":"2022-08-13T05:54:19.303681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = data.iloc[:train.shape[0],:]\ntest = data.iloc[train.shape[0]:,:]\nprint(train.shape, test.shape)\n\ngroups = train.product_code\nX = train\ny = target","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:54:19.307379Z","iopub.execute_input":"2022-08-13T05:54:19.307740Z","iopub.status.idle":"2022-08-13T05:54:19.332590Z","shell.execute_reply.started":"2022-08-13T05:54:19.307707Z","shell.execute_reply":"2022-08-13T05:54:19.331334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Thanks to @MAXSARMENTO \nwoe_encoder = WoEEncoder(variables=['attribute_0'])\nwoe_encoder.fit(X, y)\nX = woe_encoder.transform(X)\ntest = woe_encoder.transform(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:54:19.334201Z","iopub.execute_input":"2022-08-13T05:54:19.334532Z","iopub.status.idle":"2022-08-13T05:54:19.405221Z","shell.execute_reply.started":"2022-08-13T05:54:19.334501Z","shell.execute_reply":"2022-08-13T05:54:19.404031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"select_feature = ['loading',\n                  'attribute_0',\n                  'measurement_17',\n                  'measurement_0',\n                  'measurement_1',\n                  'measurement_2',\n                  'area',\n                  'm3_missing',\n                  'm5_missing',\n                  'measurement_avg']","metadata":{"execution":{"iopub.status.busy":"2022-08-13T05:54:19.406782Z","iopub.execute_input":"2022-08-13T05:54:19.407140Z","iopub.status.idle":"2022-08-13T05:54:19.412380Z","shell.execute_reply.started":"2022-08-13T05:54:19.407108Z","shell.execute_reply":"2022-08-13T05:54:19.411112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CTGAN","metadata":{}},{"cell_type":"code","source":"all_df = pd.concat([X, y], axis=1)\nall_codes = X.product_code.unique()\nctgans = []","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:06:30.436187Z","iopub.execute_input":"2022-08-13T06:06:30.436575Z","iopub.status.idle":"2022-08-13T06:06:30.449578Z","shell.execute_reply.started":"2022-08-13T06:06:30.436545Z","shell.execute_reply":"2022-08-13T06:06:30.448689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for product_code in tqdm(all_codes):\n    ctgan = CTGANSynthesizer(epochs=10)\n    ctgan.fit(all_df[(all_df.product_code == product_code) & (all_df.failure == 1.)][select_feature], select_feature)\n    ctgans.append(ctgan)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:06:31.624504Z","iopub.execute_input":"2022-08-13T06:06:31.624907Z","iopub.status.idle":"2022-08-13T06:07:02.560797Z","shell.execute_reply.started":"2022-08-13T06:06:31.624862Z","shell.execute_reply":"2022-08-13T06:07:02.559757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nr_samples = int(0.6 * len(y) - len(y[y == 1.]))\nprint(nr_samples)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:09:29.520414Z","iopub.execute_input":"2022-08-13T06:09:29.520866Z","iopub.status.idle":"2022-08-13T06:09:29.528519Z","shell.execute_reply.started":"2022-08-13T06:09:29.520820Z","shell.execute_reply":"2022-08-13T06:09:29.527503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"lr_oof = np.zeros(len(train))\nlr_test = np.zeros(len(test))\nlr_auc = 0\n\nkf = StratifiedGroupKFold(n_splits=5)\nfor fold_idx, (train_idx, val_idx) in enumerate(kf.split(X, y, groups=train.product_code )):\n    x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n    y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n    \n    x_train = pd.concat([x_train[select_feature], ctgans[fold_idx].sample(nr_samples)], axis=0)\n    y_train = pd.concat([y_train, pd.DataFrame([1. for i in range(nr_samples)])])\n    x_train, y_train = shuffle(x_train, y_train)\n    \n    x_train, x_val, x_test = _scale(x_train, x_val, test, select_feature)\n\n    model = linear_model.LogisticRegression(max_iter=200, C=0.0001, penalty='l2', solver='newton-cg')\n    model.fit(x_train, y_train)\n\n    val_preds = model.predict_proba(x_val[select_feature])[:, 1]\n    print(\"FOLD: \", fold_idx+1, \" ROC-AUC:\", round(roc_auc_score(y_val, val_preds), 5))\n    lr_auc += roc_auc_score(y_val, val_preds) / 5\n    lr_test += model.predict_proba(x_test[select_feature])[:, 1] / 5\n    lr_oof[val_idx] = val_preds\n\nprint(f\"\\n{Fore.GREEN}{Style.BRIGHT}Average auc = {round(lr_auc, 5)}{Style.RESET_ALL}\")\nprint(f\"{Fore.BLUE}{Style.BRIGHT}OOF auc     = {round(roc_auc_score(y, lr_oof), 5)}{Style.RESET_ALL}\\n\")\n\nsubmission['failure'] = lr_test\nsubmission.to_csv(f\"./submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T06:23:08.170835Z","iopub.execute_input":"2022-08-13T06:23:08.171443Z","iopub.status.idle":"2022-08-13T06:23:31.770332Z","shell.execute_reply.started":"2022-08-13T06:23:08.171394Z","shell.execute_reply":"2022-08-13T06:23:31.768761Z"},"trusted":true},"execution_count":null,"outputs":[]}]}