{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.utils import class_weight\nfrom sklearn.impute import KNNImputer\nfrom sklearn.model_selection import KFold, GroupKFold\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-09T03:37:33.199774Z","iopub.execute_input":"2022-08-09T03:37:33.200254Z","iopub.status.idle":"2022-08-09T03:37:33.207122Z","shell.execute_reply.started":"2022-08-09T03:37:33.200214Z","shell.execute_reply":"2022-08-09T03:37:33.206207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"train_path = \"../input/tabular-playground-series-aug-2022/train.csv\"\ntest_path = \"../input/tabular-playground-series-aug-2022/test.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:33.310617Z","iopub.execute_input":"2022-08-09T03:37:33.311285Z","iopub.status.idle":"2022-08-09T03:37:33.316337Z","shell.execute_reply.started":"2022-08-09T03:37:33.311249Z","shell.execute_reply":"2022-08-09T03:37:33.315347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:33.411583Z","iopub.execute_input":"2022-08-09T03:37:33.412048Z","iopub.status.idle":"2022-08-09T03:37:33.593841Z","shell.execute_reply.started":"2022-08-09T03:37:33.412008Z","shell.execute_reply":"2022-08-09T03:37:33.592593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['product_code']=='C']","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:33.596840Z","iopub.execute_input":"2022-08-09T03:37:33.597220Z","iopub.status.idle":"2022-08-09T03:37:33.635972Z","shell.execute_reply.started":"2022-08-09T03:37:33.597187Z","shell.execute_reply":"2022-08-09T03:37:33.634874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.drop(['id', 'product_code'], axis=1, inplace=True)\n\ntrain_df.drop(['id', 'product_code'], axis=1, inplace=True)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:33.637938Z","iopub.execute_input":"2022-08-09T03:37:33.638294Z","iopub.status.idle":"2022-08-09T03:37:33.672926Z","shell.execute_reply.started":"2022-08-09T03:37:33.638262Z","shell.execute_reply":"2022-08-09T03:37:33.672015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Categorical to Numerical","metadata":{}},{"cell_type":"code","source":"materials = {\n    \"material_5\" : 0, \n    \"material_6\" : 1,\n    \"material_7\" : 2,\n    \"material_8\" : 3\n}\n\natt2_3 = {\n    4 : 0,\n    5 : 1,\n    6 : 2,\n    8 : 3,\n    9 : 4\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:33.684388Z","iopub.execute_input":"2022-08-09T03:37:33.685321Z","iopub.status.idle":"2022-08-09T03:37:33.690860Z","shell.execute_reply.started":"2022-08-09T03:37:33.685277Z","shell.execute_reply":"2022-08-09T03:37:33.689771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['attribute_0'].replace(materials, inplace=True)\ntrain_df['attribute_1'].replace(materials, inplace=True)\ntrain_df['attribute_2'].replace(att2_3, inplace=True)\ntrain_df['attribute_3'].replace(att2_3, inplace=True)\n\n\ntest_df['attribute_0'].replace(materials, inplace=True)\ntest_df['attribute_1'].replace(materials, inplace=True)\ntest_df['attribute_2'].replace(att2_3, inplace=True)\ntest_df['attribute_3'].replace(att2_3, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:33.797258Z","iopub.execute_input":"2022-08-09T03:37:33.798083Z","iopub.status.idle":"2022-08-09T03:37:33.854025Z","shell.execute_reply.started":"2022-08-09T03:37:33.798037Z","shell.execute_reply":"2022-08-09T03:37:33.852781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Missing Data","metadata":{}},{"cell_type":"code","source":"missingValues = train_df.isna().sum()\nmissingValues = missingValues[missingValues>0]\nmissingCols = missingValues.index\nplt.figure(figsize=(22,6))\nplt.bar(missingValues.index, height=missingValues.values, width=0.8)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:33.869190Z","iopub.execute_input":"2022-08-09T03:37:33.869847Z","iopub.status.idle":"2022-08-09T03:37:34.156981Z","shell.execute_reply.started":"2022-08-09T03:37:33.869811Z","shell.execute_reply":"2022-08-09T03:37:34.155586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missingImputer = KNNImputer(n_neighbors=13)\nmissingImputer.fit(train_df[missingCols])\n\ntrain_df[missingCols] = missingImputer.transform(train_df[missingCols])\ntest_df[missingCols] = missingImputer.transform(test_df[missingCols])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:37:34.159150Z","iopub.execute_input":"2022-08-09T03:37:34.159515Z","iopub.status.idle":"2022-08-09T03:38:41.169568Z","shell.execute_reply.started":"2022-08-09T03:37:34.159483Z","shell.execute_reply":"2022-08-09T03:38:41.167890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adding Features","metadata":{}},{"cell_type":"code","source":"measurementsCols = [f\"measurement_{colI}\" for colI in range(3, 18)]\nmeasure2Cols = [f\"measurement_{i:d}\" for i in list(range(5, 9))]\nattCols = [f\"attribute_{colI}\" for colI in range(0, 4)]\n\ntrain_df[\"measure-mean\"] = train_df[measurementsCols[:-1]].mean(axis=1)\ntrain_df[\"measure-std\"] = train_df[measurementsCols[:-1]].std(axis=1)\ntrain_df[\"measure-sum\"] = train_df[measurementsCols[:-1]].sum(axis=1)\n\ntest_df[\"measure-mean\"] = test_df[measurementsCols[:-1]].mean(axis=1)\ntest_df[\"measure-std\"] = test_df[measurementsCols[:-1]].std(axis=1)\ntest_df[\"measure-sum\"] = test_df[measurementsCols[:-1]].sum(axis=1)\n\ntrain_df['measure2-mean'] = np.mean(train_df[measure2Cols], axis=1)\ntest_df['measure2-mean'] = np.mean(test_df[measure2Cols], axis=1)\n\n\ntrain_df['meas17/measure2-mean'] = train_df['measurement_17'] / train_df['measure2-mean']\ntest_df['meas17/measure2-mean'] = test_df['measurement_17'] / test_df['measure2-mean']\n\n\n\ntrain_df['attribute_0*1'] = train_df['attribute_0'] * train_df['attribute_1']\ntest_df['attribute_0*1'] = test_df['attribute_0'] * test_df['attribute_1']\ntrain_df['attribute_2*3'] = train_df['attribute_2'] * train_df['attribute_3']\ntest_df['attribute_2*3'] = test_df['attribute_2'] * test_df['attribute_3']\n\n\ntrain_df[\"attribute-sum\"] = train_df[attCols].sum(axis=1)\ntest_df[\"attribute-sum\"] = test_df[attCols].sum(axis=1)\n\n\nfor i in range(len(measurementsCols)):\n    train_df[f'loadXmeas{i}'] = train_df['loading'] * train_df[f'measurement_{i}']\n    test_df[f'loadXmeas{i}'] = test_df['loading'] * test_df[f'measurement_{i}']","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.171842Z","iopub.execute_input":"2022-08-09T03:38:41.173102Z","iopub.status.idle":"2022-08-09T03:38:41.276522Z","shell.execute_reply.started":"2022-08-09T03:38:41.173045Z","shell.execute_reply":"2022-08-09T03:38:41.275542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Clipping","metadata":{}},{"cell_type":"code","source":"train_df = train_df[(train_df.measurement_0<20) & (train_df.measurement_1<20) & (train_df.measurement_2<15)]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.279003Z","iopub.execute_input":"2022-08-09T03:38:41.279699Z","iopub.status.idle":"2022-08-09T03:38:41.298653Z","shell.execute_reply.started":"2022-08-09T03:38:41.279657Z","shell.execute_reply":"2022-08-09T03:38:41.297648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seperate Features & target","metadata":{}},{"cell_type":"code","source":"y = train_df['failure']\nx = train_df.drop(['failure'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.300024Z","iopub.execute_input":"2022-08-09T03:38:41.300564Z","iopub.status.idle":"2022-08-09T03:38:41.310245Z","shell.execute_reply.started":"2022-08-09T03:38:41.300508Z","shell.execute_reply":"2022-08-09T03:38:41.308953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Normalization","metadata":{}},{"cell_type":"code","source":"cols2Normalize = [\n                'loading', 'measurement_0', 'measurement_1', 'measurement_2', 'measurement_3',\n                'measurement_4', 'measurement_5', 'measurement_6', 'measurement_7',\n                'measurement_8', 'measurement_9', 'measurement_10', 'measurement_11',\n                'measurement_12', 'measurement_13', 'measurement_14', 'measurement_15',\n                'measurement_16', 'measurement_17', 'measure-mean', 'measure-std',\n                'measure-sum', 'loadXmeas0', 'loadXmeas1', 'loadXmeas2', 'loadXmeas3',\n                'loadXmeas4', 'loadXmeas5', 'loadXmeas6', 'loadXmeas7', 'loadXmeas8',\n                'loadXmeas9', 'loadXmeas10', 'loadXmeas11', 'loadXmeas12',\n                'loadXmeas13', 'loadXmeas14'\n                 ]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.312605Z","iopub.execute_input":"2022-08-09T03:38:41.313553Z","iopub.status.idle":"2022-08-09T03:38:41.323702Z","shell.execute_reply.started":"2022-08-09T03:38:41.313480Z","shell.execute_reply":"2022-08-09T03:38:41.322221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(x[cols2Normalize])\n\nx[cols2Normalize] = scaler.transform(x[cols2Normalize])\ntest_df[cols2Normalize] = scaler.transform(test_df[cols2Normalize])","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.325477Z","iopub.execute_input":"2022-08-09T03:38:41.326825Z","iopub.status.idle":"2022-08-09T03:38:41.405564Z","shell.execute_reply.started":"2022-08-09T03:38:41.326781Z","shell.execute_reply":"2022-08-09T03:38:41.404331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Unbalanced class","metadata":{}},{"cell_type":"code","source":"targetCounts = y.value_counts()\ntargetCounts = (targetCounts / targetCounts.sum()) * 100\nplt.figure(figsize=(7,4))\nplt.bar(targetCounts.index, height=targetCounts.values, width=0.8)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.407145Z","iopub.execute_input":"2022-08-09T03:38:41.407836Z","iopub.status.idle":"2022-08-09T03:38:41.616157Z","shell.execute_reply.started":"2022-08-09T03:38:41.407792Z","shell.execute_reply":"2022-08-09T03:38:41.614889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Weight","metadata":{}},{"cell_type":"code","source":"class_weights = class_weight.compute_class_weight(class_weight='balanced', classes=np.unique(y), y=y)\nclass_weights = {k:v for k, v in enumerate(class_weights)}\nprint(f\"Weights of class [0] - {class_weights[0]}\\nWeights of class [1] - {class_weights[1]}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.621691Z","iopub.execute_input":"2022-08-09T03:38:41.622494Z","iopub.status.idle":"2022-08-09T03:38:41.634687Z","shell.execute_reply.started":"2022-08-09T03:38:41.622447Z","shell.execute_reply":"2022-08-09T03:38:41.633381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from imblearn.over_sampling import SMOTE\n\nsm = SMOTE(k_neighbors=143)\nx, y = sm.fit_resample(x,y)\n\ngroups = x['attribute-sum'].values ","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:41.636835Z","iopub.execute_input":"2022-08-09T03:38:41.637810Z","iopub.status.idle":"2022-08-09T03:38:42.656674Z","shell.execute_reply.started":"2022-08-09T03:38:41.637762Z","shell.execute_reply":"2022-08-09T03:38:42.654916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ANN Model Architecture","metadata":{}},{"cell_type":"code","source":"def buildModel():\n    model = tf.keras.Sequential()\n    model.add(layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.L1(l1=0.001), input_shape=[x.shape[-1]]))\n    model.add(layers.Dense(32, activation='relu'))    \n    model.add(layers.Dense(64, activation='relu'))   \n    model.add(layers.Dense(128, activation='relu')) \n    model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(256, activation='relu'))\n    model.add(layers.Dropout(0.2))\n    model.add(layers.Dense(128, activation='relu'))\n    model.add(layers.Dense(1, activation='sigmoid'))    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:42.717066Z","iopub.execute_input":"2022-08-09T03:38:42.717620Z","iopub.status.idle":"2022-08-09T03:38:42.727762Z","shell.execute_reply.started":"2022-08-09T03:38:42.717591Z","shell.execute_reply":"2022-08-09T03:38:42.726859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"x = x.values\ny = y.values","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:42.729611Z","iopub.execute_input":"2022-08-09T03:38:42.730420Z","iopub.status.idle":"2022-08-09T03:38:42.762991Z","shell.execute_reply.started":"2022-08-09T03:38:42.730374Z","shell.execute_reply":"2022-08-09T03:38:42.761880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model","metadata":{}},{"cell_type":"code","source":"kf = GroupKFold(n_splits=3)\n\nfor fold, (train_idx, test_idx) in enumerate(kf.split(x, y, groups)):\n    print(f\"\\n\\nTraining fold ---- {fold+1}\")\n    X_train, X_valid = x[train_idx], x[test_idx]\n    y_train, y_valid = y[train_idx], y[test_idx]\n    \n    \n    model = buildModel()\n    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5),\n             loss=tf.keras.losses.BinaryCrossentropy(),\n             metrics=['acc', tf.keras.metrics.AUC(name=\"roc\")])\n    \n    \n    callback0 = [tf.keras.callbacks.ModelCheckpoint(f\"Model_{fold+1}.h5\", \n                                               monitor='val_roc', mode=\"max\", save_best_only=True, verbose=1)]\n    callbacks = callback0 + callback1\n    \n    model.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, callbacks=callbacks)    ","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:38:42.764565Z","iopub.execute_input":"2022-08-09T03:38:42.765157Z","iopub.status.idle":"2022-08-09T03:52:19.033139Z","shell.execute_reply.started":"2022-08-09T03:38:42.765122Z","shell.execute_reply":"2022-08-09T03:52:19.031389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models_paths = [\"./Model_1.h5\", \"./Model_2.h5\", \"./Model_3.h5\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:54:24.994342Z","iopub.execute_input":"2022-08-09T03:54:24.995139Z","iopub.status.idle":"2022-08-09T03:54:25.000447Z","shell.execute_reply.started":"2022-08-09T03:54:24.995097Z","shell.execute_reply":"2022-08-09T03:54:24.999370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\n\nfor i, path in enumerate(models_paths):\n    models.append(tf.keras.models.load_model(path))\n","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:54:25.553988Z","iopub.execute_input":"2022-08-09T03:54:25.555075Z","iopub.status.idle":"2022-08-09T03:54:25.953390Z","shell.execute_reply.started":"2022-08-09T03:54:25.555023Z","shell.execute_reply":"2022-08-09T03:54:25.952166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pres = []\n\nfor i in range(3):\n    pre = models[i].predict(test_df)\n    pres.append(pre)\n\npres = np.array(pres)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:54:33.458706Z","iopub.execute_input":"2022-08-09T03:54:33.459193Z","iopub.status.idle":"2022-08-09T03:54:37.061102Z","shell.execute_reply.started":"2022-08-09T03:54:33.459156Z","shell.execute_reply":"2022-08-09T03:54:37.059854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"presMedian = np.median(pres.T[0], axis=1)\npresMean = np.mean(pres.T[0], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:54:37.064061Z","iopub.execute_input":"2022-08-09T03:54:37.064658Z","iopub.status.idle":"2022-08-09T03:54:37.073557Z","shell.execute_reply.started":"2022-08-09T03:54:37.064606Z","shell.execute_reply":"2022-08-09T03:54:37.072333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_file = pd.read_csv(\"../input/tabular-playground-series-aug-2022/sample_submission.csv\")\nsub_file['failure'] = presMean\nsub_file.to_csv(\"submission0.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:54:37.075209Z","iopub.execute_input":"2022-08-09T03:54:37.075758Z","iopub.status.idle":"2022-08-09T03:54:37.135995Z","shell.execute_reply.started":"2022-08-09T03:54:37.075710Z","shell.execute_reply":"2022-08-09T03:54:37.135014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_file = pd.read_csv(\"../input/tabular-playground-series-aug-2022/sample_submission.csv\")\nsub_file['failure'] = presMedian\nsub_file.to_csv(\"submission1.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T03:54:39.297303Z","iopub.execute_input":"2022-08-09T03:54:39.298617Z","iopub.status.idle":"2022-08-09T03:54:39.349157Z","shell.execute_reply.started":"2022-08-09T03:54:39.298566Z","shell.execute_reply":"2022-08-09T03:54:39.347748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}