{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! pip install -qq --ignore-installed --pre pycaret","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:10:59.440698Z","iopub.execute_input":"2022-08-06T15:10:59.441112Z","iopub.status.idle":"2022-08-06T15:14:06.000796Z","shell.execute_reply.started":"2022-08-06T15:10:59.441079Z","shell.execute_reply":"2022-08-06T15:14:05.998164Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nfrom pycaret.regression import *\n\nfrom sklearn.preprocessing import LabelEncoder\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-06T15:14:06.007607Z","iopub.execute_input":"2022-08-06T15:14:06.008967Z","iopub.status.idle":"2022-08-06T15:14:10.616829Z","shell.execute_reply.started":"2022-08-06T15:14:06.008889Z","shell.execute_reply":"2022-08-06T15:14:10.615454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reference\n[desalegngeb - Feature Engineering Notebook](https://www.kaggle.com/code/desalegngeb/tps08-logisticregression-and-some-fe)","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv', index_col ='id')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv', index_col ='id')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:14:10.620998Z","iopub.execute_input":"2022-08-06T15:14:10.621453Z","iopub.status.idle":"2022-08-06T15:14:10.969993Z","shell.execute_reply.started":"2022-08-06T15:14:10.621419Z","shell.execute_reply":"2022-08-06T15:14:10.968448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"!git clone --quiet https://github.com/analokmaus/kuma_utils.git\nsys.path.append(\"kuma_utils/\")\nfrom kuma_utils.preprocessing.imputer import LGBMImputer","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:14:10.975314Z","iopub.execute_input":"2022-08-06T15:14:10.976621Z","iopub.status.idle":"2022-08-06T15:14:12.633266Z","shell.execute_reply.started":"2022-08-06T15:14:10.976551Z","shell.execute_reply":"2022-08-06T15:14:12.632080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = train.pop('failure')\nfloat_cols = [col for col in train.columns if train[col].dtypes == 'float64']\nobject_cols = [col for col in train.columns if train[col].dtypes == 'object']\nint_object_cols = [col for col in train.columns[:-1] if (train[col].dtypes == 'object' or train[col].dtypes == 'int64')]\nnullValue_cols = [col for col in train.columns if train[col].isnull().sum()!=0]","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:14:12.634984Z","iopub.execute_input":"2022-08-06T15:14:12.635699Z","iopub.status.idle":"2022-08-06T15:14:12.661127Z","shell.execute_reply.started":"2022-08-06T15:14:12.635661Z","shell.execute_reply":"2022-08-06T15:14:12.660020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_A = train[train['product_code']=='A']\ndf_B = train[train['product_code']=='B']\ndf_C = train[train['product_code']=='C']\ndf_D = train[train['product_code']=='D']\ndf_E = train[train['product_code']=='E']\n\ndf_F_t = test[test['product_code']=='F']\ndf_G_t = test[test['product_code']=='G']\ndf_H_t = test[test['product_code']=='H']\ndf_I_t = test[test['product_code']=='I']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:14:12.662647Z","iopub.execute_input":"2022-08-06T15:14:12.663815Z","iopub.status.idle":"2022-08-06T15:14:12.821185Z","shell.execute_reply.started":"2022-08-06T15:14:12.663781Z","shell.execute_reply":"2022-08-06T15:14:12.820195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_imtr = LGBMImputer(cat_features=object_cols, n_iter=250)\n\n# train datset\ntrain_iterimp_A = lgbm_imtr.fit_transform(df_A[nullValue_cols])\ntrain_iterimp_B = lgbm_imtr.fit_transform(df_B[nullValue_cols])\ntrain_iterimp_C = lgbm_imtr.fit_transform(df_C[nullValue_cols])\ntrain_iterimp_D = lgbm_imtr.fit_transform(df_D[nullValue_cols])\ntrain_iterimp_E = lgbm_imtr.fit_transform(df_E[nullValue_cols])\n\n# tests data\ntest_iterimp_F = lgbm_imtr.fit_transform(df_F_t[nullValue_cols])\ntest_iterimp_G = lgbm_imtr.fit_transform(df_G_t[nullValue_cols])\ntest_iterimp_H = lgbm_imtr.fit_transform(df_H_t[nullValue_cols])\ntest_iterimp_I = lgbm_imtr.fit_transform(df_I_t[nullValue_cols])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:14:12.823271Z","iopub.execute_input":"2022-08-06T15:14:12.824532Z","iopub.status.idle":"2022-08-06T15:16:06.661407Z","shell.execute_reply.started":"2022-08-06T15:14:12.824483Z","shell.execute_reply":"2022-08-06T15:16:06.659660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"none_na_cols = [col for col in train.columns if col not in nullValue_cols]\ndf_train = train[none_na_cols]\ndf_test = test[none_na_cols]\n\ntrain_ = pd.concat([train_iterimp_A, train_iterimp_B,train_iterimp_C,train_iterimp_D,train_iterimp_E], axis=0)\ntrain = pd.concat([df_train, train_], axis=1)\n\ntest_ = pd.concat([test_iterimp_F, test_iterimp_G,test_iterimp_H,test_iterimp_I], axis=0)\ntest = pd.concat([df_test, test_], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.663490Z","iopub.execute_input":"2022-08-06T15:16:06.664017Z","iopub.status.idle":"2022-08-06T15:16:06.697169Z","shell.execute_reply.started":"2022-08-06T15:16:06.663968Z","shell.execute_reply":"2022-08-06T15:16:06.695509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['attribute_2*3'] = train['attribute_2'] * train['attribute_3']\ntest['attribute_2*3'] = test['attribute_2'] * test['attribute_3']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.699199Z","iopub.execute_input":"2022-08-06T15:16:06.699618Z","iopub.status.idle":"2022-08-06T15:16:06.710330Z","shell.execute_reply.started":"2022-08-06T15:16:06.699583Z","shell.execute_reply":"2022-08-06T15:16:06.708939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meas_gr1_cols = [f\"measurement_{i:d}\" for i in list(range(3, 5)) + list(range(9, 17)) ]\ntrain['meas_gr1_avg'] = np.mean(train[meas_gr1_cols], axis=1)\ntrain['meas_gr1_std'] = np.std(train[meas_gr1_cols], axis=1)\n\ntest['meas_gr1_avg'] = np.mean(test[meas_gr1_cols], axis=1)\ntest['meas_gr1_std'] = np.std(test[meas_gr1_cols], axis=1) \n\nmeas_gr2_cols = [f\"measurement_{i:d}\" for i in list(range(5, 9))]\ntrain['meas_gr2_avg'] = np.mean(train[meas_gr2_cols], axis=1)\ntest['meas_gr2_avg'] = np.mean(test[meas_gr2_cols], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.715114Z","iopub.execute_input":"2022-08-06T15:16:06.715970Z","iopub.status.idle":"2022-08-06T15:16:06.787980Z","shell.execute_reply.started":"2022-08-06T15:16:06.715928Z","shell.execute_reply":"2022-08-06T15:16:06.786876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['meas17/meas_gr2_avg'] = train['measurement_17'] / train['meas_gr2_avg']\ntest['meas17/meas_gr2_avg'] = test['measurement_17'] / test['meas_gr2_avg']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.789402Z","iopub.execute_input":"2022-08-06T15:16:06.789929Z","iopub.status.idle":"2022-08-06T15:16:06.798255Z","shell.execute_reply.started":"2022-08-06T15:16:06.789897Z","shell.execute_reply":"2022-08-06T15:16:06.796441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_use = ['measurement_0', 'measurement_1', 'measurement_2', 'attribute_0', 'attribute_1',\n               'meas_gr1_avg', 'meas_gr1_std', 'attribute_2*3', 'loading', 'measurement_17', 'meas17/meas_gr2_avg']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.800851Z","iopub.execute_input":"2022-08-06T15:16:06.802732Z","iopub.status.idle":"2022-08-06T15:16:06.810061Z","shell.execute_reply.started":"2022-08-06T15:16:06.802694Z","shell.execute_reply":"2022-08-06T15:16:06.808929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ = train[cols_to_use]\ntest_ = test[cols_to_use]","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.812133Z","iopub.execute_input":"2022-08-06T15:16:06.812987Z","iopub.status.idle":"2022-08-06T15:16:06.842449Z","shell.execute_reply.started":"2022-08-06T15:16:06.812940Z","shell.execute_reply":"2022-08-06T15:16:06.840981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoder = LabelEncoder()\ntrain_le = train_.copy()\ntest_le = test_.copy()\n\nfor col in ['attribute_0', 'attribute_1']:\n    train_le[col] = label_encoder.fit_transform(train_[col])\n    test_le[col] = label_encoder.fit_transform(test_[col]) \n        \ntrain_ = train_le\ntest_ = test_le","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.847180Z","iopub.execute_input":"2022-08-06T15:16:06.847554Z","iopub.status.idle":"2022-08-06T15:16:06.884203Z","shell.execute_reply.started":"2022-08-06T15:16:06.847521Z","shell.execute_reply":"2022-08-06T15:16:06.882907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, test, y = train_, test_, target ","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.886085Z","iopub.execute_input":"2022-08-06T15:16:06.886515Z","iopub.status.idle":"2022-08-06T15:16:06.891739Z","shell.execute_reply.started":"2022-08-06T15:16:06.886482Z","shell.execute_reply":"2022-08-06T15:16:06.890577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([X, y], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:06.893259Z","iopub.execute_input":"2022-08-06T15:16:06.893679Z","iopub.status.idle":"2022-08-06T15:16:06.908191Z","shell.execute_reply.started":"2022-08-06T15:16:06.893645Z","shell.execute_reply":"2022-08-06T15:16:06.906843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PyCaret","metadata":{}},{"cell_type":"code","source":"reg = setup(data=data, target='failure', session_id=123,\n            normalize = True, transformation = True, transform_target = True, \n            remove_multicollinearity = True, multicollinearity_threshold = 0.95) ","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:16:24.645364Z","iopub.execute_input":"2022-08-06T15:16:24.646480Z","iopub.status.idle":"2022-08-06T15:16:28.848963Z","shell.execute_reply.started":"2022-08-06T15:16:24.646442Z","shell.execute_reply":"2022-08-06T15:16:28.847569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top3 = compare_models(fold=5, n_select=3)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:36:43.976933Z","iopub.execute_input":"2022-08-06T15:36:43.978770Z","iopub.status.idle":"2022-08-06T15:38:10.150611Z","shell.execute_reply.started":"2022-08-06T15:36:43.978716Z","shell.execute_reply":"2022-08-06T15:38:10.149080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top3","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:38:20.009217Z","iopub.execute_input":"2022-08-06T15:38:20.009675Z","iopub.status.idle":"2022-08-06T15:38:20.022439Z","shell.execute_reply.started":"2022-08-06T15:38:20.009640Z","shell.execute_reply":"2022-08-06T15:38:20.021517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuned_top3 = [tune_model(i, fold=5) for i in top3]","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:38:23.709461Z","iopub.execute_input":"2022-08-06T15:38:23.709911Z","iopub.status.idle":"2022-08-06T15:40:40.298787Z","shell.execute_reply.started":"2022-08-06T15:38:23.709870Z","shell.execute_reply":"2022-08-06T15:40:40.297107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bagged_top3 = [ensemble_model(i, fold=5) for i in tuned_top3]","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:43:08.903220Z","iopub.execute_input":"2022-08-06T15:43:08.903671Z","iopub.status.idle":"2022-08-06T15:45:08.183616Z","shell.execute_reply.started":"2022-08-06T15:43:08.903638Z","shell.execute_reply":"2022-08-06T15:45:08.181857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prep = reg.pipeline.transform(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:49:50.971013Z","iopub.execute_input":"2022-08-06T15:49:50.971740Z","iopub.status.idle":"2022-08-06T15:49:51.039600Z","shell.execute_reply.started":"2022-08-06T15:49:50.971693Z","shell.execute_reply":"2022-08-06T15:49:51.038062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat = np.array([bagged_top3[i].predict(test_prep) for i in range(len(bagged_top3))])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:51:03.587194Z","iopub.execute_input":"2022-08-06T15:51:03.587638Z","iopub.status.idle":"2022-08-06T15:51:04.553617Z","shell.execute_reply.started":"2022-08-06T15:51:03.587604Z","shell.execute_reply":"2022-08-06T15:51:04.552216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(y_hat, axis=0)\nsub.to_csv('submission_mean.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:52:06.218986Z","iopub.execute_input":"2022-08-06T15:52:06.219791Z","iopub.status.idle":"2022-08-06T15:52:06.282732Z","shell.execute_reply.started":"2022-08-06T15:52:06.219736Z","shell.execute_reply":"2022-08-06T15:52:06.281387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['failure'] = (0.5 * y_hat[0]) + (0.3 * y_hat[1]) + (0.2 * y_hat[2])\nsub.to_csv('submission_weights.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T15:52:57.524208Z","iopub.execute_input":"2022-08-06T15:52:57.524667Z","iopub.status.idle":"2022-08-06T15:52:57.586840Z","shell.execute_reply.started":"2022-08-06T15:52:57.524627Z","shell.execute_reply":"2022-08-06T15:52:57.585665Z"},"trusted":true},"execution_count":null,"outputs":[]}]}