{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -qq git+https://github.com/keras-team/keras-tuner.git\n!pip install -qq autokeras","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:16:40.844418Z","iopub.execute_input":"2022-08-06T16:16:40.845388Z","iopub.status.idle":"2022-08-06T16:18:23.749512Z","shell.execute_reply.started":"2022-08-06T16:16:40.845244Z","shell.execute_reply":"2022-08-06T16:18:23.748072Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport pandas as pd\nimport numpy as np\nimport autokeras as ak\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import mean_squared_error\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-06T16:56:39.448315Z","iopub.execute_input":"2022-08-06T16:56:39.449376Z","iopub.status.idle":"2022-08-06T16:56:39.456788Z","shell.execute_reply.started":"2022-08-06T16:56:39.449330Z","shell.execute_reply":"2022-08-06T16:56:39.454810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reference\n[desalegngeb - Feature Engineering Notebook](https://www.kaggle.com/code/desalegngeb/tps08-logisticregression-and-some-fe)","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv', index_col ='id')\ntest = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv', index_col ='id')\nsub = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:18:53.370664Z","iopub.execute_input":"2022-08-06T16:18:53.371414Z","iopub.status.idle":"2022-08-06T16:18:53.672589Z","shell.execute_reply.started":"2022-08-06T16:18:53.371374Z","shell.execute_reply":"2022-08-06T16:18:53.671620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"!git clone --quiet https://github.com/analokmaus/kuma_utils.git\nsys.path.append(\"kuma_utils/\")\nfrom kuma_utils.preprocessing.imputer import LGBMImputer","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:18:55.291015Z","iopub.execute_input":"2022-08-06T16:18:55.292455Z","iopub.status.idle":"2022-08-06T16:18:58.231879Z","shell.execute_reply.started":"2022-08-06T16:18:55.292415Z","shell.execute_reply":"2022-08-06T16:18:58.230622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = train.pop('failure')\nfloat_cols = [col for col in train.columns if train[col].dtypes == 'float64']\nobject_cols = [col for col in train.columns if train[col].dtypes == 'object']\nint_object_cols = [col for col in train.columns[:-1] if (train[col].dtypes == 'object' or train[col].dtypes == 'int64')]\nnullValue_cols = [col for col in train.columns if train[col].isnull().sum()!=0]","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:18:58.234464Z","iopub.execute_input":"2022-08-06T16:18:58.234854Z","iopub.status.idle":"2022-08-06T16:18:58.256356Z","shell.execute_reply.started":"2022-08-06T16:18:58.234819Z","shell.execute_reply":"2022-08-06T16:18:58.255236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_A = train[train['product_code']=='A']\ndf_B = train[train['product_code']=='B']\ndf_C = train[train['product_code']=='C']\ndf_D = train[train['product_code']=='D']\ndf_E = train[train['product_code']=='E']\n\ndf_F_t = test[test['product_code']=='F']\ndf_G_t = test[test['product_code']=='G']\ndf_H_t = test[test['product_code']=='H']\ndf_I_t = test[test['product_code']=='I']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:18:58.258435Z","iopub.execute_input":"2022-08-06T16:18:58.258920Z","iopub.status.idle":"2022-08-06T16:18:58.301714Z","shell.execute_reply.started":"2022-08-06T16:18:58.258885Z","shell.execute_reply":"2022-08-06T16:18:58.300622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_imtr = LGBMImputer(cat_features=object_cols, n_iter=250)\n\n# train datset\ntrain_iterimp_A = lgbm_imtr.fit_transform(df_A[nullValue_cols])\ntrain_iterimp_B = lgbm_imtr.fit_transform(df_B[nullValue_cols])\ntrain_iterimp_C = lgbm_imtr.fit_transform(df_C[nullValue_cols])\ntrain_iterimp_D = lgbm_imtr.fit_transform(df_D[nullValue_cols])\ntrain_iterimp_E = lgbm_imtr.fit_transform(df_E[nullValue_cols])\n\n# tests data\ntest_iterimp_F = lgbm_imtr.fit_transform(df_F_t[nullValue_cols])\ntest_iterimp_G = lgbm_imtr.fit_transform(df_G_t[nullValue_cols])\ntest_iterimp_H = lgbm_imtr.fit_transform(df_H_t[nullValue_cols])\ntest_iterimp_I = lgbm_imtr.fit_transform(df_I_t[nullValue_cols])","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:18:58.305293Z","iopub.execute_input":"2022-08-06T16:18:58.305811Z","iopub.status.idle":"2022-08-06T16:20:28.416645Z","shell.execute_reply.started":"2022-08-06T16:18:58.305759Z","shell.execute_reply":"2022-08-06T16:20:28.415801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"none_na_cols = [col for col in train.columns if col not in nullValue_cols]\ndf_train = train[none_na_cols]\ndf_test = test[none_na_cols]\n\ntrain_ = pd.concat([train_iterimp_A, train_iterimp_B,train_iterimp_C,train_iterimp_D,train_iterimp_E], axis=0)\ntrain = pd.concat([df_train, train_], axis=1)\n\ntest_ = pd.concat([test_iterimp_F, test_iterimp_G,test_iterimp_H,test_iterimp_I], axis=0)\ntest = pd.concat([df_test, test_], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.418238Z","iopub.execute_input":"2022-08-06T16:20:28.418572Z","iopub.status.idle":"2022-08-06T16:20:28.447787Z","shell.execute_reply.started":"2022-08-06T16:20:28.418539Z","shell.execute_reply":"2022-08-06T16:20:28.446532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['attribute_2*3'] = train['attribute_2'] * train['attribute_3']\ntest['attribute_2*3'] = test['attribute_2'] * test['attribute_3']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.449505Z","iopub.execute_input":"2022-08-06T16:20:28.450379Z","iopub.status.idle":"2022-08-06T16:20:28.458693Z","shell.execute_reply.started":"2022-08-06T16:20:28.450334Z","shell.execute_reply":"2022-08-06T16:20:28.457490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meas_gr1_cols = [f\"measurement_{i:d}\" for i in list(range(3, 5)) + list(range(9, 17)) ]\ntrain['meas_gr1_avg'] = np.mean(train[meas_gr1_cols], axis=1)\ntrain['meas_gr1_std'] = np.std(train[meas_gr1_cols], axis=1)\n\ntest['meas_gr1_avg'] = np.mean(test[meas_gr1_cols], axis=1)\ntest['meas_gr1_std'] = np.std(test[meas_gr1_cols], axis=1) \n\nmeas_gr2_cols = [f\"measurement_{i:d}\" for i in list(range(5, 9))]\ntrain['meas_gr2_avg'] = np.mean(train[meas_gr2_cols], axis=1)\ntest['meas_gr2_avg'] = np.mean(test[meas_gr2_cols], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.460309Z","iopub.execute_input":"2022-08-06T16:20:28.461411Z","iopub.status.idle":"2022-08-06T16:20:28.521984Z","shell.execute_reply.started":"2022-08-06T16:20:28.461375Z","shell.execute_reply":"2022-08-06T16:20:28.520672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['meas17/meas_gr2_avg'] = train['measurement_17'] / train['meas_gr2_avg']\ntest['meas17/meas_gr2_avg'] = test['measurement_17'] / test['meas_gr2_avg']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.523514Z","iopub.execute_input":"2022-08-06T16:20:28.523914Z","iopub.status.idle":"2022-08-06T16:20:28.532567Z","shell.execute_reply.started":"2022-08-06T16:20:28.523879Z","shell.execute_reply":"2022-08-06T16:20:28.531491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_use = ['measurement_0', 'measurement_1', 'measurement_2', 'attribute_0', 'attribute_1',\n               'meas_gr1_avg', 'meas_gr1_std', 'attribute_2*3', 'loading', 'measurement_17', 'meas17/meas_gr2_avg']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.533914Z","iopub.execute_input":"2022-08-06T16:20:28.534815Z","iopub.status.idle":"2022-08-06T16:20:28.543625Z","shell.execute_reply.started":"2022-08-06T16:20:28.534779Z","shell.execute_reply":"2022-08-06T16:20:28.542452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ = train[cols_to_use]\ntest_ = test[cols_to_use]","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.547339Z","iopub.execute_input":"2022-08-06T16:20:28.548336Z","iopub.status.idle":"2022-08-06T16:20:28.576356Z","shell.execute_reply.started":"2022-08-06T16:20:28.548284Z","shell.execute_reply":"2022-08-06T16:20:28.575358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoder = LabelEncoder()\ntrain_le = train_.copy()\ntest_le = test_.copy()\n\nfor col in ['attribute_0', 'attribute_1']:\n    train_le[col] = label_encoder.fit_transform(train_[col])\n    test_le[col] = label_encoder.fit_transform(test_[col]) \n        \ntrain_ = train_le\ntest_ = test_le","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.578244Z","iopub.execute_input":"2022-08-06T16:20:28.578698Z","iopub.status.idle":"2022-08-06T16:20:28.616790Z","shell.execute_reply.started":"2022-08-06T16:20:28.578653Z","shell.execute_reply":"2022-08-06T16:20:28.615929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, test, y = train_, test_, target ","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:20:28.617986Z","iopub.execute_input":"2022-08-06T16:20:28.618908Z","iopub.status.idle":"2022-08-06T16:20:28.625266Z","shell.execute_reply.started":"2022-08-06T16:20:28.618873Z","shell.execute_reply":"2022-08-06T16:20:28.624121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = X.select_dtypes(int).columns\nnum_cols = set(X.columns) - set(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:47:53.649018Z","iopub.execute_input":"2022-08-06T16:47:53.649933Z","iopub.status.idle":"2022-08-06T16:47:53.658142Z","shell.execute_reply.started":"2022-08-06T16:47:53.649886Z","shell.execute_reply":"2022-08-06T16:47:53.656771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_dummies = pd.get_dummies(pd.concat([X[cat_cols], test[cat_cols]], axis=0), columns=cat_cols)\nX_cat = all_dummies[:len(X)]\ntest_cat = all_dummies[len(X):]","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:47:54.009527Z","iopub.execute_input":"2022-08-06T16:47:54.010700Z","iopub.status.idle":"2022-08-06T16:47:54.066374Z","shell.execute_reply.started":"2022-08-06T16:47:54.010657Z","shell.execute_reply":"2022-08-06T16:47:54.065233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.concat([X_cat, X[num_cols]], axis=1)\ntest_data = pd.concat([test_cat, test[num_cols]], axis=1)\n\ndummy_cat = set(X_train.columns) - set(X_train[num_cols].columns)\nall_cols = list(dummy_cat) + list(num_cols)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T16:52:27.432895Z","iopub.execute_input":"2022-08-06T16:52:27.434231Z","iopub.status.idle":"2022-08-06T16:52:27.452000Z","shell.execute_reply.started":"2022-08-06T16:52:27.434190Z","shell.execute_reply":"2022-08-06T16:52:27.450813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Autokeras","metadata":{}},{"cell_type":"code","source":"n_folds = 5\noof_preds = np.zeros((n_folds, len(test)))\n\nskf = StratifiedKFold(n_splits=n_folds)\n\nfor i,(train_index, test_index) in enumerate(skf.split(data, y)):\n    \n    X_train, X_test = data.iloc[train_index], data.iloc[test_index]\n    y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n    \n    scaler = StandardScaler().fit(pd.concat([X_train[num_cols], X_test[num_cols]], axis=0))\n    X_nums = scaler.transform(X_train[num_cols])\n    X_test_nums = scaler.transform(X_test[num_cols])\n    test_nums = scaler.transform(test[num_cols])\n    \n    X_train = pd.DataFrame(np.hstack([X_train[dummy_cat].values,X_nums]), columns=all_cols)\n    X_test = pd.DataFrame(np.hstack([X_test[dummy_cat].values,X_test_nums]), columns=all_cols)\n    test_df = pd.DataFrame(np.hstack([test_data[dummy_cat].values,test_nums]), columns=all_cols)\n    \n    reg = ak.StructuredDataRegressor(max_trials=1, seed=i, overwrite=True)\n    reg.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=1)\n    \n    y_hat = reg.predict(X_test)\n    rmse = mean_squared_error(y_test, y_hat, squared=True)\n    \n    oof_preds[i] = reg.predict(test_df).reshape(-1)\n    \n    print('-----------------------------------------------------------')\n    print('Fold:%2d, RMSE:%1.3f' % (i+1, rmse))","metadata":{"execution":{"iopub.status.busy":"2022-08-06T17:15:14.212425Z","iopub.execute_input":"2022-08-06T17:15:14.212878Z","iopub.status.idle":"2022-08-06T17:39:01.826212Z","shell.execute_reply.started":"2022-08-06T17:15:14.212841Z","shell.execute_reply":"2022-08-06T17:39:01.824552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"sub['failure'] = np.mean(oof_preds, axis=0)\nsub.to_csv('submission_weights.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T17:40:19.401833Z","iopub.execute_input":"2022-08-06T17:40:19.402271Z","iopub.status.idle":"2022-08-06T17:40:19.460500Z","shell.execute_reply.started":"2022-08-06T17:40:19.402236Z","shell.execute_reply":"2022-08-06T17:40:19.459585Z"},"trusted":true},"execution_count":null,"outputs":[]}]}