{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Tabular Playground Series - Aug 2022**","metadata":{}},{"cell_type":"markdown","source":"The August 2022 edition of the Tabular Playground Series is an opportunity to help the fictional company Keep It Dry improve its main product Super Soaker. The product is used in factories to absorb spills and leaks.\n\nThe company has just completed a large testing study for different product prototypes. Can you use this data to build a model that predicts product failures?","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport category_encoders\nimport optuna\n\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\n\nfrom sklearn.metrics import roc_auc_score, roc_curve, f1_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold, KFold\n\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom catboost import CatBoostClassifier\n\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer, KNNImputer\n\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, PowerTransformer\n\nfrom sklearn import cluster\nfrom sklearn import metrics\nfrom sklearn import decomposition\nimport umap\nfrom sklearn.manifold import TSNE","metadata":{"execution":{"iopub.status.busy":"2022-08-06T09:26:56.554352Z","iopub.execute_input":"2022-08-06T09:26:56.555155Z","iopub.status.idle":"2022-08-06T09:26:57.040317Z","shell.execute_reply.started":"2022-08-06T09:26:56.555112Z","shell.execute_reply":"2022-08-06T09:26:57.039088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:27:48.166754Z","iopub.execute_input":"2022-08-06T07:27:48.167557Z","iopub.status.idle":"2022-08-06T07:27:48.174168Z","shell.execute_reply.started":"2022-08-06T07:27:48.167518Z","shell.execute_reply":"2022-08-06T07:27:48.172117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Read datasets**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/train.csv', index_col='id')\ntest = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/test.csv', index_col='id')\nsubmit = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/sample_submission.csv')\nprint('Train:', train.shape)\nprint('Test:', test.shape)\nprint('Submit:', submit.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T10:06:06.677466Z","iopub.execute_input":"2022-08-06T10:06:06.677934Z","iopub.status.idle":"2022-08-06T10:06:06.893697Z","shell.execute_reply.started":"2022-08-06T10:06:06.677901Z","shell.execute_reply":"2022-08-06T10:06:06.892543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.563189Z","iopub.execute_input":"2022-08-05T14:44:39.563566Z","iopub.status.idle":"2022-08-05T14:44:39.602355Z","shell.execute_reply.started":"2022-08-05T14:44:39.563531Z","shell.execute_reply":"2022-08-05T14:44:39.600916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.604240Z","iopub.execute_input":"2022-08-05T14:44:39.604696Z","iopub.status.idle":"2022-08-05T14:44:39.635285Z","shell.execute_reply.started":"2022-08-05T14:44:39.604656Z","shell.execute_reply":"2022-08-05T14:44:39.634009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.636807Z","iopub.execute_input":"2022-08-05T14:44:39.637190Z","iopub.status.idle":"2022-08-05T14:44:39.666514Z","shell.execute_reply.started":"2022-08-05T14:44:39.637155Z","shell.execute_reply":"2022-08-05T14:44:39.665286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.668552Z","iopub.execute_input":"2022-08-05T14:44:39.669404Z","iopub.status.idle":"2022-08-05T14:44:39.688597Z","shell.execute_reply.started":"2022-08-05T14:44:39.669356Z","shell.execute_reply":"2022-08-05T14:44:39.687232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [col for col in train.columns if train[col].dtypes == object]\nint_features = [col for col in train.columns if (train[col].dtypes == 'int64' and col != 'failure')]\nfloat_features = [col for col in train.columns if train[col].dtypes == 'float64']\nprint('Checking:', len(cat_features) + len(int_features) + len(float_features) == len(train.columns)-1)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:27:48.466839Z","iopub.execute_input":"2022-08-06T07:27:48.467700Z","iopub.status.idle":"2022-08-06T07:27:48.481665Z","shell.execute_reply.started":"2022-08-06T07:27:48.467652Z","shell.execute_reply":"2022-08-06T07:27:48.480391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr = train.drop(columns='failure')\ntr['tr/te'] = 'train'\nte = test.copy()\nte['tr/te'] = 'test'\ndf = pd.concat([tr, te], axis=0, join='inner')\nlen(df)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:27:48.483635Z","iopub.execute_input":"2022-08-06T07:27:48.484456Z","iopub.status.idle":"2022-08-06T07:27:48.522869Z","shell.execute_reply.started":"2022-08-06T07:27:48.484410Z","shell.execute_reply":"2022-08-06T07:27:48.521754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Checking nulls**","metadata":{}},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.737002Z","iopub.execute_input":"2022-08-05T14:44:39.737472Z","iopub.status.idle":"2022-08-05T14:44:39.756289Z","shell.execute_reply.started":"2022-08-05T14:44:39.737435Z","shell.execute_reply":"2022-08-05T14:44:39.754969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.758130Z","iopub.execute_input":"2022-08-05T14:44:39.758994Z","iopub.status.idle":"2022-08-05T14:44:39.774714Z","shell.execute_reply.started":"2022-08-05T14:44:39.758945Z","shell.execute_reply":"2022-08-05T14:44:39.773359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Missing categorical values:', train[cat_features].isnull().sum().sum())\nprint('Missing integer values:', train[int_features].isnull().sum().sum())\nprint('Missing float values:', train[float_features].isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.776113Z","iopub.execute_input":"2022-08-05T14:44:39.776431Z","iopub.status.idle":"2022-08-05T14:44:39.795850Z","shell.execute_reply.started":"2022-08-05T14:44:39.776401Z","shell.execute_reply":"2022-08-05T14:44:39.794154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Missing categorical values:', test[cat_features].isnull().sum().sum())\nprint('Missing integer values:', test[int_features].isnull().sum().sum())\nprint('Missing float values:', test[float_features].isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.797495Z","iopub.execute_input":"2022-08-05T14:44:39.797859Z","iopub.status.idle":"2022-08-05T14:44:39.819250Z","shell.execute_reply.started":"2022-08-05T14:44:39.797826Z","shell.execute_reply":"2022-08-05T14:44:39.817614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_tr = train[float_features].isnull().sum(axis=1)\nsumma = 0\nfor i in sorted(temp_tr.unique()):\n    n_miss = len(train[temp_tr == i])\n    print(i, 'missing values:', n_miss, '    \\tPercent:', round(100*n_miss/len(train),2))\n    summa += n_miss*i\nprint('Checking:', summa == train[float_features].isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.820967Z","iopub.execute_input":"2022-08-05T14:44:39.821588Z","iopub.status.idle":"2022-08-05T14:44:39.845691Z","shell.execute_reply.started":"2022-08-05T14:44:39.821552Z","shell.execute_reply":"2022-08-05T14:44:39.844485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_te = test[float_features].isnull().sum(axis=1)\nsumma = 0\nfor i in sorted(temp_te.unique()):\n    n_miss = len(test[temp_te == i])\n    print(i, 'missing values:', n_miss, '   \\tPercent:', round(100*n_miss/len(test),2))\n    summa += n_miss*i\nprint('Checking:', summa == test[float_features].isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.848370Z","iopub.execute_input":"2022-08-05T14:44:39.848814Z","iopub.status.idle":"2022-08-05T14:44:39.875341Z","shell.execute_reply.started":"2022-08-05T14:44:39.848779Z","shell.execute_reply":"2022-08-05T14:44:39.874114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 1:** float_features columns have some missing values, measurement_17 column has maximum nulls, but less than 9% of all data. About a half of the float values have no missing values. Maximum 6 missing values in one row.","metadata":{}},{"cell_type":"markdown","source":"# **Statistical Analysis**","metadata":{}},{"cell_type":"markdown","source":"**Integer values**","metadata":{}},{"cell_type":"code","source":"train[int_features].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.878860Z","iopub.execute_input":"2022-08-05T14:44:39.880123Z","iopub.status.idle":"2022-08-05T14:44:39.918744Z","shell.execute_reply.started":"2022-08-05T14:44:39.880065Z","shell.execute_reply":"2022-08-05T14:44:39.917746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[int_features].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.920201Z","iopub.execute_input":"2022-08-05T14:44:39.921109Z","iopub.status.idle":"2022-08-05T14:44:39.952248Z","shell.execute_reply.started":"2022-08-05T14:44:39.921071Z","shell.execute_reply":"2022-08-05T14:44:39.951003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(3,2, sharex=False, sharey=False, figsize=(12,8), constrained_layout=True)\nfig.suptitle('Integer features (boxplot)', fontsize=25)\n\nfor i, col in enumerate(int_features):\n    sns.boxplot(data=df[int_features + [df.columns[-1]]], ax=ax[i//2,i%2], y=col, x='tr/te')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:39.953824Z","iopub.execute_input":"2022-08-05T14:44:39.954219Z","iopub.status.idle":"2022-08-05T14:44:41.076485Z","shell.execute_reply.started":"2022-08-05T14:44:39.954184Z","shell.execute_reply":"2022-08-05T14:44:41.075002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(3,2, sharex=False, sharey=False, figsize=(14,10), constrained_layout=True)\nfig.suptitle('Integer features (countplot)', fontsize=25)\n\nfor i, col in enumerate(int_features):\n    sns.countplot(data=df[int_features + [df.columns[-1]]], ax=ax[i//2,i%2], x=col, hue='tr/te')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:41.078537Z","iopub.execute_input":"2022-08-05T14:44:41.079000Z","iopub.status.idle":"2022-08-05T14:44:43.388909Z","shell.execute_reply.started":"2022-08-05T14:44:41.078955Z","shell.execute_reply":"2022-08-05T14:44:43.387238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Float values**","metadata":{}},{"cell_type":"code","source":"train[float_features].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:43.391239Z","iopub.execute_input":"2022-08-05T14:44:43.391656Z","iopub.status.idle":"2022-08-05T14:44:43.482402Z","shell.execute_reply.started":"2022-08-05T14:44:43.391621Z","shell.execute_reply":"2022-08-05T14:44:43.481041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[float_features].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:43.484397Z","iopub.execute_input":"2022-08-05T14:44:43.484815Z","iopub.status.idle":"2022-08-05T14:44:43.565589Z","shell.execute_reply.started":"2022-08-05T14:44:43.484783Z","shell.execute_reply":"2022-08-05T14:44:43.564811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(4,4, sharex=False, sharey=False, figsize=(14,12), constrained_layout=True)\nfig.suptitle('Float features', fontsize=25)\n\nfor i, col in enumerate(float_features):\n    sns.boxplot(data=df[float_features + [df.columns[-1]]], ax=ax[i//4,i%4], y=col, x='tr/te')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:43.566638Z","iopub.execute_input":"2022-08-05T14:44:43.567838Z","iopub.status.idle":"2022-08-05T14:44:46.801681Z","shell.execute_reply.started":"2022-08-05T14:44:43.567800Z","shell.execute_reply":"2022-08-05T14:44:46.800411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Categorical values**","metadata":{}},{"cell_type":"code","source":"train[cat_features].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:46.803421Z","iopub.execute_input":"2022-08-05T14:44:46.804553Z","iopub.status.idle":"2022-08-05T14:44:46.832857Z","shell.execute_reply.started":"2022-08-05T14:44:46.804504Z","shell.execute_reply":"2022-08-05T14:44:46.831592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[cat_features].describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:46.834336Z","iopub.execute_input":"2022-08-05T14:44:46.835204Z","iopub.status.idle":"2022-08-05T14:44:46.862563Z","shell.execute_reply.started":"2022-08-05T14:44:46.835168Z","shell.execute_reply":"2022-08-05T14:44:46.861387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(2,2, sharex=False, sharey=False, figsize=(12,8), constrained_layout=True)\nfig.suptitle('Categorical features', fontsize=25)\n\nfor i, col in enumerate(cat_features):\n    sns.countplot(data=df[cat_features + [df.columns[-1]]], ax=ax[i//2,i%2], x=col, hue='tr/te')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:46.864954Z","iopub.execute_input":"2022-08-05T14:44:46.865591Z","iopub.status.idle":"2022-08-05T14:44:47.954041Z","shell.execute_reply.started":"2022-08-05T14:44:46.865554Z","shell.execute_reply":"2022-08-05T14:44:47.952739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Unique Values**","metadata":{}},{"cell_type":"code","source":"for column in train.columns:\n    temp = train[column].unique()\n    print(column, ' \\t\\t(', len(temp), '): ', temp, sep='')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:47.955852Z","iopub.execute_input":"2022-08-05T14:44:47.956309Z","iopub.status.idle":"2022-08-05T14:44:47.991067Z","shell.execute_reply.started":"2022-08-05T14:44:47.956262Z","shell.execute_reply":"2022-08-05T14:44:47.989511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in test.columns:\n    temp = test[column].unique()\n    print(column, ' \\t\\t(', len(temp), '): ', temp, sep='')","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:47.999530Z","iopub.execute_input":"2022-08-05T14:44:47.999956Z","iopub.status.idle":"2022-08-05T14:44:48.038150Z","shell.execute_reply.started":"2022-08-05T14:44:47.999922Z","shell.execute_reply":"2022-08-05T14:44:48.036837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 2:** All the attribute_0-3 look like categorical features. Measurement_0-2 can be ordinal. All the number features are greater than or equal to zero. Statistical variables of the train and test float features are close each other. They should have the same distributions. ","metadata":{}},{"cell_type":"markdown","source":"**Some other visualizations**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8,6))\nplt.title('Checking a Balance of target', fontsize=16)\nsns.countplot(x='failure', data=train)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:48.039905Z","iopub.execute_input":"2022-08-05T14:44:48.040670Z","iopub.status.idle":"2022-08-05T14:44:48.189773Z","shell.execute_reply.started":"2022-08-05T14:44:48.040632Z","shell.execute_reply":"2022-08-05T14:44:48.187998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 3:** The dataset is unbalanced.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(4,2, sharex=False, sharey=False, figsize=(14,12), constrained_layout=True)\nfig.suptitle('Frequency Analysis', fontsize=25)\n\ntemp_features = cat_features + int_features\nfor i, col in enumerate(temp_features):\n    sns.countplot(data=train[temp_features + ['failure']], ax=ax[i//2,i%2], x=col, hue='failure')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:48.192794Z","iopub.execute_input":"2022-08-05T14:44:48.194092Z","iopub.status.idle":"2022-08-05T14:44:50.617004Z","shell.execute_reply.started":"2022-08-05T14:44:48.194005Z","shell.execute_reply":"2022-08-05T14:44:50.615749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 4:** Categorical features do not change the proportion of unbalancing.","metadata":{}},{"cell_type":"code","source":"enc_hot = category_encoders.one_hot.OneHotEncoder(cols=cat_features+int_features[:2]).fit(train)\ntrain_hot = enc_hot.transform(train)\n\nenc_ord = category_encoders.ordinal.OrdinalEncoder(cols=cat_features+int_features[:2],\n    mapping=[{'col': 'product_code', 'mapping': {None: 0, 'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5}},\n             {'col': 'attribute_0', 'mapping': {None: 0, 'material_5': 5, 'material_7': 7}},\n             {'col': 'attribute_1', 'mapping': {None: 0, 'material_5': 5, 'material_6': 6, 'material_8': 8}}]).fit(train)\ntrain_ord = enc_ord.transform(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:28:50.262496Z","iopub.execute_input":"2022-08-06T07:28:50.263453Z","iopub.status.idle":"2022-08-06T07:28:50.656709Z","shell.execute_reply.started":"2022-08-06T07:28:50.263408Z","shell.execute_reply":"2022-08-06T07:28:50.655473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_hot.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:51.102806Z","iopub.execute_input":"2022-08-05T14:44:51.103286Z","iopub.status.idle":"2022-08-05T14:44:51.124070Z","shell.execute_reply.started":"2022-08-05T14:44:51.103245Z","shell.execute_reply":"2022-08-05T14:44:51.123182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_hot.corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:51.125481Z","iopub.execute_input":"2022-08-05T14:44:51.126002Z","iopub.status.idle":"2022-08-05T14:44:51.287195Z","shell.execute_reply.started":"2022-08-05T14:44:51.125966Z","shell.execute_reply":"2022-08-05T14:44:51.285952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,10))\nsns.heatmap(train_hot.corr())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:51.288710Z","iopub.execute_input":"2022-08-05T14:44:51.289076Z","iopub.status.idle":"2022-08-05T14:44:52.465564Z","shell.execute_reply.started":"2022-08-05T14:44:51.289017Z","shell.execute_reply":"2022-08-05T14:44:52.464507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enc_hot_test = category_encoders.one_hot.OneHotEncoder(cols=cat_features+int_features[:2]).fit(test)\ntest_hot = enc_hot_test.transform(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:29:49.073473Z","iopub.execute_input":"2022-08-06T07:29:49.073900Z","iopub.status.idle":"2022-08-06T07:29:49.279433Z","shell.execute_reply.started":"2022-08-06T07:29:49.073863Z","shell.execute_reply":"2022-08-06T07:29:49.278155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_hot.corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:52.745007Z","iopub.execute_input":"2022-08-05T14:44:52.745393Z","iopub.status.idle":"2022-08-05T14:44:52.870789Z","shell.execute_reply.started":"2022-08-05T14:44:52.745359Z","shell.execute_reply":"2022-08-05T14:44:52.869315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,10))\nsns.heatmap(test_hot.corr())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:52.872561Z","iopub.execute_input":"2022-08-05T14:44:52.873440Z","iopub.status.idle":"2022-08-05T14:44:53.907084Z","shell.execute_reply.started":"2022-08-05T14:44:52.873400Z","shell.execute_reply":"2022-08-05T14:44:53.905673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ord.corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:53.909111Z","iopub.execute_input":"2022-08-05T14:44:53.909515Z","iopub.status.idle":"2022-08-05T14:44:54.000826Z","shell.execute_reply.started":"2022-08-05T14:44:53.909470Z","shell.execute_reply":"2022-08-05T14:44:53.999537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,10))\nsns.heatmap(train_ord.corr())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:54.003286Z","iopub.execute_input":"2022-08-05T14:44:54.004242Z","iopub.status.idle":"2022-08-05T14:44:54.822153Z","shell.execute_reply.started":"2022-08-05T14:44:54.004182Z","shell.execute_reply":"2022-08-05T14:44:54.820712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 5:** Correlation heatmap (OHE) shows strong linear dependence between the product_code and the attributes. It means they give the same information and we can just drop product_code. Anyway the product_code is helpless for us because train and test have different values. We can also see the next linear dependences with the coef=1:\n\nattribute_0 and attribute_2_2 (material 8), \n\nattribute_1_3 (material 6) and attribute_3_4 (material 9), \n\nattribute_2_1 (material 9) and attribute_3_1 (material 5). \n\nIf we look at correlations in test set we will see the same linear dependence between:\n\nattribute_1_2 (material 7) and attribute_2_3 (material 7) and attribute_3_3 (material 9),\n\nattribute 2_1 (material 6) and attribute_3_1 (material 4),\n\nattribute_1_3 (material 5) and attribute_3_4 (material 5).\n\nWe have to untangle these relationships in order to use attributes as features in our models because train and test can have different materials in their attributes. For a start the attribute_0 looks quite well. It has the same values in train and test and the linear correlation with other attributes.\n\nThe measurements_3-9 don't have significant correlation between other features excluding measurement_17. The other features have weak correlations with each other and with the measurements_0-2 a bit stronger. The target has some correlation with the loading.","metadata":{}},{"cell_type":"code","source":"%%time\nsns.pairplot(df.iloc[:,6:], hue='tr/te')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:44:54.825764Z","iopub.execute_input":"2022-08-05T14:44:54.826268Z","iopub.status.idle":"2022-08-05T14:59:11.025851Z","shell.execute_reply.started":"2022-08-05T14:44:54.826230Z","shell.execute_reply":"2022-08-05T14:59:11.022917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsns.pairplot(train.iloc[:,6:], hue='failure')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T14:59:11.029076Z","iopub.execute_input":"2022-08-05T14:59:11.030917Z","iopub.status.idle":"2022-08-05T15:07:51.706241Z","shell.execute_reply.started":"2022-08-05T14:59:11.030828Z","shell.execute_reply":"2022-08-05T15:07:51.703181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Analysis of features**","metadata":{}},{"cell_type":"code","source":"train.groupby(['product_code','attribute_0', 'attribute_1', 'attribute_2', 'attribute_3']).count()['failure']","metadata":{"execution":{"iopub.status.busy":"2022-08-05T17:09:32.090629Z","iopub.execute_input":"2022-08-05T17:09:32.091296Z","iopub.status.idle":"2022-08-05T17:09:32.124254Z","shell.execute_reply.started":"2022-08-05T17:09:32.091241Z","shell.execute_reply":"2022-08-05T17:09:32.122710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.groupby(['product_code','attribute_0','attribute_1','attribute_2','attribute_3']).count()['measurement_0']","metadata":{"execution":{"iopub.status.busy":"2022-08-05T16:44:19.080746Z","iopub.execute_input":"2022-08-05T16:44:19.081366Z","iopub.status.idle":"2022-08-05T16:44:19.115456Z","shell.execute_reply.started":"2022-08-05T16:44:19.081317Z","shell.execute_reply":"2022-08-05T16:44:19.114141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.distplot(train[train['failure'] == 0]['loading'],hist=False)\nsns.distplot(train[train['failure'] == 1]['loading'],hist=False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T16:57:26.064352Z","iopub.execute_input":"2022-08-05T16:57:26.066079Z","iopub.status.idle":"2022-08-05T16:57:26.523811Z","shell.execute_reply.started":"2022-08-05T16:57:26.065987Z","shell.execute_reply":"2022-08-05T16:57:26.522135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Resume:** we have unbalanced dataset with different types of features for a binary classification problem. Float values in our dataset have some missings. We have disharmony between train and test in categorical features as well. Some of those categorical features have strong linear dependence between each other. Actually each product consists of different materials. The product may have multilayer structure and the attributes may mean layers (4 layers or 2 layers with 2 characteristics), we don't know what it is, but we can try the next feature: a number of materials in a product.\n\nThe only feature having the weak correlation with target is loading. We can see that after about loading=140 failure will be more often equal to 1. So we can use this logical condition as a new categorical feature.","metadata":{}},{"cell_type":"markdown","source":"# **Imputing**","metadata":{}},{"cell_type":"markdown","source":"**Zero-Imputing**","metadata":{}},{"cell_type":"code","source":"random_state = 17\nss = StandardScaler()\nrs = RobustScaler()\nmms = MinMaxScaler()\npt = PowerTransformer()\n\nclassifiers = [DecisionTreeClassifier(random_state=random_state),\n               LogisticRegression(random_state=random_state),\n               KNeighborsClassifier(),\n               RandomForestClassifier(random_state=random_state),\n               GradientBoostingClassifier(random_state=random_state),\n               XGBClassifier(random_state=random_state),\n               LGBMClassifier(random_state=random_state),\n               CatBoostClassifier(random_state=random_state, verbose=0)]\nclf_names = ['DT', 'LR', 'KNN', 'RF', 'GBDT', 'XGB', 'LGBM', 'CatBoost']","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:30:18.968100Z","iopub.execute_input":"2022-08-06T07:30:18.968523Z","iopub.status.idle":"2022-08-06T07:30:18.982452Z","shell.execute_reply.started":"2022-08-06T07:30:18.968485Z","shell.execute_reply":"2022-08-06T07:30:18.981609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['product_code', 'attribute_1', 'attribute_2', 'attribute_3', 'failure'], axis=1)\nX_ord.fillna(0.0, inplace=True)\nX_ord_ss = ss.fit_transform(X_ord)\nX_ord_rs = rs.fit_transform(X_ord)\nX_ord_mms = mms.fit_transform(X_ord)\nX_ord_pt = pt.fit_transform(X_ord)\n\nX_hot = train_hot.drop(['product_code_1', 'product_code_2','product_code_3', 'product_code_4','product_code_5',\n                        'attribute_1_1', 'attribute_1_2', 'attribute_1_3', \n                        'attribute_2_1', 'attribute_2_2', 'attribute_2_3', 'attribute_2_4', \n                        'attribute_3_1', 'attribute_3_2', 'attribute_3_3', 'attribute_3_4', 'failure'], axis=1)\nX_hot.fillna(0.0, inplace=True)\nX_hot_ss = ss.fit_transform(X_hot)\nX_hot_rs = rs.fit_transform(X_hot)\nX_hot_mms = mms.fit_transform(X_hot)\nX_hot_pt = pt.fit_transform(X_hot)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T18:59:24.685755Z","iopub.execute_input":"2022-08-05T18:59:24.686279Z","iopub.status.idle":"2022-08-05T18:59:26.196569Z","shell.execute_reply.started":"2022-08-05T18:59:24.686239Z","shell.execute_reply":"2022-08-05T18:59:26.195117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as ordinal","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nprint('StandardScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_ss, y):\n        X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\n    \nprint()\nprint('RobustScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_rs, y):\n        X_train, y_train = X_ord_rs[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_rs[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\n\nprint()\nprint('MinMaxScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_mms, y):\n        X_train, y_train = X_ord_mms[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_mms[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\n    \nprint()\nprint('PowerTransformer')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_pt, y):\n        X_train, y_train = X_ord_pt[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_pt[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T18:40:25.249145Z","iopub.execute_input":"2022-08-05T18:40:25.249990Z","iopub.status.idle":"2022-08-05T18:48:20.107641Z","shell.execute_reply.started":"2022-08-05T18:40:25.249926Z","shell.execute_reply":"2022-08-05T18:48:20.105861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as OHE","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nprint('StandardScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_ss, y):\n        X_train, y_train = X_hot_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\nprint()\n\nprint('RobustScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_rs, y):\n        X_train, y_train = X_hot_rs[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_rs[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\nprint()\n\nprint('MinMaxScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_mms, y):\n        X_train, y_train = X_hot_mms[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_mms[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\nprint()\nprint('PowerTransformer')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_pt, y):\n        X_train, y_train = X_hot_pt[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_pt[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T18:48:20.110081Z","iopub.execute_input":"2022-08-05T18:48:20.111183Z","iopub.status.idle":"2022-08-05T18:56:06.677084Z","shell.execute_reply.started":"2022-08-05T18:48:20.111140Z","shell.execute_reply":"2022-08-05T18:56:06.675467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 6:** There is no significant difference between scalers and encoders. The best model is LogisticRegression but GradientBoostingClassifier also looks not bad.","metadata":{}},{"cell_type":"markdown","source":"**Mean-Imputing**","metadata":{}},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['product_code', 'attribute_1', 'attribute_2', 'attribute_3', 'failure'], axis=1)\nX_ord.fillna(X_ord.mean(), inplace=True)\nX_ord_ss = ss.fit_transform(X_ord)\nX_ord_rs = rs.fit_transform(X_ord)\nX_ord_mms = mms.fit_transform(X_ord)\nX_ord_pt = pt.fit_transform(X_ord)\n\nX_hot = train_hot.drop(['product_code_1', 'product_code_2','product_code_3', 'product_code_4','product_code_5',\n                        'attribute_1_1', 'attribute_1_2', 'attribute_1_3', \n                        'attribute_2_1', 'attribute_2_2', 'attribute_2_3', 'attribute_2_4', \n                        'attribute_3_1', 'attribute_3_2', 'attribute_3_3', 'attribute_3_4', 'failure'], axis=1)\nX_hot.fillna(X_hot.mean(), inplace=True)\nX_hot_ss = ss.fit_transform(X_hot)\nX_hot_rs = rs.fit_transform(X_hot)\nX_hot_mms = mms.fit_transform(X_hot)\nX_hot_pt = pt.fit_transform(X_hot)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:42:24.170844Z","iopub.execute_input":"2022-08-06T07:42:24.171295Z","iopub.status.idle":"2022-08-06T07:42:25.514151Z","shell.execute_reply.started":"2022-08-06T07:42:24.171256Z","shell.execute_reply":"2022-08-06T07:42:25.513141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as ordinal","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nprint('StandardScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_ss, y):\n        X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T15:07:51.910736Z","iopub.execute_input":"2022-08-05T15:07:51.911191Z","iopub.status.idle":"2022-08-05T15:13:36.639072Z","shell.execute_reply.started":"2022-08-05T15:07:51.911149Z","shell.execute_reply":"2022-08-05T15:13:36.637293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as OHE","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nprint('StandardScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_ss, y):\n        X_train, y_train = X_hot_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T15:13:36.641445Z","iopub.execute_input":"2022-08-05T15:13:36.641874Z","iopub.status.idle":"2022-08-05T15:19:10.167113Z","shell.execute_reply.started":"2022-08-05T15:13:36.641826Z","shell.execute_reply":"2022-08-05T15:19:10.165547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 7:** Mean-Imputing looks a little bit better (if we look at third digit after point).","metadata":{}},{"cell_type":"markdown","source":"**Median-imputing**","metadata":{}},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['product_code', 'attribute_1', 'attribute_2', 'attribute_3', 'failure'], axis=1)\nX_ord.fillna(X_ord.median(), inplace=True)\nX_ord_ss = ss.fit_transform(X_ord)\nX_ord_rs = rs.fit_transform(X_ord)\nX_ord_mms = mms.fit_transform(X_ord)\nX_ord_pt = pt.fit_transform(X_ord)\n\nX_hot = train_hot.drop(['product_code_1', 'product_code_2','product_code_3', 'product_code_4','product_code_5',\n                        'attribute_1_1', 'attribute_1_2', 'attribute_1_3', \n                        'attribute_2_1', 'attribute_2_2', 'attribute_2_3', 'attribute_2_4', \n                        'attribute_3_1', 'attribute_3_2', 'attribute_3_3', 'attribute_3_4', 'failure'], axis=1)\nX_hot.fillna(X_hot.median(), inplace=True)\nX_hot_ss = ss.fit_transform(X_hot)\nX_hot_rs = rs.fit_transform(X_hot)\nX_hot_mms = mms.fit_transform(X_hot)\nX_hot_pt = pt.fit_transform(X_hot)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:55:33.609030Z","iopub.execute_input":"2022-08-06T07:55:33.609757Z","iopub.status.idle":"2022-08-06T07:55:35.102361Z","shell.execute_reply.started":"2022-08-06T07:55:33.609721Z","shell.execute_reply":"2022-08-06T07:55:35.101422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as ordinal","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nprint('StandardScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_ss, y):\n        X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:55:50.653947Z","iopub.execute_input":"2022-08-06T07:55:50.654374Z","iopub.status.idle":"2022-08-06T07:57:36.256431Z","shell.execute_reply.started":"2022-08-06T07:55:50.654336Z","shell.execute_reply":"2022-08-06T07:57:36.255309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as OHE","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nprint('StandardScaler')\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_ss, y):\n        X_train, y_train = X_hot_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:57:36.257951Z","iopub.execute_input":"2022-08-06T07:57:36.258877Z","iopub.status.idle":"2022-08-06T07:59:18.872400Z","shell.execute_reply.started":"2022-08-06T07:57:36.258845Z","shell.execute_reply":"2022-08-06T07:59:18.871525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 8:** Median-Imputing is no different from Mean-Imputing.","metadata":{}},{"cell_type":"markdown","source":"**Iterative Imputer**","metadata":{}},{"cell_type":"markdown","source":"Categories as ordinal","metadata":{}},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['product_code', 'attribute_1', 'attribute_2', 'attribute_3', 'failure'], axis=1)\nimp_mean = IterativeImputer(random_state=0)\nX_ord = imp_mean.fit_transform(X_ord)\nX_ord_ss = ss.fit_transform(X_ord)\nX_ord_rs = rs.fit_transform(X_ord)\nX_ord_mms = mms.fit_transform(X_ord)\nX_ord_pt = pt.fit_transform(X_ord)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T07:46:53.000097Z","iopub.execute_input":"2022-08-06T07:46:53.000486Z","iopub.status.idle":"2022-08-06T07:47:05.289486Z","shell.execute_reply.started":"2022-08-06T07:46:53.000455Z","shell.execute_reply":"2022-08-06T07:47:05.288488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_ss, y):\n        X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T15:19:19.890576Z","iopub.execute_input":"2022-08-05T15:19:19.890954Z","iopub.status.idle":"2022-08-05T15:21:17.950459Z","shell.execute_reply.started":"2022-08-05T15:19:19.890903Z","shell.execute_reply":"2022-08-05T15:21:17.949084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as OHE","metadata":{}},{"cell_type":"code","source":"X_hot = train_hot.drop(['product_code_1', 'product_code_2','product_code_3', 'product_code_4','product_code_5',\n                        'attribute_1_1', 'attribute_1_2', 'attribute_1_3', \n                        'attribute_2_1', 'attribute_2_2', 'attribute_2_3', 'attribute_2_4', \n                        'attribute_3_1', 'attribute_3_2', 'attribute_3_3', 'attribute_3_4', 'failure'], axis=1)\nimp_mean = IterativeImputer(random_state=0)\nX_hot = imp_mean.fit_transform(X_hot)\nX_hot_ss = ss.fit_transform(X_hot)\nX_hot_rs = rs.fit_transform(X_hot)\nX_hot_mms = mms.fit_transform(X_hot)\nX_hot_pt = pt.fit_transform(X_hot)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:00:35.707318Z","iopub.execute_input":"2022-08-05T19:00:35.707780Z","iopub.status.idle":"2022-08-05T19:00:49.834069Z","shell.execute_reply.started":"2022-08-05T19:00:35.707743Z","shell.execute_reply":"2022-08-05T19:00:49.833115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_ss, y):\n        X_train, y_train = X_hot_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T15:21:33.801861Z","iopub.execute_input":"2022-08-05T15:21:33.802863Z","iopub.status.idle":"2022-08-05T15:23:29.695849Z","shell.execute_reply.started":"2022-08-05T15:21:33.802800Z","shell.execute_reply":"2022-08-05T15:23:29.693165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 9:** I don't see any difference from the previous results.","metadata":{}},{"cell_type":"markdown","source":"**KNN Imputer**","metadata":{}},{"cell_type":"markdown","source":"Categories as ordinal","metadata":{}},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['product_code', 'attribute_1', 'attribute_2', 'attribute_3', 'failure'], axis=1)\nimp_mean = KNNImputer()\nX_ord = imp_mean.fit_transform(X_ord)\nX_ord_ss = ss.fit_transform(X_ord)\nX_ord_rs = rs.fit_transform(X_ord)\nX_ord_mms = mms.fit_transform(X_ord)\nX_ord_pt = pt.fit_transform(X_ord)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T08:07:31.158744Z","iopub.execute_input":"2022-08-06T08:07:31.160389Z","iopub.status.idle":"2022-08-06T08:08:04.923908Z","shell.execute_reply.started":"2022-08-06T08:07:31.160340Z","shell.execute_reply":"2022-08-06T08:08:04.922480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord_ss, y):\n        X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T15:24:06.782924Z","iopub.execute_input":"2022-08-05T15:24:06.783833Z","iopub.status.idle":"2022-08-05T15:26:05.067340Z","shell.execute_reply.started":"2022-08-05T15:24:06.783781Z","shell.execute_reply":"2022-08-05T15:26:05.065930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Categories as OHE","metadata":{}},{"cell_type":"code","source":"X_hot = train_hot.drop(['product_code_1', 'product_code_2','product_code_3', 'product_code_4','product_code_5',\n                        'attribute_1_1', 'attribute_1_2', 'attribute_1_3', \n                        'attribute_2_1', 'attribute_2_2', 'attribute_2_3', 'attribute_2_4', \n                        'attribute_3_1', 'attribute_3_2', 'attribute_3_3', 'attribute_3_4', 'failure'], axis=1)\nimp_mean = KNNImputer()\nX_hot = imp_mean.fit_transform(X_hot)\nX_hot_ss = ss.fit_transform(X_hot)\nX_hot_rs = rs.fit_transform(X_hot)\nX_hot_mms = mms.fit_transform(X_hot)\nX_hot_pt = pt.fit_transform(X_hot)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T19:01:32.999070Z","iopub.execute_input":"2022-08-05T19:01:32.999575Z","iopub.status.idle":"2022-08-05T19:02:11.682751Z","shell.execute_reply.started":"2022-08-05T19:01:32.999527Z","shell.execute_reply":"2022-08-05T19:02:11.681462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nfor i, clf in enumerate(classifiers):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_hot_ss, y):\n        X_train, y_train = X_hot_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_hot_ss[idx_val,:], y[idx_val]\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print(clf_names[i] + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-05T15:26:43.218667Z","iopub.execute_input":"2022-08-05T15:26:43.219109Z","iopub.status.idle":"2022-08-05T15:28:36.216637Z","shell.execute_reply.started":"2022-08-05T15:26:43.219071Z","shell.execute_reply":"2022-08-05T15:28:36.215533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking a number of neighbors","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=3)\nfor i in range(2,20):\n    j = 0\n    for idx_tr, idx_val in skf.split(X_ord, y):\n        imp_mean = KNNImputer(n_neighbors=i)\n        X_ord = imp_mean.fit_transform(X_ord)\n        X_ord_ss = ss.fit_transform(X_ord)\n        \n        X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n        clf = LogisticRegression(random_state=random_state)\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print('n_neighbors = ' + str(i) + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T08:09:20.763403Z","iopub.execute_input":"2022-08-06T08:09:20.763791Z","iopub.status.idle":"2022-08-06T08:09:25.408259Z","shell.execute_reply.started":"2022-08-06T08:09:20.763759Z","shell.execute_reply":"2022-08-06T08:09:25.406717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Conclusion 10:** KNN-Imputer seems a little-little bit better but the number of neighbors doesn't matter.","metadata":{}},{"cell_type":"markdown","source":"**Checking a couple of hypotheses**","metadata":{}},{"cell_type":"markdown","source":"Checking influence of categorical features","metadata":{}},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['failure'], axis=1)\nimp_mean = KNNImputer()\nX_ord = imp_mean.fit_transform(X_ord)\nX_ord_ss = ss.fit_transform(X_ord)\n\nskf = StratifiedKFold(n_splits=3)\nj = 0\nfor idx_tr, idx_val in skf.split(X_ord, y):\n    X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n    X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n    clf = LogisticRegression(random_state=random_state)\n    clf.fit(X_train, y_train)\n    y_pred = clf.predict_proba(X_valid)\n    score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n    print('n_neighbors = ' + str(i) + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n    j += 1\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T08:22:47.989500Z","iopub.execute_input":"2022-08-06T08:22:47.989807Z","iopub.status.idle":"2022-08-06T08:22:48.337490Z","shell.execute_reply.started":"2022-08-06T08:22:47.989779Z","shell.execute_reply":"2022-08-06T08:22:48.335996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_hot['failure']\nX_hot = train_hot.drop(['failure'], axis=1)\nimp_mean = KNNImputer()\nX_hot = imp_mean.fit_transform(X_hot)\nX_hot_ss = ss.fit_transform(X_hot)\n\nskf = StratifiedKFold(n_splits=3)\nj = 0\nfor idx_tr, idx_val in skf.split(X_hot, y):\n    X_train, y_train = X_hot_ss[idx_tr,:], y[idx_tr]\n    X_valid, y_valid = X_hot_ss[idx_val,:], y[idx_val]\n    clf = LogisticRegression(random_state=random_state)\n    clf.fit(X_train, y_train)\n    y_pred = clf.predict_proba(X_valid)\n    score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n    print('n_neighbors = ' + str(i) + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n    j += 1\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T08:27:37.106520Z","iopub.execute_input":"2022-08-06T08:27:37.107676Z","iopub.status.idle":"2022-08-06T08:28:35.464738Z","shell.execute_reply.started":"2022-08-06T08:27:37.107629Z","shell.execute_reply":"2022-08-06T08:28:35.463252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Adding a new feature","metadata":{}},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['failure'], axis=1)\nX_ord['load_more'] = X_ord['loading'] > 140\nimp_mean = KNNImputer()\nX_ord = imp_mean.fit_transform(X_ord)\nX_ord_ss = ss.fit_transform(X_ord)\n\nskf = StratifiedKFold(n_splits=3)\nj = 0\nfor idx_tr, idx_val in skf.split(X_ord, y):\n    X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n    X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n    clf = LogisticRegression(random_state=random_state)\n    clf.fit(X_train, y_train)\n    y_pred = clf.predict_proba(X_valid)\n    score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n    print('n_neighbors = ' + str(i) + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n    j += 1\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T08:24:27.833735Z","iopub.execute_input":"2022-08-06T08:24:27.834157Z","iopub.status.idle":"2022-08-06T08:25:03.064931Z","shell.execute_reply.started":"2022-08-06T08:24:27.834122Z","shell.execute_reply":"2022-08-06T08:25:03.063439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train_hot['failure']\nX_hot = train_hot.drop(['failure'], axis=1)\nX_hot['load_more'] = X_hot['loading'] > 140\nimp_mean = KNNImputer()\nX_hot = imp_mean.fit_transform(X_hot)\nX_hot_ss = ss.fit_transform(X_hot)\n\nskf = StratifiedKFold(n_splits=3)\nj = 0\nfor idx_tr, idx_val in skf.split(X_hot, y):\n    X_train, y_train = X_hot_ss[idx_tr,:], y[idx_tr]\n    X_valid, y_valid = X_hot_ss[idx_val,:], y[idx_val]\n    clf = LogisticRegression(random_state=random_state)\n    clf.fit(X_train, y_train)\n    y_pred = clf.predict_proba(X_valid)\n    score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n    print('n_neighbors = ' + str(i) + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n    j += 1\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T08:28:53.566502Z","iopub.execute_input":"2022-08-06T08:28:53.566897Z","iopub.status.idle":"2022-08-06T08:29:47.623035Z","shell.execute_reply.started":"2022-08-06T08:28:53.566863Z","shell.execute_reply":"2022-08-06T08:29:47.621576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Checking product_code","metadata":{}},{"cell_type":"code","source":"for i in range(1, 6):\n    XA, yA = train_ord[train_ord['product_code'] == i], np.array(y[train_ord['product_code'] == i])\n\n    XA = XA.drop(['product_code', 'failure'], axis=1)\n    XA['load_more'] = XA['loading'] > 140\n    imp_mean = KNNImputer()\n    XA = imp_mean.fit_transform(XA)\n    XA_ss = ss.fit_transform(XA)\n\n    skf = StratifiedKFold(n_splits=3)\n    j = 0\n    for idx_tr, idx_val in skf.split(XA, yA):\n        X_train, y_train = XA_ss[idx_tr,:], yA[idx_tr]\n        X_valid, y_valid = XA_ss[idx_val,:], yA[idx_val]\n        clf = LogisticRegression(random_state=random_state)\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        print('product_code = ' + str(i) + ' Fold ' + str(j), ' ROC AUC: ', score_auc)\n        j += 1\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-06T08:59:08.942068Z","iopub.execute_input":"2022-08-06T08:59:08.943236Z","iopub.status.idle":"2022-08-06T08:59:17.425163Z","shell.execute_reply.started":"2022-08-06T08:59:08.943196Z","shell.execute_reply":"2022-08-06T08:59:17.421173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Resume:** The results are not changed significantly after removing or saving all the categorical features. Although the results for different groups of product_code differ from each other, so the future analysis may include an investigation of relationship between product_codes in train and test sets.","metadata":{}},{"cell_type":"markdown","source":"**Hyperparameters tuning**","metadata":{}},{"cell_type":"code","source":"y = train_ord['failure']\nX_ord = train_ord.drop(['product_code', 'attribute_1', 'attribute_2', 'attribute_3', 'failure'], axis=1)\nX_ord['load_more'] = X_ord['loading'] > 140\nimp_mean = KNNImputer()\nX_ord = imp_mean.fit_transform(X_ord)\nX_ord_ss = ss.fit_transform(X_ord)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T09:45:37.829252Z","iopub.execute_input":"2022-08-06T09:45:37.829657Z","iopub.status.idle":"2022-08-06T09:46:15.372631Z","shell.execute_reply.started":"2022-08-06T09:45:37.829625Z","shell.execute_reply":"2022-08-06T09:46:15.371459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(trial):\n    params = {\n        'l1_ratio': trial.suggest_float(\"l1_ratio\", 0.0, 1.0),\n        'C': trial.suggest_float(\"C\", 0.001, 100.0)\n    }\n\n    skf = StratifiedKFold(n_splits=5)\n    list_auc = []\n    for idx_tr, idx_val in skf.split(X_ord, y):\n        X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n        X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n        clf = LogisticRegression(random_state=17, penalty='elasticnet', solver='saga', **params)\n        clf.fit(X_train, y_train)\n        y_pred = clf.predict_proba(X_valid)\n        score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n        list_auc.append(score_auc)\n\n    preds = np.mean(np.column_stack(list_auc), axis=1)\n    return preds","metadata":{"execution":{"iopub.status.busy":"2022-08-06T09:48:09.545528Z","iopub.execute_input":"2022-08-06T09:48:09.545945Z","iopub.status.idle":"2022-08-06T09:48:09.554470Z","shell.execute_reply.started":"2022-08-06T09:48:09.545908Z","shell.execute_reply":"2022-08-06T09:48:09.553652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = optuna.create_study(direction=\"maximize\")\nstudy.optimize(run, n_trials=500)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T09:48:10.636730Z","iopub.execute_input":"2022-08-06T09:48:10.637533Z","iopub.status.idle":"2022-08-06T09:50:10.166178Z","shell.execute_reply.started":"2022-08-06T09:48:10.637469Z","shell.execute_reply":"2022-08-06T09:50:10.164618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Best trial:', study.best_trial.params)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T09:50:10.168617Z","iopub.execute_input":"2022-08-06T09:50:10.169087Z","iopub.status.idle":"2022-08-06T09:50:10.177728Z","shell.execute_reply.started":"2022-08-06T09:50:10.169025Z","shell.execute_reply":"2022-08-06T09:50:10.176692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enc_test_ord = category_encoders.ordinal.OrdinalEncoder(cols=['attribute_0'],\n    mapping=[{'col': 'attribute_0', 'mapping': {None: 0, 'material_5': 5, 'material_7': 7}}]).fit(test)\ntest_ord = enc_test_ord.transform(test)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T09:53:13.411459Z","iopub.execute_input":"2022-08-06T09:53:13.412779Z","iopub.status.idle":"2022-08-06T09:53:13.467544Z","shell.execute_reply.started":"2022-08-06T09:53:13.412726Z","shell.execute_reply":"2022-08-06T09:53:13.466373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_ord = test_ord.drop(['product_code', 'attribute_1', 'attribute_2', 'attribute_3'], axis=1)\nX_test_ord['load_more'] = X_test_ord['loading'] > 140\nimp_mean = KNNImputer()\nX_test_ord = imp_mean.fit_transform(X_test_ord)\nX_test_ord_ss = ss.fit_transform(X_test_ord)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T09:58:28.644664Z","iopub.execute_input":"2022-08-06T09:58:28.645134Z","iopub.status.idle":"2022-08-06T09:58:49.573583Z","shell.execute_reply.started":"2022-08-06T09:58:28.645097Z","shell.execute_reply":"2022-08-06T09:58:49.572726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5)\nlist_auc = []\ntest_auc = []\nfor idx_tr, idx_val in skf.split(X_ord, y):\n    X_train, y_train = X_ord_ss[idx_tr,:], y[idx_tr]\n    X_valid, y_valid = X_ord_ss[idx_val,:], y[idx_val]\n    clf = LogisticRegression(random_state=17, penalty='elasticnet', solver='saga', **study.best_trial.params)\n    clf.fit(X_train, y_train)\n    y_pred = clf.predict_proba(X_valid)\n    score_auc = roc_auc_score(y_valid, y_pred[:,-1])\n    list_auc.append(score_auc)\n    \n    y_test = clf.predict_proba(X_test_ord_ss)\n    test_auc.append(y_test[:,-1])\n\npreds = np.mean(np.column_stack(list_auc), axis=1)\nprint(preds)","metadata":{"execution":{"iopub.status.busy":"2022-08-06T10:01:24.566421Z","iopub.execute_input":"2022-08-06T10:01:24.566841Z","iopub.status.idle":"2022-08-06T10:01:26.035485Z","shell.execute_reply.started":"2022-08-06T10:01:24.566805Z","shell.execute_reply":"2022-08-06T10:01:26.033758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = np.mean(np.column_stack(test_auc), axis=1)\nresult","metadata":{"execution":{"iopub.status.busy":"2022-08-06T10:01:33.300745Z","iopub.execute_input":"2022-08-06T10:01:33.301352Z","iopub.status.idle":"2022-08-06T10:01:33.309595Z","shell.execute_reply.started":"2022-08-06T10:01:33.301303Z","shell.execute_reply":"2022-08-06T10:01:33.308581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit['failure'] = result\nsubmit.to_csv(\"submission1.csv\", index=False)\nsubmit","metadata":{"execution":{"iopub.status.busy":"2022-08-06T10:08:45.250467Z","iopub.execute_input":"2022-08-06T10:08:45.251504Z","iopub.status.idle":"2022-08-06T10:08:45.326389Z","shell.execute_reply.started":"2022-08-06T10:08:45.251447Z","shell.execute_reply":"2022-08-06T10:08:45.325139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}