{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

TPS AUG 22

\n

EDA + Baseline

\n

Process of classification

","metadata":{}},{"cell_type":"markdown","source":"This notebook is designed to describe the process of exploring and gaining insight into the dataset of TPS AUG 22. It also provides baseline code for simple machine learning models for binary classification. \nI referred to the following notes for creating this notebook. \n1. https://www.kaggle.com/code/ambrosm/tpsaug22-eda-which-makes-sense \n2. https://www.kaggle.com/code/thedevastator/tps-aug-simple-baseline/notebook?scriptVersionId=102278024 \n3. https://www.kaggle.com/code/cv13j0/tps-aug22-binary-classification \n \nThank you!","metadata":{}},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n# πŸ“Œ Import Modules","metadata":{}},{"cell_type":"markdown","source":"### Modules","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nfrom IPython.core.display import HTML\nimport missingno as msno\n\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder, MinMaxScaler, StandardScaler\nfrom sklearn.model_selection import StratifiedKFold, cross_val_score, GridSearchCV\nfrom sklearn.ensemble import RandomForestClassifier as rfc, GradientBoostingClassifier as gbc\nfrom xgboost import XGBClassifier as xgb\nfrom lightgbm import LGBMClassifier as lgb\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score\n\nwarnings.filterwarnings('ignore')\npd.set_option('max_columns', 5000)\n# sns.set_style('darkgrid')\ncolor_red = '#ff7878'\ncolor_blue = '#4ca0f5'\ncolor_green = '#90fcb6'\ncolor_yellow = '#fffc42'\ncolor_orange = '#f0ca73'\nPALETTE_RED = ['#ff2e2e', '#fa4848', '#f55b5b', '#ff7a7a', '#f58c8c', '#f5a9a9', '#ffcccc', '#ffdede', '#f5e9e9']\nPALETTE_MISS = ['#f73a2d', '#f26f66', '#f79892']\nPALETTE_BIN = ['lightskyblue', '#ff6d59']\nPALETTE_CAT = [color_red, color_blue, color_green, color_yellow, color_orange]\nBACKCOLOR = '#f6f5f5'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-04T09:34:48.340176Z","iopub.execute_input":"2022-08-04T09:34:48.340570Z","iopub.status.idle":"2022-08-04T09:34:48.354145Z","shell.execute_reply.started":"2022-08-04T09:34:48.340538Z","shell.execute_reply":"2022-08-04T09:34:48.352781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n### User Modules","metadata":{}},{"cell_type":"code","source":"def multi_table(table_list):\n return HTML(\n f\" {''.join(['' for table in table_list])}
' + table._repr_html_() + '
\")","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.000498Z","iopub.execute_input":"2022-08-04T09:24:49.000958Z","iopub.status.idle":"2022-08-04T09:24:49.007530Z","shell.execute_reply.started":"2022-08-04T09:24:49.000916Z","shell.execute_reply":"2022-08-04T09:24:49.005462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def go(start, cnt, rlt):\n if cnt == 2:\n list_temp.append(pd.crosstab([df_train[rlt[0]], df_train['failure']], df_train[rlt[1]],margins=True).style.background_gradient())\n return\n for i in range(start, 5):\n rlt.append(col_norminal[i])\n go(i+1, cnt+1, rlt)\n rlt.pop()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.009700Z","iopub.execute_input":"2022-08-04T09:24:49.010111Z","iopub.status.idle":"2022-08-04T09:24:49.021593Z","shell.execute_reply.started":"2022-08-04T09:24:49.010058Z","shell.execute_reply":"2022-08-04T09:24:49.020320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def temp(att):\n df_att = pd.DataFrame({'code': [], 'cat': [], 'back': [], 'ratio': []})\n for code in df_train['product_code'].unique():\n for cat in df_total[att].unique():\n df_std = df_train[(df_train['product_code']==code)]\n df_temp = df_train[(df_train['product_code']==code) & (df_train[att]==cat)]\n ratio = df_temp.shape[0] / df_std.shape[0] * 100\n df_att.loc[df_att.shape[0]] = [code, cat, 100, ratio]\n for code in df_test['product_code'].unique():\n for cat in df_total[att].unique():\n df_std = df_test[(df_test['product_code']==code)]\n df_temp = df_test[(df_test['product_code']==code) & (df_test[att]==cat)]\n ratio = df_temp.shape[0] / df_std.shape[0] * 100\n df_att.loc[df_att.shape[0]] = [code, cat, 100, ratio]\n return df_att","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.025628Z","iopub.execute_input":"2022-08-04T09:24:49.025989Z","iopub.status.idle":"2022-08-04T09:24:49.037816Z","shell.execute_reply.started":"2022-08-04T09:24:49.025959Z","shell.execute_reply":"2022-08-04T09:24:49.036571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n# πŸ“Œ Read Data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ndf_test = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\ndf_total = pd.concat((df_train, df_test), axis=0).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.040306Z","iopub.execute_input":"2022-08-04T09:24:49.040763Z","iopub.status.idle":"2022-08-04T09:24:49.319231Z","shell.execute_reply.started":"2022-08-04T09:24:49.040721Z","shell.execute_reply":"2022-08-04T09:24:49.318159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n# πŸ“Œ EDA","metadata":{}},{"cell_type":"markdown","source":"## 1. Check DataπŸ”Ž","metadata":{}},{"cell_type":"markdown","source":"We can check the data structure by loading the train and test data and checking it directly. The following code checks the top 10 rows of train, test data, and provides the number of objects and variables in each data.","metadata":{}},{"cell_type":"code","source":"df_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.320664Z","iopub.execute_input":"2022-08-04T09:24:49.321055Z","iopub.status.idle":"2022-08-04T09:24:49.362443Z","shell.execute_reply.started":"2022-08-04T09:24:49.321017Z","shell.execute_reply":"2022-08-04T09:24:49.361249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.364236Z","iopub.execute_input":"2022-08-04T09:24:49.364937Z","iopub.status.idle":"2022-08-04T09:24:49.403780Z","shell.execute_reply.started":"2022-08-04T09:24:49.364892Z","shell.execute_reply":"2022-08-04T09:24:49.402487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.405821Z","iopub.execute_input":"2022-08-04T09:24:49.406279Z","iopub.status.idle":"2022-08-04T09:24:49.417032Z","shell.execute_reply.started":"2022-08-04T09:24:49.406238Z","shell.execute_reply":"2022-08-04T09:24:49.415174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n πŸ”‘insight \n\n`Train data`: The object of the train data is `26570`, and the number of variables is `26`. \n \n`Test data`: The object of the test data is `20775` and the number of variables is `25`. \n \n`Variable Description`: This data represents the results of a test study of a large product. All products have a unique value of `Product Code`. `Attributes 0 to 3` are fixed according to the Product Code. All products will have `measurement 0 to 17` through testing. Our goal is to use these variables to predict the failure of a particular product.\n
","metadata":{}},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n## 2. DescriptionπŸ”Ž \nCheck the storage type (object / numerical), data type (nominal / ordinal / continuous), and missing value for each variable.","metadata":{}},{"cell_type":"markdown","source":"First, you can use info to determine the type of storage and the number of missing values for each variable. The product_code is of object type and looks more likely to be normal. It's interesting that attributes_0 and 1 are objects and 2 and 3 are int. Measurement_0~2 is int and others are float format. While the total data size is 47345, you can see that there are many variables that are less than that.","metadata":{}},{"cell_type":"code","source":"df_total.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:24:49.418792Z","iopub.execute_input":"2022-08-04T09:24:49.419237Z","iopub.status.idle":"2022-08-04T09:24:49.455415Z","shell.execute_reply.started":"2022-08-04T09:24:49.419202Z","shell.execute_reply":"2022-08-04T09:24:49.454179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following tables and graphs provide the number and percentage of missing values for each variable. The measurements in float format are different in number but all have missing values. The variable with the most missing values is measurement_17.","metadata":{"execution":{"iopub.status.busy":"2022-08-01T23:30:51.532715Z","iopub.execute_input":"2022-08-01T23:30:51.533621Z","iopub.status.idle":"2022-08-01T23:30:51.541082Z","shell.execute_reply.started":"2022-08-01T23:30:51.533585Z","shell.execute_reply":"2022-08-01T23:30:51.539560Z"}}},{"cell_type":"code","source":"df_miss_info = pd.DataFrame({'feature':[], 'dataset': [], 'miss percent': []})\nfor col in df_total.columns:\n if col == 'failure':\n continue\n for dataset_name, dataset in [('total', df_total), ('train', df_train), ('test', df_test)]:\n miss_percent = dataset[col].isnull().sum() / dataset[col].shape[0] * 100\n df_miss_info.loc[df_miss_info.shape[0]] = [col, dataset_name, miss_percent]\ndf_miss_info = df_miss_info.sort_values(by='miss percent', ascending=False)\ndf_miss_info = df_miss_info[df_miss_info['miss percent'] != 0].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:18.121266Z","iopub.execute_input":"2022-08-04T09:26:18.121984Z","iopub.status.idle":"2022-08-04T09:26:18.334072Z","shell.execute_reply.started":"2022-08-04T09:26:18.121947Z","shell.execute_reply":"2022-08-04T09:26:18.332949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(figsize=(35, 7))\nsns.barplot(x='feature', y='miss percent', hue='dataset', data=df_miss_info, palette=PALETTE_MISS)\nplt.ylim(0, 50)\nax.spines[['top','right']].set_visible(False)\nax.set_facecolor(BACKCOLOR)\nax.axhline(30, color=color_red, ls='--')\nax.set_title('Missing value ratios', size=25)\nax.set_xlabel('Features', size=20)\nax.set_ylabel('Miss percent(%)', size=20)\nplt.text(1, 32, 'threshold', size=20, color=color_red)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:20.727918Z","iopub.execute_input":"2022-08-04T09:26:20.728325Z","iopub.status.idle":"2022-08-04T09:26:21.280129Z","shell.execute_reply.started":"2022-08-04T09:26:20.728291Z","shell.execute_reply":"2022-08-04T09:26:21.279146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following code uses value_counts to determine the actual value of each variable. We can use this method to specify the data type of each variable separately from the storage format. For example, attributes_2 and 3 are nominal variables because they were int-formatted but do not have consecutive individual values.","metadata":{}},{"cell_type":"code","source":"multi_table([pd.DataFrame(df_total[col].value_counts()) for col in df_total.columns])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:24.860416Z","iopub.execute_input":"2022-08-04T09:26:24.860824Z","iopub.status.idle":"2022-08-04T09:26:24.977344Z","shell.execute_reply.started":"2022-08-04T09:26:24.860792Z","shell.execute_reply":"2022-08-04T09:26:24.976179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_continuous = df_total.columns.values[((df_total.dtypes == 'int64') | (df_total.dtypes == 'float64')) & (df_total.columns != 'failure') & (df_total.columns != 'id') & (df_total.columns != 'attribute_2') & (df_total.columns != 'attribute_3')]\ncol_norminal = list(df_total.columns.values[(df_total.dtypes == 'object') ]) + ['attribute_2', 'attribute_3']\ncol_target = 'failure'","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:25.747790Z","iopub.execute_input":"2022-08-04T09:26:25.748191Z","iopub.status.idle":"2022-08-04T09:26:25.757823Z","shell.execute_reply.started":"2022-08-04T09:26:25.748157Z","shell.execute_reply":"2022-08-04T09:26:25.756476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_total.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:29.497918Z","iopub.execute_input":"2022-08-04T09:26:29.498510Z","iopub.status.idle":"2022-08-04T09:26:29.636317Z","shell.execute_reply.started":"2022-08-04T09:26:29.498474Z","shell.execute_reply":"2022-08-04T09:26:29.635032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n πŸ”‘insight \n\n\nWe've gained insight into each variable. \n`id`: Unique value of each object. \n`product_code`: This is a nominal variable. \n`loading`: This is a continuous variable. A small number of missing values exist. \n`attributes 0 to 3`: These are nominal variables. Fixed according to product_code. \n`measurement 0 to 17`: These are continuous variables. Some missing values exist. \n`failure`: This is a dependent variable and has two values.\n
","metadata":{"execution":{"iopub.status.busy":"2022-08-01T23:37:14.373247Z","iopub.execute_input":"2022-08-01T23:37:14.373667Z","iopub.status.idle":"2022-08-01T23:37:14.381638Z","shell.execute_reply.started":"2022-08-01T23:37:14.373634Z","shell.execute_reply":"2022-08-01T23:37:14.380115Z"}}},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n## 3. Detail Explore \nExplore in detail for each variable.","metadata":{}},{"cell_type":"code","source":"df_total['dataset']='train'\ndf_total.loc[df_train.shape[0]:, 'dataset'] = 'test'","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:33.711578Z","iopub.execute_input":"2022-08-04T09:26:33.711969Z","iopub.status.idle":"2022-08-04T09:26:33.721441Z","shell.execute_reply.started":"2022-08-04T09:26:33.711937Z","shell.execute_reply":"2022-08-04T09:26:33.720232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Failure(Dependent, Nominal) \nThis variable is the target variable for this issue. It is divided into steady state (0) and fault condition (1). Most products (78.7%) are normal, but some products (21.3%) have failed. This means 'data imbalance'. Classification models may have different performance depending on the training data configuration. To evaluate the performance of the model more fairly, 'Strategized KFold' can be used.","metadata":{}},{"cell_type":"code","source":"plt.subplots(figsize=(25, 10))\nplt.pie(df_train[col_target].value_counts(), shadow=True, explode=[.03,.03], autopct='%1.1f%%', textprops={'fontsize': 20, 'color': 'white'}, colors=PALETTE_BIN)\nplt.title('Failure distribution', size=20)\nplt.legend(['0', '1'], loc='best', fontsize=12)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:36.184012Z","iopub.execute_input":"2022-08-04T09:26:36.184579Z","iopub.status.idle":"2022-08-04T09:26:36.610260Z","shell.execute_reply.started":"2022-08-04T09:26:36.184537Z","shell.execute_reply":"2022-08-04T09:26:36.609085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### loading(Continuous) ","metadata":{}},{"cell_type":"code","source":"f = plt.figure(figsize=(25, 10))\ngs = f.add_gridspec(2, 4)\nax1 = f.add_subplot(gs[0,0])\nax2 = f.add_subplot(gs[0,1])\nax3 = f.add_subplot(gs[0,2])\nax4 = f.add_subplot(gs[0,3])\nax5 = f.add_subplot(gs[1,:])\naxs = [ax1, ax2, ax3, ax4, ax5]\n\nf.set_facecolor(BACKCOLOR)\n[ax.set_facecolor(BACKCOLOR) for ax in axs]\n[ax.spines[['top', 'right']].set_visible(False) for ax in axs]\n\n#-------------------------------------------------------------------------------------------------------------------\n\nsns.histplot(data=df_total, x='loading', hue='dataset', element='step', palette=PALETTE_BIN, ax=ax1)\nsns.histplot(data=df_train, x='loading', hue='failure', element='step', palette=PALETTE_BIN, ax=ax2)\nsns.violinplot(data=df_train, x='failure', y='loading', palette=PALETTE_BIN, ax=ax3)\nsns.stripplot(data=df_train, x='failure', y='loading', palette=PALETTE_BIN, ax=ax4)\n\n\ndf_target = df_train[['loading', 'failure']].sort_values('loading').reset_index(drop=True)\ndf_target['rolling_mean'] = df_target['failure'].rolling(1000, center=True).mean()\n# sns.scatterplot(x=df_target['loading'], y=df_target['rolling_mean'], ax=ax5)\nsns.regplot(x=df_target['loading'], y=df_target['rolling_mean'], color=color_red, ax=ax5)\n\n#-------------------------------------------------------------------------------------------------------------------\n\nax1.set_title('(a) Train / Test Distribution', size=15)\nax2.set_title('(b) Failure 0 / 1 Distribution_1', size=15)\nax3.set_title('(c) Failure 0 / 1 Distribution_2', size=15)\nax4.set_title('(d) Failure 0 / 1 Distribution_3', size=15)\nax5.set_title('(e) Failure Rolling Mean', size=15)\nf.suptitle('Loading Information', size=25)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:26:39.758291Z","iopub.execute_input":"2022-08-04T09:26:39.758728Z","iopub.status.idle":"2022-08-04T09:26:42.970900Z","shell.execute_reply.started":"2022-08-04T09:26:39.758683Z","shell.execute_reply":"2022-08-04T09:26:42.969710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n πŸ”‘insight \n \n`(a)`: The distribution of loading is almost the same in train and test. This distribution is slightly skewed to the left. \n`(b)`: There seems to be no significant difference in the distribution of loading according to the value of Failure. \n`(c)`: When Failure is 1, the value of Loading is slightly larger. \n`(d)`: There is a point that deviates from the center. \n`(e)`: As the loading value increases, the ratio of failure tends to increase. \n=> \nSince it is a continuous variable, the missing value can be replaced by the mean. It is expected that an influence would not be large because the difference in the distribution of loading according to the value of failure is not large, but as the loading value increases, the ratio of failure increases, so loading would be necessary to solve this problem.\n
","metadata":{}},{"cell_type":"markdown","source":"### Product code and Attribute (Norminal) \n","metadata":{}},{"cell_type":"markdown","source":"Product code and attribute are nominal variables. Therefore, you can use a bar graph to see the distribution. As previously described, the value of the attribute is fixed according to the product code. For example, a product with product code of 0 will have attribute 0 fixed to 7. This can be found in the crosstab table below. This shows that the product code contains the meaning of the attribute combination.","metadata":{}},{"cell_type":"code","source":"f = plt.figure(figsize=(25, 15))\ngs = f.add_gridspec(3, 4)\nax1 = f.add_subplot(gs[0,0])\nax2 = f.add_subplot(gs[0,1])\nax7 = f.add_subplot(gs[0,2])\nax8 = f.add_subplot(gs[0,3])\nax3 = f.add_subplot(gs[1,:2])\nax4 = f.add_subplot(gs[1,2:4])\nax5 = f.add_subplot(gs[2,:2])\nax6 = f.add_subplot(gs[2,2:4])\n\naxs = [ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8]\n\nf.set_facecolor(BACKCOLOR)\n[ax.set_facecolor(BACKCOLOR) for ax in axs]\n[ax.spines[['top', 'right']].set_visible(False) for ax in axs]\n\n#-------------------------------------------------------------------------------------------------\n\nax1.pie(df_train['product_code'].value_counts(), shadow=True, explode=[.03]*df_train['product_code'].nunique(), autopct='%1.1f%%', textprops={'fontsize': 20, 'color': 'white'}, colors=PALETTE_CAT)\nax1.legend(df_train['product_code'].unique(), loc='best', fontsize=12)\n\nax2.pie(df_test['product_code'].value_counts(), shadow=True, explode=[.03]*df_test['product_code'].nunique(), autopct='%1.1f%%', textprops={'fontsize': 20, 'color': 'white'}, colors=PALETTE_CAT)\nax2.legend(df_test['product_code'].unique(), loc='best', fontsize=12)\n\nax7.pie(df_train['attribute_2'].value_counts(), shadow=True, explode=[.03]*df_train['attribute_2'].nunique(), autopct='%1.1f%%', textprops={'fontsize': 20, 'color': 'white'}, colors=PALETTE_CAT)\nax7.legend(df_train['attribute_2'].unique(), loc='best', fontsize=12)\n\nax8.pie(df_test['attribute_2'].value_counts(), shadow=True, explode=[.03]*df_test['attribute_2'].nunique(), autopct='%1.1f%%', textprops={'fontsize': 20, 'color': 'white'}, colors=PALETTE_CAT)\nax8.legend(df_test['attribute_2'].unique(), loc='best', fontsize=12)\n\ndf_att0 = temp('attribute_0')\ndf_att1 = temp('attribute_1')\ndf_att2 = temp('attribute_2')\ndf_att3 = temp('attribute_3')\n\n\nsns.barplot(data=df_att0, x='code', y='back', hue='cat', palette=['lightgray'], alpha=.5, ax=ax3)\nsns.barplot(data=df_att0, x='code', y='ratio', hue='cat', palette=[color_blue], ax=ax3)\n\nsns.barplot(data=df_att1, x='code', y='back', hue='cat', palette=['lightgray'], alpha=.5, ax=ax4)\nsns.barplot(data=df_att1, x='code', y='ratio', hue='cat', palette=[color_blue], ax=ax4)\n\nsns.barplot(data=df_att2, x='code', y='back', hue='cat', palette=['lightgray'], alpha=.5, ax=ax5)\nsns.barplot(data=df_att2, x='code', y='ratio', hue='cat', palette=[color_blue], ax=ax5)\n\nsns.barplot(data=df_att3, x='code', y='back', hue='cat', palette=['lightgray'], alpha=.5, ax=ax6)\nsns.barplot(data=df_att3, x='code', y='ratio', hue='cat', palette=[color_blue], ax=ax6)\n\n#-------------------------------------------------------------------------------------------------\n\n[ax.get_legend().remove() for ax in [ax3, ax4, ax5, ax6]]\n\nax1.set_title('Train Code Distribution')\nax2.set_title('Test Code Distribution')\nax7.set_title('Train attribution2 Distribution')\nax8.set_title('Test attribution2 Distribution')\nax3.set_title('Attribute 0 (material_7 / material_5)')\nax4.set_title('Attribute 1 (material_8 / material_5 / material_6 / material_7)')\nax5.set_title('Attribute 2 (5, 6, 7, 8, 9)')\nax6.set_title('Attribute 3 (4, 5, 6, 7, 8, 9)')\n\nfor ax in [ax3, ax4, ax5, ax6]:\n for patch in ax.patches:\n patch.set_edgecolor('lightgray')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:31:54.619711Z","iopub.execute_input":"2022-08-04T09:31:54.620172Z","iopub.status.idle":"2022-08-04T09:31:58.814369Z","shell.execute_reply.started":"2022-08-04T09:31:54.620103Z","shell.execute_reply":"2022-08-04T09:31:58.813168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(1, 5, figsize=(30, 5))\nfor i, col in enumerate(col_norminal):\n sns.countplot(df_total[col], ax=ax[i], palette=PALETTE_RED)\nplt.show()\n\nlist_df_attbycode =[\n pd.crosstab([df_total['attribute_0']], df_train['product_code'],margins=True),\n pd.crosstab([df_total['attribute_1']], df_train['product_code'],margins=True),\n pd.crosstab([df_total['attribute_2']], df_train['product_code'],margins=True),\n pd.crosstab([df_total['attribute_3']], df_train['product_code'],margins=True),\n]\nmulti_table(list_df_attbycode)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:32:17.712904Z","iopub.execute_input":"2022-08-04T09:32:17.713308Z","iopub.status.idle":"2022-08-04T09:32:18.715080Z","shell.execute_reply.started":"2022-08-04T09:32:17.713275Z","shell.execute_reply":"2022-08-04T09:32:18.713838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can check the distribution of each variable by dividing it according to the status of the product. As shown in the bar graph, there is no difference in the distribution of values for each variable depending on the product's status. To understand more accurately, we created a pivot table. For example, the failure rate according to the value of product_code is 23%. It was confirmed that there was no significant difference between 20%, 21%, 22%, and 21%.","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(1, 5, figsize=(30, 5))\nfor i, col in enumerate(col_norminal):\n sns.countplot(df_train[col], hue='failure', data=df_train, palette=PALETTE_BIN, ax=ax[i])\nplt.show()\ndf_train.pivot_table(index='product_code', values='failure', aggfunc=['count', 'sum', 'mean']).style.background_gradient()\n\nmulti_table(\n [\n df_train.pivot_table(index=col, values='failure', aggfunc=['count', 'sum', 'mean']).style.background_gradient() for col in col_norminal\n ])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:32:25.442231Z","iopub.execute_input":"2022-08-04T09:32:25.442618Z","iopub.status.idle":"2022-08-04T09:32:26.583778Z","shell.execute_reply.started":"2022-08-04T09:32:25.442573Z","shell.execute_reply":"2022-08-04T09:32:26.582488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Measurement (continuous) \n","metadata":{}},{"cell_type":"markdown","source":"These are all continuous variables. Therefore, you can see the distribution with a histogram or KDE plot. The measurement variables are all normally distributed.","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(4, 5, figsize=(23, 15))\nfor i, col in enumerate(col_continuous):\n if df_total[col].isnull().sum():\n sns.distplot(df_total[col], ax=ax[i//5][i%5], color=color_red)\n else:\n sns.distplot(df_total[col], ax=ax[i//5][i%5])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:32:40.346661Z","iopub.execute_input":"2022-08-04T09:32:40.347041Z","iopub.status.idle":"2022-08-04T09:32:49.878165Z","shell.execute_reply.started":"2022-08-04T09:32:40.347012Z","shell.execute_reply":"2022-08-04T09:32:49.877144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In addition, you can check the distribution of measurements according to the product status. It's hard to see any obvious difference here.","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(4, 5, figsize=(23, 15))\nfor i, col in enumerate(col_continuous):\n sns.histplot(data=df_train, x=col, hue='failure', element='step', ax=ax[i//5][i%5], palette=PALETTE_BIN)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:32:55.390806Z","iopub.execute_input":"2022-08-04T09:32:55.391243Z","iopub.status.idle":"2022-08-04T09:32:58.904870Z","shell.execute_reply.started":"2022-08-04T09:32:55.391208Z","shell.execute_reply":"2022-08-04T09:32:58.903875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This time, I marked the product failure rate in various sections of these variables. For example, you can see that the higher the loading value, the higher the failure rate.","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(4, 5, figsize=(23, 15))\nfor i, col in enumerate(col_continuous):\n failure_ratio = []\n bins = np.linspace(df_total[col].min(), df_total[col].max(), 100)\n for j in range(len(bins)):\n if j < len(bins) - 1:\n failure_ratio.append(df_train.loc[(df_train[col] < bins[j+1]) & (df_train[col] >= bins[j]), 'failure'].mean())\n sns.regplot(x=bins[:-1], y=failure_ratio, ax=ax[i//5][i%5], color=PALETTE_BIN[0], marker='+')\n ax[i//5][i%5].set_title(col)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:33:03.361989Z","iopub.execute_input":"2022-08-04T09:33:03.362804Z","iopub.status.idle":"2022-08-04T09:33:09.201019Z","shell.execute_reply.started":"2022-08-04T09:33:03.362755Z","shell.execute_reply":"2022-08-04T09:33:09.199744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(15, 15))\nsns.heatmap(\n df_train.corr(),\n square=True,\n annot=True,\n fmt='.2f',\n center=0,\n linewidth=2,\n cbar=False,\n cmap=sns.diverging_palette(240, 10, as_cmap=True)\n)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:33:41.008468Z","iopub.execute_input":"2022-08-04T09:33:41.009595Z","iopub.status.idle":"2022-08-04T09:33:43.367095Z","shell.execute_reply.started":"2022-08-04T09:33:41.009555Z","shell.execute_reply":"2022-08-04T09:33:43.366051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n# πŸ“Œ Preprocessing \nLet's move on with the preprocessing. The preprocessing process is as follows:","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('../input/tabular-playground-series-aug-2022/train.csv')\ndf_test = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv')\ndf_total = pd.concat((df_train, df_test), axis=0).reset_index(drop=True)\nsubmission = pd.read_csv('/kaggle/input/tabular-playground-series-aug-2022/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:37:16.642455Z","iopub.execute_input":"2022-08-04T09:37:16.643076Z","iopub.status.idle":"2022-08-04T09:37:16.926940Z","shell.execute_reply.started":"2022-08-04T09:37:16.643009Z","shell.execute_reply":"2022-08-04T09:37:16.925798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imputing missing value\ndf_total[col_continuous] = df_total[col_continuous].fillna(df_total[col_continuous].mean())","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:37:17.680825Z","iopub.execute_input":"2022-08-04T09:37:17.681923Z","iopub.status.idle":"2022-08-04T09:37:17.712761Z","shell.execute_reply.started":"2022-08-04T09:37:17.681880Z","shell.execute_reply":"2022-08-04T09:37:17.711417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Encoding\ndf_total['product_code'] = LabelEncoder().fit_transform(df_total['product_code'])\n# df_total = pd.get_dummies(df_total, columns=['product_code'])","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:37:18.607800Z","iopub.execute_input":"2022-08-04T09:37:18.608186Z","iopub.status.idle":"2022-08-04T09:37:18.625020Z","shell.execute_reply.started":"2022-08-04T09:37:18.608154Z","shell.execute_reply":"2022-08-04T09:37:18.623803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_total = df_total.drop(['product_code']+[f'attribute_{i}' for i in range(4)], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:37:19.558312Z","iopub.execute_input":"2022-08-04T09:37:19.560453Z","iopub.status.idle":"2022-08-04T09:37:19.571383Z","shell.execute_reply.started":"2022-08-04T09:37:19.560400Z","shell.execute_reply":"2022-08-04T09:37:19.570209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train / test split\ndf_train = df_total[:df_train.shape[0]]\ndf_test = df_total[df_train.shape[0]:]\n\nX_train = df_train.drop(['id', 'failure'], axis=1)\ny_train = df_train['failure']\nX_test = df_test.drop(['id', 'failure'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:37:24.076129Z","iopub.execute_input":"2022-08-04T09:37:24.076730Z","iopub.status.idle":"2022-08-04T09:37:24.087115Z","shell.execute_reply.started":"2022-08-04T09:37:24.076695Z","shell.execute_reply":"2022-08-04T09:37:24.085849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\nX_train = pd.DataFrame(scaler.fit_transform(X_train), columns=X_train.columns)\nX_test = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:38:20.028562Z","iopub.execute_input":"2022-08-04T09:38:20.029195Z","iopub.status.idle":"2022-08-04T09:38:20.066733Z","shell.execute_reply.started":"2022-08-04T09:38:20.029092Z","shell.execute_reply":"2022-08-04T09:38:20.065279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-------------------------------------------------------------------------------\n# πŸ“Œ Train Model and Predict \nrefer: https://www.kaggle.com/code/cv13j0/tps-aug22-binary-classification","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nSEED = 18\nX_train_temp, X_valid, y_train_temp, y_valid = train_test_split(X_train, y_train, test_size = 0.15, random_state = SEED)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:38:22.939638Z","iopub.execute_input":"2022-08-04T09:38:22.940168Z","iopub.status.idle":"2022-08-04T09:38:22.962313Z","shell.execute_reply.started":"2022-08-04T09:38:22.940108Z","shell.execute_reply":"2022-08-04T09:38:22.961151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'n_estimators': 4096,\n 'max_depth': 7,\n 'learning_rate': 0.15,\n 'subsample': 0.95,\n 'colsample_bytree': 0.60,\n 'reg_lambda': 1.50,\n 'reg_alpha': 6.10,\n 'gamma': 1.40,\n 'random_state': SEED,\n 'objective': 'binary:logistic',\n 'tree_method': 'gpu_hist',\n }\n\nmodel_xgb = xgb(**params)\nmodel_xgb.fit(X_train_temp, y_train_temp, eval_set = [(X_valid, y_valid)], eval_metric = ['auc'], early_stopping_rounds = 256, verbose = 50)\npred_y_valid = model_xgb.predict_proba(X_valid)[:,1]\npred_y_test = model_xgb.predict_proba(X_test)[:,1]\n\nauc_score = roc_auc_score(y_valid, pred_y_valid)\nprint(f'AUC Score: {auc_score: .3f}') ","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:38:52.701356Z","iopub.execute_input":"2022-08-04T09:38:52.701723Z","iopub.status.idle":"2022-08-04T09:38:54.282116Z","shell.execute_reply.started":"2022-08-04T09:38:52.701695Z","shell.execute_reply":"2022-08-04T09:38:54.281297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['failure'] = pred_y_test\nsubmission.to_csv('submission.csv', index = False)\nsubmission.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-08-04T09:38:59.226927Z","iopub.execute_input":"2022-08-04T09:38:59.227663Z","iopub.status.idle":"2022-08-04T09:38:59.280228Z","shell.execute_reply.started":"2022-08-04T09:38:59.227625Z","shell.execute_reply":"2022-08-04T09:38:59.279000Z"},"trusted":true},"execution_count":null,"outputs":[]}]}