{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-11T15:07:45.062142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/tabular-playground-series-aug-2022/train.csv\")\ntest = pd.read_csv(\"../input/tabular-playground-series-aug-2022/test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Different Values for product code for train and test data","metadata":{}},{"cell_type":"code","source":"train['product_code'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['product_code'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['attribute_0'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['attribute_0'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Different values for train and test data for attribute 1","metadata":{}},{"cell_type":"code","source":"train['attribute_1'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['attribute_1'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Different values for train and test data for attribute 2 ","metadata":{}},{"cell_type":"code","source":"train['attribute_2'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['attribute_2'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Different values for train and test data for attribute 3","metadata":{}},{"cell_type":"code","source":"train['attribute_3'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['attribute_3'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How to approach different values for train and test data:\n* Delete features that differ (produt_code, attribute_1, attribute_2, attribute_3)\n* combine train and test data then split\n* Convert values to dummy features and remove all dummy features that are not on both sets of data\n\nWill most likely choose the third approach since it will save as much information as possible without causing bias when fitting the model on the training data","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Distribution of Numerical Features","metadata":{}},{"cell_type":"markdown","source":"Most features follow a normal distribution, can most likely replace null values with mean.","metadata":{}},{"cell_type":"code","source":"cols = ['loading',\n    'attribute_3',\n     'measurement_0',\n     'measurement_1',\n     'measurement_2',\n     'measurement_3',\n     'measurement_4',\n     'measurement_5',\n     'measurement_6',\n     'measurement_7',\n     'measurement_8',\n     'measurement_9',\n     'measurement_10',\n     'measurement_11',\n     'measurement_12',\n     'measurement_13',\n     'measurement_14',\n     'measurement_15',\n     'measurement_16',\n     'measurement_17']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=0\nwhile i < len(cols):\n   \n    fig, axs = plt.subplots(1,4, figsize=(10,3))\n    sns.histplot(data=train,x=cols[i], kde=True, color=\"skyblue\",ax=axs[0])\n    axs[0].set_title(f'{cols[i]}')\n    sns.histplot(data=train,x=cols[i+1], kde=True, color=\"skyblue\",ax=axs[1])\n    axs[1].set_title(f'{cols[i+1]}')\n    sns.histplot(data=train,x=cols[i+2], kde=True, color=\"skyblue\",ax=axs[2])\n    axs[2].set_title(f'{cols[i+2]}')\n    sns.histplot(data=train,x=cols[i+3], kde=True, color=\"skyblue\",ax=axs[3])\n    axs[3].set_title(f'{cols[i+3]}')\n\n    i+=4\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Heat Maps","metadata":{}},{"cell_type":"code","source":"corr_df1 = train.drop(['id','product_code','attribute_0','attribute_1','attribute_2','attribute_3'],axis=1)\ncorr_df2 = train.drop(['id','loading','measurement_0',\n    'measurement_1',\n     'measurement_2',\n     'measurement_3',\n     'measurement_4',\n     'measurement_5',\n     'measurement_6',\n     'measurement_7',\n     'measurement_8',\n     'measurement_9',\n     'measurement_10',\n     'measurement_11',\n     'measurement_12',\n     'measurement_13',\n     'measurement_14',\n     'measurement_15',\n     'measurement_16',\n     'measurement_17'], axis=1)\ncorr_df3 = train.drop(['id'],axis=1)\n\ntest_corr_df = test.drop(['id'],axis=1)\ntest_corr_df2 = test.drop(['id','loading','measurement_0',\n    'measurement_1',\n     'measurement_2',\n     'measurement_3',\n     'measurement_4',\n     'measurement_5',\n     'measurement_6',\n     'measurement_7',\n     'measurement_8',\n     'measurement_9',\n     'measurement_10',\n     'measurement_11',\n     'measurement_12',\n     'measurement_13',\n     'measurement_14',\n     'measurement_15',\n     'measurement_16',\n     'measurement_17'], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = corr_df1.corr()\n\nplt.figure(figsize=(16,9))\nsns.heatmap(corr,vmin=-1,vmax=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_df2['attribute_0'] = corr_df2['attribute_0'].astype(str)\ncorr_df2['attribute_1'] = corr_df2['attribute_1'].astype(str)\ncorr_df2['attribute_2'] = corr_df2['attribute_2'].astype(str)\ncorr_df2['attribute_3'] = corr_df2['attribute_3'].astype(str)\n\ncorr_df2 = pd.get_dummies(corr_df2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_df2.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr2 = corr_df2.corr()\n\nplt.figure(figsize=(16,9))\nsns.heatmap(corr2,vmin=-1,vmax=1)\n\n#No distinct correlation between failure and any specific value for the categorical features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_df3['attribute_0'] = corr_df3['attribute_0'].astype(str)\ncorr_df3['attribute_1'] = corr_df3['attribute_1'].astype(str)\ncorr_df3['attribute_2'] = corr_df3['attribute_2'].astype(str)\ncorr_df3['attribute_3'] = corr_df3['attribute_3'].astype(str)\n\ncorr_df3 = pd.get_dummies(corr_df3)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr3 = corr_df3.corr()\n\nplt.figure(figsize=(16,9))\nsns.heatmap(corr3,vmin=-1,vmax=1)\n\n#There is a lot of positive and negative correlation among the categorical featues","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_corr_df['attribute_0'] = test_corr_df['attribute_0'].astype(str)\ntest_corr_df['attribute_1'] = test_corr_df['attribute_1'].astype(str)\ntest_corr_df['attribute_2'] = test_corr_df['attribute_2'].astype(str)\ntest_corr_df['attribute_3'] = test_corr_df['attribute_3'].astype(str)\n\ntest_corr_df2['attribute_0'] = test_corr_df2['attribute_0'].astype(str)\ntest_corr_df2['attribute_1'] = test_corr_df2['attribute_1'].astype(str)\ntest_corr_df2['attribute_2'] = test_corr_df2['attribute_2'].astype(str)\ntest_corr_df2['attribute_3'] = test_corr_df2['attribute_3'].astype(str)\n\ntest_corr_df = pd.get_dummies(test_corr_df)\ntest_corr_df2 = pd.get_dummies(test_corr_df2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_corr = test_corr_df.corr()\ntest_corr2 = test_corr_df2.corr()\n\nplt.figure(figsize=(16,9))\nsns.heatmap(test_corr,vmin=-1,vmax=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax =plt.subplots(1,2,figsize=(10,5))\n\nsns.heatmap(corr2, ax=ax[0])\nax[0].set_title('Train Categorical Features')\nsns.heatmap(test_corr2, ax=ax[1])\nax[1].set_title('Test Categorical Features')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Selection","metadata":{}},{"cell_type":"code","source":"from sklearn import preprocessing\n\n#le = preprocessing.LabelEncoder()\n#product_code = le.fit_transform(train['product_code'])\n#attribute_0 = le.fit_transform(train['attribute_0'])\n#attribute_1 = le.fit_transform(train['attribute_1'])\ntrain['attribute_0'] = train['attribute_0'].astype(str)\ntrain['attribute_1'] = train['attribute_1'].astype(str)\ntrain['attribute_2'] = train['attribute_2'].astype(str)\ntrain['attribute_3'] = train['attribute_3'].astype(str)\n\ntrain = pd.get_dummies(train)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train.columns:\n    mean_value = train[col].mean()\n    train[col].fillna(value=mean_value, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train['failure']\ntrain = train.drop(['failure'],axis=1)\ntrain = train.set_index('id')\ntest = test.set_index('id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_regression\n\nmi_scores = mutual_info_regression(train, y_train)\nmi_scores = pd.Series(mi_scores, name=\"MI_score\", index=train.columns)\nmi_scores = mi_scores.sort_values(ascending=False)\ndf_mi_scores1 = pd.DataFrame(mi_scores).reset_index().rename(columns={'index':'feature'})\ndf_mi_scores1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import eli5\nfrom catboost import CatBoostClassifier\nfrom eli5.sklearn import PermutationImportance\n\nmodel1 = CatBoostClassifier(logging_level='Silent', random_state=42, eval_metric='AUC',loss_function='Logloss').fit(train,y_train)\n\nperm1 = PermutationImportance(model1, random_state=1).fit(train, y_train)\neli5.show_weights(perm1, feature_names = train.columns.tolist(), top=None)\n\n#None of the categorical features were marked as important using the permutation importance method","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Buidling Data Set","metadata":{}},{"cell_type":"code","source":"#Removing all categorical columns that are not in both the train and test data\n\ntest['attribute_0'] = test['attribute_0'].astype(str)\ntest['attribute_1'] = test['attribute_1'].astype(str)\ntest['attribute_2'] = test['attribute_2'].astype(str)\ntest['attribute_3'] = test['attribute_3'].astype(str)\n\ntest = pd.get_dummies(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cols = test.columns\ntrain_cols = train.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_cols = train_cols.intersection(test_cols)\nprint(final_cols)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del_cols_train = [i for i in train_cols if i not in final_cols]\ndel_cols_test = [i for i in test_cols if i not in final_cols]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.drop(del_cols_train, axis=1)\ntest = test.drop(del_cols_test, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['failure'] = y_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train.csv', index=True)\ntest.to_csv('test.csv', index=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}