{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0. Preparation","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nfrom itertools import cycle, islice\nimport matplotlib.pylab as plt\nimport seaborn as sns\nfrom sklearn.model_selection import StratifiedShuffleSplit,StratifiedKFold\nfrom sklearn.model_selection import train_test_split\nfrom xgboost import XGBClassifier, XGBRegressor\nfrom sklearn.metrics import roc_auc_score\nfrom lightgbm import LGBMClassifier,LGBMRegressor\nfrom sklearn import preprocessing\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.model_selection import GroupKFold\n\nplt.style.use(\"ggplot\")\ncolor_pal = plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"]\ncolor_cycle = cycle(plt.rcParams[\"axes.prop_cycle\"].by_key()[\"color\"])","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-08T12:57:50.653325Z","iopub.execute_input":"2022-08-08T12:57:50.653984Z","iopub.status.idle":"2022-08-08T12:57:50.662402Z","shell.execute_reply.started":"2022-08-08T12:57:50.653946Z","shell.execute_reply":"2022-08-08T12:57:50.661284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/tabular-playground-series-aug-2022/train.csv\",index_col='id')\ntest_df = pd.read_csv('../input/tabular-playground-series-aug-2022/test.csv',index_col='id')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:50.672100Z","iopub.execute_input":"2022-08-08T12:57:50.672576Z","iopub.status.idle":"2022-08-08T12:57:50.794393Z","shell.execute_reply.started":"2022-08-08T12:57:50.672548Z","shell.execute_reply":"2022-08-08T12:57:50.793400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Exploring Data","metadata":{}},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:50.796185Z","iopub.execute_input":"2022-08-08T12:57:50.796569Z","iopub.status.idle":"2022-08-08T12:57:50.835641Z","shell.execute_reply.started":"2022-08-08T12:57:50.796533Z","shell.execute_reply":"2022-08-08T12:57:50.834666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Product Code\n\n* Test set has totally different product codes from train dataset. It might be a good idea to predict a similar model type from trainingset.\n\n*  Product C has slightly more samples than the others but generally they\nre balanced. Failure ratios about 21%.","metadata":{}},{"cell_type":"code","source":"train_df.product_code.value_counts(), test_df.product_code.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:50.837372Z","iopub.execute_input":"2022-08-08T12:57:50.837844Z","iopub.status.idle":"2022-08-08T12:57:50.850603Z","shell.execute_reply.started":"2022-08-08T12:57:50.837804Z","shell.execute_reply":"2022-08-08T12:57:50.849619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1,2, figsize=(12,5),sharey=True)\n\ntrain_df.groupby('product_code')['loading'].count().plot(kind='bar', ax=axs[0],title=\"Train Product Type\",color=color_pal);\ntest_df.groupby('product_code')['loading'].count().plot(kind='bar', ax=axs[1],title=\"Test Procut Type\",color=color_pal[5:] +color_pal);","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-08T12:57:50.853605Z","iopub.execute_input":"2022-08-08T12:57:50.854545Z","iopub.status.idle":"2022-08-08T12:57:51.762123Z","shell.execute_reply.started":"2022-08-08T12:57:50.854498Z","shell.execute_reply":"2022-08-08T12:57:51.761009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#making color cycle\ncolors = list(islice(cycle(['b','r']),None, len(train_df.product_code.unique()) *2))\n\ntrain_df.groupby(['product_code','failure'])['loading'].count().plot(kind='bar',color=colors,figsize=(12, 5),title= 'Failure Ratio by Product Category');","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-08T12:57:51.763987Z","iopub.execute_input":"2022-08-08T12:57:51.764374Z","iopub.status.idle":"2022-08-08T12:57:51.991284Z","shell.execute_reply.started":"2022-08-08T12:57:51.764338Z","shell.execute_reply":"2022-08-08T12:57:51.990171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate ratio of failures\ntrain_df.groupby('product_code')['failure'].value_counts(normalize=True).mul(100)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:51.993305Z","iopub.execute_input":"2022-08-08T12:57:51.994002Z","iopub.status.idle":"2022-08-08T12:57:52.009109Z","shell.execute_reply.started":"2022-08-08T12:57:51.993962Z","shell.execute_reply":"2022-08-08T12:57:52.008253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading \n\n#### Loading shows absorbed amount of fluid. It seems very important to predict failure ratio.\n\n* C succeeded absorbing up to 374.33 liquid. That is probably the reason why C was tested more than the others.\n\n* H was tested like C according to the maximum loading data but the number of samples are least in the test data product categories.","metadata":{}},{"cell_type":"code","source":"ax =train_df[train_df['failure']==0].groupby(['product_code']).agg({'loading': ['mean', 'min', 'max']}).plot(kind='bar',figsize=(12, 7), title=\"Success Data by Product Type\");\nax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.10),\n          fancybox=True, shadow=True, ncol=5);","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.010626Z","iopub.execute_input":"2022-08-08T12:57:52.010988Z","iopub.status.idle":"2022-08-08T12:57:52.262449Z","shell.execute_reply.started":"2022-08-08T12:57:52.010953Z","shell.execute_reply":"2022-08-08T12:57:52.261514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax =train_df[train_df['failure']==1].groupby(['product_code']).agg({'loading': ['mean', 'min', 'max']}).plot(kind='bar',figsize=(12, 7), title=\"Failure Data by Product\");\nax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.10),\n          fancybox=True, shadow=True, ncol=5);","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.263888Z","iopub.execute_input":"2022-08-08T12:57:52.265476Z","iopub.status.idle":"2022-08-08T12:57:52.520156Z","shell.execute_reply.started":"2022-08-08T12:57:52.265431Z","shell.execute_reply":"2022-08-08T12:57:52.519200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['failure']==0].groupby(['product_code','failure']).agg({'loading': ['mean', 'min', 'max']})","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.524096Z","iopub.execute_input":"2022-08-08T12:57:52.525089Z","iopub.status.idle":"2022-08-08T12:57:52.548667Z","shell.execute_reply.started":"2022-08-08T12:57:52.525050Z","shell.execute_reply":"2022-08-08T12:57:52.547598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['failure']==1].groupby(['product_code','failure']).agg({'loading': ['mean', 'min', 'max']})","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.550246Z","iopub.execute_input":"2022-08-08T12:57:52.550622Z","iopub.status.idle":"2022-08-08T12:57:52.583066Z","shell.execute_reply.started":"2022-08-08T12:57:52.550586Z","shell.execute_reply":"2022-08-08T12:57:52.582036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby('product_code').agg({'loading': ['mean', 'min', 'max']})","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.584707Z","iopub.execute_input":"2022-08-08T12:57:52.585064Z","iopub.status.idle":"2022-08-08T12:57:52.603482Z","shell.execute_reply.started":"2022-08-08T12:57:52.585029Z","shell.execute_reply":"2022-08-08T12:57:52.602353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax =test_df.groupby(['product_code']).agg({'loading': ['mean', 'min', 'max']}).plot(kind='bar',figsize=(12, 7), title=\"Failure Data by Product\");\nax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.10),\n          fancybox=True, shadow=True, ncol=5);","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.605020Z","iopub.execute_input":"2022-08-08T12:57:52.605617Z","iopub.status.idle":"2022-08-08T12:57:52.878390Z","shell.execute_reply.started":"2022-08-08T12:57:52.605582Z","shell.execute_reply":"2022-08-08T12:57:52.877266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading data have some missing ","metadata":{}},{"cell_type":"code","source":"train_df[train_df['loading'].isna()].groupby(['product_code','failure'])['attribute_0'].count()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.879803Z","iopub.execute_input":"2022-08-08T12:57:52.880397Z","iopub.status.idle":"2022-08-08T12:57:52.895000Z","shell.execute_reply.started":"2022-08-08T12:57:52.880367Z","shell.execute_reply":"2022-08-08T12:57:52.893797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" test_df[test_df['loading'].isna()].groupby(['product_code'])['attribute_0'].count()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.896803Z","iopub.execute_input":"2022-08-08T12:57:52.897165Z","iopub.status.idle":"2022-08-08T12:57:52.908402Z","shell.execute_reply.started":"2022-08-08T12:57:52.897128Z","shell.execute_reply":"2022-08-08T12:57:52.907165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading vs Failure","metadata":{}},{"cell_type":"code","source":"train_df['loading'].plot(kind='hist',alpha=0.4,label='train');\ntest_df['loading'].plot(kind='hist',alpha=0.4,label = 'test');\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:52.910294Z","iopub.execute_input":"2022-08-08T12:57:52.910676Z","iopub.status.idle":"2022-08-08T12:57:53.163983Z","shell.execute_reply.started":"2022-08-08T12:57:52.910642Z","shell.execute_reply":"2022-08-08T12:57:53.162997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(train_df,x='loading',hue='failure')\nplt.title('Loading and Failure');","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:53.165629Z","iopub.execute_input":"2022-08-08T12:57:53.166294Z","iopub.status.idle":"2022-08-08T12:57:53.781019Z","shell.execute_reply.started":"2022-08-08T12:57:53.166254Z","shell.execute_reply":"2022-08-08T12:57:53.780017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Attributes 0-3\n\n#### There are four columns about attributes. The first two are about materials and the last two are just categorical numbers. \n\n* Looks like Product code was difined by the Attributes combinations. Each product code has only one attribute combinations.","metadata":{}},{"cell_type":"code","source":"train_df.groupby(['product_code','attribute_0','attribute_1','attribute_2','attribute_3']).size()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:53.782666Z","iopub.execute_input":"2022-08-08T12:57:53.783299Z","iopub.status.idle":"2022-08-08T12:57:53.802005Z","shell.execute_reply.started":"2022-08-08T12:57:53.783261Z","shell.execute_reply":"2022-08-08T12:57:53.801127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.groupby(['product_code','attribute_0','attribute_1','attribute_2','attribute_3']).size()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:53.803306Z","iopub.execute_input":"2022-08-08T12:57:53.803658Z","iopub.status.idle":"2022-08-08T12:57:53.820887Z","shell.execute_reply.started":"2022-08-08T12:57:53.803624Z","shell.execute_reply":"2022-08-08T12:57:53.819885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Mesurement 0-17\n\n#### There are 18 columns about mesurements. Mesurements except for the first two mesurement have some missing values. \n\n* Mesurement 0-2 seems essential for testing and some of meaurements after them are skipped for some conditions. Latter mesurements are skipped more.\n","metadata":{}},{"cell_type":"code","source":"measurements = [a for a in train_df.columns if a.startswith('measurement')]\ntrain_df[measurements].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:53.822420Z","iopub.execute_input":"2022-08-08T12:57:53.823192Z","iopub.status.idle":"2022-08-08T12:57:53.835592Z","shell.execute_reply.started":"2022-08-08T12:57:53.823154Z","shell.execute_reply":"2022-08-08T12:57:53.834271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def missing_plot(df,num,name):\n    a = train_df[measurements].notna().sum().plot.bar(title=f'Mesurement Missing Values Distribution {name}',ax=axs[num])\n    MAX = max(train_df[measurements].notna().sum())\n    MIN = min(train_df[measurements].notna().sum())\n    a.set_ybound(MIN-300,MAX+300)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:53.837019Z","iopub.execute_input":"2022-08-08T12:57:53.838175Z","iopub.status.idle":"2022-08-08T12:57:53.844620Z","shell.execute_reply.started":"2022-08-08T12:57:53.838139Z","shell.execute_reply":"2022-08-08T12:57:53.843547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1,2, figsize=(12,5),sharey=True)\n\nmissing_plot(train_df,0,'Train')\nmissing_plot(test_df,1,'Test')\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:53.847017Z","iopub.execute_input":"2022-08-08T12:57:53.848124Z","iopub.status.idle":"2022-08-08T12:57:54.287356Z","shell.execute_reply.started":"2022-08-08T12:57:53.848088Z","shell.execute_reply":"2022-08-08T12:57:54.286236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['isTrain'] = True\ntest_df['isTrain'] = False\nfull = pd.concat([train_df,test_df])\n\nfig, axes = plt.subplots(6, 3, figsize=(16,30))\n\n# Set the ticks and ticklabels for all axes\nplt.setp(axes, xticks=[],\n        yticks=[])\nfig.subplots_adjust(hspace=0.5)\n\n\nfor i,measurement in enumerate(measurements):\n    ax= fig.add_subplot(6,3,i+1)\n    sns.histplot(full,x=measurement,hue='isTrain',binwidth=0.8,alpha=0.2,ax=ax);\n    ax.set_title(measurement);","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:57:54.289439Z","iopub.execute_input":"2022-08-08T12:57:54.290141Z","iopub.status.idle":"2022-08-08T12:58:05.677361Z","shell.execute_reply.started":"2022-08-08T12:57:54.290079Z","shell.execute_reply":"2022-08-08T12:58:05.676400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df= train_df.drop('isTrain',axis=1)\ntest_df= test_df.drop('isTrain',axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.683670Z","iopub.execute_input":"2022-08-08T12:58:05.684648Z","iopub.status.idle":"2022-08-08T12:58:05.695150Z","shell.execute_reply.started":"2022-08-08T12:58:05.684620Z","shell.execute_reply":"2022-08-08T12:58:05.694250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isna().sum(axis=1).unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.696623Z","iopub.execute_input":"2022-08-08T12:58:05.697025Z","iopub.status.idle":"2022-08-08T12:58:05.713788Z","shell.execute_reply.started":"2022-08-08T12:58:05.696983Z","shell.execute_reply":"2022-08-08T12:58:05.712815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data with six missing values\ntrain_df[train_df.isna().sum(axis=1)==6]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.715458Z","iopub.execute_input":"2022-08-08T12:58:05.716438Z","iopub.status.idle":"2022-08-08T12:58:05.745658Z","shell.execute_reply.started":"2022-08-08T12:58:05.716403Z","shell.execute_reply":"2022-08-08T12:58:05.744743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data with five missing values\ntrain_df[train_df.isna().sum(axis=1)==5]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.746962Z","iopub.execute_input":"2022-08-08T12:58:05.747725Z","iopub.status.idle":"2022-08-08T12:58:05.785909Z","shell.execute_reply.started":"2022-08-08T12:58:05.747690Z","shell.execute_reply":"2022-08-08T12:58:05.785011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Cleaning Data\n\n* Clean the data based on what I found on Data Exploration.\n\n* https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342126 There is a great discussion by des. I will tray attribute_2 * attribute_3 here. ","metadata":{}},{"cell_type":"markdown","source":"Attributes define product types. ","metadata":{}},{"cell_type":"code","source":"# remove attribution\n#def no_attribute(df):\n#    return df[[a for a in df.columns if not a.startswith('attribute')]]\n\n#train_df = no_attribute(train_df)\n#test_df = no_attribute(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.787304Z","iopub.execute_input":"2022-08-08T12:58:05.788604Z","iopub.status.idle":"2022-08-08T12:58:05.792858Z","shell.execute_reply.started":"2022-08-08T12:58:05.788565Z","shell.execute_reply":"2022-08-08T12:58:05.791964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['area'] = train_df['attribute_2'] * train_df['attribute_3']\ntest_df['area'] = test_df['attribute_2'] * test_df['attribute_3']\n\ntrain_df = train_df.drop(['attribute_2','attribute_3'], axis=1)\ntest_df = test_df.drop(['attribute_2','attribute_3'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.794163Z","iopub.execute_input":"2022-08-08T12:58:05.795379Z","iopub.status.idle":"2022-08-08T12:58:05.812906Z","shell.execute_reply.started":"2022-08-08T12:58:05.795336Z","shell.execute_reply":"2022-08-08T12:58:05.811852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove material_ \n\ntrain_df['attribute_0'] = train_df['attribute_0'].str.extract(r'(\\d)').astype(int)\ntrain_df['attribute_1'] = train_df['attribute_1'].str.extract(r'(\\d)').astype(int)\n\ntest_df['attribute_0'] = test_df['attribute_0'].str.extract(r'(\\d)').astype(int)\ntest_df['attribute_1'] = test_df['attribute_1'].str.extract(r'(\\d)').astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.815044Z","iopub.execute_input":"2022-08-08T12:58:05.815600Z","iopub.status.idle":"2022-08-08T12:58:05.953560Z","shell.execute_reply.started":"2022-08-08T12:58:05.815566Z","shell.execute_reply":"2022-08-08T12:58:05.952618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['isTrain'] = True\ntest_df['isTrain'] = False\n\nfull = pd.concat([train_df,test_df])\n\nattributes = ['attribute_0','attribute_1']\n\nfor attribute in attributes:\n    le = preprocessing.LabelEncoder()\n    full[attribute] = le.fit_transform(full[attribute])\n    \ntrain_df = full[full['isTrain']==True].drop('isTrain',axis=1)\ntest_df = full[full['isTrain']== False].drop(['isTrain','failure'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.955123Z","iopub.execute_input":"2022-08-08T12:58:05.955530Z","iopub.status.idle":"2022-08-08T12:58:05.986561Z","shell.execute_reply.started":"2022-08-08T12:58:05.955494Z","shell.execute_reply":"2022-08-08T12:58:05.985574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add Missing Values Counts for Each Row","metadata":{}},{"cell_type":"code","source":"def missing_counts(df):\n    df['missing_count'] = df[measurements].isna().sum(axis=1)\n    df['isM3'] = df.measurement_3.isna()\n    df['isM5'] = df.measurement_5.isna()\n    return df\nmissing_counts(train_df)\nmissing_counts(test_df)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:05.988171Z","iopub.execute_input":"2022-08-08T12:58:05.988622Z","iopub.status.idle":"2022-08-08T12:58:06.032905Z","shell.execute_reply.started":"2022-08-08T12:58:05.988581Z","shell.execute_reply":"2022-08-08T12:58:06.031906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filling nan with product imputer","metadata":{}},{"cell_type":"code","source":"#referred to https://stackoverflow.com/questions/19966018/pandas-filling-missing-values-by-mean-in-each-group\n#for column in train_df.drop(['product_code','failure'], axis=1).columns:\n#    train_df[column] = train_df.groupby('product_code')[column].transform(lambda x: x.fillna(x.mean()))\n#    test_df[column] = test_df.groupby('product_code')[column].transform(lambda x: x.fillna(x.mean()))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.034391Z","iopub.execute_input":"2022-08-08T12:58:06.034718Z","iopub.status.idle":"2022-08-08T12:58:06.039241Z","shell.execute_reply.started":"2022-08-08T12:58:06.034685Z","shell.execute_reply":"2022-08-08T12:58:06.038237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# impute by product category \nfeatures = [a for a in train_df.columns if a =='loading' or a.startswith('measurement')]\nframes = []\n\ntrain_df['isTrain'] = True\ntest_df['isTrain'] = False\n\nfull = pd.concat([train_df,test_df])\n\nfor code in full.product_code.unique():\n    df = full[full.product_code==code] \n    # mean, median, most_frequesnt, constant\n    imputer = SimpleImputer(strategy='most_frequent')\n    imputer.fit(df[features])\n    df[features] = imputer.transform(df[features])\n    frames.append(df)\nimputed = pd.concat(frames)\n\n\ntrain_df = imputed[imputed['isTrain']==True].drop('isTrain',axis=1)\ntest_df = imputed[imputed['isTrain']==False].drop(['isTrain','failure'],axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.040697Z","iopub.execute_input":"2022-08-08T12:58:06.041285Z","iopub.status.idle":"2022-08-08T12:58:06.182249Z","shell.execute_reply.started":"2022-08-08T12:58:06.041249Z","shell.execute_reply":"2022-08-08T12:58:06.181117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#features = [a for a in train_df.columns if a =='loading' or a.startswith('measurement')]\n\n\n\n#for df in [train_df,test_df]:\n#    imputer = KNNImputer(n_neighbors=3) \n#    imputer.fit(df[features])\n#    df[features] = imputer.transform(df[features])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.183866Z","iopub.execute_input":"2022-08-08T12:58:06.184266Z","iopub.status.idle":"2022-08-08T12:58:06.189057Z","shell.execute_reply.started":"2022-08-08T12:58:06.184226Z","shell.execute_reply":"2022-08-08T12:58:06.187904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Give failure rate to each rounded loading.","metadata":{}},{"cell_type":"code","source":"train_df['rounded'] = train_df['loading'].apply(lambda x: round(x,-1))\ntest_df['rounded'] =  test_df['loading'].apply(lambda x: round(x,-1))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.190869Z","iopub.execute_input":"2022-08-08T12:58:06.191239Z","iopub.status.idle":"2022-08-08T12:58:06.235805Z","shell.execute_reply.started":"2022-08-08T12:58:06.191184Z","shell.execute_reply":"2022-08-08T12:58:06.234958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num = train_df.groupby('rounded')['failure'].count()\nfailure = train_df.groupby('rounded')['failure'].sum()\nfailure_rate = failure/num\n(failure_rate).plot(kind='bar')\nplt.title('Failure Rate by Loading');","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.237372Z","iopub.execute_input":"2022-08-08T12:58:06.237942Z","iopub.status.idle":"2022-08-08T12:58:06.569376Z","shell.execute_reply.started":"2022-08-08T12:58:06.237907Z","shell.execute_reply":"2022-08-08T12:58:06.568468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df['failure_rate'] = train_df['rounded'].map(failure_rate)\n#test_df['failure_rate'] = test_df['rounded'].map(failure_rate)\n\ntrain_df = train_df.drop('rounded', axis=1)\ntest_df = test_df.drop('rounded', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.570710Z","iopub.execute_input":"2022-08-08T12:58:06.571163Z","iopub.status.idle":"2022-08-08T12:58:06.584498Z","shell.execute_reply.started":"2022-08-08T12:58:06.571126Z","shell.execute_reply":"2022-08-08T12:58:06.583549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add stats by product_code","metadata":{}},{"cell_type":"code","source":"numericals = [a for a in train_df.columns if a.startswith('me') or a.startswith('loading')]\ntrain_stats = train_df.groupby('product_code')[numericals].agg(['min','max','std'])\ntest_stats = test_df.groupby('product_code')[numericals].agg(['min','max','std'])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.586050Z","iopub.execute_input":"2022-08-08T12:58:06.586493Z","iopub.status.idle":"2022-08-08T12:58:06.660821Z","shell.execute_reply.started":"2022-08-08T12:58:06.586458Z","shell.execute_reply":"2022-08-08T12:58:06.659772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_stats","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.662499Z","iopub.execute_input":"2022-08-08T12:58:06.662882Z","iopub.status.idle":"2022-08-08T12:58:06.728153Z","shell.execute_reply.started":"2022-08-08T12:58:06.662837Z","shell.execute_reply":"2022-08-08T12:58:06.727257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#merge stats to the original dataframe\npd.options.display.max_columns = 100\nmerged_train = train_df.merge(train_stats, left_on='product_code',right_on='product_code')\nmerged_test  = test_df.merge(test_stats, left_on='product_code',right_on='product_code')\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.729800Z","iopub.execute_input":"2022-08-08T12:58:06.730596Z","iopub.status.idle":"2022-08-08T12:58:06.763542Z","shell.execute_reply.started":"2022-08-08T12:58:06.730554Z","shell.execute_reply":"2022-08-08T12:58:06.762421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# referred to https://stackoverflow.com/questions/39658574/how-to-drop-columns-which-have-same-values-in-all-rows-via-pandas-or-spark-dataf\n# renove the coloumn have all the same values\ndef drop_nunique(df):\n    nunique= df.nunique()\n    drop_cols = nunique[nunique== 1].index\n    df =df.drop(drop_cols,axis=1)\n    return df\ntrain_df = drop_nunique(merged_train)\ntest  = drop_nunique(merged_test)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.765128Z","iopub.execute_input":"2022-08-08T12:58:06.765526Z","iopub.status.idle":"2022-08-08T12:58:06.854757Z","shell.execute_reply.started":"2022-08-08T12:58:06.765490Z","shell.execute_reply":"2022-08-08T12:58:06.853750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make five folds by Product code\n\nhttps://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/341070 and https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/341896 I checked these discussions by AmbrosM","metadata":{}},{"cell_type":"code","source":"#train_list = []\n#test_list = []\n#for code in train_df.product_code.unique(): \n#    train_list.append(train_df[train_df['product_code'] != code])\n#    test_list.append(train_df[train_df['product_code'] == code]) ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.856158Z","iopub.execute_input":"2022-08-08T12:58:06.856559Z","iopub.status.idle":"2022-08-08T12:58:06.861621Z","shell.execute_reply.started":"2022-08-08T12:58:06.856521Z","shell.execute_reply":"2022-08-08T12:58:06.860626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split the row keeping the ratio of product category and failure rate\n# referred to https://stackoverflow.com/questions/45516424/sklearn-train-test-split-on-pandas-stratify-by-multiple-columns\n\ntrain_list = []\ntest_list = []\ntests = []\n\ntrain_df['cat_failure'] = train_df['product_code'].astype(str)  + train_df['failure'].astype(str)\n\nkfold = StratifiedKFold(n_splits=5,random_state=42, shuffle=True)\nX = train_df.drop('cat_failure',axis=1)\ny = train_df.cat_failure\nfor train_index,test_index in kfold.split(X,y):\n     \n    \n     X_train, X_test = X.iloc[train_index],X.iloc[test_index]\n     y_train, y_test = X.iloc[train_index].failure, X.iloc[test_index].failure\n     \n     # add failure rate by roudned loading    \n     X_train['rounded'] = X_train['loading'].apply(lambda x: round(x,-1))\n     X_test['rounded'] =  X_test['loading'].apply(lambda x: round(x,-1))\n     ts = test.copy()\n     ts['rounded'] = ts['loading'].apply(lambda x: round(x,-1))\n     \n     num = X_train.groupby('rounded')['failure'].count()\n     failure = X_train.groupby('rounded')['failure'].sum()\n     failure_rate = failure/num\n    \n     X_train['failure_rate'] = X_train['rounded'].map(failure_rate)\n     X_test['failure_rate'] =  X_test['rounded'].map(failure_rate)\n     ts['failure_rate'] = ts['rounded'].map(failure_rate)\n\n     X_train = X_train.drop('rounded', axis=1)\n     X_test = X_test.drop('rounded', axis=1)\n     ts = ts.drop('rounded',axis=1)\n     \n     \n    \n     train_list.append((X_train,y_train))\n     test_list.append((X_test,y_test))\n     tests.append(ts)\n    \n\n     \n\n#tr, ts = train_test_split(train, test_size=0.2, random_state=0, stratify=train[['cat_failure']])\n","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:06.863015Z","iopub.execute_input":"2022-08-08T12:58:06.863994Z","iopub.status.idle":"2022-08-08T12:58:07.316720Z","shell.execute_reply.started":"2022-08-08T12:58:06.863958Z","shell.execute_reply":"2022-08-08T12:58:07.315682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"markdown","source":"### CatBoostRegressor","metadata":{}},{"cell_type":"code","source":"from catboost import CatBoostRegressor\n\nscores =[]\ntest_predictions  = []\nfor i in range(5):\n   \n    y_train = train_list[i][1]\n    X_train = train_list[i][0].drop(['product_code','failure',], axis=1)\n    \n    y_test = test_list[i][1]\n    X_test = test_list[i][0].drop(['product_code','failure'],axis=1)\n    \n    model = CatBoostRegressor()\n    # Fit model\n    model.fit(X_train, y_train,verbose=0)\n    # Get predictions\n    score = roc_auc_score(y_test,model.predict(X_test))\n    scores.append(score)\n    \n    test_predictions.append(model.predict(tests[i].drop('product_code',axis=1)))\n    \n    print(f'FOLD {i}: {score}')\n    \nprint(np.mean(scores))\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:58:07.318399Z","iopub.execute_input":"2022-08-08T12:58:07.318817Z","iopub.status.idle":"2022-08-08T12:59:19.984284Z","shell.execute_reply.started":"2022-08-08T12:58:07.318768Z","shell.execute_reply":"2022-08-08T12:59:19.983178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LGBMReggressor","metadata":{}},{"cell_type":"code","source":"scores =[]\ntest_predictions  = []\n\n\n# lightbgm are not happy with tuples in columns \nimport re\ndef rename_df(df):\n    return df.rename(columns = lambda x: x[0] + '_' + x[1] if type(x)==tuple else x)\n\nfor i in range(5):\n    y_train = train_list[i][1]\n    X_train = rename_df(train_list[i][0].drop(['product_code','failure'], axis=1))\n    \n    y_test = test_list[i][1]\n    X_test = rename_df(test_list[i][0].drop(['product_code','failure'],axis=1))\n    \n    model = LGBMRegressor(boosting='dart',n_estimators=115,learning_rate=0.12)\n    # Fit model\n    model.fit(X_train, y_train,verbose=0)\n    # Get predictions\n    score = roc_auc_score(y_test,model.predict(X_test))\n    scores.append(score)\n    \n    test_predictions.append(model.predict(rename_df(tests[i].drop('product_code',axis=1))))\n    \n    print(f'FOLD {i}: {score}')\n    \nprint(np.mean(scores))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:59:19.985891Z","iopub.execute_input":"2022-08-08T12:59:19.986261Z","iopub.status.idle":"2022-08-08T12:59:30.038770Z","shell.execute_reply.started":"2022-08-08T12:59:19.986225Z","shell.execute_reply":"2022-08-08T12:59:30.037984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### XGBReggressor","metadata":{}},{"cell_type":"markdown","source":"* booster gblinear works really well with this dataset https://xgboost.readthedocs.io/en/stable/parameter.html#parameters-for-linear-booster-booster-gblinear","metadata":{}},{"cell_type":"code","source":"#tree_method='gpu_hist',booster='gblinear',reg_lambda=0.8,updater='coord_descent',feature_selector='random'\n#tree_method='gpu_hist',booster='gblinear',reg_lambda=0.8,updater='coord_descent',feature_selector='thrifty'\n\nscores =[]\ntest_predictions  = []\nfor i in range(5):\n    y_train = train_list[i][1]\n    X_train = train_list[i][0].drop(['product_code','failure'], axis=1)\n    \n    y_test = test_list[i][1]\n    X_test = test_list[i][0].drop(['product_code','failure'],axis=1)\n    \n    model = XGBRegressor(tree_method='gpu_hist',booster='gblinear',reg_lambda=0.9,updater='coord_descent',feature_selector='thrifty')\n    # Fit model\n    model.fit(X_train, y_train,verbose=0)\n    # Get predictions\n    score = roc_auc_score(y_test,model.predict(X_test))\n    scores.append(score)\n    \n    test_predictions.append(model.predict(tests[i].drop('product_code',axis=1)))\n    \n    print(f'FOLD {i}: {score}')\n\nprint('')\nprint(f' Total Average: {np.mean(scores)}')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:59:30.042605Z","iopub.execute_input":"2022-08-08T12:59:30.044375Z","iopub.status.idle":"2022-08-08T12:59:50.846184Z","shell.execute_reply.started":"2022-08-08T12:59:30.044339Z","shell.execute_reply":"2022-08-08T12:59:50.845437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Making Submissionfile","metadata":{}},{"cell_type":"code","source":"# get average of each element of test_predictions\n\npredictions= [np.mean(a) for a in zip(*test_predictions)]","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:59:50.849690Z","iopub.execute_input":"2022-08-08T12:59:50.851580Z","iopub.status.idle":"2022-08-08T12:59:51.086362Z","shell.execute_reply.started":"2022-08-08T12:59:50.851547Z","shell.execute_reply":"2022-08-08T12:59:51.085354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/tabular-playground-series-aug-2022/sample_submission.csv',index_col='id')\nsubmission['failure'] = predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:59:51.087935Z","iopub.execute_input":"2022-08-08T12:59:51.088320Z","iopub.status.idle":"2022-08-08T12:59:51.107557Z","shell.execute_reply.started":"2022-08-08T12:59:51.088282Z","shell.execute_reply":"2022-08-08T12:59:51.106613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission_14')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T12:59:51.109161Z","iopub.execute_input":"2022-08-08T12:59:51.109550Z","iopub.status.idle":"2022-08-08T12:59:51.150673Z","shell.execute_reply.started":"2022-08-08T12:59:51.109515Z","shell.execute_reply":"2022-08-08T12:59:51.149647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n\n1. https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342126 There is a great discussion by des. I will tray attribute_2 * attribute_3 here.\n\n2. https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/341070 ,https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/341896, and https://www.kaggle.com/competitions/tabular-playground-series-aug-2022/discussion/342319 I checked these discussions by AmbrosM","metadata":{}}]}