{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Tabular Playgroud Regression\n**OBJECTIVE** : Explore statistic and visualize to understand the data and Build a baseline ML model to predict the target value base on numerical features and categorical features without any domain knowledge to select and/or feature engineering because there are no the meta-data of the dataset.  \n\n**evaluation metric** : root mean squared error","metadata":{}},{"cell_type":"markdown","source":"### Import","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-07T18:07:49.339636Z","iopub.execute_input":"2022-07-07T18:07:49.340714Z","iopub.status.idle":"2022-07-07T18:07:49.351380Z","shell.execute_reply.started":"2022-07-07T18:07:49.340671Z","shell.execute_reply":"2022-07-07T18:07:49.350047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data\n","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/tabular-playground-series-feb-2021/train.csv')\ntest_df = pd.read_csv('../input/tabular-playground-series-feb-2021/test.csv')\n\nprint(train_df.shape)\nprint(test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:49.370090Z","iopub.execute_input":"2022-07-07T18:07:49.370707Z","iopub.status.idle":"2022-07-07T18:07:52.304991Z","shell.execute_reply.started":"2022-07-07T18:07:49.370674Z","shell.execute_reply":"2022-07-07T18:07:52.303758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:52.307236Z","iopub.execute_input":"2022-07-07T18:07:52.307621Z","iopub.status.idle":"2022-07-07T18:07:52.335455Z","shell.execute_reply.started":"2022-07-07T18:07:52.307587Z","shell.execute_reply":"2022-07-07T18:07:52.334138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:52.337302Z","iopub.execute_input":"2022-07-07T18:07:52.338099Z","iopub.status.idle":"2022-07-07T18:07:52.682514Z","shell.execute_reply.started":"2022-07-07T18:07:52.338034Z","shell.execute_reply":"2022-07-07T18:07:52.681680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"seperate the features and target and drop id column  \nthere are 2 type of feature, categorical and numeric, I will seperate them and store the features name in a list for future use.","metadata":{}},{"cell_type":"code","source":"features = [col for col in train_df.columns if col not in ['id','target']]\ncat_features = [cat for cat in features if train_df[cat].dtype == 'object']\nnum_features = [num for num in features if train_df[num].dtype == 'float64']\nprint(f'{len(features)} features : {features}\\n')\nprint(f'{len(cat_features)} categorical features : {cat_features}\\n')\nprint(f'{len(num_features)} numerical features : {num_features}')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:52.684573Z","iopub.execute_input":"2022-07-07T18:07:52.685536Z","iopub.status.idle":"2022-07-07T18:07:52.693382Z","shell.execute_reply.started":"2022-07-07T18:07:52.685498Z","shell.execute_reply":"2022-07-07T18:07:52.692511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, looking for missing and duplicate entries  \nthey are none.","metadata":{}},{"cell_type":"code","source":"print(f'there are : {train_df.isna().sum().sum()} missing value')\nprint(f'          : {train_df.duplicated().sum()} duplicate entries')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:52.695415Z","iopub.execute_input":"2022-07-07T18:07:52.696499Z","iopub.status.idle":"2022-07-07T18:07:53.827163Z","shell.execute_reply.started":"2022-07-07T18:07:52.696458Z","shell.execute_reply":"2022-07-07T18:07:53.825738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA\ntarget is a bimodal-distribution and there are outliers ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nsns.histplot(train_df.target, kde=True)\nplt.subplot(1,2,2)\nsns.boxplot(data=train_df, x='target')\nplt.suptitle('Original Distribution', fontsize=16)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:53.831469Z","iopub.execute_input":"2022-07-07T18:07:53.831842Z","iopub.status.idle":"2022-07-07T18:07:56.061953Z","shell.execute_reply.started":"2022-07-07T18:07:53.831811Z","shell.execute_reply":"2022-07-07T18:07:56.060626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's remove these outlier to help some learning algorithm learn better  \nand re-visualize the distribution there are 353 entries removed.","metadata":{}},{"cell_type":"code","source":"#calculate the upper boundary and lower boundary\np_25, p_75 = np.percentile(train_df['target'], [25, 75])\niqr = p_75 - p_25\nupper_bound = (p_75 + 1.5 * iqr).round(2)\nlower_bound  = (p_25 - 1.5 * iqr).round(2)\nprint(upper_bound)\nprint(lower_bound)\n\n#remove the value under lowerbound and over upperbound\ntrain_df = train_df[train_df.target > lower_bound][train_df.target < upper_bound].reset_index()\nnum_removed = 300000 - train_df.shape[0]\nprint(f'{num_removed} entries have been removed.')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:56.063663Z","iopub.execute_input":"2022-07-07T18:07:56.064061Z","iopub.status.idle":"2022-07-07T18:07:56.299178Z","shell.execute_reply.started":"2022-07-07T18:07:56.064027Z","shell.execute_reply":"2022-07-07T18:07:56.298296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.subplot(1,2,1)\nsns.histplot(train_df.target, kde=True)\nplt.subplot(1,2,2)\nsns.boxplot(data=train_df, x='target')\nplt.suptitle('Distribution after remove outliers', fontsize=16)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:56.300483Z","iopub.execute_input":"2022-07-07T18:07:56.301017Z","iopub.status.idle":"2022-07-07T18:07:58.227099Z","shell.execute_reply.started":"2022-07-07T18:07:56.300984Z","shell.execute_reply":"2022-07-07T18:07:58.225367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.target.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:58.229007Z","iopub.execute_input":"2022-07-07T18:07:58.229409Z","iopub.status.idle":"2022-07-07T18:07:58.254282Z","shell.execute_reply.started":"2022-07-07T18:07:58.229374Z","shell.execute_reply":"2022-07-07T18:07:58.253361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's move on to numerical variable distribution there are 14 of them  \nthey're all multimodal and in the same value range, looks like the data has been scaled already.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(25,10))\nfor i,col in enumerate(num_features):\n    plt.subplot(2, 7, i+1)\n    sns.histplot(train_df[col], kde=True, color='m') #change color for your eyes\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:07:58.259403Z","iopub.execute_input":"2022-07-07T18:07:58.260082Z","iopub.status.idle":"2022-07-07T18:08:21.082642Z","shell.execute_reply.started":"2022-07-07T18:07:58.260029Z","shell.execute_reply":"2022-07-07T18:08:21.081334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[num_features].describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:08:21.084212Z","iopub.execute_input":"2022-07-07T18:08:21.084846Z","iopub.status.idle":"2022-07-07T18:08:21.432510Z","shell.execute_reply.started":"2022-07-07T18:08:21.084808Z","shell.execute_reply":"2022-07-07T18:08:21.431155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"correlation : there are little linear correlation between numeric features and target","metadata":{}},{"cell_type":"code","source":"cor = train_df[num_features+['target']].corr()\nmask = np.zeros_like(cor)\nmask[np.triu_indices_from(mask)] = True\n\n#plot\nplt.figure(figsize=(15,10))\nsns.heatmap(cor, annot=True,vmax=1, vmin=-1, fmt='.2f', mask=mask, cmap=sns.diverging_palette(220, 20, as_cmap=True))\n#seaborn heatmap : https://seaborn.pydata.org/generated/seaborn.heatmap.html\n#color palettes : https://seaborn.pydata.org/tutorial/color_palettes.html","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:08:21.434126Z","iopub.execute_input":"2022-07-07T18:08:21.434626Z","iopub.status.idle":"2022-07-07T18:08:22.423093Z","shell.execute_reply.started":"2022-07-07T18:08:21.434577Z","shell.execute_reply":"2022-07-07T18:08:22.421844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"categorical features","metadata":{}},{"cell_type":"code","source":"#encode categorical features\nfrom sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\n\nfor cat in cat_features:\n    train_df[cat] = le.fit_transform(train_df[cat])\n\n\nfor cat in cat_features:\n    test_df[cat] = le.fit_transform(test_df[cat])\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:08:22.424917Z","iopub.execute_input":"2022-07-07T18:08:22.425317Z","iopub.status.idle":"2022-07-07T18:08:23.986965Z","shell.execute_reply.started":"2022-07-07T18:08:22.425282Z","shell.execute_reply":"2022-07-07T18:08:23.985584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(25,10))\nfor i,col in enumerate(cat_features):\n    plt.subplot(2, 5, i+1)\n    sns.histplot(train_df[col], kde=True, color='g')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:08:23.988514Z","iopub.execute_input":"2022-07-07T18:08:23.988865Z","iopub.status.idle":"2022-07-07T18:08:38.953039Z","shell.execute_reply.started":"2022-07-07T18:08:23.988833Z","shell.execute_reply":"2022-07-07T18:08:38.951824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cor = train_df[cat_features+['target']].corr()\nmask = np.zeros_like(cor)\nmask[np.triu_indices_from(mask)] = True\n\n#plot\nplt.figure(figsize=(15,10))\nsns.heatmap(cor, annot=True,vmax=1, vmin=-1, fmt='.2f', mask=mask, cmap='coolwarm')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:08:38.954631Z","iopub.execute_input":"2022-07-07T18:08:38.954994Z","iopub.status.idle":"2022-07-07T18:08:39.625907Z","shell.execute_reply.started":"2022-07-07T18:08:38.954962Z","shell.execute_reply":"2022-07-07T18:08:39.622654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Mutual information cont4 has 0.0 score, drop this feature might help our model's performance but for now let's use all features","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import mutual_info_regression\nmi = mutual_info_regression(train_df[features], train_df.target)\nmi","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:08:39.627828Z","iopub.execute_input":"2022-07-07T18:08:39.631634Z","iopub.status.idle":"2022-07-07T18:10:30.806402Z","shell.execute_reply.started":"2022-07-07T18:08:39.631582Z","shell.execute_reply":"2022-07-07T18:10:30.805237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mi_scores = pd.DataFrame(data=mi, index=features, columns=['mi_scores']).sort_values(by=['mi_scores']).reset_index()\nsns.barplot(data=mi_scores, x='mi_scores', y='index')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:10:30.808758Z","iopub.execute_input":"2022-07-07T18:10:30.809570Z","iopub.status.idle":"2022-07-07T18:10:31.508282Z","shell.execute_reply.started":"2022-07-07T18:10:30.809500Z","shell.execute_reply":"2022-07-07T18:10:31.507220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Generate baseline random forest model that use all features ","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nSEED = 42\nX = train_df[['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat6', 'cat7', 'cat8', 'cat9', 'cont0', 'cont1', 'cont2', 'cont3', 'cont5', 'cont6', 'cont7', 'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13']]\ny = train_df.target\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=SEED)\n\n#model\nrfr = RandomForestRegressor(n_estimators=300, n_jobs=-1) #no tuning at all\nrfr.fit(X_train, y_train)\nrfr_pred = rfr.predict(X_val)\nrfr_mse = mean_squared_error(y_val, rfr_pred) #squared=False, I forgot\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:10:31.509662Z","iopub.execute_input":"2022-07-07T18:10:31.510641Z","iopub.status.idle":"2022-07-07T18:23:09.844057Z","shell.execute_reply.started":"2022-07-07T18:10:31.510586Z","shell.execute_reply":"2022-07-07T18:23:09.842874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'RMSE: {np.sqrt(rfr_mse)}')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:23:09.845588Z","iopub.execute_input":"2022-07-07T18:23:09.845952Z","iopub.status.idle":"2022-07-07T18:23:09.851958Z","shell.execute_reply.started":"2022-07-07T18:23:09.845919Z","shell.execute_reply":"2022-07-07T18:23:09.850652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Actually, I have to retrain with the whole dataset but for now let's use this to predict the y_test","metadata":{}},{"cell_type":"code","source":"rfr.fit(X,y)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:30:35.715172Z","iopub.execute_input":"2022-07-07T18:30:35.715703Z","iopub.status.idle":"2022-07-07T18:47:14.142497Z","shell.execute_reply.started":"2022-07-07T18:30:35.715667Z","shell.execute_reply":"2022-07-07T18:47:14.140610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = test_df[['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat6', 'cat7', 'cat8', 'cat9', 'cont0', 'cont1', 'cont2', 'cont3', 'cont5', 'cont6', 'cont7', 'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13']]\ny_pred = rfr.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:47:14.145466Z","iopub.execute_input":"2022-07-07T18:47:14.146354Z","iopub.status.idle":"2022-07-07T18:47:31.996309Z","shell.execute_reply.started":"2022-07-07T18:47:14.146316Z","shell.execute_reply":"2022-07-07T18:47:31.994660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"let's submit the baseline prediction","metadata":{}},{"cell_type":"code","source":"#see what submission file looks like\nsubmission = pd.read_csv('../input/tabular-playground-series-feb-2021/sample_submission.csv')\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:47:31.998549Z","iopub.execute_input":"2022-07-07T18:47:31.999049Z","iopub.status.idle":"2022-07-07T18:47:32.079100Z","shell.execute_reply.started":"2022-07-07T18:47:31.998999Z","shell.execute_reply":"2022-07-07T18:47:32.077551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.DataFrame({'id':test_df.id, 'target':y_pred})\noutput.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:47:32.081801Z","iopub.execute_input":"2022-07-07T18:47:32.082291Z","iopub.status.idle":"2022-07-07T18:47:32.097446Z","shell.execute_reply.started":"2022-07-07T18:47:32.082252Z","shell.execute_reply":"2022-07-07T18:47:32.095709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:47:32.101138Z","iopub.execute_input":"2022-07-07T18:47:32.101636Z","iopub.status.idle":"2022-07-07T18:47:32.760575Z","shell.execute_reply.started":"2022-07-07T18:47:32.101597Z","shell.execute_reply":"2022-07-07T18:47:32.759294Z"},"trusted":true},"execution_count":null,"outputs":[]}]}