{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\n\nfrom sklearn.impute import SimpleImputer\nfrom scipy.stats import spearmanr\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\nfrom sklearn.impute import KNNImputer\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.preprocessing import StandardScaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-01T11:46:41.676055Z","iopub.execute_input":"2022-08-01T11:46:41.676433Z","iopub.status.idle":"2022-08-01T11:46:41.682676Z","shell.execute_reply.started":"2022-08-01T11:46:41.676401Z","shell.execute_reply":"2022-08-01T11:46:41.681733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_path = Path('/kaggle/input/tabular-playground-series-jun-2022/')\ndata = pd.read_csv(input_path / 'data.csv', index_col = 'row_id')\nsubmission = pd.read_csv(input_path / 'sample_submission.csv', index_col = 'row-col')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T11:46:41.745968Z","iopub.execute_input":"2022-08-01T11:46:41.746877Z","iopub.status.idle":"2022-08-01T11:46:55.679974Z","shell.execute_reply.started":"2022-08-01T11:46:41.746838Z","shell.execute_reply":"2022-08-01T11:46:55.679037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T11:46:55.681484Z","iopub.execute_input":"2022-08-01T11:46:55.681936Z","iopub.status.idle":"2022-08-01T11:46:55.709370Z","shell.execute_reply.started":"2022-08-01T11:46:55.681894Z","shell.execute_reply":"2022-08-01T11:46:55.707803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T11:46:55.711052Z","iopub.execute_input":"2022-08-01T11:46:55.712103Z","iopub.status.idle":"2022-08-01T11:46:59.913310Z","shell.execute_reply.started":"2022-08-01T11:46:55.712056Z","shell.execute_reply":"2022-08-01T11:46:59.912372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T11:46:59.915336Z","iopub.execute_input":"2022-08-01T11:46:59.915795Z","iopub.status.idle":"2022-08-01T11:47:00.086611Z","shell.execute_reply.started":"2022-08-01T11:46:59.915757Z","shell.execute_reply":"2022-08-01T11:47:00.085623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.columns","metadata":{"execution":{"iopub.status.busy":"2022-08-01T11:47:00.087890Z","iopub.execute_input":"2022-08-01T11:47:00.088256Z","iopub.status.idle":"2022-08-01T11:47:00.095457Z","shell.execute_reply.started":"2022-08-01T11:47:00.088196Z","shell.execute_reply":"2022-08-01T11:47:00.094445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number of missing values in each column\nprint('Number of missing values in each column: ', data.isna().sum(), '\\n')\n\n# total number of missing values\nprint('Total number of missing values: ', data.isna().sum().sum(), '\\n')\n\n# percentage of missing values\nprint('Percentage of missing values: ', data.isna().sum().sum() / (data.shape[0] * data.shape[1]))","metadata":{"execution":{"iopub.status.busy":"2022-08-01T11:47:00.096996Z","iopub.execute_input":"2022-08-01T11:47:00.097899Z","iopub.status.idle":"2022-08-01T11:47:00.543836Z","shell.execute_reply.started":"2022-08-01T11:47:00.097851Z","shell.execute_reply":"2022-08-01T11:47:00.542816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# standardizing numeric columns\nstandardizer = StandardScaler()\ndata[:] = standardizer.fit_transform(data)\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T11:47:00.545477Z","iopub.execute_input":"2022-08-01T11:47:00.545956Z","iopub.status.idle":"2022-08-01T11:47:23.930384Z","shell.execute_reply.started":"2022-08-01T11:47:00.545910Z","shell.execute_reply":"2022-08-01T11:47:23.929296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The two following tables show absolute values of Pearson and Spearman correlation \n# coefficients between all pairs of numeric variables sorted from highest \n# to lowest correlation.\n\n# Pearson\nPearson_corr = data[[x for x in data.columns]].corr().abs()\nPearson_corr = Pearson_corr.unstack()\nPearson_corr = Pearson_corr.sort_values(kind = \"quicksort\", ascending = False)\nPearson_corr = Pearson_corr[Pearson_corr != 1].drop_duplicates()\n\nfeature1 = []\nfor pair in list(Pearson_corr.index):\n    feature1.append(pair[0])\n    \nfeature2 = []\nfor pair in list(Pearson_corr.index):\n    feature2.append(pair[1])\n    \nPearson_corr = pd.DataFrame(Pearson_corr)\nPearson_corr.reset_index(drop = True, inplace = True)\nPearson_corr.rename(columns = {0: \"Pearson\"}, inplace = True)\nPearson_corr['Feature1'] = feature1\nPearson_corr['Feature2'] = feature2    \nPearson_corr = Pearson_corr[['Feature1', 'Feature2', 'Pearson']]\nprint(Pearson_corr[:10])\n\n# Spearman\nSpearman_corr = []\nfor i in [x for x in data.columns]:\n    for j in [x for x in data.columns]:\n        if i != j:\n            \n            coef, p = spearmanr(data[i], data[j])\n            #calculate Spearmann correlation coefficient \n            Spearman_corr.append([i, j, abs(coef)])\n            \nSpearman_corr = pd.DataFrame(Spearman_corr)\nSpearman_corr.rename(columns = {0: \"Feature1\", 1: \"Feature2\", 2: \"Spearman\"}, inplace = True)\nSpearman_corr.sort_values('Spearman', ascending = False, inplace = True)\nSpearman_corr.reset_index(drop = True, inplace = True)\nSpearman_corr = Spearman_corr.iloc[::2, :]\nprint('\\n', Spearman_corr[:10])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T10:32:03.638183Z","iopub.execute_input":"2022-07-28T10:32:03.638986Z","iopub.status.idle":"2022-07-28T10:35:09.726999Z","shell.execute_reply.started":"2022-07-28T10:32:03.638952Z","shell.execute_reply":"2022-07-28T10:35:09.725716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data1 = data2 = data3 = data4 = data","metadata":{"execution":{"iopub.status.busy":"2022-07-28T11:42:39.068845Z","iopub.execute_input":"2022-07-28T11:42:39.069157Z","iopub.status.idle":"2022-07-28T11:42:39.073909Z","shell.execute_reply.started":"2022-07-28T11:42:39.069128Z","shell.execute_reply":"2022-07-28T11:42:39.073044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Iterative imputer\nimp = IterativeImputer(max_iter = 10, random_state = 0)\ndata1[:] = imp.fit_transform(data)\n\n# Submission 1\nfor i in tqdm(submission.index):\n    row = int(i.split('-')[0])\n    col = i.split('-')[1]\n    submission.loc[i, 'value'] = data1.loc[row, col]\n\nsubmission.to_csv('submission1.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T11:42:44.346496Z","iopub.execute_input":"2022-07-28T11:42:44.346964Z","iopub.status.idle":"2022-07-28T13:45:56.706031Z","shell.execute_reply.started":"2022-07-28T11:42:44.346924Z","shell.execute_reply":"2022-07-28T13:45:56.704844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  Nearest neighbors imputation\nimp = KNNImputer(n_neighbors = 6, weights = \"uniform\") \n# weights = 'distance'\n# change n_neighbors\ndata2[:] = imp.fit_transform(data)\n\n# Submission 2\nfor i in tqdm(submission.index):\n    row = int(i.split('-')[0])\n    col = i.split('-')[1]\n    submission.loc[i, 'value'] = data2.loc[row, col]\n\nsubmission.to_csv('submission2.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:49:41.348425Z","iopub.execute_input":"2022-07-28T13:49:41.349708Z","iopub.status.idle":"2022-07-28T13:51:32.077402Z","shell.execute_reply.started":"2022-07-28T13:49:41.349659Z","shell.execute_reply":"2022-07-28T13:51:32.076222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IterativeImputer with RandomForestRegressor\nimp = IterativeImputer(estimator = RandomForestRegressor())\ndata3[:] = imp.fit_transform(data)\n\n# Submission 3\nfor i in tqdm(submission.index):\n    row = int(i.split('-')[0])\n    col = i.split('-')[1]\n    submission.loc[i, 'value'] = data3.loc[row, col]\n\nsubmission.to_csv('submission3.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:53:48.271769Z","iopub.execute_input":"2022-07-28T13:53:48.272818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IterativeImputer with KNeighborsRegressor\nimp = IterativeImputer(estimator = KNeighborsRegressor(n_neighbors = 5, weights = 'uniform'))\ndata4[:] = imp.fit_transform(data)\n\n# Submission 4\nfor i in tqdm(submission.index):\n    row = int(i.split('-')[0])\n    col = i.split('-')[1]\n    submission.loc[i, 'value'] = data4.loc[row, col]\n\nsubmission.to_csv('submission4.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}