{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1 Loading and Exploring Data","metadata":{}},{"cell_type":"markdown","source":"# 1.1  Loading the libraries and data","metadata":{}},{"cell_type":"markdown","source":"Loading Python packages","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:10.632494Z","iopub.execute_input":"2022-08-07T11:46:10.632975Z","iopub.status.idle":"2022-08-07T11:46:11.307649Z","shell.execute_reply.started":"2022-08-07T11:46:10.632881Z","shell.execute_reply":"2022-08-07T11:46:11.306425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loading test dataset from parquet files.","metadata":{}},{"cell_type":"code","source":"test=pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\")","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:11.309870Z","iopub.execute_input":"2022-08-07T11:46:11.310372Z","iopub.status.idle":"2022-08-07T11:46:46.964031Z","shell.execute_reply.started":"2022-08-07T11:46:11.310285Z","shell.execute_reply":"2022-08-07T11:46:46.962949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:46.965590Z","iopub.execute_input":"2022-08-07T11:46:46.965953Z","iopub.status.idle":"2022-08-07T11:46:47.086670Z","shell.execute_reply.started":"2022-08-07T11:46:46.965918Z","shell.execute_reply":"2022-08-07T11:46:47.085349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1.2 Data size and structure","metadata":{}},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:47.091236Z","iopub.execute_input":"2022-08-07T11:46:47.092302Z","iopub.status.idle":"2022-08-07T11:46:47.102811Z","shell.execute_reply.started":"2022-08-07T11:46:47.092262Z","shell.execute_reply":"2022-08-07T11:46:47.101580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Below is a glimps of the test data","metadata":{}},{"cell_type":"code","source":"pd.set_option(\"display.max_columns\",1000)\npd.set_option(\"display.max_rows\",1000)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:47.104476Z","iopub.execute_input":"2022-08-07T11:46:47.105098Z","iopub.status.idle":"2022-08-07T11:46:47.112784Z","shell.execute_reply.started":"2022-08-07T11:46:47.105053Z","shell.execute_reply":"2022-08-07T11:46:47.111631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:47.114342Z","iopub.execute_input":"2022-08-07T11:46:47.114677Z","iopub.status.idle":"2022-08-07T11:46:47.259805Z","shell.execute_reply.started":"2022-08-07T11:46:47.114647Z","shell.execute_reply":"2022-08-07T11:46:47.258416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thus there are 189 features(without customer_ID) out of which ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68'] are categorical features.","metadata":{}},{"cell_type":"markdown","source":"# 2 Missing data, label encoding, and factorizing variables","metadata":{}},{"cell_type":"markdown","source":"First of all we will look for missing values.","metadata":{}},{"cell_type":"code","source":"test.isnull().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:47.261213Z","iopub.execute_input":"2022-08-07T11:46:47.261584Z","iopub.status.idle":"2022-08-07T11:46:51.542406Z","shell.execute_reply.started":"2022-08-07T11:46:47.261550Z","shell.execute_reply":"2022-08-07T11:46:51.541144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.1 Dropping Columns","metadata":{}},{"cell_type":"markdown","source":"Drop D_87, D_88, D_110, D_111, B_39, D_73, B_42, D_134, B_29, D_132, D_76, D_142, D_53, D_50, D_105, D_56 as they have been droped in trainning set.","metadata":{}},{"cell_type":"code","source":"dropCols=[\"D_87\", \"D_88\", \"D_110\", \"D_111\", \"B_39\", \"D_73\", \"B_42\", \"D_134\", \"B_29\", \"D_132\", \"D_76\", \"D_142\", \"D_53\", \"D_50\", \"D_105\", \"D_56\"]\ntest.drop(labels=dropCols,axis=1,inplace=True)\ntest.drop(labels=[\"S_3\",\"S_22\"],axis=1,inplace=True)\ntest.drop(labels=[\"D_42\",\"D_77\"],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:46:51.544106Z","iopub.execute_input":"2022-08-07T11:46:51.544649Z","iopub.status.idle":"2022-08-07T11:47:02.867056Z","shell.execute_reply.started":"2022-08-07T11:46:51.544607Z","shell.execute_reply":"2022-08-07T11:47:02.865440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.2 Imputing Missing Values","metadata":{}},{"cell_type":"markdown","source":"The operations done on the columns are same as that of the operations done in the training set","metadata":{}},{"cell_type":"markdown","source":"# 2.2.1 Spend Variables","metadata":{}},{"cell_type":"code","source":"test[\"S_7\"].fillna(value=test[\"S_7\"].mean(),inplace=True)\ntest[\"S_24\"].fillna(value=test[\"S_24\"].mean(),inplace=True)\ntest[\"S_9\"].fillna(value=test[\"S_9\"].mean(),inplace=True)\ntest[\"S_25\"].fillna(value=test[\"S_25\"].mean(),inplace=True)\ntest[\"S_26\"].fillna(value=test[\"S_26\"].mean(),inplace=True)\ntest[\"S_23\"].fillna(value=test[\"S_23\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:02.871960Z","iopub.execute_input":"2022-08-07T11:47:02.872494Z","iopub.status.idle":"2022-08-07T11:47:03.328362Z","shell.execute_reply.started":"2022-08-07T11:47:02.872441Z","shell.execute_reply":"2022-08-07T11:47:03.327238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.2.2 Risk Variables","metadata":{}},{"cell_type":"code","source":"test[\"R_7\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:03.333912Z","iopub.execute_input":"2022-08-07T11:47:03.334354Z","iopub.status.idle":"2022-08-07T11:47:03.656020Z","shell.execute_reply.started":"2022-08-07T11:47:03.334296Z","shell.execute_reply":"2022-08-07T11:47:03.654651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"R_7 has maximum values as 0. So we need to scale the larger values down.","metadata":{}},{"cell_type":"code","source":"test[\"R_7\"]=test[\"R_7\"]/test[\"R_7\"].max()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:03.657582Z","iopub.execute_input":"2022-08-07T11:47:03.658263Z","iopub.status.idle":"2022-08-07T11:47:03.730551Z","shell.execute_reply.started":"2022-08-07T11:47:03.658227Z","shell.execute_reply":"2022-08-07T11:47:03.729380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"R_14\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:03.732039Z","iopub.execute_input":"2022-08-07T11:47:03.732792Z","iopub.status.idle":"2022-08-07T11:47:04.012554Z","shell.execute_reply.started":"2022-08-07T11:47:03.732756Z","shell.execute_reply":"2022-08-07T11:47:04.011336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"R_14 has maximum values as 0. So we need to scale the larger values down.","metadata":{}},{"cell_type":"code","source":"test[\"R_14\"]=test[\"R_14\"]/test[\"R_14\"].max()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:04.013895Z","iopub.execute_input":"2022-08-07T11:47:04.014300Z","iopub.status.idle":"2022-08-07T11:47:04.083998Z","shell.execute_reply.started":"2022-08-07T11:47:04.014265Z","shell.execute_reply":"2022-08-07T11:47:04.082678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we need to fill the null values.","metadata":{"execution":{"iopub.status.busy":"2022-07-30T15:41:23.145553Z","iopub.execute_input":"2022-07-30T15:41:23.146777Z","iopub.status.idle":"2022-07-30T15:41:23.152747Z","shell.execute_reply.started":"2022-07-30T15:41:23.146719Z","shell.execute_reply":"2022-07-30T15:41:23.151717Z"}}},{"cell_type":"code","source":"test[\"R_7\"].fillna(value=test[\"R_7\"].median(),inplace=True)\ntest[\"R_7\"].fillna(value=test[\"R_12\"].mean(),inplace=True)\ntest[\"R_7\"].fillna(value=test[\"R_14\"].median(),inplace=True)\ntest[\"R_7\"].fillna(value=test[\"R_27\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:04.085905Z","iopub.execute_input":"2022-08-07T11:47:04.086770Z","iopub.status.idle":"2022-08-07T11:47:04.468755Z","shell.execute_reply.started":"2022-08-07T11:47:04.086722Z","shell.execute_reply":"2022-08-07T11:47:04.467115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4.2.3 Balance Variables","metadata":{}},{"cell_type":"code","source":"test.loc[test[\"B_37\"].isnull() & ~test[\"B_25\"].isnull(),\"B_37\"]=(test[\"B_37\"].corr(test[\"B_25\"])*test[\"B_37\"].std()/test[\"B_25\"].std())*(test.loc[test[\"B_37\"].isnull() & ~test[\"B_25\"].isnull(),\"B_25\"]-test[\"B_25\"].mean())+test[\"B_37\"].mean()\ntest.loc[~test[\"B_37\"].isnull() & test[\"B_25\"].isnull(),\"B_25\"]=(test[\"B_37\"].corr(test[\"B_25\"])*test[\"B_25\"].std()/test[\"B_37\"].std())*(test.loc[~test[\"B_37\"].isnull() & test[\"B_25\"].isnull(),\"B_37\"]-test[\"B_37\"].mean())+test[\"B_25\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:04.473925Z","iopub.execute_input":"2022-08-07T11:47:04.474383Z","iopub.status.idle":"2022-08-07T11:47:05.854888Z","shell.execute_reply.started":"2022-08-07T11:47:04.474345Z","shell.execute_reply":"2022-08-07T11:47:05.853570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"B_37\"].fillna(test[\"B_37\"].mean(),inplace=True)\ntest[\"B_25\"].fillna(test[\"B_25\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:05.856573Z","iopub.execute_input":"2022-08-07T11:47:05.857311Z","iopub.status.idle":"2022-08-07T11:47:05.984295Z","shell.execute_reply.started":"2022-08-07T11:47:05.857264Z","shell.execute_reply":"2022-08-07T11:47:05.982960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.loc[test[\"B_13\"].isnull() & ~test[\"B_15\"].isnull(),\"B_13\"]=(test[\"B_13\"].corr(test[\"B_15\"])*test[\"B_13\"].std()/test[\"B_15\"].std())*(test.loc[test[\"B_13\"].isnull() & ~test[\"B_15\"].isnull(),\"B_15\"]-test[\"B_15\"].mean())+test[\"B_13\"].mean()\ntest.loc[~test[\"B_13\"].isnull() & test[\"B_15\"].isnull(),\"B_15\"]=(test[\"B_13\"].corr(test[\"B_15\"])*test[\"B_15\"].std()/test[\"B_13\"].std())*(test.loc[~test[\"B_13\"].isnull() & test[\"B_15\"].isnull(),\"B_13\"]-test[\"B_13\"].mean())+test[\"B_15\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:05.986206Z","iopub.execute_input":"2022-08-07T11:47:05.987828Z","iopub.status.idle":"2022-08-07T11:47:07.344853Z","shell.execute_reply.started":"2022-08-07T11:47:05.987776Z","shell.execute_reply":"2022-08-07T11:47:07.343712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fill the remaning null values.","metadata":{}},{"cell_type":"code","source":"test[\"B_13\"].fillna(test[\"B_13\"].mean(),inplace=True)\ntest[\"B_15\"].fillna(test[\"B_15\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:07.346466Z","iopub.execute_input":"2022-08-07T11:47:07.347132Z","iopub.status.idle":"2022-08-07T11:47:07.475510Z","shell.execute_reply.started":"2022-08-07T11:47:07.347087Z","shell.execute_reply":"2022-08-07T11:47:07.474253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thus B_13 and B_15 is handled.","metadata":{}},{"cell_type":"markdown","source":"Calculating Values of B_2 and B_3 from B_37","metadata":{}},{"cell_type":"code","source":"test.loc[test[\"B_2\"].isnull() & ~test[\"B_37\"].isnull(),\"B_2\"]=(test[\"B_2\"].corr(test[\"B_37\"])*test[\"B_2\"].std()/test[\"B_37\"].std())*(test.loc[test[\"B_2\"].isnull() & ~test[\"B_37\"].isnull(),\"B_37\"]-test[\"B_37\"].mean())+test[\"B_2\"].mean()\ntest.loc[test[\"B_3\"].isnull() & ~test[\"B_37\"].isnull(),\"B_3\"]=(test[\"B_3\"].corr(test[\"B_37\"])*test[\"B_3\"].std()/test[\"B_37\"].std())*(test.loc[test[\"B_3\"].isnull() & ~test[\"B_37\"].isnull(),\"B_37\"]-test[\"B_37\"].mean())+test[\"B_3\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:07.477179Z","iopub.execute_input":"2022-08-07T11:47:07.478154Z","iopub.status.idle":"2022-08-07T11:47:08.774967Z","shell.execute_reply.started":"2022-08-07T11:47:07.478103Z","shell.execute_reply":"2022-08-07T11:47:08.773664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculating B_17 from B_2","metadata":{}},{"cell_type":"code","source":"test.loc[test[\"B_17\"].isnull() & ~test[\"B_2\"].isnull(),\"B_17\"]=(test[\"B_17\"].corr(test[\"B_2\"])*test[\"B_17\"].std()/test[\"B_2\"].std())*(test.loc[test[\"B_17\"].isnull() & ~test[\"B_2\"].isnull(),\"B_2\"]-test[\"B_2\"].mean())+test[\"B_17\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:08.776849Z","iopub.execute_input":"2022-08-07T11:47:08.777369Z","iopub.status.idle":"2022-08-07T11:47:09.672398Z","shell.execute_reply.started":"2022-08-07T11:47:08.777308Z","shell.execute_reply":"2022-08-07T11:47:09.671069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.2.4 Payment Variables","metadata":{}},{"cell_type":"code","source":"test[\"P_2\"].fillna(value=test[\"P_2\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:09.674095Z","iopub.execute_input":"2022-08-07T11:47:09.674582Z","iopub.status.idle":"2022-08-07T11:47:09.733934Z","shell.execute_reply.started":"2022-08-07T11:47:09.674528Z","shell.execute_reply":"2022-08-07T11:47:09.732540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"P_2 variable is handled","metadata":{}},{"cell_type":"code","source":"test[\"P_3\"].fillna(value=test[\"P_3\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:09.735851Z","iopub.execute_input":"2022-08-07T11:47:09.737029Z","iopub.status.idle":"2022-08-07T11:47:09.803945Z","shell.execute_reply.started":"2022-08-07T11:47:09.736981Z","shell.execute_reply":"2022-08-07T11:47:09.802232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"P_3 column is free of any null values","metadata":{}},{"cell_type":"markdown","source":"# 2.2.5 Delinquency Variables","metadata":{}},{"cell_type":"code","source":"test.loc[test[\"D_43\"].isnull() & ~test[\"R_3\"].isnull(),\"D_43\"]=(test[\"D_43\"].corr(test[\"R_3\"])*test[\"D_43\"].std()/test[\"R_3\"].std())*(test.loc[test[\"D_43\"].isnull() & ~test[\"R_3\"].isnull(),\"R_3\"]-test[\"R_3\"].mean())+test[\"D_43\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:09.805951Z","iopub.execute_input":"2022-08-07T11:47:09.806496Z","iopub.status.idle":"2022-08-07T11:47:10.541297Z","shell.execute_reply.started":"2022-08-07T11:47:09.806447Z","shell.execute_reply":"2022-08-07T11:47:10.540061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"D_46\"].fillna(value=test[\"D_46\"].mean(),inplace=True)\ntest[\"D_62\"].fillna(value=test[\"D_62\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:10.542914Z","iopub.execute_input":"2022-08-07T11:47:10.543246Z","iopub.status.idle":"2022-08-07T11:47:10.707240Z","shell.execute_reply.started":"2022-08-07T11:47:10.543217Z","shell.execute_reply":"2022-08-07T11:47:10.706105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.loc[test[\"D_48\"].isnull() & ~test[\"P_2\"].isnull(),\"D_48\"]=(test[\"D_48\"].corr(test[\"P_2\"])*test[\"D_48\"].std()/test[\"P_2\"].std())*(test.loc[test[\"D_48\"].isnull() & ~test[\"P_2\"].isnull(),\"P_2\"]-test[\"P_2\"].mean())+test[\"D_48\"].mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:10.708789Z","iopub.execute_input":"2022-08-07T11:47:10.709123Z","iopub.status.idle":"2022-08-07T11:47:11.484339Z","shell.execute_reply.started":"2022-08-07T11:47:10.709093Z","shell.execute_reply":"2022-08-07T11:47:11.483159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"D_61\"]=test[\"D_61\"]/(test[\"D_61\"].max()-test[\"D_61\"].min())","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:11.486249Z","iopub.execute_input":"2022-08-07T11:47:11.486760Z","iopub.status.idle":"2022-08-07T11:47:11.609181Z","shell.execute_reply.started":"2022-08-07T11:47:11.486714Z","shell.execute_reply":"2022-08-07T11:47:11.608119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"D_61\"].fillna(value=test[\"D_61\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:11.610468Z","iopub.execute_input":"2022-08-07T11:47:11.611264Z","iopub.status.idle":"2022-08-07T11:47:11.684869Z","shell.execute_reply.started":"2022-08-07T11:47:11.611205Z","shell.execute_reply":"2022-08-07T11:47:11.683750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"D_69\"]=test[\"D_69\"]/(test[\"D_69\"].max()-test[\"D_69\"].min())","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:11.690445Z","iopub.execute_input":"2022-08-07T11:47:11.690841Z","iopub.status.idle":"2022-08-07T11:47:11.800707Z","shell.execute_reply.started":"2022-08-07T11:47:11.690806Z","shell.execute_reply":"2022-08-07T11:47:11.799265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"D_69\"].fillna(value=test[\"D_69\"].mean(),inplace=True)\ntest[\"D_55\"].fillna(value=test[\"D_55\"].mean(),inplace=True)\ntest[\"D_118\"].fillna(value=test[\"D_118\"].mean(),inplace=True)\ntest[\"D_121\"].fillna(value=test[\"D_121\"].mean(),inplace=True)\ntest[\"D_115\"].fillna(value=test[\"D_115\"].mean(),inplace=True)\ntest[\"D_119\"].fillna(value=test[\"D_119\"].mean(),inplace=True)\ntest[\"D_131\"].fillna(value=0,inplace=True)\ntest[\"D_130\"].fillna(value=0,inplace=True)\ntest[\"D_104\"].fillna(value=0,inplace=True)\ntest[\"D_141\"].fillna(value=0,inplace=True)\ntest[\"D_128\"].fillna(value=0,inplace=True)\ntest[\"D_133\"].fillna(value=test[\"D_133\"].mean(),inplace=True)\ntest[\"D_144\"].fillna(value=test[\"D_144\"].mean(),inplace=True)\ntest[\"D_102\"].fillna(value=test[\"D_102\"].mean(),inplace=True)\ntest[\"D_52\"].fillna(value=test[\"D_52\"].mean(),inplace=True)\ntest[\"D_112\"].fillna(value=1,inplace=True)\ntest[\"D_45\"].fillna(value=test[\"D_45\"].mean(),inplace=True)\ntest[\"D_54\"].fillna(value=1,inplace=True)\ntest[\"D_41\"].fillna(value=0,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:11.802528Z","iopub.execute_input":"2022-08-07T11:47:11.802860Z","iopub.status.idle":"2022-08-07T11:47:12.502588Z","shell.execute_reply.started":"2022-08-07T11:47:11.802831Z","shell.execute_reply":"2022-08-07T11:47:12.501564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2.2.6 Remaining Unhandled Variables","metadata":{}},{"cell_type":"markdown","source":"Check for unhandled columns.","metadata":{}},{"cell_type":"code","source":"test.isnull().sum().sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:12.504074Z","iopub.execute_input":"2022-08-07T11:47:12.504441Z","iopub.status.idle":"2022-08-07T11:47:16.298382Z","shell.execute_reply.started":"2022-08-07T11:47:12.504409Z","shell.execute_reply":"2022-08-07T11:47:16.297356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"S_27\"].fillna(value=test[\"S_27\"].mean(),inplace=True)\ntest[\"R_27\"].fillna(value=test[\"R_27\"].mean(),inplace=True)\ntest[\"R_27\"].fillna(value=0,inplace=True)\ntest[\"B_27\"].fillna(value=test[\"B_27\"].mean(),inplace=True)\ntest[\"B_26\"].fillna(value=test[\"B_26\"].mean(),inplace=True)\ntest[\"B_6\"].fillna(value=test[\"B_6\"].mean(),inplace=True)\ntest[\"B_8\"].fillna(value=test[\"B_8\"].mean(),inplace=True)\ntest[\"B_40\"].fillna(value=test[\"B_40\"].mean(),inplace=True)\ntest[\"R_12\"].fillna(value=test[\"R_12\"].mean(),inplace=True)\ntest[\"R_14\"].fillna(value=test[\"R_14\"].mean(),inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:16.299551Z","iopub.execute_input":"2022-08-07T11:47:16.299860Z","iopub.status.idle":"2022-08-07T11:47:16.788100Z","shell.execute_reply.started":"2022-08-07T11:47:16.299832Z","shell.execute_reply":"2022-08-07T11:47:16.786834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:16.789544Z","iopub.execute_input":"2022-08-07T11:47:16.789878Z","iopub.status.idle":"2022-08-07T11:47:16.939477Z","shell.execute_reply.started":"2022-08-07T11:47:16.789849Z","shell.execute_reply":"2022-08-07T11:47:16.938143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3 Upload the Data","metadata":{}},{"cell_type":"markdown","source":"We will upload the data and download it in chuncks as there is memory constrains.","metadata":{}},{"cell_type":"code","source":"test.to_csv(\"test.csv.gzip\",compression=\"gzip\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T11:47:16.940751Z","iopub.execute_input":"2022-08-07T11:47:16.941099Z","iopub.status.idle":"2022-08-07T13:23:53.387383Z","shell.execute_reply.started":"2022-08-07T11:47:16.941045Z","shell.execute_reply":"2022-08-07T13:23:53.382040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test","metadata":{"execution":{"iopub.status.busy":"2022-08-07T13:23:53.393577Z","iopub.execute_input":"2022-08-07T13:23:53.394466Z","iopub.status.idle":"2022-08-07T13:23:53.712523Z","shell.execute_reply.started":"2022-08-07T13:23:53.394402Z","shell.execute_reply":"2022-08-07T13:23:53.711329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect() ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T13:23:53.714387Z","iopub.execute_input":"2022-08-07T13:23:53.716118Z","iopub.status.idle":"2022-08-07T13:23:54.197353Z","shell.execute_reply.started":"2022-08-07T13:23:53.716062Z","shell.execute_reply":"2022-08-07T13:23:54.195607Z"},"trusted":true},"execution_count":null,"outputs":[]}]}