{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook to create smaller and manageable train,test data for future experimentation","metadata":{}},{"cell_type":"markdown","source":"In this notebook, I use libraries Pandas and Pickle to reduce the size of the training and test data which can be used by the community to work on this competition.\n\n**If you like my work, please upvote the kernel !!**","metadata":{}},{"cell_type":"code","source":"#All imports goes here\nimport numpy as np \nimport pandas as pd \nimport os\nimport pickle\nimport glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-25T18:21:29.603736Z","iopub.execute_input":"2022-05-25T18:21:29.604163Z","iopub.status.idle":"2022-05-25T18:21:29.608611Z","shell.execute_reply.started":"2022-05-25T18:21:29.604129Z","shell.execute_reply":"2022-05-25T18:21:29.607542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = \"../input/amex-default-prediction/train_data.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:21:30.419907Z","iopub.execute_input":"2022-05-25T18:21:30.420428Z","iopub.status.idle":"2022-05-25T18:21:30.424479Z","shell.execute_reply.started":"2022-05-25T18:21:30.420396Z","shell.execute_reply":"2022-05-25T18:21:30.423451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_df = pd.read_csv(fn,nrows=100)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:26:19.556734Z","iopub.execute_input":"2022-05-25T18:26:19.558238Z","iopub.status.idle":"2022-05-25T18:26:19.600735Z","shell.execute_reply.started":"2022-05-25T18:26:19.558136Z","shell.execute_reply":"2022-05-25T18:26:19.599691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_cols = train_df.columns.to_list()\ncat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nstr_cols = [\"customer_ID\"]\ndate_cols = [\"S_2\"]\n\ndf_dtype = {col:\"float16\" for col in all_cols if col not in cat_cols + str_cols + date_cols}\n\nfor col in cat_cols:\n    df_dtype[col] = \"category\"\n    \ndf_dtype[\"customer_ID\"] = \"str\"","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:26:24.681116Z","iopub.execute_input":"2022-05-25T18:26:24.681630Z","iopub.status.idle":"2022-05-25T18:26:24.687554Z","shell.execute_reply.started":"2022-05-25T18:26:24.681588Z","shell.execute_reply":"2022-05-25T18:26:24.686289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I got the idea and the code from the below link: \n\nhttps://stackoverflow.com/questions/25962114/how-do-i-read-a-large-csv-file-with-pandas","metadata":{}},{"cell_type":"code","source":"%%time\n\nout_path = \"/kaggle/working\" #Path to save the pickle files to\nchunk_size = 400000 #size of chunks relies on your available memory\n\n\nreader = pd.read_csv(fn,chunksize=chunk_size, low_memory=False,dtype=df_dtype,parse_dates=[1])    \n\n\nfor i, chunk in enumerate(reader):\n    out_file = out_path + \"/data_{}.pkl\".format(i+1)\n    with open(out_file, \"wb\") as f:\n        pickle.dump(chunk,f,pickle.HIGHEST_PROTOCOL)\n        print(f\"Written chunk {i+1}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:28:09.654783Z","iopub.execute_input":"2022-05-25T18:28:09.655383Z","iopub.status.idle":"2022-05-25T18:36:49.335934Z","shell.execute_reply.started":"2022-05-25T18:28:09.655350Z","shell.execute_reply":"2022-05-25T18:36:49.333529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle_path = \"/kaggle/working\" #Same Path as out_path i.e. where the pickle files are\n\ndata_p_files=[]\nfor name in glob.glob(pickle_path + \"/data_*.pkl\"):\n    data_p_files.append(name)\n\n\ntrain_df = pd.DataFrame([])\nfor i in range(len(data_p_files)):\n    train_df = train_df.append(pd.read_pickle(data_p_files[i]),ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:37:54.109279Z","iopub.execute_input":"2022-05-25T18:37:54.110222Z","iopub.status.idle":"2022-05-25T18:38:13.910293Z","shell.execute_reply.started":"2022-05-25T18:37:54.110124Z","shell.execute_reply":"2022-05-25T18:38:13.909063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:38:23.216098Z","iopub.execute_input":"2022-05-25T18:38:23.216658Z","iopub.status.idle":"2022-05-25T18:38:23.223558Z","shell.execute_reply.started":"2022-05-25T18:38:23.216625Z","shell.execute_reply":"2022-05-25T18:38:23.222568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(\"../input/amex-default-prediction/train_labels.csv\")\ntrain_labels.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:39:22.825197Z","iopub.execute_input":"2022-05-25T18:39:22.825687Z","iopub.status.idle":"2022-05-25T18:39:23.780374Z","shell.execute_reply.started":"2022-05-25T18:39:22.825653Z","shell.execute_reply":"2022-05-25T18:39:23.779177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:48:28.601520Z","iopub.execute_input":"2022-05-25T18:48:28.602236Z","iopub.status.idle":"2022-05-25T18:48:28.608474Z","shell.execute_reply.started":"2022-05-25T18:48:28.602198Z","shell.execute_reply":"2022-05-25T18:48:28.607482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_df = train_df.merge(train_labels,on=\"customer_ID\",how=\"left\")\ntrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:53:07.590885Z","iopub.execute_input":"2022-05-25T18:53:07.591335Z","iopub.status.idle":"2022-05-25T18:54:52.957460Z","shell.execute_reply.started":"2022-05-25T18:53:07.591300Z","shell.execute_reply":"2022-05-25T18:54:52.956184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_pickle(\"amex_train_data.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T18:57:24.279013Z","iopub.execute_input":"2022-05-25T18:57:24.279444Z","iopub.status.idle":"2022-05-25T18:57:29.250317Z","shell.execute_reply.started":"2022-05-25T18:57:24.279412Z","shell.execute_reply":"2022-05-25T18:57:29.249232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Clean up the output area\n\nfor i in range(14):\n    _ = os.system(f\"rm /kaggle/working/data_{i+1}.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:00:24.892004Z","iopub.execute_input":"2022-05-25T19:00:24.892425Z","iopub.status.idle":"2022-05-25T19:00:24.929665Z","shell.execute_reply.started":"2022-05-25T19:00:24.892392Z","shell.execute_reply":"2022-05-25T19:00:24.928729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfn_test = \"../input/amex-default-prediction/test_data.csv\"\ntest_df = pd.read_csv(fn_test,nrows=100)\nprint(test_df.shape)\ntest_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:02:07.052440Z","iopub.execute_input":"2022-05-25T19:02:07.052978Z","iopub.status.idle":"2022-05-25T19:02:07.110726Z","shell.execute_reply.started":"2022-05-25T19:02:07.052940Z","shell.execute_reply":"2022-05-25T19:02:07.109867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nout_path = \"/kaggle/working\" #Path to save the pickle files to\nchunk_size = 400000 #size of chunks relies on your available memory\n\n\nreader = pd.read_csv(fn_test,chunksize=chunk_size, low_memory=False,dtype=df_dtype,parse_dates=[1])    \n\n\nfor i, chunk in enumerate(reader):\n    out_file = out_path + \"/data_test{}.pkl\".format(i+1)\n    with open(out_file, \"wb\") as f:\n        pickle.dump(chunk,f,pickle.HIGHEST_PROTOCOL)\n        print(f\"Written chunk {i+1}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:07:40.569695Z","iopub.execute_input":"2022-05-25T19:07:40.570199Z","iopub.status.idle":"2022-05-25T19:26:07.276294Z","shell.execute_reply.started":"2022-05-25T19:07:40.570163Z","shell.execute_reply":"2022-05-25T19:26:07.275037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pickle_path = \"/kaggle/working\" #Same Path as out_path i.e. where the pickle files are\n\ndata_p_files=[]\nfor name in glob.glob(pickle_path + \"/data_test*.pkl\"):\n    data_p_files.append(name)\n\n\ntest_df = pd.DataFrame([])\nfor i in range(len(data_p_files)):\n    test_df = test_df.append(pd.read_pickle(data_p_files[i]),ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:26:24.371006Z","iopub.execute_input":"2022-05-25T19:26:24.371605Z","iopub.status.idle":"2022-05-25T19:27:51.075675Z","shell.execute_reply.started":"2022-05-25T19:26:24.371568Z","shell.execute_reply":"2022-05-25T19:27:51.074389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:27:58.603406Z","iopub.execute_input":"2022-05-25T19:27:58.603869Z","iopub.status.idle":"2022-05-25T19:27:58.610838Z","shell.execute_reply.started":"2022-05-25T19:27:58.603829Z","shell.execute_reply":"2022-05-25T19:27:58.609824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_pickle(\"amex_test_data.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:28:04.153658Z","iopub.execute_input":"2022-05-25T19:28:04.154575Z","iopub.status.idle":"2022-05-25T19:28:20.116274Z","shell.execute_reply.started":"2022-05-25T19:28:04.154526Z","shell.execute_reply":"2022-05-25T19:28:20.115161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Clean up the output area\n\nfor i in range(29):\n    _ = os.system(f\"rm /kaggle/working/data_test{i+1}.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-05-25T19:28:36.561432Z","iopub.execute_input":"2022-05-25T19:28:36.561855Z","iopub.status.idle":"2022-05-25T19:28:36.754437Z","shell.execute_reply.started":"2022-05-25T19:28:36.561822Z","shell.execute_reply":"2022-05-25T19:28:36.753702Z"},"trusted":true},"execution_count":null,"outputs":[]}]}