{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\n\nfrom pathlib import Path\nimport pyarrow as pa\nimport pyarrow.parquet as pq","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-26T19:13:58.381733Z","iopub.execute_input":"2022-06-26T19:13:58.382325Z","iopub.status.idle":"2022-06-26T19:13:58.387942Z","shell.execute_reply.started":"2022-06-26T19:13:58.382286Z","shell.execute_reply":"2022-06-26T19:13:58.387229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = Path(\"../input/amex-default-prediction\")\nTRAIN_CSV = Path(\"train_data.csv\")\nTRAIN_LABEL_CSV = Path(\"train_labels.csv\")\nTEST_CSV = Path('test_data.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:13:59.004885Z","iopub.execute_input":"2022-06-26T19:13:59.005318Z","iopub.status.idle":"2022-06-26T19:13:59.010398Z","shell.execute_reply.started":"2022-06-26T19:13:59.005281Z","shell.execute_reply":"2022-06-26T19:13:59.009533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(ROOT_DIR / TRAIN_CSV, chunksize=1)\ncat = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\ncont_vars = list(set(df.__next__().columns).difference(set(cat)))\ncont_vars.remove('customer_ID')\ncont_vars.remove('S_2')","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:13:59.828008Z","iopub.execute_input":"2022-06-26T19:13:59.828576Z","iopub.status.idle":"2022-06-26T19:13:59.854111Z","shell.execute_reply.started":"2022-06-26T19:13:59.828543Z","shell.execute_reply":"2022-06-26T19:13:59.853089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_to_parquet(input_path, output_path, chunksize = 15000):\n   \n    pq_writer = None\n    \n    for idx, df_chunk in enumerate(pd.read_csv(input_path, chunksize=chunksize)):\n        print(f\"id: {idx} Chunk size {df_chunk.shape}\")\n        df_chunk[cont_vars] = df_chunk[cont_vars].astype('float32')\n\n        # change the hash string to integer. Reference :https://www.kaggle.com/competitions/amex-default-prediction/discussion/328054\n        df_chunk.customer_ID = df_chunk.customer_ID.apply(lambda x: int(x[-16:],16)).astype('int64')\n\n        #convert categorical data to type categorical in pandas.\n        df_chunk[cat] = df_chunk[cat].astype('category')\n\n        df_chunk['S_2'] = pd.to_datetime(df_chunk['S_2'])\n\n        table = pa.Table.from_pandas(df_chunk)\n        if idx == 0:\n            pq_writer = pq.ParquetWriter(output_path, table.schema, compression = 'snappy')\n        \n        pq_writer.write_table(table)\n        \n\n        # Removing current chunk from meory to free up memory\n        del df_chunk\n        del table\n        gc.collect()\n        \n  \n    if pq_writer:\n        pq_writer.close()\n    gc.collect()\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:14:00.693745Z","iopub.execute_input":"2022-06-26T19:14:00.694188Z","iopub.status.idle":"2022-06-26T19:14:00.705755Z","shell.execute_reply.started":"2022-06-26T19:14:00.694153Z","shell.execute_reply":"2022-06-26T19:14:00.70479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convert_to_parquet(input_path=  ROOT_DIR / TRAIN_CSV, output_path='./train.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:11:53.676987Z","iopub.execute_input":"2022-06-26T19:11:53.677748Z","iopub.status.idle":"2022-06-26T19:12:06.544721Z","shell.execute_reply.started":"2022-06-26T19:11:53.677682Z","shell.execute_reply":"2022-06-26T19:12:06.543489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"convert_to_parquet(input_path=  ROOT_DIR / TEST_CSV, output_path='./test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-06-26T19:01:15.84948Z","iopub.execute_input":"2022-06-26T19:01:15.850302Z","iopub.status.idle":"2022-06-26T19:01:15.870173Z","shell.execute_reply.started":"2022-06-26T19:01:15.850246Z","shell.execute_reply":"2022-06-26T19:01:15.869095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_label_df = pd.read_csv(ROOT_DIR / TRAIN_LABEL_CSV)\ntrain_label_df.customer_ID = train_label_df.customer_ID.apply(lambda x: int(x[-16:],16)).astype('int64')\ntrain_label_df.to_parquet('./train_label.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-06-26T18:46:04.715634Z","iopub.status.idle":"2022-06-26T18:46:04.716307Z","shell.execute_reply.started":"2022-06-26T18:46:04.716038Z","shell.execute_reply":"2022-06-26T18:46:04.716064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}