{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport gc\n\ndef chunks(lst, n):\n    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n    for i in range(0, len(lst), n):\n        yield lst[i:i + n]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-04T10:25:45.000345Z","iopub.execute_input":"2022-06-04T10:25:45.000960Z","iopub.status.idle":"2022-06-04T10:25:45.033334Z","shell.execute_reply.started":"2022-06-04T10:25:45.000877Z","shell.execute_reply":"2022-06-04T10:25:45.032299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NB_CID_PER_CHUNK  = 100_000\nTRAIN_FPATH       = '../input/amex-data-integer-dtypes-parquet-format/train.parquet'\nTEST_FPATH        = '../input/amex-data-integer-dtypes-parquet-format/test.parquet'\ncolumns           = ['customer_ID', 'S_2', 'P_2', 'D_39', 'B_1', 'B_2', 'R_1', 'S_3', 'D_41', 'B_3', 'D_42', 'D_43', 'D_44', 'B_4', 'D_45', 'B_5', 'R_2', 'D_46', 'D_47', 'D_48', 'D_49', 'B_6', 'B_7', 'B_8', 'D_50', 'D_51', 'B_9', 'R_3', 'D_52', 'P_3', 'B_10', 'D_53', 'S_5', 'B_11', 'S_6', 'D_54', 'R_4', 'S_7', 'B_12', 'S_8', 'D_55', 'D_56', 'B_13', 'R_5', 'D_58', 'S_9', 'B_14', 'D_59', 'D_60', 'D_61', 'B_15', 'S_11', 'D_62', 'D_63', 'D_64', 'D_65', 'B_16', 'B_17', 'B_18', 'B_19', 'D_66', 'B_20', 'D_68', 'S_12', 'R_6', 'S_13', 'B_21', 'D_69', 'B_22', 'D_70', 'D_71', 'D_72', 'S_15', 'B_23', 'D_73', 'P_4', 'D_74', 'D_75', 'D_76', 'B_24', 'R_7', 'D_77', 'B_25', 'B_26', 'D_78', 'D_79', 'R_8', 'R_9', 'S_16', 'D_80', 'R_10', 'R_11', 'B_27', 'D_81', 'D_82', 'S_17', 'R_12', 'B_28', 'R_13', 'D_83', 'R_14', 'R_15', 'D_84', 'R_16', 'B_29', 'B_30', 'S_18', 'D_86', 'D_87', 'R_17', 'R_18', 'D_88', 'B_31', 'S_19', 'R_19', 'B_32', 'S_20', 'R_20', 'R_21', 'B_33', 'D_89', 'R_22', 'R_23', 'D_91', 'D_92', 'D_93', 'D_94', 'R_24', 'R_25', 'D_96', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'D_102', 'D_103', 'D_104', 'D_105', 'D_106', 'D_107', 'B_36', 'B_37', 'R_26', 'R_27', 'B_38', 'D_108', 'D_109', 'D_110', 'D_111', 'B_39', 'D_112', 'B_40', 'S_27', 'D_113', 'D_114', 'D_115', 'D_116', 'D_117', 'D_118', 'D_119', 'D_120', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_127', 'D_128', 'D_129', 'B_41', 'B_42', 'D_130', 'D_131', 'D_132', 'D_133', 'R_28', 'D_134', 'D_135', 'D_136', 'D_137', 'D_138', 'D_139', 'D_140', 'D_141', 'D_142', 'D_143', 'D_144', 'D_145']\ncid_column        = columns[0:1]\nfeatures_columns  = columns[1:]","metadata":{"execution":{"iopub.status.busy":"2022-06-04T10:25:45.067826Z","iopub.execute_input":"2022-06-04T10:25:45.068174Z","iopub.status.idle":"2022-06-04T10:25:45.081901Z","shell.execute_reply.started":"2022-06-04T10:25:45.068127Z","shell.execute_reply":"2022-06-04T10:25:45.081168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def chunkify(train_or_test = 'train'):\n    if train_or_test == 'train':\n        fpath = TRAIN_FPATH\n    elif train_or_test == 'test':\n        fpath = TEST_FPATH\n    else:\n        raise Exception('train_or_test = \"train\" or \"test\"')\n    \n    # Read the parquet to get all_cids\n    cids = pd.read_parquet(fpath, columns = cid_column)[cid_column[0]].unique()\n    \n    # Split each cids into chunks of size NB_CID_PER_CHUNK\n    for i, cid_chunk in enumerate(chunks(cids, NB_CID_PER_CHUNK)):\n        dfs = []\n        for features in chunks(features_columns, 30):\n            df = pd.read_parquet(fpath, columns=cid_column+features)\n            df = df.query('customer_ID in @cid_chunk')\n            dfs.append(df)\n\n        chunk_df = pd.concat(dfs, axis=1)\n        # Remove duplicates columns\n        chunk_df = chunk_df.loc[:,~chunk_df.columns.duplicated()]\n        chunk_df.to_parquet(f'{train_or_test}_chunk_{i}.parquet')\n        del df, dfs, chunk_df\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-04T10:25:45.126201Z","iopub.execute_input":"2022-06-04T10:25:45.126655Z","iopub.status.idle":"2022-06-04T10:25:45.133050Z","shell.execute_reply.started":"2022-06-04T10:25:45.126632Z","shell.execute_reply":"2022-06-04T10:25:45.132321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunkify('train')\nchunkify('test')","metadata":{"execution":{"iopub.status.busy":"2022-06-04T10:25:45.173655Z","iopub.execute_input":"2022-06-04T10:25:45.174413Z","iopub.status.idle":"2022-06-04T10:34:42.106182Z","shell.execute_reply.started":"2022-06-04T10:25:45.174367Z","shell.execute_reply":"2022-06-04T10:34:42.104970Z"},"trusted":true},"execution_count":null,"outputs":[]}]}