{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Objectives\n\nIn order to train [this model](https://www.kaggle.com/code/ragnar123/amex-lgbm-dart-cv-0-7977) on Kaggle Kernels with 16GB RAM instead of seeking for 32GB RAM machine, I performed some optimizations, hope this will help guys with limited resources.\n\n[Part 2](https://www.kaggle.com/code/pham0030/amex-outofmemory-fe-with-lgb-sequence-part2-train) is about training dataset preparation for lgb.Sequence training out of memory.\n\n[Part 3 ](https://www.kaggle.com/code/pham0030/amex-outofmemory-fe-with-lgb-sequence-part3-final) is lgb.Sequence training out of memory.\n\nThank you very much for your work [@ragnar](https://www.kaggle.com/ragnar123).","metadata":{}},{"cell_type":"markdown","source":"## Part 1: Test Data Processing","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport pandas as pd\nimport numpy as np\nfrom tqdm.auto import tqdm\n# add progress_apply method to DataFrame objects\ntqdm.pandas()\n\n## hardcoded features to reduce memory read\nfeatures = [\"customer_ID\", \"S_2\", \"P_2\", \"D_39\", \"B_1\", \"B_2\", \"R_1\", \"S_3\", \"D_41\", \"B_3\", \"D_42\", \"D_43\", \n            \"D_44\", \"B_4\", \"D_45\", \"B_5\", \"R_2\", \"D_46\", \"D_47\", \"D_48\", \"D_49\", \"B_6\", \n            \"B_7\", \"B_8\", \"D_50\", \"D_51\", \"B_9\", \"R_3\", \"D_52\", \"P_3\", \"B_10\", \"D_53\", \n            \"S_5\", \"B_11\", \"S_6\", \"D_54\", \"R_4\", \"S_7\", \"B_12\", \"S_8\", \"D_55\", \"D_56\", \n            \"B_13\", \"R_5\", \"D_58\", \"S_9\", \"B_14\", \"D_59\", \"D_60\", \"D_61\", \"B_15\", \"S_11\", \n            \"D_62\", \"D_63\", \"D_64\", \"D_65\", \"B_16\", \"B_17\", \"B_18\", \"B_19\", \"D_66\", \"B_20\", \n            \"D_68\", \"S_12\", \"R_6\", \"S_13\", \"B_21\", \"D_69\", \"B_22\", \"D_70\", \"D_71\", \"D_72\", \n            \"S_15\", \"B_23\", \"D_73\", \"P_4\", \"D_74\", \"D_75\", \"D_76\", \"B_24\", \"R_7\", \"D_77\", \n            \"B_25\", \"B_26\", \"D_78\", \"D_79\", \"R_8\", \"R_9\", \"S_16\", \"D_80\", \"R_10\", \"R_11\", \n            \"B_27\", \"D_81\", \"D_82\", \"S_17\", \"R_12\", \"B_28\", \"R_13\", \"D_83\", \"R_14\", \"R_15\", \n            \"D_84\", \"R_16\", \"B_29\", \"B_30\", \"S_18\", \"D_86\", \"D_87\", \"R_17\", \"R_18\", \"D_88\", \n            \"B_31\", \"S_19\", \"R_19\", \"B_32\", \"S_20\", \"R_20\", \"R_21\", \"B_33\", \"D_89\", \"R_22\", \n            \"R_23\", \"D_91\", \"D_92\", \"D_93\", \"D_94\", \"R_24\", \"R_25\", \"D_96\", \"S_22\", \"S_23\", \n            \"S_24\", \"S_25\", \"S_26\", \"D_102\", \"D_103\", \"D_104\", \"D_105\", \"D_106\", \"D_107\", \n            \"B_36\", \"B_37\", \"R_26\", \"R_27\", \"B_38\", \"D_108\", \"D_109\", \"D_110\", \"D_111\", \n            \"B_39\", \"D_112\", \"B_40\", \"S_27\", \"D_113\", \"D_114\", \"D_115\", \"D_116\", \"D_117\", \n            \"D_118\", \"D_119\", \"D_120\", \"D_121\", \"D_122\", \"D_123\", \"D_124\", \"D_125\", \"D_126\", \n            \"D_127\", \"D_128\", \"D_129\", \"B_41\", \"B_42\", \"D_130\", \"D_131\", \"D_132\", \"D_133\", \n            \"R_28\", \"D_134\", \"D_135\", \"D_136\", \"D_137\", \"D_138\", \"D_139\", \"D_140\", \"D_141\", \n            \"D_142\", \"D_143\", \"D_144\", \"D_145\"]\n\nfeatures.remove('customer_ID')\nfeatures.remove('S_2')\ncat_features = [\n    \"B_30\",\n    \"B_38\",\n    \"D_114\",\n    \"D_116\",\n    \"D_117\",\n    \"D_120\",\n    \"D_126\",\n    \"D_63\",\n    \"D_64\",\n    \"D_66\",\n    \"D_68\",\n]\nnum_features = [col for col in features if col not in cat_features]\nprint(f'Number of original features (excludes customer_ID, S_2): {len(features)}')\nprint(f'Number of categorical features: {len(cat_features)}')\nprint(f'Number of numeric features: {len(num_features)}')","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:45:20.705530Z","iopub.execute_input":"2022-07-13T15:45:20.706236Z","iopub.status.idle":"2022-07-13T15:45:20.857285Z","shell.execute_reply.started":"2022-07-13T15:45:20.706115Z","shell.execute_reply":"2022-07-13T15:45:20.856484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If you load the whole test dataset into pandas dataframe and perform aggregration, you will overflow 16GB memory. My strategy is to perform aggregration with separate chunks of columns, save the results to temporary parquet files, reload and merge the results after clear unused dataframe from memory.","metadata":{}},{"cell_type":"markdown","source":"### Numeric features engineering","metadata":{}},{"cell_type":"markdown","source":"We perform aggregration and feature engineerings operations on chunk of columns (with chunks_size=10 in this example), instead of the whole datasets.","metadata":{}},{"cell_type":"code","source":"%%time\nchunks_size = 10  # number of columns in on chunk\nchunks = [num_features[x:x+chunks_size] for x in range(0, len(num_features), chunks_size)]\n\nfor chunk_idx, feature_chunk in enumerate(chunks):\n    print('-'*50)\n    print(f'Processing chunk{chunk_idx}: {feature_chunk}...')\n    print('-'*50)\n    cols = feature_chunk + ['customer_ID']\n    test_part = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet', columns=cols)\n    \n    # Aggregration features\n    print(f'Get the aggregration features...')\n    test_num_agg = test_part.groupby(\"customer_ID\")[cols].agg(['mean', 'std', 'min', 'max', 'last'])\n    test_num_agg.columns = ['_'.join(x) for x in test_num_agg.columns]\n    # Transform float64 columns to float32 to reduce memory usage\n    cols = list(test_num_agg.dtypes[test_num_agg.dtypes == 'float64'].index)\n    print(f'Transform float64 columns to float32: {cols}')\n    test_num_agg.loc[:,cols] = test_num_agg.loc[:,cols].progress_apply(lambda x: x.astype(np.float32))\n    print(f'Chunk shape {test_num_agg.shape}')\n    print(f'Save test_num_agg_chunk{chunk_idx}.parquet...')\n    test_num_agg.to_parquet(f'test_num_agg_chunk{chunk_idx}.parquet')\n    del test_num_agg\n    gc.collect()\n    \n    # Difference (diff) features\n    cols = [col + '_diff1' for col in feature_chunk]\n    print(f'Get the diff1 features of cols: {cols}...')\n    test_num_diff = test_part.groupby(['customer_ID']).progress_apply(lambda x: np.diff(x.values[-2:,:], axis = 0).squeeze().astype(np.float32))\n    index = test_num_diff.index\n    test_num_diff = pd.DataFrame(test_num_diff.values.tolist(), columns=cols)\n    test_num_diff['customer_ID'] = index\n    test_num_diff = test_num_diff.set_index('customer_ID')\n    print(f'Chunk shape {test_num_diff.shape}')\n    print(f'Save test_num_diff_chunk{chunk_idx}.parquet...')\n    test_num_diff.to_parquet(f'test_num_diff_chunk{chunk_idx}.parquet')    \n    del test_num_diff\n\n    del test_part\n    del index\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T15:47:51.225825Z","iopub.execute_input":"2022-07-13T15:47:51.226542Z","iopub.status.idle":"2022-07-13T16:17:43.720490Z","shell.execute_reply.started":"2022-07-13T15:47:51.226505Z","shell.execute_reply":"2022-07-13T16:17:43.718958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Categorical features engineering","metadata":{}},{"cell_type":"markdown","source":"Similar to categorical features engineering","metadata":{}},{"cell_type":"code","source":"%%time\nchunks = [cat_features[x:x+chunks_size] for x in range(0, len(cat_features), chunks_size)]\nfor chunk_idx, feature_chunk in enumerate(chunks):\n    print('-'*50)\n    print(f'Processing chunk{chunk_idx}: {feature_chunk}...')\n    print('-'*50)\n    cols = feature_chunk + ['customer_ID']\n    test_part = pd.read_parquet('../input/amex-data-integer-dtypes-parquet-format/test.parquet', columns=cols)\n    \n    # Aggregration features\n    print(f'Get the aggregration features...')\n    test_cat_agg = test_part.groupby(\"customer_ID\").agg(['count', 'last', 'nunique'])\n    test_cat_agg.columns = ['_'.join(x) for x in test_cat_agg.columns]\n    # Transform int64 columns to int32 to reduce memory usage\n    cols = list(test_cat_agg.dtypes[test_cat_agg.dtypes == 'int64'].index)\n    print(f'Transform int64 columns to int32: {cols}')\n    test_cat_agg.loc[:,cols] = test_cat_agg.loc[:,cols].progress_apply(lambda x: x.astype(np.int32))\n    print(f'Save test_cat_agg_chunk{chunk_idx}.parquet...')\n    test_cat_agg.to_parquet(f'test_cat_agg_chunk{chunk_idx}.parquet')\n    \n    del test_cat_agg\n    del test_part\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:18:11.835810Z","iopub.execute_input":"2022-07-13T16:18:11.836248Z","iopub.status.idle":"2022-07-13T16:18:41.930115Z","shell.execute_reply.started":"2022-07-13T16:18:11.836214Z","shell.execute_reply":"2022-07-13T16:18:41.924460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Combine parquet files\nAfter saving the intermediate parquet files, we will process to merge the results. If you encounter out of memory in this part (on the first run), simply restart the kernels, and run the script from here with the intermediate files saved. We will reload the intermediate parquet files to memory and merge into single parquet file.","metadata":{}},{"cell_type":"markdown","source":"Merge the test_num_agg first, be aware of the merging so that we have matching order later with training set","metadata":{}},{"cell_type":"code","source":"!pip install natsort\nfrom natsort import natsorted","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:23:35.470658Z","iopub.execute_input":"2022-07-13T16:23:35.471025Z","iopub.status.idle":"2022-07-13T16:23:35.503915Z","shell.execute_reply.started":"2022-07-13T16:23:35.470982Z","shell.execute_reply":"2022-07-13T16:23:35.502867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npath ='.'\n\nfiles = [file for file in os.listdir(path) if file.startswith('test_num_agg_chunk')]\nfiles = natsorted(files)  # have to loaded in order matched with train data\n\nprint(f'Load FIRST chunk: {files[0]}...')\ntest = pd.read_parquet(files[0])\n\nfor file in files:\n    if file != files[0]:\n        print(f'Merge {file}...')\n        temp = pd.read_parquet(file)\n        test = test.merge(temp, on='customer_ID', how='left')\n        del temp\n        gc.collect()\n\n\n# print('Load test_num_agg_chunk0.parquet...')\n# test = pd.read_parquet('test_num_agg_chunk0.parquet')\n\n# for file in os.listdir(path):\n#     if file != 'test_num_agg_chunk0.parquet':\n#         if file.startswith('test_num_agg_chunk'):      \n#             print(f'Merge {file}...')\n#             temp = pd.read_parquet(file)\n#             test = test.merge(temp, on='customer_ID', how='left')\n#             del temp\n#             gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:26:33.165318Z","iopub.execute_input":"2022-07-13T16:26:33.165736Z","iopub.status.idle":"2022-07-13T16:26:41.711845Z","shell.execute_reply.started":"2022-07-13T16:26:33.165686Z","shell.execute_reply":"2022-07-13T16:26:41.710759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merge test_num_diff","metadata":{}},{"cell_type":"code","source":"%%time\n\nfiles = [file for file in os.listdir(path) if file.startswith('test_num_diff_chunk')]\nfiles = natsorted(files)\n\nfor file in files:\n    print(f'Merge {file}...')\n    temp = pd.read_parquet(file)\n    test = test.merge(temp, on='customer_ID', how='left')\n    del temp\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:28:04.117114Z","iopub.execute_input":"2022-07-13T16:28:04.117558Z","iopub.status.idle":"2022-07-13T16:28:13.455184Z","shell.execute_reply.started":"2022-07-13T16:28:04.117523Z","shell.execute_reply":"2022-07-13T16:28:13.453767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merge test_cat_agg","metadata":{}},{"cell_type":"code","source":"%%time\n\nfiles = [file for file in os.listdir(path) if file.startswith('test_cat_agg_chunk')]\nfiles = natsorted(files)\n\nfor file in files:\n    print(f'Merge {file}...')\n    temp = pd.read_parquet(file)\n    test = test.merge(temp, on='customer_ID', how='left')\n    del temp\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:28:15.805167Z","iopub.execute_input":"2022-07-13T16:28:15.805881Z","iopub.status.idle":"2022-07-13T16:28:20.717504Z","shell.execute_reply.started":"2022-07-13T16:28:15.805845Z","shell.execute_reply":"2022-07-13T16:28:20.716114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-07-13T16:28:21.779895Z","iopub.execute_input":"2022-07-13T16:28:21.780279Z","iopub.status.idle":"2022-07-13T16:28:21.999789Z","shell.execute_reply.started":"2022-07-13T16:28:21.780249Z","shell.execute_reply":"2022-07-13T16:28:21.998676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saving\nSaving to csv file so that we could read rows of chunks later to predict, yes, even the predicting operation will be out of 16GB RAM if you load everything in one single run.","metadata":{}},{"cell_type":"code","source":"%%time\nprint('Saving preprocessed test data to csv...')\ntest.to_csv('test_fe.csv.gzip', compression='gzip')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Saving to parquet files if you like","metadata":{}},{"cell_type":"code","source":"%%time\nprint('Saving preprocessed test data to parquet...')\ntest.to_parquet('test_fe.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:58:48.654393Z","iopub.execute_input":"2022-07-12T14:58:48.655194Z","iopub.status.idle":"2022-07-12T14:59:31.215472Z","shell.execute_reply.started":"2022-07-12T14:58:48.655146Z","shell.execute_reply":"2022-07-12T14:59:31.214149Z"},"trusted":true},"execution_count":null,"outputs":[]}]}