{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Introduction**","metadata":{}},{"cell_type":"markdown","source":"This notebook is intended to show how can we read and reduce size of a large dataset.\n\nBelow are my observations: -\n* Saving dataset to parquet format without any transformation reduces dataet size to almost 50%\n* Float64 columns can be converted to Float32 simply by reading CSV with explicitly specifying dtypes\n* Customer ID, S_2 (date column), D_63 and D_64 (categorical columns) are encoded\n* Dataset size reduced to 4.55 GB after all transformations\n\nChallenges: -\n* Code optimization\n* Converting categorical columns to int8\n\n\n","metadata":{}},{"cell_type":"markdown","source":"# **Setup**","metadata":{}},{"cell_type":"code","source":"import dask.dataframe as dd\nimport pandas as pd\nimport numpy as np\nfrom sklearn import preprocessing\nfrom dask.array import from_array as fa","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-08T01:37:27.178627Z","iopub.execute_input":"2022-07-08T01:37:27.179601Z","iopub.status.idle":"2022-07-08T01:37:29.041136Z","shell.execute_reply.started":"2022-07-08T01:37:27.179490Z","shell.execute_reply":"2022-07-08T01:37:29.039741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_missing(ddf):\n    for c in ddf.columns:\n        if ddf[c].dtype == 'float32':\n            ddf[c] = ddf[c].fillna(0.0)\n    return ddf\n\ndef save_parquet(ddf):\n    ddf.to_parquet('./train_parquet/')  ","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:37:31.193865Z","iopub.execute_input":"2022-07-08T01:37:31.194266Z","iopub.status.idle":"2022-07-08T01:37:31.200811Z","shell.execute_reply.started":"2022-07-08T01:37:31.194234Z","shell.execute_reply":"2022-07-08T01:37:31.199723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Read Dataset**","metadata":{}},{"cell_type":"markdown","source":"Map columns to their respective data types.","metadata":{}},{"cell_type":"code","source":"train_dtypes = {'customer_ID': object, 'S_2': object,'P_2': np.float32, 'P_3': np.float32, 'P_4': np.float32,'S_3': np.float32, 'S_5': np.float32, 'S_6': np.float32, 'S_7': np.float32, 'S_8': np.float32, 'S_9': np.float32, 'S_11': np.float32, 'S_12': np.float32, 'S_13': np.float32, 'S_15': np.float32, 'S_16': np.float32, 'S_17': np.float32, 'S_18': np.float32, 'S_19': np.float32, 'S_20': np.float32, 'S_22': np.float32, 'S_23': np.float32, 'S_24': np.float32, 'S_25': np.float32, 'S_26': np.float32, 'S_27': np.float32,'R_1': np.float32, 'R_2': np.float32, 'R_3': np.float32, 'R_4': np.float32, 'R_5': np.float32, 'R_6': np.float32, 'R_7': np.float32, 'R_8': np.float32, 'R_9': np.float32, 'R_10': np.float32, 'R_11': np.float32, 'R_12': np.float32, 'R_13': np.float32, 'R_14': np.float32, 'R_15': np.float32, 'R_16': np.float32, 'R_17': np.float32, 'R_18': np.float32, 'R_19': np.float32, 'R_20': np.float32, 'R_21': np.float32, 'R_22': np.float32, 'R_23': np.float32, 'R_24': np.float32, 'R_25': np.float32, 'R_26': np.float32, 'R_27': np.float32, 'R_28': np.float32,'B_1': np.float32, 'B_2': np.float32, 'B_3': np.float32, 'B_4': np.float32, 'B_5': np.float32, 'B_6': np.float32, 'B_7': np.float32, 'B_8': np.float32, 'B_9': np.float32, 'B_10': np.float32, 'B_11': np.float32, 'B_12': np.float32, 'B_13': np.float32, 'B_14': np.float32, 'B_15': np.float32, 'B_16': np.float32, 'B_17': np.float32, 'B_18': np.float32, 'B_19': np.float32, 'B_20': np.float32, 'B_21': np.float32, 'B_22': np.float32, 'B_23': np.float32, 'B_24': np.float32, 'B_25': np.float32, 'B_26': np.float32, 'B_27': np.float32, 'B_28': np.float32, 'B_29': np.float32, 'B_30': np.float32, 'B_31': np.float32, 'B_32': np.float32, 'B_33': np.float32, 'B_36': np.float32, 'B_37': np.float32, 'B_38': np.float32, 'B_39': np.float32, 'B_40': np.float32, 'B_41': np.float32, 'B_42': np.float32,'D_39': np.float32, 'D_41': np.float32, 'D_42': np.float32, 'D_43': np.float32, 'D_44': np.float32, 'D_45': np.float32, 'D_46': np.float32, 'D_47': np.float32, 'D_48': np.float32, 'D_49': np.float32, 'D_50': np.float32, 'D_51': np.float32, 'D_52': np.float32, 'D_53': np.float32, 'D_54': np.float32, 'D_55': np.float32, 'D_56': np.float32, 'D_58': np.float32, 'D_59': np.float32, 'D_60': np.float32, 'D_61': np.float32, 'D_62': np.float32, 'D_65': np.float32, 'D_69': np.float32, 'D_70': np.float32, 'D_71': np.float32, 'D_72': np.float32, 'D_73': np.float32, 'D_74': np.float32, 'D_75': np.float32, 'D_76': np.float32, 'D_77': np.float32, 'D_78': np.float32, 'D_79': np.float32, 'D_80': np.float32, 'D_81': np.float32, 'D_82': np.float32, 'D_83': np.float32, 'D_84': np.float32, 'D_86': np.float32, 'D_87': np.float32, 'D_88': np.float32, 'D_89': np.float32, 'D_91': np.float32, 'D_92': np.float32, 'D_93': np.float32, 'D_94': np.float32, 'D_96': np.float32, 'D_102': np.float32, 'D_103': np.float32, 'D_104': np.float32, 'D_105': np.float32, 'D_106': np.float32, 'D_107': np.float32, 'D_108': np.float32, 'D_109': np.float32, 'D_110': np.float32, 'D_111': np.float32, 'D_112': np.float32, 'D_113': np.float32, 'D_115': np.float32, 'D_118': np.float32, 'D_119': np.float32, 'D_121': np.float32, 'D_122': np.float32, 'D_123': np.float32, 'D_124': np.float32, 'D_125': np.float32, 'D_127': np.float32, 'D_128': np.float32, 'D_129': np.float32, 'D_130': np.float32, 'D_131': np.float32, 'D_132': np.float32, 'D_133': np.float32, 'D_134': np.float32, 'D_135': np.float32, 'D_136': np.float32, 'D_137': np.float32, 'D_138': np.float32, 'D_139': np.float32, 'D_140': np.float32, 'D_141': np.float32, 'D_142': np.float32, 'D_143': np.float32, 'D_144': np.float32, 'D_145': np.float32, 'D_114': np.float32, 'D_116': np.float32, 'D_117': np.float32, 'D_120': np.float32, 'D_126': np.float32, 'D_66': np.float32, 'D_68': np.float32, 'D_63': object, 'D_64': object}","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:37:39.159295Z","iopub.execute_input":"2022-07-08T01:37:39.159699Z","iopub.status.idle":"2022-07-08T01:37:39.192726Z","shell.execute_reply.started":"2022-07-08T01:37:39.159666Z","shell.execute_reply":"2022-07-08T01:37:39.191192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_path = '../input/amex-default-prediction/train_data.csv'\n\ntrain_data = dd.read_csv(train_data_path, storage_options={'anon': True}, assume_missing=True,dtype=train_dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:37:47.431234Z","iopub.execute_input":"2022-07-08T01:37:47.431645Z","iopub.status.idle":"2022-07-08T01:37:47.538009Z","shell.execute_reply.started":"2022-07-08T01:37:47.431608Z","shell.execute_reply":"2022-07-08T01:37:47.536829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.head(10))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:37:49.846847Z","iopub.execute_input":"2022-07-08T01:37:49.847237Z","iopub.status.idle":"2022-07-08T01:37:51.527800Z","shell.execute_reply.started":"2022-07-08T01:37:49.847206Z","shell.execute_reply":"2022-07-08T01:37:51.526541Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Fill Missing Values**","metadata":{}},{"cell_type":"code","source":"fill_missing(train_data)","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:38:10.286251Z","iopub.execute_input":"2022-07-08T01:38:10.286693Z","iopub.status.idle":"2022-07-08T01:38:18.804189Z","shell.execute_reply.started":"2022-07-08T01:38:10.286627Z","shell.execute_reply":"2022-07-08T01:38:18.802800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Encode Customer ID**","metadata":{}},{"cell_type":"code","source":"le = preprocessing.LabelEncoder()\nddf = fa(le.fit_transform(train_data['customer_ID']))\nddf.compute()\nddf2 = dd.io.from_dask_array(ddf, columns=['new_ID'])\nddf2.compute()\ntrain_data = train_data.merge(ddf2, 'left')\nddf_customer = train_data[['customer_ID','new_ID']]\ntrain_data = train_data.drop(columns={'customer_ID'})","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:38:28.758163Z","iopub.execute_input":"2022-07-08T01:38:28.758600Z","iopub.status.idle":"2022-07-08T01:43:27.302250Z","shell.execute_reply.started":"2022-07-08T01:38:28.758560Z","shell.execute_reply":"2022-07-08T01:43:27.301195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Encode Date Feature**","metadata":{}},{"cell_type":"code","source":"train_data['S_2'] = train_data['S_2'].str.replace('-','').astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:43:40.227707Z","iopub.execute_input":"2022-07-08T01:43:40.228293Z","iopub.status.idle":"2022-07-08T01:43:40.286994Z","shell.execute_reply.started":"2022-07-08T01:43:40.228241Z","shell.execute_reply":"2022-07-08T01:43:40.285440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Encode Non-Numeric Cateorical Columns**","metadata":{}},{"cell_type":"code","source":"ddf = fa(le.fit_transform(train_data['D_63']))\nddf.compute()\ntrain_data = train_data.drop(columns={'D_63'})\nddf2 = dd.io.from_dask_array(ddf, columns=['D_63'])\nddf2.compute()\ntrain_data = train_data.merge(ddf2, 'left')\n\nddf = fa(le.fit_transform(train_data['D_64']))\nddf.compute()\ntrain_data = train_data.drop(columns={'D_64'})\nddf2 = dd.io.from_dask_array(ddf, columns=['D_64'])\nddf2.compute()\ntrain_data = train_data.merge(ddf2, 'left')","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:43:44.143287Z","iopub.execute_input":"2022-07-08T01:43:44.144038Z","iopub.status.idle":"2022-07-08T01:53:03.945912Z","shell.execute_reply.started":"2022-07-08T01:43:44.143994Z","shell.execute_reply":"2022-07-08T01:53:03.944726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.head(10))","metadata":{"execution":{"iopub.status.busy":"2022-07-08T01:53:36.755109Z","iopub.execute_input":"2022-07-08T01:53:36.755593Z","iopub.status.idle":"2022-07-08T01:53:41.428862Z","shell.execute_reply.started":"2022-07-08T01:53:36.755556Z","shell.execute_reply":"2022-07-08T01:53:41.427681Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Enable below line to save dataset in parquet format\n\n#save_parquet(train_data)","metadata":{},"execution_count":null,"outputs":[]}]}