{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This notebook explains how to convert a CSV to feather file format and other downcasting approach.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List of imports\n\nimport pandas as pd \nimport numpy as np\nimport tqdm\nimport gc\n\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-01T09:52:36.163754Z","iopub.execute_input":"2022-06-01T09:52:36.164463Z","iopub.status.idle":"2022-06-01T09:52:36.189377Z","shell.execute_reply.started":"2022-06-01T09:52:36.164321Z","shell.execute_reply":"2022-06-01T09:52:36.188675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = '../input/amex-default-prediction/'","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:36.351021Z","iopub.execute_input":"2022-06-01T09:52:36.351643Z","iopub.status.idle":"2022-06-01T09:52:36.356195Z","shell.execute_reply.started":"2022-06-01T09:52:36.351592Z","shell.execute_reply":"2022-06-01T09:52:36.355539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for file in os.listdir(PATH):\n    print('{} has size {} mb'.format(file , round(os.stat(os.path.join(PATH, file)).st_size/(1024*1024),3)))","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:36.482344Z","iopub.execute_input":"2022-06-01T09:52:36.483146Z","iopub.status.idle":"2022-06-01T09:52:36.493073Z","shell.execute_reply.started":"2022-06-01T09:52:36.483093Z","shell.execute_reply":"2022-06-01T09:52:36.492444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Good old way \n!ls -lh {PATH}","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:36.579266Z","iopub.execute_input":"2022-06-01T09:52:36.580009Z","iopub.status.idle":"2022-06-01T09:52:37.342591Z","shell.execute_reply.started":"2022-06-01T09:52:36.579971Z","shell.execute_reply":"2022-06-01T09:52:37.341577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only load the first 5 rows to get an idea of what the data look like\ndf_temp = pd.read_csv(f'{PATH}train_data.csv', nrows=5)\ndf_temp.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:37.344703Z","iopub.execute_input":"2022-06-01T09:52:37.345319Z","iopub.status.idle":"2022-06-01T09:52:37.412067Z","shell.execute_reply.started":"2022-06-01T09:52:37.345280Z","shell.execute_reply":"2022-06-01T09:52:37.411193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Information on Datatype\ndf_temp.info()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:37.413514Z","iopub.execute_input":"2022-06-01T09:52:37.413953Z","iopub.status.idle":"2022-06-01T09:52:37.446078Z","shell.execute_reply.started":"2022-06-01T09:52:37.413905Z","shell.execute_reply":"2022-06-01T09:52:37.444964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float_cols = df_temp.select_dtypes(include=['float'])\nint_cols = df_temp.select_dtypes(include=['int'])\ncat_cols = df_temp.select_dtypes(include=['object'])\n\nfor cols in float_cols.columns:\n    df_temp[cols] = pd.to_numeric(df_temp[cols], downcast='float')\n    \nfor cols in int_cols.columns:\n    df_temp[cols] = pd.to_numeric(df_temp[cols], downcast='integer')\n\n    \nprint(df_temp.info())","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:37.448629Z","iopub.execute_input":"2022-06-01T09:52:37.449703Z","iopub.status.idle":"2022-06-01T09:52:37.534684Z","shell.execute_reply.started":"2022-06-01T09:52:37.449652Z","shell.execute_reply":"2022-06-01T09:52:37.532826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the dtype ( pandas ) to be used while importing \n# Downcasting float64 -> float32\n# Converting Object -> category format\n\ndtypes = {\n    'customer_ID': \"object\",\n     'S_2': \"object\", # This is a date object . Can be taken care by parse_dates while importing\n     'P_2': 'float16',\n     'D_39': 'float16',\n     'B_1': 'float16',\n     'B_2': 'float16',\n     'R_1': 'float16',\n     'S_3': 'float16',\n     'D_41': 'float16',\n     'B_3': 'float16',\n     'D_42': 'float16',\n     'D_43': 'float16',\n     'D_44': 'float16',\n     'B_4': 'float16',\n     'D_45': 'float16',\n     'B_5': 'float16',\n     'R_2': 'float16',\n     'D_46': 'float16',\n     'D_47': 'float16',\n     'D_48': 'float16',\n     'D_49': 'float16',\n     'B_6': 'float16',\n     'B_7': 'float16',\n     'B_8': 'float16',\n     'D_50': 'float16',\n     'D_51': 'float16',\n     'B_9': 'float16',\n     'R_3': 'float16',\n     'D_52': 'float16',\n     'P_3': 'float16',\n     'B_10': 'float16',\n     'D_53': 'float16',\n     'S_5': 'float16',\n     'B_11': 'float16',\n     'S_6': 'float16',\n     'D_54': 'float16',\n     'R_4': 'float16',\n     'S_7': 'float16',\n     'B_12': 'float16',\n     'S_8': 'float16',\n     'D_55': 'float16',\n     'D_56': 'float16',\n     'B_13': 'float16',\n     'R_5': 'float16',\n     'D_58': 'float16',\n     'S_9': 'float16',\n     'B_14': 'float16',\n     'D_59': 'float16',\n     'D_60': 'float16',\n     'D_61': 'float16',\n     'B_15': 'float16',\n     'S_11': 'float16',\n     'D_62': 'float16',\n     'D_63': 'category', # Define as category datatype\n     'D_64': 'category',  # Define as category datatype\n     'D_65': 'float16',\n     'B_16': 'float16',\n     'B_17': 'float16',\n     'B_18': 'float16',\n     'B_19': 'float16',\n     'D_66': 'float16',\n     'B_20': 'float16',\n     'D_68': 'float16',\n     'S_12': 'float16',\n     'R_6': 'float16',\n     'S_13': 'float16',\n     'B_21': 'float16',\n     'D_69': 'float16',\n     'B_22': 'float16',\n     'D_70': 'float16',\n     'D_71': 'float16',\n     'D_72': 'float16',\n     'S_15': 'float16',\n     'B_23': 'float16',\n     'D_73': 'float16',\n     'P_4': 'float16',\n     'D_74': 'float16',\n     'D_75': 'float16',\n     'D_76': 'float16',\n     'B_24': 'float16',\n     'R_7': 'float16',\n     'D_77': 'float16',\n     'B_25': 'float16',\n     'B_26': 'float16',\n     'D_78': 'float16',\n     'D_79': 'float16',\n     'R_8': 'float16',\n     'R_9': 'float16',\n     'S_16': 'float16',\n     'D_80': 'float16',\n     'R_10': 'float16',\n     'R_11': 'float16',\n     'B_27': 'float16',\n     'D_81': 'float16',\n     'D_82': 'float16',\n     'S_17': 'float16',\n     'R_12': 'float16',\n     'B_28': 'float16',\n     'R_13': 'float16',\n     'D_83': 'float16',\n     'R_14': 'float16',\n     'R_15': 'float16',\n     'D_84': 'float16',\n     'R_16': 'float16',\n     'B_29': 'float16',\n     'B_30': 'float16',\n     'S_18': 'float16',\n     'D_86': 'float16',\n     'D_87': 'float16',\n     'R_17': 'float16',\n     'R_18': 'float16',\n     'D_88': 'float16',\n     'B_31': 'int64',\n     'S_19': 'float16',\n     'R_19': 'float16',\n     'B_32': 'float16',\n     'S_20': 'float16',\n     'R_20': 'float16',\n     'R_21': 'float16',\n     'B_33': 'float16',\n     'D_89': 'float16',\n     'R_22': 'float16',\n     'R_23': 'float16',\n     'D_91': 'float16',\n     'D_92': 'float16',\n     'D_93': 'float16',\n     'D_94': 'float16',\n     'R_24': 'float16',\n     'R_25': 'float16',\n     'D_96': 'float16',\n     'S_22': 'float16',\n     'S_23': 'float16',\n     'S_24': 'float16',\n     'S_25': 'float16',\n     'S_26': 'float16',\n     'D_102': 'float16',\n     'D_103': 'float16',\n     'D_104': 'float16',\n     'D_105': 'float16',\n     'D_106': 'float16',\n     'D_107': 'float16',\n     'B_36': 'float16',\n     'B_37': 'float16',\n     'R_26': 'float16',\n     'R_27': 'float16',\n     'B_38': 'float16',\n     'D_108': 'float16',\n     'D_109': 'float16',\n     'D_110': 'float16',\n     'D_111': 'float16',\n     'B_39': 'float16',\n     'D_112': 'float16',\n     'B_40': 'float16',\n     'S_27': 'float16',\n     'D_113': 'float16',\n     'D_114': 'float16',\n     'D_115': 'float16',\n     'D_116': 'float16',\n     'D_117': 'float16',\n     'D_118': 'float16',\n     'D_119': 'float16',\n     'D_120': 'float16',\n     'D_121': 'float16',\n     'D_122': 'float16',\n     'D_123': 'float16',\n     'D_124': 'float16',\n     'D_125': 'float16',\n     'D_126': 'float16',\n     'D_127': 'float16',\n     'D_128': 'float16',\n     'D_129': 'float16',\n     'B_41': 'float16',\n     'B_42': 'float16',\n     'D_130': 'float16',\n     'D_131': 'float16',\n     'D_132': 'float16',\n     'D_133': 'float16',\n     'R_28': 'float16',\n     'D_134': 'float16',\n     'D_135': 'float16',\n     'D_136': 'float16',\n     'D_137': 'float16',\n     'D_138': 'float16',\n     'D_139': 'float16',\n     'D_140': 'float16',\n     'D_141': 'float16',\n     'D_142': 'float16',\n     'D_143': 'float16',\n     'D_144': 'float16',\n     'D_145': 'float16'\n}\n\ncol_names = list(dtypes.keys())","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:37.536564Z","iopub.execute_input":"2022-06-01T09:52:37.536955Z","iopub.status.idle":"2022-06-01T09:52:37.571587Z","shell.execute_reply.started":"2022-06-01T09:52:37.536919Z","shell.execute_reply":"2022-06-01T09:52:37.570799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex_file = [\n    'train_data.csv'\n]","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:37.572633Z","iopub.execute_input":"2022-06-01T09:52:37.573477Z","iopub.status.idle":"2022-06-01T09:52:37.589057Z","shell.execute_reply.started":"2022-06-01T09:52:37.573443Z","shell.execute_reply":"2022-06-01T09:52:37.587855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to be used in case multiple files are to be concatenated\ndf_list = []\n\nfor i in tqdm.tqdm(amex_file):\n    df = pd.read_csv(f'{PATH}'+i, parse_dates = True,  usecols=col_names,dtype=dtypes)\n    df_list.append(df)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:52:37.590126Z","iopub.execute_input":"2022-06-01T09:52:37.590971Z","iopub.status.idle":"2022-06-01T09:58:57.492686Z","shell.execute_reply.started":"2022-06-01T09:52:37.590923Z","shell.execute_reply":"2022-06-01T09:58:57.491907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"amex = pd.concat(df_list)\n\ndel df_list\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:58:57.494070Z","iopub.execute_input":"2022-06-01T09:58:57.494988Z","iopub.status.idle":"2022-06-01T09:59:01.559179Z","shell.execute_reply.started":"2022-06-01T09:58:57.494950Z","shell.execute_reply":"2022-06-01T09:59:01.557981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save to feather so we can use it in other kernels\namex.reset_index(drop=True).to_feather(f'amex.feather')","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:59:01.560698Z","iopub.execute_input":"2022-06-01T09:59:01.561096Z","iopub.status.idle":"2022-06-01T09:59:12.667163Z","shell.execute_reply.started":"2022-06-01T09:59:01.561064Z","shell.execute_reply":"2022-06-01T09:59:12.666435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -lh ","metadata":{"execution":{"iopub.status.busy":"2022-06-01T09:59:12.671559Z","iopub.execute_input":"2022-06-01T09:59:12.672521Z","iopub.status.idle":"2022-06-01T09:59:13.650863Z","shell.execute_reply.started":"2022-06-01T09:59:12.672458Z","shell.execute_reply":"2022-06-01T09:59:13.649498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}