{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-22T10:17:19.652309Z","iopub.execute_input":"2024-12-22T10:17:19.652690Z","iopub.status.idle":"2024-12-22T10:17:20.050932Z","shell.execute_reply.started":"2024-12-22T10:17:19.652649Z","shell.execute_reply":"2024-12-22T10:17:20.049809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's Find out all the datatypes present in the file so that we can optimize them accordingly which can drastically reduce our file size.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\ncsv_file = '/kaggle/input/amex-default-prediction/train_data.csv'\nchunks = pd.read_csv(csv_file, chunksize = 50000)\nfor chunk in chunks:\n    print(chunk.dtypes.unique())  # Print column data types for each chunk\n    break  # Break after the first chunk to avoid loading the entire file","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T10:24:24.147902Z","iopub.execute_input":"2024-12-22T10:24:24.148265Z","iopub.status.idle":"2024-12-22T10:24:26.266363Z","shell.execute_reply.started":"2024-12-22T10:24:24.148234Z","shell.execute_reply":"2024-12-22T10:24:26.265154Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As we have cofirmed there are just 3 type of dataype present in the file we need to find all the respective columns","metadata":{}},{"cell_type":"code","source":"float_cols = chunk.select_dtypes(include = 'float64').columns.to_list()\nint_cols =  chunk.select_dtypes(include = 'int64').columns.to_list()\ncat_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T10:17:32.093560Z","iopub.execute_input":"2024-12-22T10:17:32.094040Z","iopub.status.idle":"2024-12-22T10:17:32.137577Z","shell.execute_reply.started":"2024-12-22T10:17:32.093976Z","shell.execute_reply":"2024-12-22T10:17:32.136271Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Dynamic Optimization. This will reduce the file size considerably.","metadata":{}},{"cell_type":"code","source":"optimized_dtype = {col : 'float32' for col in float_cols}\noptimized_dtype.update({col:'int32' for col in int_cols})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T10:24:58.550343Z","iopub.execute_input":"2024-12-22T10:24:58.550738Z","iopub.status.idle":"2024-12-22T10:24:58.556081Z","shell.execute_reply.started":"2024-12-22T10:24:58.550693Z","shell.execute_reply":"2024-12-22T10:24:58.554809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now Let's divide the files in chunks and optimize them while uploading it from the source CSV.","metadata":{}},{"cell_type":"code","source":"chunk_size = 500000\n\nchunk_list = []\n\nfor i, chunk in enumerate(pd.read_csv(csv_file , chunksize = chunk_size, dtype = optimized_dtype)):\n\n    chunk.to_feather(f'optimized file chunk {i}.ftr')\n\n    print(f'chunk{i} saved as feather format')\n\n\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-22T10:18:10.574104Z","iopub.execute_input":"2024-12-22T10:18:10.574455Z","iopub.status.idle":"2024-12-22T10:24:24.145664Z","shell.execute_reply.started":"2024-12-22T10:18:10.574426Z","shell.execute_reply":"2024-12-22T10:24:24.144122Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Once all chunks are saved we can concatenate this feather file into one single file and load it into pandas as a dataframe for further processing.","metadata":{}},{"cell_type":"code","source":"feather_files = [f'optimized file chunk {i}.ftr' for i in range(12)]\ndataframe_files = [pd.read_feather(file) for file in feather_files]\ndataframe = pd.concat(dataframe_files, ignore_index = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:13:41.842339Z","iopub.execute_input":"2024-12-20T04:13:41.844599Z","iopub.status.idle":"2024-12-20T04:14:03.903021Z","shell.execute_reply.started":"2024-12-20T04:13:41.844520Z","shell.execute_reply":"2024-12-20T04:14:03.901375Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's check the shape of the resultant Dataframe. As we can see we have whole data oploaded in the resultant dataframe.","metadata":{}},{"cell_type":"code","source":"dataframe.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:14:17.381334Z","iopub.execute_input":"2024-12-20T04:14:17.381771Z","iopub.status.idle":"2024-12-20T04:14:17.396707Z","shell.execute_reply.started":"2024-12-20T04:14:17.381743Z","shell.execute_reply":"2024-12-20T04:14:17.395353Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now let's check the difference of the data we just created. Remeber that Previous file is of ~50 GB","metadata":{}},{"cell_type":"code","source":"memory_in_bytes =dataframe.memory_usage(deep=True).sum()\n\n# Convert to megabytes (MB)\nmemory_in_mb = memory_in_bytes / (1024 ** 2)\nprint(f\"Memory usage of combined DataFrame: {memory_in_mb:.2f} MB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-20T04:24:47.302966Z","iopub.execute_input":"2024-12-20T04:24:47.303385Z","iopub.status.idle":"2024-12-20T04:24:51.442451Z","shell.execute_reply.started":"2024-12-20T04:24:47.303359Z","shell.execute_reply":"2024-12-20T04:24:51.441448Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"WOW...We have just converted the 50 GB data to just 5 GB . It's just unbelievable what all we can do with data.","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}