{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59575,"databundleVersionId":8060720,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport gc","metadata":{"execution":{"iopub.status.busy":"2024-06-18T07:24:00.686961Z","iopub.execute_input":"2024-06-18T07:24:00.687364Z","iopub.status.idle":"2024-06-18T07:24:02.219953Z","shell.execute_reply.started":"2024-06-18T07:24:00.687331Z","shell.execute_reply":"2024-06-18T07:24:02.218374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patent_data_path = '/kaggle/input/uspto-explainable-ai/patent_data'","metadata":{"execution":{"iopub.status.busy":"2024-06-18T07:24:02.222387Z","iopub.execute_input":"2024-06-18T07:24:02.223047Z","iopub.status.idle":"2024-06-18T07:24:02.229837Z","shell.execute_reply.started":"2024-06-18T07:24:02.222999Z","shell.execute_reply":"2024-06-18T07:24:02.228281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Most weighty parquet\n# # 500 MB Before loading -> 16 GB (reach) When loading -> 11.8 GB After loading\n# most_weightly_parquet = pd.read_parquet(\"/kaggle/input/uspto-explainable-ai/patent_data/2023_6.parquet\")\n# memory_usage = most_weightly_parquet.memory_usage(deep=True).sum() / (1024 ** 3)\n# print(f\"Memory usage of DataFrame (deep): {memory_usage} GB\") # 11.3 GB","metadata":{"execution":{"iopub.status.busy":"2024-06-18T07:04:42.232590Z","iopub.execute_input":"2024-06-18T07:04:42.232972Z","iopub.status.idle":"2024-06-18T07:04:42.243531Z","shell.execute_reply.started":"2024-06-18T07:04:42.232938Z","shell.execute_reply":"2024-06-18T07:04:42.242407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a list of all files in the folder\nall_files = os.listdir(patent_data_path)\n\n# Sort files in reverse order (Going from the most weighty to the smaller files)\nparquet_files_sorted = sorted(all_files, reverse=True)\n\n# Go through the files in reverse order\nfor file in parquet_files_sorted:\n    # Get the full path to the current parquet file\n    parquet_file = os.path.join(patent_data_path, file)\n            \n    # Reading data from parquet file\n    parquet_info = pd.read_parquet(parquet_file)\n    \n    # Your operations with parquet_info data here\n    \n    # Releasing the memory\n    del parquet_info\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-18T07:27:22.114549Z","iopub.execute_input":"2024-06-18T07:27:22.114946Z"},"trusted":true},"execution_count":null,"outputs":[]}]}