{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\n\n\ndef save_test_chunks():\n    \"\"\"\n    Load all the test set chunk by chunk, and dump to individual pickled files\n    This requires an extra 20 GB of storage and allows for easier manipulation of the test set:\n    - Faster load time\n    - Parallelizable over chunks\n    \"\"\"\n\n    # Create a new directory for the test chunks\n    os.makedirs(\"test_chunks\", exist_ok=True)\n\n    # There are 453653105 lines in the test_set.csv\n    # We will create ~ 200 smaller, pickled files for easy access\n    chunksize = 453653105 // 200\n\n    reader = pd.read_table(\"../input/test_set.csv\", sep=\",\", chunksize=chunksize)\n\n    # Get the first chunk\n    df_temp = next(reader)\n    for chunk_idx, df_chunk in enumerate(reader):\n        # Find out if df_chunk has an overlap with df_temp\n\n        last_id = df_temp.object_id.values[-1]\n        first_id = df_chunk.object_id.values[0]\n\n        if last_id == first_id:\n            df_temp = pd.concat(\n                [df_temp, df_chunk[df_chunk.object_id == last_id]]\n            ).reset_index(drop=True)\n\n        df_temp.to_pickle(f\"test_chunks/test_set_chunk{chunk_idx}.pickle\")\n\n        df_temp = df_chunk[df_chunk.object_id != last_id].reset_index(drop=True)\n\n    # Dump the last chunk\n    df_temp.to_pickle(f\"test_chunks/test_set_chunk{chunk_idx + 1}.pickle\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}