{"cells":[{"metadata":{},"cell_type":"markdown","source":"The zip data files are very large and therefore, it takes much time to unzip them using python or a shel script\nIf you use VM, you pay for every second the VM runs and you may want to unzip the files as quaick as possible.\n\nHere I suggest an efficient method for unzipping the full trining data. The unzipping uses all the cpu that your VM has and therefore it will take less time than unzipping the data files one by one.\n\nNote that I assume that you have already downloaded all the zip files to your VM.\n\nJust copy the following script to your VM and run it.\nGood luck"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport zipfile\nfrom multiprocessing import Pool, cpu_count\n\n# base_path is where the zip files are located. Change it to your location \nbase_path = '/data/kaggle/deepfake/input/train/'\nif not os.path.isdir(base_path):\n    base_path = 'C:/Users/itama/Documents/kaggle/deepfake/input/train/'\n\n\ndef unzip(n):\n    idx_str = str(n)\n    if len(idx_str) == 1:\n        idx_str = '0' + idx_str\n\n    zip_file = base_path + 'dfdc_train_part_' + idx_str + '.zip'\n    if not os.path.isfile(zip_file):\n        return\n\n    print('Unzipping dfdc_train_part_' + idx_str + '.zip')\n    zip_ref = zipfile.ZipFile(zip_file, 'r')\n    zip_ref.extractall(base_path)\n    zip_ref.close()\n\n    os.remove(zip_file)\n\n\nif __name__ == '__main__':\n    n_cpu = cpu_count()\n    n_cpu = min(n_cpu, 50)\n    print('unzipping the files using ' + str(n_cpu) + ' processes.')\n    pool = Pool(processes=n_cpu)\n\n    pool.map(unzip, range(50), chunksize=1)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}