{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\nimport random\nimport os\nimport numpy as np\nimport pandas as pd\n\nCLASSES_NUM = 10\n\ndef get_sampled_classes(sample_num=CLASSES_NUM):\n    random.seed(2023)\n    train_path = '/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train'\n    classes_name = [d for d in os.listdir(train_path) if os.path.isdir(os.path.join(train_path, d))]\n    print('len(classes_name): ', len(classes_name))\n    classes_name = random.sample(classes_name, sample_num)\n    return classes_name\n\nSAMPLED_CLASSES = get_sampled_classes(sample_num=CLASSES_NUM)\nprint('SAMPLED_CLASSES:', SAMPLED_CLASSES[:10])\n# SAMPLED_CLASSES = ['n02325366', 'n03938244', 'n04086273', 'n03393912', 'n02804610', 'n02268443', 'n03697007', \n#                    'n03240683', 'n04067472', 'n04536866', 'n02093991', 'n03388549', 'n02108551', 'n03791053', \n#                    'n04149813', 'n02099712', 'n07583066', 'n02747177', 'n01608432', 'n03876231']\n\nSAMPLED_CLASSES = set(SAMPLED_CLASSES)\n# len(classes_name):  1000\n# SAMPLED_CLASSES: ['n02325366', 'n03938244', 'n04086273', 'n03393912', 'n02804610', \n#                   'n02268443', 'n03697007', 'n03240683', 'n04067472', 'n04536866']","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-23T16:31:06.960382Z","iopub.execute_input":"2023-01-23T16:31:06.961239Z","iopub.status.idle":"2023-01-23T16:31:07.137443Z","shell.execute_reply.started":"2023-01-23T16:31:06.961109Z","shell.execute_reply":"2023-01-23T16:31:07.136166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_mapping_file = '/kaggle/input/imagenet-object-localization-challenge/LOC_synset_mapping.txt'\nclass_mapping_dict = {}\nfor line in open(class_mapping_file):\n    if line[:9] in SAMPLED_CLASSES:\n        real_class_name = line[10:].strip()\n        class_mapping_dict[line[:9]] = real_class_name\n        \nprint('class_mapping_dict: ', class_mapping_dict)","metadata":{"execution":{"iopub.status.busy":"2023-01-23T16:31:07.140119Z","iopub.execute_input":"2023-01-23T16:31:07.140994Z","iopub.status.idle":"2023-01-23T16:31:07.160399Z","shell.execute_reply.started":"2023-01-23T16:31:07.140927Z","shell.execute_reply":"2023-01-23T16:31:07.158946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/train/'\nnew_train_path = '/kaggle/working/training/'\nos.system(f'rm -rf {new_train_path}')\nos.system(f'mkdir {new_train_path}')\n\nfor i, name in enumerate(SAMPLED_CLASSES):\n    print('i: ', i)\n    os.system(f'cp -r {train_path + name}  {new_train_path}')\nprint('get here!!!')","metadata":{"execution":{"iopub.status.busy":"2023-01-23T16:31:07.162386Z","iopub.execute_input":"2023-01-23T16:31:07.162853Z","iopub.status.idle":"2023-01-23T16:32:16.29734Z","shell.execute_reply.started":"2023-01-23T16:31:07.162801Z","shell.execute_reply":"2023-01-23T16:32:16.29629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_path = '/kaggle/input/imagenet-object-localization-challenge/ILSVRC/Data/CLS-LOC/val/'\nnew_valid_path = '/kaggle/working/validation/'\nos.system(f'rm -rf {new_valid_path}')\nos.system(f'mkdir {new_valid_path}')\n\nprocess_start_t = time.time()\n\nos.chdir('/kaggle/working/validation/')\n\nline_cnt, write_cnt = 0, 0\nfor line in open('/kaggle/input/valprep/valprep.sh'):\n    line_cnt += 1\n    if line_cnt % 10000 == 0:\n        print('line_cnt: ', line_cnt)\n    if line.startswith('mkdir'):\n        class_name = line.strip().split(' ')[-1]\n        if class_name in SAMPLED_CLASSES:\n            write_cnt += 1\n            #file_w.write(line)\n            os.system(line.strip())\n    elif line.startswith('mv'):\n        class_name = line.strip().split(' ')[-1][:-1]\n        if class_name in SAMPLED_CLASSES:\n            write_cnt += 1\n            src_file_name = line.strip().split(' ')[1]\n            #file_w.write(f'cp {valid_path}{src_file_name} {class_name}/' + '\\n')\n            os.system(f'cp {valid_path}{src_file_name} {class_name}/')\n        \nprint('ok, get here!!! write_cnt: ', write_cnt)\n\nos.chdir('/kaggle/working/')\n\nprint(f'get here!!! cost time: {time.time()-process_start_t:.2f} sec')","metadata":{"execution":{"iopub.status.busy":"2023-01-23T16:32:16.300257Z","iopub.execute_input":"2023-01-23T16:32:16.300759Z","iopub.status.idle":"2023-01-23T16:32:21.38051Z","shell.execute_reply.started":"2023-01-23T16:32:16.300689Z","shell.execute_reply":"2023-01-23T16:32:21.379203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_files(directory):\n    filenames = []\n    class_name = directory.split('/')[-1]\n    for filename in os.listdir(directory):\n        if os.path.isfile(os.path.join(directory, filename)):\n            filenames.append(filename)\n    return filenames\n\nfor i, directory in enumerate(SAMPLED_CLASSES):\n    directory_path = new_valid_path + '/' + directory\n    filenames = get_all_files(directory_path)\n    print(f'valid class_name: {directory}', len(filenames))\n    \n    \nfor i, directory in enumerate(SAMPLED_CLASSES):\n    directory_path = new_train_path + '/' + directory\n    filenames = get_all_files(directory_path)\n    print(f'train class_name: {directory}', len(filenames))","metadata":{"execution":{"iopub.status.busy":"2023-01-23T16:32:21.381544Z","iopub.execute_input":"2023-01-23T16:32:21.381885Z","iopub.status.idle":"2023-01-23T16:32:21.491179Z","shell.execute_reply.started":"2023-01-23T16:32:21.381854Z","shell.execute_reply":"2023-01-23T16:32:21.490018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -sh ./training","metadata":{"execution":{"iopub.status.busy":"2023-01-23T16:32:21.492545Z","iopub.execute_input":"2023-01-23T16:32:21.493211Z","iopub.status.idle":"2023-01-23T16:32:22.62763Z","shell.execute_reply.started":"2023-01-23T16:32:21.493174Z","shell.execute_reply":"2023-01-23T16:32:22.626476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zip_start_t = time.time()\n!zip -r -q training.zip ./training\n!zip -r -q validation.zip ./validation\n\nprint(f'ok, get here!!! zip cost time: {time.time()-zip_start_t:.2f} sec')","metadata":{"execution":{"iopub.status.busy":"2023-01-23T16:32:22.629307Z","iopub.execute_input":"2023-01-23T16:32:22.630186Z","iopub.status.idle":"2023-01-23T16:33:28.974516Z","shell.execute_reply.started":"2023-01-23T16:32:22.630146Z","shell.execute_reply":"2023-01-23T16:33:28.973197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf ./training\n!rm -rf ./validation","metadata":{"execution":{"iopub.status.busy":"2023-01-23T16:33:28.9761Z","iopub.execute_input":"2023-01-23T16:33:28.976473Z","iopub.status.idle":"2023-01-23T16:33:32.204814Z","shell.execute_reply.started":"2023-01-23T16:33:28.976439Z","shell.execute_reply":"2023-01-23T16:33:32.203311Z"},"trusted":true},"execution_count":null,"outputs":[]}]}