{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Happy Whale Images Converted to HDF5 for Faster Batch Loading","metadata":{}},{"cell_type":"code","source":"import h5py\nimport cv2\nimport io\nfrom PIL import Image\nimport os\nimport numpy as np\nfrom tqdm import tqdm\n\n# paths\nTRAIN_IMAGES = '../input/happy-whale-and-dolphin/train_images'\nTEST_IMAGES = '../input/happy-whale-and-dolphin/test_images'\n\n\n\ndef list_files(gtdir):\n    file_list = []\n    for root, dirs, files in os.walk(gtdir):\n        for file in files:\n            file_list.append(os.path.join(root,file))\n    return file_list\n\ndef tohdf5(file_list, out_file_path='train_images.hdf5'):\n    print ('=> Converting images to hdf5')\n    print ('=> Total Images To Process : {}'.format(len(file_list)))\n    pbar = tqdm(total=len(file_list))\n    count = 0\n    with h5py.File(out_file_path, \"w\") as h5:\n        for f_ in file_list:\n            image = Image.open(f_)\n            if image.mode == 'L':\n                image = image.convert('RGB')\n            image = image.resize((224,224))\n            image = np.array(image)\n            file_name = f_.split(os.sep)[-1]\n            #print (file_name, image.shape)\n            h5.create_dataset(file_name, data=image)\n            count = count + 1\n            if count % 10 == 0:\n                pbar.update(count)\n    h5.close()\n    pbar.close()\n    print('=>  Finished Converting images to hdf5')\n       \nprint('=> ========= Converting Train Images ========= <=')\nfile_list = list_files(TRAIN_IMAGES)\ntohdf5(file_list,out_file_path='train_images.hdf5')\nprint('=> ========= Converting Test Images ========= <=')\nfile_list = list_files(TEST_IMAGES)\ntohdf5(file_list,out_file_path='test_images.hdf5')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T09:37:27.144626Z","iopub.execute_input":"2022-02-27T09:37:27.144956Z","iopub.status.idle":"2022-02-27T09:37:50.546306Z","shell.execute_reply.started":"2022-02-27T09:37:27.144923Z","shell.execute_reply":"2022-02-27T09:37:50.545502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example DataLoader and DataSet in PyTorch","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset\nimport pandas as pd\n\nTRAIN_CSV = '../input/happy-whale-and-dolphin/train.csv'\nTEST_CSV = '../input/happy-whale-and-dolphin/sample_submission.csv'\n\n# Change accordingly \n# if input\n#DATASET_ROOT = '../input/happy-whale-to-hdf5-224x224'\n\n# if output\nDATASET_ROOT = './'\n\n# Read CSV to DataFrame\ntrain_df = pd.read_csv(TRAIN_CSV)\n\n\n# Train Transforms\ntrain_transforms  = transforms.Compose([\n                transforms.ToTensor(),\n                transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) # imagenet\n        ])\n\n\n# Label Encoder\ndef get_label_encoder_decoder(unique_values):\n    label_encoder = {}\n    label_decoder = {}\n    for idx, label in enumerate(unique_values):\n        label_encoder[label] = idx\n        label_decoder[idx] = label\n    return label_encoder, label_decoder\n\nlabel_encoder_ind_id, label_decoder_ind_id = get_label_encoder_decoder(train_df['individual_id'].unique())\n\n# torch dataloader\nclass DolphinWhaleDatasetH5(Dataset):\n    \n    def __init__(self, root_dir, data_frame, is_train=True, transforms=None):\n    \n        self.image_names = data_frame['image'].values\n        self.is_train = is_train\n        if is_train:\n            self.labels = data_frame['individual_id'].values\n        else:\n            self.labels = [-1] *  len(self.image_names)\n           \n        self.transforms = transforms\n        print ('=> Reading HDF5 File...')\n        hdf5_path = os.path.join(root_dir,'{}_images.hdf5'.format('train' if is_train else 'test'))\n        self.h5 = h5py.File(hdf5_path,'r')\n        print('=> Dataset created, image hdf5 file is : {}'.format(hdf5_path))\n        \n    def __len__(self):\n        return len(self.image_names)\n    \n    def fetch_item_train(self,idx):\n        \n        # image name\n        image_name = self.image_names[idx]\n       \n        # read image \n        image = np.array(self.h5[image_name])\n        \n        # fetch and encode label\n        label = label_encoder_ind_id[self.labels[idx]]\n       \n        if self.transforms:\n            image = self.transforms(image)\n        \n        return {'image':image,\n                'label':label,}\n    \n    def fetch_item_test(self,idx):\n        image_name = self.image_names[idx]\n       \n        # read image \n        image = np.array(self.h5[image_name])  \n       \n        if self.transforms:\n            image = self.transforms(image)\n        \n        return {'image':image,\n                'image_name':image_name}\n    \n    def __getitem__(self, index):\n        if self.is_train:\n            return self.fetch_item_train(index)\n        else:\n            return self.fetch_item_test(index)\n    \n# Training and Validation Dataset\ndataset = DolphinWhaleDatasetH5(DATASET_ROOT,train_df,is_train=True, transforms=train_transforms)\n\ntrain_loader = DataLoader(\n    dataset, batch_size=4, num_workers=1)\n\nfor batch_idx, sample_ in enumerate(train_loader):\n    inputs = sample_['image']  \n    print(inputs.shape)\n    if batch_idx > 10:\n        break","metadata":{"execution":{"iopub.status.busy":"2022-02-27T09:39:11.795539Z","iopub.execute_input":"2022-02-27T09:39:11.796281Z","iopub.status.idle":"2022-02-27T09:39:11.980003Z","shell.execute_reply.started":"2022-02-27T09:39:11.796232Z","shell.execute_reply":"2022-02-27T09:39:11.977665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}