{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport pyarrow.parquet as pq\nimport matplotlib.pyplot as plt\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"H = 137\nW = 236","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## arrange train_data"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\nPATH = \"/kaggle/input/bengaliai-cv19/\"\ntrain_df = pd.read_csv(PATH+\"train.csv\")\n\ndata0 = pq.read_table(PATH+\"train_image_data_0.parquet\").to_pandas()\ndata0_id = data0[\"image_id\"]\ndata0 = data0.iloc[:, 1:].astype(np.uint8).values\ndata0 = data0.reshape(-1, H, W, 1)                    # [N, H, W, C]\n\ndata1 = pq.read_table(PATH+\"train_image_data_1.parquet\").to_pandas()\ndata1_id = data1[\"image_id\"]\ndata1 = data1.iloc[:, 1:].astype(np.uint8).values\ndata1 = data1.reshape(-1, H, W, 1)\n\ndata2 = pq.read_table(PATH+\"train_image_data_2.parquet\").to_pandas()\ndata2_id = data2[\"image_id\"]\ndata2 = data2.iloc[:, 1:].astype(np.uint8).values\ndata2 = data2.reshape(-1, H, W, 1)\n\ndata3 = pq.read_table(PATH+\"train_image_data_3.parquet\").to_pandas()\ndata3_id = data3[\"image_id\"]\ndata3 = data3.iloc[:, 1:].astype(np.uint8).values\ndata3 = data3.reshape(-1, H, W, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### concat train_data"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_full = np.vstack([data0, data1, data2, data3])\n\ndel data0, data1, data2, data3\ngc.collect()\n\nid_full = pd.concat([data0_id, data1_id, data2_id, data3_id], ignore_index=True)\ndel data0_id, data1_id, data2_id, data3_id\ngc.collect()\nprint(data_full.shape)\nprint(id_full.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### save train_data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith open(\"train_data_full.joblib\", \"wb\") as f:\n    joblib.dump(data_full, f, compress=3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check train_data using torch.utils.data.Dataset\nI like PyTorch. Therefore, check i'll if this data is available in torch.utils.data.Dataset and DataLoader."},{"metadata":{"trusted":true},"cell_type":"code","source":"class BengaliDataset(Dataset):\n    def __init__(self, data: np.ndarray, label: pd.DataFrame, train: bool = True, transform: object = None):\n        self.data: np.ndarray = data\n        self.label: pd.DataFrame = label\n        self.isTrain: bool = train\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, item):\n        img = self.data[item]\n        pil = transforms.ToPILImage()\n        \n        if self.transform is not None:\n            img = pil(img)\n            img = self.transform(img)\n        \n        if self.isTrain:\n            label1 = self.label.grapheme_root.values[item]\n            label2 = self.label.vowel_diacritic.values[item]\n            label3 = self.label.consonant_diacritic.values[item]\n\n            return img, label1, label2, label3\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = BengaliDataset(\n        data_full, \n        train_df, \n        train=True, \n        transform=transforms.Compose([\n            transforms.CenterCrop(H), transforms.ToTensor()\n        ])\n    )\n\nloader = DataLoader(ds, batch_size=2, shuffle=False)\n\nx, y1, y2, y3 = next(iter(loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(facecolor=\"azure\")\nplt.imshow(x[0].permute(1, 2, 0).squeeze().numpy(), cmap=\"gray\")\n\nprint(\"grapheme_root: \", y1[0])\nprint(\"vowel_diacritic: \", y2[0])\nprint(\"consonant_diacritic: \", y3[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del data_full, id_full, train_df, ds, loader, x, y1, y2, y3\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## arrange test_data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntest_df = pd.read_csv(PATH+\"test.csv\")\n\ndata0 = pq.read_table(PATH+\"test_image_data_0.parquet\").to_pandas()\ndata0_id = data0[\"image_id\"]\ndata0 = data0.iloc[:, 1:].astype(np.uint8).values\ndata0 = data0.reshape(-1, 137, 236, 1)\n\ndata1 = pq.read_table(PATH+\"test_image_data_1.parquet\").to_pandas()\ndata1_id = data1[\"image_id\"]\ndata1 = data1.iloc[:, 1:].astype(np.uint8).values\ndata1 = data1.reshape(-1, 137, 236, 1)\n\ndata2 = pq.read_table(PATH+\"test_image_data_2.parquet\").to_pandas()\ndata2_id = data2[\"image_id\"]\ndata2 = data2.iloc[:, 1:].astype(np.uint8).values\ndata2 = data2.reshape(-1, 137, 236, 1)\n\ndata3 = pq.read_table(PATH+\"test_image_data_3.parquet\").to_pandas()\ndata3_id = data3[\"image_id\"]\ndata3 = data3.iloc[:, 1:].astype(np.uint8).values\ndata3 = data3.reshape(-1, 137, 236, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_full = np.vstack([data0, data1, data2, data3])\n\ndel data0, data1, data2, data3\ngc.collect()\n\nid_full = pd.concat([data0_id, data1_id, data2_id, data3_id], ignore_index=True)\ndel data0_id, data1_id, data2_id, data3_id\ngc.collect()\nprint(data_full.shape)\nprint(id_full.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith open(\"test_data_full.joblib\", \"wb\") as f:\n    joblib.dump(data_full, f, compress=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = BengaliDataset(\n        data_full, \n        test_df, \n        train=False, \n        transform=transforms.Compose([\n            transforms.CenterCrop(H), transforms.ToTensor()\n        ])\n    )\nloader = DataLoader(ds, batch_size=2, shuffle=False)\n\nx = next(iter(loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(facecolor=\"azure\")\nplt.imshow(x[0].permute(1, 2, 0).squeeze().numpy(), cmap=\"gray\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}