{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Notebook using which the metadata was generated: https://www.kaggle.com/code/defcodeking/metadata-of-records-for-shuffling-val-split.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport bson\nfrom sklearn.model_selection import train_test_split\nimport os\nimport io\nfrom PIL import Image\nimport torch\nimport albumentations as A\nfrom sklearn import preprocessing","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-23T22:18:07.592254Z","iopub.execute_input":"2022-05-23T22:18:07.592707Z","iopub.status.idle":"2022-05-23T22:18:11.406681Z","shell.execute_reply.started":"2022-05-23T22:18:07.592656Z","shell.execute_reply":"2022-05-23T22:18:11.405537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"DATA_DIR = \"../input/cdiscount-image-classification-challenge\"\nMETADATA_DIR = \"../input/cdiscount-metadata-for-shuffling-val-split\"","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:30.664785Z","iopub.execute_input":"2022-05-23T22:17:30.665120Z","iopub.status.idle":"2022-05-23T22:17:30.670451Z","shell.execute_reply.started":"2022-05-23T22:17:30.665081Z","shell.execute_reply":"2022-05-23T22:17:30.669171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"An optional but good step before using the dataset is to move away from a per-product row system to a per-image row system. That is, instead of having a row for each product, we should have a row for each image. This can be easily achieved by repeating all rows where `n_imgs` is more than 1 and then adding an additional column which goes from 0 to `n_imgs - 1` for each product. Then, we can simply index the `imgs` array that is part of each record to obtain the required image.\n\nAfter this preprocessing step, we are essentially dealing with a dataset with one row per image.","metadata":{}},{"cell_type":"code","source":"filepath = os.path.join(METADATA_DIR, \"train_metadata.csv\")\ntrain_md = pd.read_csv(filepath)\ntrain_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:30.671775Z","iopub.execute_input":"2022-05-23T22:17:30.672212Z","iopub.status.idle":"2022-05-23T22:17:37.031622Z","shell.execute_reply.started":"2022-05-23T22:17:30.672174Z","shell.execute_reply":"2022-05-23T22:17:37.030643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = os.path.join(METADATA_DIR, \"test_metadata.csv\")\ntest_md = pd.read_csv(filepath)\ntest_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:37.034740Z","iopub.execute_input":"2022-05-23T22:17:37.035391Z","iopub.status.idle":"2022-05-23T22:17:38.246546Z","shell.execute_reply.started":"2022-05-23T22:17:37.035332Z","shell.execute_reply":"2022-05-23T22:17:38.245371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Repeating Rows","metadata":{}},{"cell_type":"code","source":"len(train_md), len(test_md)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:38.248243Z","iopub.execute_input":"2022-05-23T22:17:38.248589Z","iopub.status.idle":"2022-05-23T22:17:38.256266Z","shell.execute_reply.started":"2022-05-23T22:17:38.248541Z","shell.execute_reply":"2022-05-23T22:17:38.255179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_md = train_md.loc[train_md.index.repeat(train_md[\"n_imgs\"])]\ntest_md = test_md.loc[test_md.index.repeat(test_md[\"n_imgs\"])]\nlen(train_md), len(test_md)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:38.257683Z","iopub.execute_input":"2022-05-23T22:17:38.257981Z","iopub.status.idle":"2022-05-23T22:17:39.825818Z","shell.execute_reply.started":"2022-05-23T22:17:38.257947Z","shell.execute_reply":"2022-05-23T22:17:39.824601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:39.827194Z","iopub.execute_input":"2022-05-23T22:17:39.827461Z","iopub.status.idle":"2022-05-23T22:17:39.839559Z","shell.execute_reply.started":"2022-05-23T22:17:39.827429Z","shell.execute_reply":"2022-05-23T22:17:39.838277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:39.841159Z","iopub.execute_input":"2022-05-23T22:17:39.841682Z","iopub.status.idle":"2022-05-23T22:17:39.861918Z","shell.execute_reply.started":"2022-05-23T22:17:39.841605Z","shell.execute_reply":"2022-05-23T22:17:39.860461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Column","metadata":{}},{"cell_type":"code","source":"grouped_train_md = train_md.groupby(\"pid\")\ntrain_md[\"img_idx\"] = grouped_train_md.cumcount()\ntrain_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:39.863463Z","iopub.execute_input":"2022-05-23T22:17:39.863732Z","iopub.status.idle":"2022-05-23T22:17:44.166818Z","shell.execute_reply.started":"2022-05-23T22:17:39.863700Z","shell.execute_reply":"2022-05-23T22:17:44.165477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped_test_md = test_md.groupby(\"pid\")\ntest_md[\"img_idx\"] = grouped_test_md.cumcount()\ntest_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:44.169826Z","iopub.execute_input":"2022-05-23T22:17:44.170165Z","iopub.status.idle":"2022-05-23T22:17:44.892253Z","shell.execute_reply.started":"2022-05-23T22:17:44.170119Z","shell.execute_reply":"2022-05-23T22:17:44.891135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading a Random Image","metadata":{}},{"cell_type":"markdown","source":"We define a function `read_image()` which takes a file object, a start position, a length and an index. It seeks the file pointer to the start position, reads `length`-number of bytes, decodes the record and then returns the image at the correct index as PIL image.","metadata":{}},{"cell_type":"code","source":"def read_image(f, start, length, idx):\n    f.seek(start)\n    record = bson.decode(f.read(length))\n    img = record[\"imgs\"][idx][\"picture\"]\n    return Image.open(io.BytesIO(img))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:44.893998Z","iopub.execute_input":"2022-05-23T22:17:44.894393Z","iopub.status.idle":"2022-05-23T22:17:44.901005Z","shell.execute_reply.started":"2022-05-23T22:17:44.894357Z","shell.execute_reply":"2022-05-23T22:17:44.899959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training data\n# Run cell multiple times to see more images\nn_imgs = len(train_md)\nidx = np.random.choice(range(n_imgs))\nmd = train_md.iloc[idx]\n\nfilepath = os.path.join(DATA_DIR, \"train.bson\")\nwith open(filepath, \"rb\") as f:\n    img = read_image(f, md[\"start\"], md[\"length\"], md[\"img_idx\"])\n    \nimg","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:44.902351Z","iopub.execute_input":"2022-05-23T22:17:44.902769Z","iopub.status.idle":"2022-05-23T22:17:47.208549Z","shell.execute_reply.started":"2022-05-23T22:17:44.902730Z","shell.execute_reply":"2022-05-23T22:17:47.207486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test data\n# Run cell multiple times to see more images\nn_imgs = len(test_md)\nidx = np.random.choice(range(n_imgs))\nmd = test_md.iloc[idx]\n\nfilepath = os.path.join(DATA_DIR, \"test.bson\")\nwith open(filepath, \"rb\") as f:\n    img = read_image(f, md[\"start\"], md[\"length\"], md[\"img_idx\"])\n    \nimg","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:47.209920Z","iopub.execute_input":"2022-05-23T22:17:47.210852Z","iopub.status.idle":"2022-05-23T22:17:47.819908Z","shell.execute_reply.started":"2022-05-23T22:17:47.210805Z","shell.execute_reply":"2022-05-23T22:17:47.818833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Shuffling","metadata":{}},{"cell_type":"markdown","source":"Since we are now dealing with a dataset with one row per image, we can simply shuffle the dataframe to shuffle the dataset.","metadata":{}},{"cell_type":"code","source":"train_md = train_md.sample(frac=1)\ntrain_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:47.823046Z","iopub.execute_input":"2022-05-23T22:17:47.823365Z","iopub.status.idle":"2022-05-23T22:17:51.997341Z","shell.execute_reply.started":"2022-05-23T22:17:47.823323Z","shell.execute_reply":"2022-05-23T22:17:51.996412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_md = test_md.sample(frac=1)\ntest_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:17:51.999143Z","iopub.execute_input":"2022-05-23T22:17:51.999506Z","iopub.status.idle":"2022-05-23T22:17:52.656474Z","shell.execute_reply.started":"2022-05-23T22:17:51.999463Z","shell.execute_reply":"2022-05-23T22:17:52.655427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Split","metadata":{}},{"cell_type":"markdown","source":"Similar to shuffling, we can simply split the dataframe to split the dataset.","metadata":{}},{"cell_type":"code","source":"# Encode labels before splitting\nencoder = preprocessing.LabelEncoder()\ntrain_md[\"category_id\"] = train_md[\"category_id\"].astype(str)\ntrain_md[\"labels\"] = encoder.fit_transform(train_md[\"category_id\"])","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:18:19.783000Z","iopub.execute_input":"2022-05-23T22:18:19.784016Z","iopub.status.idle":"2022-05-23T22:18:38.825590Z","shell.execute_reply.started":"2022-05-23T22:18:19.783960Z","shell.execute_reply":"2022-05-23T22:18:38.824242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_md, val_md = train_test_split(train_md, test_size=0.1, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:18:49.473276Z","iopub.execute_input":"2022-05-23T22:18:49.474184Z","iopub.status.idle":"2022-05-23T22:18:59.508547Z","shell.execute_reply.started":"2022-05-23T22:18:49.474130Z","shell.execute_reply":"2022-05-23T22:18:59.507663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:19:01.754774Z","iopub.execute_input":"2022-05-23T22:19:01.755748Z","iopub.status.idle":"2022-05-23T22:19:01.769877Z","shell.execute_reply.started":"2022-05-23T22:19:01.755691Z","shell.execute_reply":"2022-05-23T22:19:01.768799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_md.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:19:03.965729Z","iopub.execute_input":"2022-05-23T22:19:03.966091Z","iopub.status.idle":"2022-05-23T22:19:03.980069Z","shell.execute_reply.started":"2022-05-23T22:19:03.966033Z","shell.execute_reply":"2022-05-23T22:19:03.978985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_md), len(val_md)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:19:11.459663Z","iopub.execute_input":"2022-05-23T22:19:11.460775Z","iopub.status.idle":"2022-05-23T22:19:11.468330Z","shell.execute_reply.started":"2022-05-23T22:19:11.460717Z","shell.execute_reply":"2022-05-23T22:19:11.467200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example PyTorch Dataset","metadata":{}},{"cell_type":"code","source":"class CDiscountDataset(torch.utils.data.Dataset):\n    def __init__(self, ds_filepath, metadata_df, has_labels=True, transforms=None):\n        self.f = open(ds_filepath, \"rb\")\n        self.md_df = metadata_df\n        self.transforms = transforms\n        self.has_labels = has_labels\n        \n    def read_image(self, record_metadata):\n        start = record_metadata[\"start\"]\n        length = record_metadata[\"length\"]\n        idx = record_metadata[\"img_idx\"]\n        \n        self.f.seek(start)\n        record = bson.decode(self.f.read(length))\n        \n        img = record[\"imgs\"][idx][\"picture\"]\n        return Image.open(io.BytesIO(img))\n        \n    def __len__(self):\n        return len(self.md_df)\n    \n    def __getitem__(self, idx):\n        metadata = self.md_df.iloc[idx]\n        img = self.read_image(metadata)\n        \n        img = np.array(img, dtype=np.float32)\n        \n        if self.transforms is not None:\n            img = self.transforms(image=img)[\"image\"]\n            \n        img = np.swapaxes(img, -1, 0)\n            \n        sample = {\"imgs\": torch.tensor(img)}\n            \n        if self.has_labels is True:\n            sample[\"labels\"] = metadata[\"labels\"]\n            \n        return sample","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:28:27.556517Z","iopub.execute_input":"2022-05-23T22:28:27.556880Z","iopub.status.idle":"2022-05-23T22:28:27.911862Z","shell.execute_reply.started":"2022-05-23T22:28:27.556833Z","shell.execute_reply":"2022-05-23T22:28:27.910848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = os.path.join(DATA_DIR, \"train.bson\")\ntrain_ds = CDiscountDataset(ds_filepath=filepath, metadata_df=train_md)\nval_ds = CDiscountDataset(ds_filepath=filepath, metadata_df=val_md)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:28:28.229439Z","iopub.execute_input":"2022-05-23T22:28:28.229947Z","iopub.status.idle":"2022-05-23T22:28:28.238442Z","shell.execute_reply.started":"2022-05-23T22:28:28.229893Z","shell.execute_reply":"2022-05-23T22:28:28.237414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath = os.path.join(DATA_DIR, \"test.bson\")\ntest_ds = CDiscountDataset(ds_filepath=filepath, metadata_df=test_md, has_labels=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:28:29.151014Z","iopub.execute_input":"2022-05-23T22:28:29.152085Z","iopub.status.idle":"2022-05-23T22:28:29.158131Z","shell.execute_reply.started":"2022-05-23T22:28:29.152021Z","shell.execute_reply":"2022-05-23T22:28:29.157176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = torch.utils.data.DataLoader(\n    dataset=train_ds,\n    batch_size=16,\n    shuffle=True,\n)\n\nval_dataloader = torch.utils.data.DataLoader(\n    dataset=val_ds,\n    batch_size=16,\n    shuffle=True,\n)\n\ntest_dataloader = torch.utils.data.DataLoader(\n    dataset=test_ds,\n    batch_size=16,\n    shuffle=True,\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:28:29.487516Z","iopub.execute_input":"2022-05-23T22:28:29.488162Z","iopub.status.idle":"2022-05-23T22:28:29.494957Z","shell.execute_reply.started":"2022-05-23T22:28:29.488117Z","shell.execute_reply":"2022-05-23T22:28:29.493847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"itrain_dl = iter(train_dataloader)\nival_dl = iter(val_dataloader)\nitest_dl = iter(test_dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:29:42.439436Z","iopub.execute_input":"2022-05-23T22:29:42.439794Z","iopub.status.idle":"2022-05-23T22:29:42.579157Z","shell.execute_reply.started":"2022-05-23T22:29:42.439756Z","shell.execute_reply":"2022-05-23T22:29:42.578075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Run cell again to get next batch\ntrain_sample = next(itrain_dl)\nval_sample = next(ival_dl)\ntest_sample = next(itest_dl)\n\ntrain_sample[\"imgs\"].shape, val_sample[\"imgs\"].shape, test_sample[\"imgs\"].shape, ","metadata":{"execution":{"iopub.status.busy":"2022-05-23T22:31:18.215072Z","iopub.execute_input":"2022-05-23T22:31:18.215605Z","iopub.status.idle":"2022-05-23T22:31:19.140569Z","shell.execute_reply.started":"2022-05-23T22:31:18.215552Z","shell.execute_reply":"2022-05-23T22:31:19.139501Z"},"trusted":true},"execution_count":null,"outputs":[]}]}