{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":77180,"databundleVersionId":9096404,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<div style='background-color: #fff7f7; border: 3px solid Red;'>\r\n    <font size=\"+1\" color=\"Red\">\r\n        <b>Please comment and upvote if you like this work!</b>\r\n    </font>\r\n</div>\r\n<div style='background-color: #fff7f7; border: 3px solid Blue;'>\r\n    <font size=\"+1\" color=\"Red\">\r\n        <b>Take a look at the other notebooks in my profile ->\r\n        <a href=\"https://www.kaggle.com/killa92/code\">  Kaggle Notebooks </a></b>\r\n    </font>\r\n</div>    \r\n<div style='background-color: #fff7f7; border: 3px solid yellow;'>\r\n    <font size=\"+1\" color=\"Red\">\r\n        <b><span style='color: green;'>Let's Support Each Other!!</span></b>\r\n    </font>\r\n</div>","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"<div style='background-color: #fff7f7; border: 2px solid '>\r\n    <font size=\"+2\" color=\"blue\" ><b>🗂️1. Custom Datasets and DataLoaders 🗂️</b></font>\r\n</div>","metadata":{}},{"cell_type":"code","source":"import os, torch, random, numpy as np, pandas as pd\nfrom glob import glob; from PIL import Image\nfrom torch.utils.data import random_split, Dataset, DataLoader\nfrom torchvision import transforms as T\ntorch.manual_seed(2024)\n\nclass CustomDataset(Dataset):\n    \n    \n    def __init__(self, root, data_type = \"train\", transformations = None, cls_names = None):\n        \n        \n        assert data_type in [\"train\", \"test\"], \"Please choose a proper data type!\"\n        \n        self.transformations, self.data_type   = transformations, data_type\n        self.cls_names, self.cls_counts = {}, {}\n        self.ims_path = \"/images/kaggle/working/Reorganized_Data/images\"\n\n        df = pd.read_csv(f\"{root}/{data_type}.csv\")        \n        self.im_paths = [f\"{root}/{self.ims_path}/{path}\" for path in list(df[\"Images\"])]\n        \n        if self.data_type == \"train\": \n            self.labels = list(df[\"Labels\"]); self.get_info()   \n\n    def get_info(self):\n\n        count = 0\n        for idx, label in enumerate(self.labels):\n            if label not in self.cls_names:  self.cls_names[label] = count; count += 1\n            if label not in self.cls_counts: self.cls_counts[label] = 1\n            else: self.cls_counts[label] += 1    \n    \n    def __len__(self): return len(self.im_paths)\n\n    def get_pos_neg_im_paths(self, qry_label):\n        \n        pos_im_paths = [self.im_paths[idx] for idx, lbl in enumerate(self.labels) if lbl == qry_label]\n        neg_im_paths = {self.im_paths[idx]: lbl for idx, lbl in enumerate(self.labels) if lbl != qry_label}\n        \n        pos_rand_int = random.randint(a = 0, b = len(pos_im_paths) - 1)\n        neg_rand_int = random.randint(a = 0, b = len(neg_im_paths) - 1)\n        \n        return pos_im_paths[pos_rand_int], list(neg_im_paths.keys())[neg_rand_int], np.unique(list(neg_im_paths.values()))[0]\n    \n    def __len__(self): return len(self.im_paths)\n\n    def __getitem__(self, idx):\n        \n        im_path = self.im_paths[idx]\n        qry_im = Image.open(im_path).convert(\"RGB\")\n        \n        if self.data_type == \"train\":       \n            \n            qry_label = self.labels[idx]\n    \n            pos_im_path, neg_im_path, neg_gt = self.get_pos_neg_im_paths(qry_label = qry_label)\n            pos_im, neg_im = Image.open(pos_im_path).convert(\"RGB\"), Image.open(neg_im_path).convert(\"RGB\")\n    \n            qry_gt = self.cls_names[qry_label]\n    \n            if self.transformations is not None: qry_im = self.transformations(qry_im); pos_im = self.transformations(pos_im); neg_im = self.transformations(neg_im)\n    \n            data = {}\n    \n            data[\"qry_im\"] = qry_im\n            data[\"qry_gt\"] = qry_gt\n            data[\"pos_im\"] = pos_im\n            data[\"neg_im\"] = neg_im\n            data[\"neg_gt\"] = neg_gt\n                \n            return data\n        \n        else: return qry_im\n    \ndef get_dls(root, transformations, bs, ns = 4):\n    \n    ds                    = CustomDataset(root = root, data_type = \"train\", transformations = transformations)\n    cls_names, cls_counts = ds.cls_names, ds.cls_counts\n    \n    all_len = len(ds); tr_len = int(all_len * 0.9); vl_len = all_len - tr_len\n    tr_ds, vl_ds = random_split(dataset = ds, lengths = [tr_len, vl_len])\n    \n    ts_ds                 = CustomDataset(root = root, data_type = \"test\", transformations = transformations,  cls_names = cls_names)\n        \n    tr_dl, val_dl, ts_dl = DataLoader(tr_ds, batch_size = bs, shuffle = True, num_workers = ns), DataLoader(vl_ds, batch_size = bs, shuffle = False, num_workers = ns), DataLoader(ts_ds, batch_size = 1, shuffle = False, num_workers = ns)\n    \n    return tr_dl, val_dl, ts_dl, cls_names, cls_counts \n\nroot = \"/kaggle/input/d-4-computer-vision\"\nmean, std, size, bs = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225], 224, 32\ntfs = T.Compose([T.ToTensor(), T.Resize(size = (size, size), antialias = False), T.Normalize(mean = mean, std = std)])\ntr_dl, val_dl, ts_dl, classes, cls_counts = get_dls(root = root, transformations = tfs, bs = bs)\n\nprint(len(tr_dl)); print(len(val_dl)); print(len(ts_dl)); print(classes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T02:25:33.042176Z","iopub.execute_input":"2024-11-13T02:25:33.042645Z","iopub.status.idle":"2024-11-13T02:25:33.093475Z","shell.execute_reply.started":"2024-11-13T02:25:33.042601Z","shell.execute_reply":"2024-11-13T02:25:33.092027Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":" <div style='background-color: #fff7f7; border: 2px solid '>\r\n    <font size=\"+2\" color=\"blue\" ><b>📈 2. Data Analysis 📈</b></font>\r\n</div>","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom matplotlib import pyplot as plt\nfrom torchvision import transforms as T\n\nclass Visualization:\n\n    def __init__(self, vis_datas, n_ims, rows, cmap = None, cls_names = None, cls_counts = None, t_type = \"rgb\"):\n\n        self.n_ims, self.rows = n_ims, rows\n        self.t_type, self.cmap,  = t_type, cmap\n        self.cls_names = cls_names\n        \n        data_names = [\"train\", \"val\", \"test\"]\n        self.vis_datas = {data_names[i]: vis_datas[i] for i in range(len(vis_datas))} \n        if isinstance(cls_counts, list): self.analysis_datas = {data_names[i]: cls_counts[i] for i in range(len(cls_counts))} \n        else: self.analysis_datas = {\"all\": cls_counts}\n\n    def tn2np(self, t):\n        \n        gray_tfs = T.Compose([T.Normalize(mean = [ 0.], std = [1/0.5]), T.Normalize(mean = [-0.5], std = [1])])\n        rgb_tfs = T.Compose([T.Normalize(mean = [ 0., 0., 0. ], std = [ 1/0.229, 1/0.224, 1/0.225 ]), T.Normalize(mean = [ -0.485, -0.456, -0.406 ], std = [ 1., 1., 1. ])])\n        \n        invTrans = gray_tfs if self.t_type == \"gray\" else rgb_tfs \n        \n        return (invTrans(t) * 255).detach().squeeze().cpu().permute(1,2,0).numpy().astype(np.uint8) if self.t_type == \"gray\" else (invTrans(t) * 255).detach().cpu().permute(1,2,0).numpy().astype(np.uint8)\n\n    def plot(self, rows, cols, count, im, title = \"Original Image\"):\n    \n        plt.subplot(rows, cols, count)\n        plt.imshow(self.tn2np(im))\n        plt.axis(\"off\"); plt.title(title)\n        \n        return count + 1\n\n    def vis(self, data, save_name):\n\n        print(f\"{save_name.upper()} Data Visualization is in process...\\n\")\n        assert self.cmap in [\"rgb\", \"gray\"], \"Please choose rgb or gray cmap\"\n        if self.cmap == \"rgb\": cmap = \"viridis\"\n        cols = self.n_ims // self.rows; count = 1\n        \n        plt.figure(figsize = (25, 20))\n                \n        indices = [np.random.randint(low = 0, high = len(data) - 1) for _ in range(self.n_ims)]\n\n        for idx, index in enumerate(indices):\n        \n            if count == self.n_ims + 1: break\n            \n            meta_data = data[index]\n            qry_im, pos_im, neg_im, qry_lbl, neg_lbl = meta_data[\"qry_im\"], meta_data[\"pos_im\"], meta_data[\"neg_im\"], meta_data[\"qry_gt\"], meta_data[\"neg_gt\"]\n\n            # First Plot\n            count = self.plot(self.rows, cols, count, im = qry_im, title = f\"Query Image \\n Class -> {self.cls_names[qry_lbl]}\")\n\n            # Second Plot\n            count = self.plot(self.rows, cols, count, im = pos_im, title = f\"Positive Image \\n Class -> {self.cls_names[qry_lbl]}\")\n\n            # Third Plot\n            count = self.plot(self.rows, cols, count, im = neg_im, title = f\"Negative Image \\n Class -> {self.cls_names[neg_lbl]}\")\n        \n        plt.show()\n\n    def data_analysis(self, cls_counts, save_name):\n\n        print(\"Data analysis is in process...\\n\")\n        \n        width, text_width, text_height = 0.7, 0.05, 2\n        cls_names = list(cls_counts.keys()); counts = list(cls_counts.values())\n        \n        _, ax = plt.subplots(figsize = (20, 10))\n        indices = np.arange(len(counts))\n\n        ax.bar(indices, counts, width, color = \"firebrick\")\n        ax.set_xlabel(\"Class Names\", color = \"red\")\n        ax.set_xticklabels(cls_names)\n        ax.set(xticks = indices, xticklabels = cls_names)\n        ax.set_ylabel(\"Data Counts\", color = \"red\")\n        ax.set_title(f\"Dataset Class Imbalance Analysis\")\n\n        for i, v in enumerate(counts): ax.text(i - text_width, v + text_height, str(v), color = \"royalblue\")\n    \n    def visualization(self): [self.vis(data.dataset, save_name) for (save_name, data) in self.vis_datas.items()]\n        \n    def analysis(self): [self.data_analysis(data, save_name) for (save_name, data) in self.analysis_datas.items()]\n        \nvis = Visualization(vis_datas = [tr_dl, val_dl], n_ims = 18, rows = 6, cmap = \"rgb\", cls_names = list(classes.keys()), cls_counts = cls_counts)\nvis.analysis()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T02:26:10.839793Z","iopub.execute_input":"2024-11-13T02:26:10.840244Z","iopub.status.idle":"2024-11-13T02:26:11.239092Z","shell.execute_reply.started":"2024-11-13T02:26:10.840202Z","shell.execute_reply":"2024-11-13T02:26:11.237542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<div style='background-color: #fff7f7; border: 2px solid '>\r\n    <font size=\"+2\" color=\"blue\" ><b>📊 3. Data Visualization 📊</b></font>\r\n</div>","metadata":{}},{"cell_type":"code","source":"vis.visualization()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-13T02:26:12.236507Z","iopub.execute_input":"2024-11-13T02:26:12.236927Z","iopub.status.idle":"2024-11-13T02:26:37.138596Z","shell.execute_reply.started":"2024-11-13T02:26:12.236888Z","shell.execute_reply":"2024-11-13T02:26:37.137239Z"}},"outputs":[],"execution_count":null}]}