{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is my first attempt on a competition\n## If my understanding is correct we have to train an object detector on underwater images (video) to detect starfish, let's try to use DeTr (Facebook).","metadata":{}},{"cell_type":"markdown","source":"This notebook was adaptaded from [https://www.kaggle.com/tanulsingh077/end-to-end-object-detection-with-transformers-detr].","metadata":{}},{"cell_type":"markdown","source":"Firstly, clone the DeTr Repo.","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/facebookresearch/detr.git   #cloning github repo of detr to import its unique loss","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:25.030781Z","iopub.execute_input":"2021-12-31T15:18:25.031159Z","iopub.status.idle":"2021-12-31T15:18:25.739876Z","shell.execute_reply.started":"2021-12-31T15:18:25.031103Z","shell.execute_reply":"2021-12-31T15:18:25.738968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Import libraries/modules","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nfrom datetime import datetime\nimport time\nimport random\nfrom tqdm import tqdm\npd.options.mode.chained_assignment = None\nfrom pandas.io.json import json_normalize\nimport ast \n\n#Torch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nimport torchvision\n#sklearn\nfrom sklearn.model_selection import StratifiedKFold,train_test_split\n#CV\nimport cv2\n\n################# DETR FUCNTIONS FOR LOSS######################## \nimport sys\nsys.path.append('./detr/')\n\nfrom detr.models.matcher import HungarianMatcher\nfrom detr.models.detr import SetCriterion\n#################################################################\n\n#Albumenatations\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\n\n#Glob\nfrom glob import glob","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:25.742276Z","iopub.execute_input":"2021-12-31T15:18:25.742563Z","iopub.status.idle":"2021-12-31T15:18:25.751373Z","shell.execute_reply.started":"2021-12-31T15:18:25.742525Z","shell.execute_reply":"2021-12-31T15:18:25.750531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load the dataset, the annotations column contain the bounding boxes for each image as a string, we will transform then to literal and only keep the ones with BB.","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/tensorflow-great-barrier-reef/train.csv\")\ndf_train['image_path'] = \"/kaggle/input/tensorflow-great-barrier-reef/train_images/video_\" + df_train['video_id'].astype(str) + '/' + df_train['video_frame'].astype(str) + '.jpg'\ndf_train=df_train.loc[df_train[\"annotations\"].astype(str) != \"[]\"]\ndf_train['annotations'] = df_train['annotations'].apply(ast.literal_eval)\ndf_train.reset_index(inplace=True)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:25.752881Z","iopub.execute_input":"2021-12-31T15:18:25.753156Z","iopub.status.idle":"2021-12-31T15:18:26.085486Z","shell.execute_reply.started":"2021-12-31T15:18:25.753121Z","shell.execute_reply":"2021-12-31T15:18:26.084662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train['annotations'].apply(len) >=5 ]\n#df_train.iloc[3902].annotations","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:26.087596Z","iopub.execute_input":"2021-12-31T15:18:26.087920Z","iopub.status.idle":"2021-12-31T15:18:26.126728Z","shell.execute_reply.started":"2021-12-31T15:18:26.087883Z","shell.execute_reply":"2021-12-31T15:18:26.126063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## To maintain the needed format (based on MR_KNOWNOTHING notebook), we will extract the bounding boxes from each image and separate them.","metadata":{}},{"cell_type":"code","source":"len_df = len(df_train)\nrow_df = pd.DataFrame()\nannot_df = pd.DataFrame()\nfor i in range(len_df):\n    row = df_train.iloc[i]\n    for annot in row['annotations']:\n        row_annot = pd.json_normalize(annot)\n        row_attr = row[['video_id','sequence','video_frame','sequence_frame','image_id','image_path']].to_frame().transpose()\n        row_df = row_df.append(row_attr, ignore_index=True)\n        annot_df = annot_df.append(row_annot, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:26.127805Z","iopub.execute_input":"2021-12-31T15:18:26.128043Z","iopub.status.idle":"2021-12-31T15:18:54.321992Z","shell.execute_reply.started":"2021-12-31T15:18:26.128010Z","shell.execute_reply":"2021-12-31T15:18:54.321279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.concat([row_df, annot_df], axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.323133Z","iopub.execute_input":"2021-12-31T15:18:54.325049Z","iopub.status.idle":"2021-12-31T15:18:54.331800Z","shell.execute_reply.started":"2021-12-31T15:18:54.325018Z","shell.execute_reply":"2021-12-31T15:18:54.331087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.333084Z","iopub.execute_input":"2021-12-31T15:18:54.333380Z","iopub.status.idle":"2021-12-31T15:18:54.350706Z","shell.execute_reply.started":"2021-12-31T15:18:54.333339Z","shell.execute_reply":"2021-12-31T15:18:54.349896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Some images have BB that overpass the image size (720,1280), we will drop them to avoid future problems.","metadata":{}},{"cell_type":"code","source":"df_train.drop(df_train[df_train[['x','width']].sum(axis=1) > 1280].index, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.352116Z","iopub.execute_input":"2021-12-31T15:18:54.352379Z","iopub.status.idle":"2021-12-31T15:18:54.362463Z","shell.execute_reply.started":"2021-12-31T15:18:54.352350Z","shell.execute_reply":"2021-12-31T15:18:54.361768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.drop(df_train[df_train[['y','height']].sum(axis=1) > 720].index, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.364201Z","iopub.execute_input":"2021-12-31T15:18:54.364912Z","iopub.status.idle":"2021-12-31T15:18:54.373942Z","shell.execute_reply.started":"2021-12-31T15:18:54.364875Z","shell.execute_reply":"2021-12-31T15:18:54.373023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's define the basic configuration for this model.","metadata":{}},{"cell_type":"code","source":"n_folds = 5\nseed = 42\nnum_classes = 2\nnum_queries = 20\nnull_class_coef = 0.2\nBATCH_SIZE = 8 # 16\nLR = 1e-4   #2e-5\nEPOCHS = 1","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.378174Z","iopub.execute_input":"2021-12-31T15:18:54.378501Z","iopub.status.idle":"2021-12-31T15:18:54.383520Z","shell.execute_reply.started":"2021-12-31T15:18:54.378474Z","shell.execute_reply":"2021-12-31T15:18:54.382072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.384803Z","iopub.execute_input":"2021-12-31T15:18:54.385303Z","iopub.status.idle":"2021-12-31T15:18:54.392142Z","shell.execute_reply.started":"2021-12-31T15:18:54.385248Z","shell.execute_reply":"2021-12-31T15:18:54.391447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.393514Z","iopub.execute_input":"2021-12-31T15:18:54.394388Z","iopub.status.idle":"2021-12-31T15:18:54.401094Z","shell.execute_reply.started":"2021-12-31T15:18:54.394353Z","shell.execute_reply":"2021-12-31T15:18:54.400393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Separate on folds","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=seed)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.403998Z","iopub.execute_input":"2021-12-31T15:18:54.404186Z","iopub.status.idle":"2021-12-31T15:18:54.409825Z","shell.execute_reply.started":"2021-12-31T15:18:54.404163Z","shell.execute_reply":"2021-12-31T15:18:54.409091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_folds = df_train[['image_path']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_path').count()\ndf_folds.loc[:, 'video_id'] = df_train[['image_path', 'video_id']].groupby('image_path').min()['video_id']\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['video_id'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\ndf_folds.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\ndf_folds.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.412613Z","iopub.execute_input":"2021-12-31T15:18:54.412836Z","iopub.status.idle":"2021-12-31T15:18:54.823614Z","shell.execute_reply.started":"2021-12-31T15:18:54.412803Z","shell.execute_reply":"2021-12-31T15:18:54.822701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_folds[\"bbox_count\"].max()","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.824948Z","iopub.execute_input":"2021-12-31T15:18:54.825280Z","iopub.status.idle":"2021-12-31T15:18:54.831652Z","shell.execute_reply.started":"2021-12-31T15:18:54.825244Z","shell.execute_reply":"2021-12-31T15:18:54.830996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Transformations to perform on the images.","metadata":{}},{"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([A.OneOf([\n                    A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2, val_shift_limit=0.2, p=0.9),\n                      A.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=0.9)],p=0.9),\n                      \n                      A.ToGray(p=0.01),\n                      \n                      A.HorizontalFlip(p=0.5),\n                      \n                      A.VerticalFlip(p=0.5),\n                      \n                      #A.Resize(height=512, width=512, p=1),\n                      \n                      #A.Cutout(num_holes=8, max_h_size=32, max_w_size=32, fill_value=0, p=0.5),\n                      \n                      ToTensorV2(p=1.0)],\n                      \n                      p=1.0,\n                     \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels']),\n                      )\n                      \n\ndef get_valid_transforms():\n    return A.Compose([ToTensorV2(p=1.0)], \n                      p=1.0, \n                      bbox_params=A.BboxParams(format='coco',min_area=0, min_visibility=0,label_fields=['labels']),\n                      )\n\n#A.Resize(height=512, width=512, p=1.0),","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.832776Z","iopub.execute_input":"2021-12-31T15:18:54.833572Z","iopub.status.idle":"2021-12-31T15:18:54.843168Z","shell.execute_reply.started":"2021-12-31T15:18:54.833528Z","shell.execute_reply":"2021-12-31T15:18:54.842461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass StarfishDataset(Dataset):\n    def __init__(self,image_ids,dataframe,transforms=None):\n        self.image_ids = image_ids\n        self.df = dataframe\n        self.transforms = transforms\n          \n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n    \n    def __getitem__(self,index):\n        image_id = self.image_ids[index]\n        records = self.df[self.df['image_path'] == image_id]\n        \n        image = cv2.imread(f'{image_id}', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        \n        # DETR takes in data in coco format \n        boxes = records[['x', 'y', 'width', 'height']].values\n        #Area of bb\n        area = boxes[:,2]*boxes[:,3]\n        area = torch.as_tensor(area, dtype=torch.float32)\n        \n        # AS pointed out by PRVI It works better if the main class is labelled as zero\n        labels =  np.zeros(len(boxes), dtype=np.int32)\n        \n        if self.transforms:\n            sample = {\n                'image': image,\n                'bboxes': boxes,\n                'labels': labels\n            }\n            sample = self.transforms(**sample)\n            image = sample['image']\n            boxes = sample['bboxes']\n            labels = sample['labels']\n                    \n        \n                                \n        #Normalizing BBOXES\n        \n        _,h,w = image.shape\n        boxes = A.augmentations.bbox_utils.normalize_bboxes(sample['bboxes'],rows=h,cols=w)\n        target = {}\n        target['boxes'] = torch.as_tensor(boxes,dtype=torch.float32)\n        target['labels'] = torch.as_tensor(labels,dtype=torch.long)\n        target['image_id'] = torch.tensor([index])\n        target['area'] = area\n        \n        return image, target, image_id","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.844223Z","iopub.execute_input":"2021-12-31T15:18:54.844490Z","iopub.status.idle":"2021-12-31T15:18:54.858505Z","shell.execute_reply.started":"2021-12-31T15:18:54.844456Z","shell.execute_reply":"2021-12-31T15:18:54.857788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class DETRModel(nn.Module):\n    def __init__(self,num_classes,num_queries):\n        super(DETRModel,self).__init__()\n        self.num_classes = num_classes\n        self.num_queries = num_queries\n        \n        self.model = torch.hub.load('facebookresearch/detr', 'detr_resnet50', pretrained=True)\n        self.in_features = self.model.class_embed.in_features\n        \n        self.model.class_embed = nn.Linear(in_features=self.in_features,out_features=self.num_classes)\n        self.model.num_queries = self.num_queries\n        \n    def forward(self,images):\n        return self.model(images)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.859772Z","iopub.execute_input":"2021-12-31T15:18:54.860219Z","iopub.status.idle":"2021-12-31T15:18:54.870218Z","shell.execute_reply.started":"2021-12-31T15:18:54.860184Z","shell.execute_reply":"2021-12-31T15:18:54.869472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ncode taken from github repo detr , 'code present in engine.py'\n'''\n\nmatcher = HungarianMatcher()\n\nweight_dict = weight_dict = {'loss_ce': 0.5, 'loss_bbox': 0.5 , 'loss_giou': 0.5}\n\nlosses = ['labels', 'boxes', 'cardinality']","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.871130Z","iopub.execute_input":"2021-12-31T15:18:54.873583Z","iopub.status.idle":"2021-12-31T15:18:54.882618Z","shell.execute_reply.started":"2021-12-31T15:18:54.873517Z","shell.execute_reply":"2021-12-31T15:18:54.881845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function","metadata":{}},{"cell_type":"code","source":"def train_fn(data_loader,model,criterion,optimizer,device,scheduler,epoch):\n    model.train()\n    criterion.train()\n    \n    summary_loss = AverageMeter()\n    \n    tk0 = tqdm(data_loader, total=len(data_loader))\n    \n    for step, (images, targets, image_ids) in enumerate(tk0):\n        \n        \n        images = list(image.to(device) for image in images)\n        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n        \n        output = model(images)\n        \n        loss_dict = criterion(output, targets)\n        weight_dict = criterion.weight_dict\n                \n        losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n        \n        optimizer.zero_grad()\n\n        losses.backward()\n        optimizer.step()\n        if scheduler is not None:\n            scheduler.step()\n        \n        summary_loss.update(losses.item(),BATCH_SIZE)\n        tk0.set_postfix(loss=summary_loss.avg)\n                \n\n    return summary_loss","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.883598Z","iopub.execute_input":"2021-12-31T15:18:54.883780Z","iopub.status.idle":"2021-12-31T15:18:54.894921Z","shell.execute_reply.started":"2021-12-31T15:18:54.883759Z","shell.execute_reply":"2021-12-31T15:18:54.894206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eval Function","metadata":{}},{"cell_type":"code","source":"def eval_fn(data_loader, model,criterion, device):\n    model.eval()\n    criterion.eval()\n    summary_loss = AverageMeter()\n    \n    with torch.no_grad():\n        \n        tk0 = tqdm(data_loader, total=len(data_loader))\n        for step, (images, targets, image_ids) in enumerate(tk0):\n            \n            images = list(image.to(device) for image in images)\n            targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n\n            output = model(images)\n        \n            loss_dict = criterion(output, targets)\n            weight_dict = criterion.weight_dict\n            \n    \n        \n            losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict)\n            \n            summary_loss.update(losses.item(),BATCH_SIZE)\n            tk0.set_postfix(loss=summary_loss.avg)\n        \n    \n    return summary_loss","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.897628Z","iopub.execute_input":"2021-12-31T15:18:54.898137Z","iopub.status.idle":"2021-12-31T15:18:54.907957Z","shell.execute_reply.started":"2021-12-31T15:18:54.898100Z","shell.execute_reply":"2021-12-31T15:18:54.907117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engine","metadata":{}},{"cell_type":"code","source":"def collate_fn(batch):\n    return tuple(zip(*batch))","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.909094Z","iopub.execute_input":"2021-12-31T15:18:54.909824Z","iopub.status.idle":"2021-12-31T15:18:54.919383Z","shell.execute_reply.started":"2021-12-31T15:18:54.909787Z","shell.execute_reply":"2021-12-31T15:18:54.918685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.920873Z","iopub.execute_input":"2021-12-31T15:18:54.921380Z","iopub.status.idle":"2021-12-31T15:18:54.928640Z","shell.execute_reply.started":"2021-12-31T15:18:54.921343Z","shell.execute_reply":"2021-12-31T15:18:54.927886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run","metadata":{}},{"cell_type":"code","source":"def run(fold):\n    \n    df_extrain = df_folds[df_folds['fold'] != fold]\n    df_valid = df_folds[df_folds['fold'] == fold]\n    \n    train_dataset = StarfishDataset(\n    image_ids=df_extrain.index.values,\n    dataframe=df_train,\n    transforms=get_train_transforms()\n    )\n\n    valid_dataset = StarfishDataset(\n    image_ids=df_valid.index.values,\n    dataframe=df_train,\n    transforms=get_valid_transforms()\n    )\n    \n    train_data_loader = DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=2, #4\n    collate_fn=collate_fn\n    )\n\n    valid_data_loader = DataLoader(\n    valid_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=False,\n    num_workers=2, #4\n    collate_fn=collate_fn\n    )\n    \n    device = torch.device('cuda')\n    model = DETRModel(num_classes=num_classes,num_queries=num_queries)\n    model = model.to(device)\n    criterion = SetCriterion(num_classes-1, matcher, weight_dict, eos_coef = null_class_coef, losses=losses)\n    criterion = criterion.to(device)\n    \n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n    \n    best_loss = 10**5\n    for epoch in range(EPOCHS):\n        train_loss = train_fn(train_data_loader, model,criterion, optimizer,device,scheduler=None,epoch=epoch)\n        valid_loss = eval_fn(valid_data_loader, model,criterion, device)\n                \n        print('|EPOCH {}| TRAIN_LOSS {}| VALID_LOSS {}|'.format(epoch+1,train_loss.avg,valid_loss.avg))\n        \n        if valid_loss.avg < best_loss:\n            best_loss = valid_loss.avg\n            print('Best model found for Fold {} in Epoch {}........Saving Model'.format(fold,epoch+1))\n            torch.save(model.state_dict(), f'detr_best_{fold}.pth')","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.930009Z","iopub.execute_input":"2021-12-31T15:18:54.930549Z","iopub.status.idle":"2021-12-31T15:18:54.942700Z","shell.execute_reply.started":"2021-12-31T15:18:54.930504Z","shell.execute_reply":"2021-12-31T15:18:54.941922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run on the 0 fold and save the best model.","metadata":{}},{"cell_type":"code","source":"run(fold=0)\n#run(df_to_train)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:18:54.945479Z","iopub.execute_input":"2021-12-31T15:18:54.945976Z","iopub.status.idle":"2021-12-31T15:27:51.618504Z","shell.execute_reply.started":"2021-12-31T15:18:54.945943Z","shell.execute_reply":"2021-12-31T15:27:51.617517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## View on sample","metadata":{}},{"cell_type":"code","source":"def view_sample(df_valid,model,device):\n    '''\n    Code taken from Peter's Kernel \n    https://www.kaggle.com/pestipeti/pytorch-starter-fasterrcnn-train\n    '''\n    valid_dataset = StarfishDataset(image_ids=df_valid.index.values,\n                                 dataframe=df_train,\n                                 transforms=get_valid_transforms()\n                                )\n     \n    valid_data_loader = DataLoader(\n                                    valid_dataset,\n                                    batch_size=BATCH_SIZE,\n                                    shuffle=True,\n                                   num_workers=2,\n                                   collate_fn=collate_fn)\n    \n    images, targets, image_ids = next(iter(valid_data_loader))\n    \n\n    _,h,w = images[0].shape # for de normalizing images\n    \n    images = list(img.to(device) for img in images)\n    targets = [{k: v.to(device) for k, v in t.items()} for t in targets]\n    \n    boxes = targets[0]['boxes'].cpu().numpy()\n    boxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(boxes,h,w)]\n    sample = images[0].permute(1,2,0).cpu().numpy()\n    \n    model.eval()\n    model.to(device)\n    cpu_device = torch.device(\"cpu\")\n    \n    with torch.no_grad():\n        outputs = model(images)\n        \n    outputs = [{k: v.to(cpu_device) for k, v in outputs.items()}]\n    \n    fig, ax = plt.subplots(1, 1, figsize=(16, 8))\n\n    for box in boxes:\n        cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2]+box[0], box[3]+box[1]),\n                  (220, 0, 0), 3)\n\n    oboxes = outputs[0]['pred_boxes'][0].detach().cpu().numpy()\n    oboxes = [np.array(box).astype(np.int32) for box in A.augmentations.bbox_utils.denormalize_bboxes(oboxes,h,w)]\n    prob   = outputs[0]['pred_logits'][0].softmax(1).detach().cpu().numpy()[:,0]\n    for box,p in zip(oboxes,prob):\n        if p >0.2:\n            print(p)\n            color = (0,0,220) #if p>0.5 else (0,0,0)\n            cv2.rectangle(sample,\n                  (box[0], box[1]),\n                  (box[2]+box[0], box[3]+box[1]),\n                  color, 3)\n    \n    ax.set_axis_off()\n    ax.imshow(sample)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:30:20.742843Z","iopub.execute_input":"2021-12-31T15:30:20.743120Z","iopub.status.idle":"2021-12-31T15:30:20.761608Z","shell.execute_reply.started":"2021-12-31T15:30:20.743089Z","shell.execute_reply":"2021-12-31T15:30:20.760887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DETRModel(num_classes=num_classes,num_queries=10)\nmodel.load_state_dict(torch.load(\"./detr_best_0.pth\"))\nview_sample(df_folds[df_folds['fold'] == 0],model=model,device=torch.device('cuda'))","metadata":{"execution":{"iopub.status.busy":"2021-12-31T15:30:32.851875Z","iopub.execute_input":"2021-12-31T15:30:32.852492Z","iopub.status.idle":"2021-12-31T15:30:35.827619Z","shell.execute_reply.started":"2021-12-31T15:30:32.852451Z","shell.execute_reply":"2021-12-31T15:30:35.824459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# - - - - - ","metadata":{}}]}