{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### OBJECTIVE\nAs PoC (Proof of Concept), show that active learning can help improve the efficacy of crowd source data annotations.\n\n### POC TODO\n- understand and play around the codebase (`AI-MDN` repo)\n    - set up dataloader\n    - run `train` (included testing evaluation metrics)\n    - run `active_learning_cycle`\n    - (set up VOC 2007 + 2012 datasets)\n    - (try train it end to end on VOC for reproducing results)\n- build pipeline customised to the competition\n    - determine number of classes\n    - set up dataloader\n    - visualise sample annotations\n- determine the best learning rate and hyperparam, batch size per cycle\n- validate the effectiveness with the annotated data we have (WITH v.s. WITHOUT active learning)\n\n### References\n1. [**Active Learning for Deep Object Detection via Probabilistic Modeling](https://github.com/NVlabs/AL-MDN)\n2. [Active Learning algorithms for classification, object detection, human pose estimation and semantic segmentation](https://github.com/superannotateai/active_learning)","metadata":{}},{"cell_type":"markdown","source":"#### Notes on `train_ssd_gmm_active_learning pipeline.main`\n1. load GMM model\n2. load VOC test dataset\n3. evaluate unbiased performance on test set\n4. load VOC unlabelled dataset \n5. apply active learning to select top samples from (4)\n6. load VOC labelled dataset\n7. train on existing dataset plus the addition","metadata":{"execution":{"iopub.status.busy":"2022-03-05T13:04:08.098173Z","iopub.execute_input":"2022-03-05T13:04:08.098447Z","iopub.status.idle":"2022-03-05T13:04:08.105976Z","shell.execute_reply.started":"2022-03-05T13:04:08.098417Z","shell.execute_reply":"2022-03-05T13:04:08.104795Z"}}},{"cell_type":"code","source":"%%capture\n!pip install pycocotools==2.0.4\n!git clone https://github.com/riven314/AL-MDN.git\n%cd AL-MDN","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:16:49.196507Z","iopub.execute_input":"2022-03-05T18:16:49.197174Z","iopub.status.idle":"2022-03-05T18:17:21.895983Z","shell.execute_reply.started":"2022-03-05T18:16:49.197077Z","shell.execute_reply":"2022-03-05T18:17:21.895087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\n\nimport torch\nimport torch.utils.data as data\nfrom torch.utils.data.sampler import SubsetRandomSampler, SequentialSampler\nfrom PIL import Image\n\n# util pulled from my repo\nfrom data import MEANS, detection_collate, BaseTransform\nfrom data.voc0712 import VOCDetection, VOCAnnotationTransform\nfrom utils.augmentations import SSDAugmentation\nfrom subset_sequential_sampler import SubsetSequentialSampler\nfrom train_ssd_gmm_active_learning import *\n\n%load_ext autoreload\n%autoreload 2","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:17:21.899865Z","iopub.execute_input":"2022-03-05T18:17:21.900071Z","iopub.status.idle":"2022-03-05T18:17:23.741019Z","shell.execute_reply.started":"2022-03-05T18:17:21.900043Z","shell.execute_reply":"2022-03-05T18:17:23.740298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 0. Config","metadata":{}},{"cell_type":"code","source":"MIN_DIM = 300\nCLASS_N = 21\nNUM_TOTAL_IMAGES=100\nNUM_INITIAL_LABELED_SET=50\nBATCH_SIZE = 8\nNUM_WORKERS = 2\nVGG_BACKBONE_PATH = '/kaggle/input/vgg16-reduced-fc-backbone/vgg16_reducedfc.pth'\nIS_CUDA = torch.cuda.is_available()\nVOC_ROOT = '/kaggle/input/pascal-voc-2007/VOCtrainval_06-Nov-2007/VOCdevkit'\n\n\nclass args:\n    resume = False\n    save_folder = os.path.dirname(VGG_BACKBONE_PATH)\n    basenet = os.path.basename(VGG_BACKBONE_PATH)\n    lr = 0.01\n    momentum = 0.9\n    weight_decay = 5e-4\n    gamma = 0.1\n    id = 1\n    cuda = IS_CUDA\n    use_cuda = IS_CUDA\n    start_iter = 0\n    \ncfg = {\n    'min_dim': MIN_DIM,\n    'num_classes': CLASS_N,\n    'max_iter': 10,\n    'lr_steps': (5, 8),\n    'name': 'testing',\n    \"num_total_images\": NUM_TOTAL_IMAGES,\n    \"acquisition_budget\": 1000\n}","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:17:23.742359Z","iopub.execute_input":"2022-03-05T18:17:23.742589Z","iopub.status.idle":"2022-03-05T18:17:23.815705Z","shell.execute_reply.started":"2022-03-05T18:17:23.742556Z","shell.execute_reply":"2022-03-05T18:17:23.814833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1. Model","metadata":{"execution":{"iopub.status.busy":"2022-03-05T16:49:11.312525Z","iopub.execute_input":"2022-03-05T16:49:11.31322Z","iopub.status.idle":"2022-03-05T16:49:11.321133Z","shell.execute_reply.started":"2022-03-05T16:49:11.313173Z","shell.execute_reply":"2022-03-05T16:49:11.320395Z"}}},{"cell_type":"code","source":"net, optimiser = load_net_optimizer_multi(cfg, args)","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:17:23.819878Z","iopub.execute_input":"2022-03-05T18:17:23.820097Z","iopub.status.idle":"2022-03-05T18:17:29.165384Z","shell.execute_reply.started":"2022-03-05T18:17:23.820072Z","shell.execute_reply":"2022-03-05T18:17:29.164670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. Set up Dataloader (VOC 2007 Dataset)","metadata":{}},{"cell_type":"code","source":"# TODO: customise this to Happy Whale competition\ndef create_loaders(\n    voc_root, batch_size, num_workers, min_dim,\n    num_total_images, num_initial_labeled_set\n):\n    num_train_images = num_total_images\n    indices = list(range(num_train_images))\n    random.shuffle(indices)\n    labeled_set = indices[:num_initial_labeled_set]\n    unlabeled_set = indices[num_initial_labeled_set:]\n\n    supervised_dataset = VOCDetection(root=voc_root, transform=SSDAugmentation(min_dim, MEANS))\n    unsupervised_dataset = VOCDetection(voc_root, [('2007', 'trainval')],\n                                        BaseTransform(300, MEANS),\n                                        VOCAnnotationTransform())\n\n    supervised_data_loader = data.DataLoader(supervised_dataset, batch_size=batch_size,\n                                             num_workers=num_workers,\n                                             sampler=SubsetRandomSampler(labeled_set),\n                                             collate_fn=detection_collate,\n                                             pin_memory=True)\n    unsupervised_data_loader = data.DataLoader(unsupervised_dataset, batch_size=1,\n                                               num_workers=num_workers,\n                                               sampler=SubsetSequentialSampler(unlabeled_set),\n                                               collate_fn=detection_collate,\n                                               pin_memory=True)\n    return supervised_dataset, supervised_data_loader, unsupervised_dataset, unsupervised_data_loader, indices, labeled_set, unlabeled_set","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:17:29.167144Z","iopub.execute_input":"2022-03-05T18:17:29.167664Z","iopub.status.idle":"2022-03-05T18:17:29.210265Z","shell.execute_reply.started":"2022-03-05T18:17:29.167611Z","shell.execute_reply":"2022-03-05T18:17:29.209447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# d = VOCDetection(root=VOC_ROOT,\n#                  image_sets=[('2007', 'trainval')],\n#                  dataset_name='VOC2007')\n# supervised_dataset = VOCDetection(root=VOC_ROOT, transform=SSDAugmentation(MIN_DIM, MEANS))\n\nsup_ds, sup_dl, unsup_ds, unsup_dl, idxs, label_set, unlabel_set = create_loaders(\n    voc_root=VOC_ROOT, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, min_dim=MIN_DIM,\n    num_total_images=NUM_TOTAL_IMAGES, num_initial_labeled_set=NUM_INITIAL_LABELED_SET\n)","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:17:29.211605Z","iopub.execute_input":"2022-03-05T18:17:29.211885Z","iopub.status.idle":"2022-03-05T18:17:29.284070Z","shell.execute_reply.started":"2022-03-05T18:17:29.211848Z","shell.execute_reply":"2022-03-05T18:17:29.283383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Train","metadata":{}},{"cell_type":"code","source":"criterion = MultiBoxLoss_GMM(cfg['num_classes'], 0.5, True, 0, True, 3, 0.5, False, args.cuda)\ncriterion = criterion.cuda()\nnet = train(label_set, sup_dl, idxs, cfg, args, criterion)","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:17:29.285354Z","iopub.execute_input":"2022-03-05T18:17:29.285612Z","iopub.status.idle":"2022-03-05T18:17:40.832488Z","shell.execute_reply.started":"2022-03-05T18:17:29.285579Z","shell.execute_reply":"2022-03-05T18:17:40.831654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4. Apply Active Learning to Rank Unseen Labels","metadata":{}},{"cell_type":"code","source":"try:\n    net.eval()\n    new_labeled_set, new_unlabeled_set = active_learning_cycle(\n        iter(unsup_dl),\n        label_set,\n        unlabel_set,\n        net,\n        cfg[\"num_classes\"],\n        acquisition_budget=cfg['acquisition_budget'],\n        num_total_images=cfg['num_total_images'],\n    )\nexcept Exception as e:\n    print(\"Failed probably because none of the prediction surpass a threshold. Need to train longer to resolve this!\")","metadata":{"execution":{"iopub.status.busy":"2022-03-05T18:23:03.036566Z","iopub.execute_input":"2022-03-05T18:23:03.037179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}