{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Detectron2 Faster R-CNN for wind damage","metadata":{}},{"cell_type":"code","source":"!pip install ../input/detectron-05/whls/pycocotools-2.0.2/dist/pycocotools-2.0.2.tar --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/fvcore-0.1.5.post20211019/fvcore-0.1.5.post20211019 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/antlr4-python3-runtime-4.8/antlr4-python3-runtime-4.8 --no-index --find-links ../input/detectron-05/whls \n!pip install ../input/detectron-05/whls/detectron2-0.5/detectron2 --no-index --find-links ../input/detectron-05/whls \n# %load_ext tensorboard.notebook  does not seem to be available","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-08-05T01:20:51.659646Z","iopub.execute_input":"2023-08-05T01:20:51.660001Z","iopub.status.idle":"2023-08-05T01:22:55.612473Z","shell.execute_reply.started":"2023-08-05T01:20:51.659956Z","shell.execute_reply":"2023-08-05T01:22:55.611350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport os\nimport random\nimport logging\nfrom ast import literal_eval\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport matplotlib.pyplot as plt\nimport torch, torchvision\nimport json\nfrom time import time\n\nfrom detectron2 import model_zoo\nfrom detectron2.data import DatasetCatalog, MetadataCatalog\nfrom detectron2.structures import BoxMode\nfrom detectron2.engine import DefaultTrainer, DefaultPredictor\nfrom detectron2.config import get_cfg","metadata":{"execution":{"iopub.status.busy":"2023-08-05T01:22:55.903950Z","iopub.execute_input":"2023-08-05T01:22:55.904775Z","iopub.status.idle":"2023-08-05T01:22:55.912214Z","shell.execute_reply.started":"2023-08-05T01:22:55.904732Z","shell.execute_reply":"2023-08-05T01:22:55.911456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configure logging to see info messages from detectron\nroot = logging.getLogger()\nroot.setLevel(logging.INFO)\nhandler = logging.StreamHandler(sys.stdout)\nhandler.setLevel(logging.INFO)\nformatter = logging.Formatter('%(levelname)s: %(message)s')\nhandler.setFormatter(formatter)\nroot.addHandler(handler)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-05T01:22:55.915000Z","iopub.execute_input":"2023-08-05T01:22:55.915648Z","iopub.status.idle":"2023-08-05T01:22:55.931264Z","shell.execute_reply.started":"2023-08-05T01:22:55.915607Z","shell.execute_reply":"2023-08-05T01:22:55.930264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATASET LOADING","metadata":{}},{"cell_type":"code","source":"#REGISTER DATASET\n\nwidth, height = [586,371] # PX\n\n# function to return competition training data in Detectron format\ndef process_data(name,imageFolder,annotationsFullPath):\n    # Opening JSON file\n    f = open(annotationsFullPath)\n    data = json.load(f)\n\n    # images\n    # {\"file_name\": \"DJI_0782_05_02.png\", \"height\": 371, \"width\": 586, \"id\": 1681}\n    \n    # annotations\n    # {'image_id': 1, 'category_id': 0, 'bbox': [0.1023895, 0.002695000000000003, 0.383959, 0.991914], 'id': 1, 'iscrowd': 0, 'area': 0.38085430752599997}\n    \n    \n    print(name,' :   ',len(data['images']),' images    ',len(data['annotations']),'annotations    ',len(data['categories']),'categories')\n\n   \n    items = []\n    annotations_index=0 #used to move around and gather annotations for images\n    for index, img_data in enumerate(data['images']):\n        boxes = []\n        while annotations_index < len(data['annotations']) and data['annotations'][annotations_index]['image_id']==img_data['id']:\n              \n            b=data['annotations'][annotations_index]['bbox']\n            boxes.append({'bbox': [round(b[0]*width), round(b[1]*height), round(b[2]*width), round(b[3]*height)],\n                          'bbox_mode': BoxMode.XYWH_ABS, \n                          'category_id': data['annotations'][annotations_index]['category_id']})\n            annotations_index+=1\n            \n        items.append({'file_name': imageFolder+img_data['file_name'],\n                      'height': height, # hardcoded they are all the same\n                      'width': width, \n                      'image_id': img_data['id'],\n                      'annotations': boxes})\n            \n    \n    #Closing json file\n    f.close()             \n    return items\n\n\n\ndef get_train_instances():\n    return process_data('instances_train','/kaggle/input/coco-annotated-wind-turbine-surface-damage/NordTank586x371/train/','/kaggle/input/coco-annotated-wind-turbine-surface-damage/NordTank586x371/annotations/instances_train.json') \n\ndef get_val_instances():\n    return process_data('instances_val','/kaggle/input/coco-annotated-wind-turbine-surface-damage/NordTank586x371/val/','/kaggle/input/coco-annotated-wind-turbine-surface-damage/NordTank586x371/annotations/instances_val.json') \n\ndef get_test_instances():\n    return process_data('instances_test','/kaggle/input/coco-annotated-wind-turbine-surface-damage/NordTank586x371/test/','/kaggle/input/coco-annotated-wind-turbine-surface-damage/NordTank586x371/annotations/instances_test.json') \n\n# register datasets\nDatasetCatalog.register('instances_train',  get_train_instances)\nDatasetCatalog.register('instances_val',  get_val_instances)\nDatasetCatalog.register('instances_test',  get_test_instances)\n\nds_train = DatasetCatalog.get('instances_train')\nds_val = DatasetCatalog.get('instances_val')\nds_test = DatasetCatalog.get('instances_test')\n\nMetadataCatalog.get('instances_train').thing_classes = [\"damage\", \"dirt\"]\nMetadataCatalog.get('instances_val').thing_classes = [\"damage\", \"dirt\"]\nMetadataCatalog.get('instances_test').thing_classes = [\"damage\", \"dirt\"]","metadata":{"execution":{"iopub.status.busy":"2023-08-05T01:22:55.932618Z","iopub.execute_input":"2023-08-05T01:22:55.933004Z","iopub.status.idle":"2023-08-05T01:22:56.254207Z","shell.execute_reply.started":"2023-08-05T01:22:55.932938Z","shell.execute_reply":"2023-08-05T01:22:56.253340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #   UNREGISTER DATASETS\n# DatasetCatalog.remove('instances_train')\n# DatasetCatalog.remove('instances_val')\n# DatasetCatalog.remove('instances_test')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(ds_train)\nd = ds_train[0]\nprint(len(d['annotations']))\nwidth, height = [586,371] # PX\nprint(width, height)\nprint(len(ds_train))","metadata":{"execution":{"iopub.status.busy":"2023-08-05T01:22:56.256456Z","iopub.execute_input":"2023-08-05T01:22:56.257019Z","iopub.status.idle":"2023-08-05T01:22:56.264887Z","shell.execute_reply.started":"2023-08-05T01:22:56.256971Z","shell.execute_reply":"2023-08-05T01:22:56.264123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## show some random images and bounding boxes from training set\nnrows, ncols = 5, 2\nfig, ax = plt.subplots(nrows, ncols, figsize=(20,31))\nsaved_sample_img=[]\nsaved_sample_img_annotated=[]\ndisplayed = 0\n#for ti in random.sample(range(23500), 10000):\nfor ti in range(len(ds_train)):\n    d = ds_train[ti]\n    if 0 == len(d['annotations']): \n        continue  # skip images without bounding boxes\n        \n    img_path = d[\"file_name\"]\n    print(img_path)\n    \n    if not os.path.exists(img_path):\n        print(f\"Error: Image file not found at {img_path}\")\n        continue\n\n    \n    img = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)\n    saved_sample_img+=[img.copy()] \n    for a in d['annotations']:\n        b = a['bbox']\n        color = (0, 200, 100) if a['category_id'] == 1 else (0, 100,200) # GREEN DAMAGE = 1 , DIRT = 0\n        cv2.rectangle(img, (b[0], b[1]), (b[0]+b[2], b[1]+b[3]), color, 2)# XYWH -> XYXY\n    saved_sample_img_annotated+=[img.copy()] \n\n    ax[displayed // ncols, displayed % ncols].grid(False)\n    ax[displayed // ncols, displayed % ncols].axis('off')\n    ax[displayed // ncols, displayed % ncols].imshow(img)\n    \n    displayed += 1\n    if nrows * ncols <= displayed: break\n        \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-05T01:22:56.266015Z","iopub.execute_input":"2023-08-05T01:22:56.266580Z","iopub.status.idle":"2023-08-05T01:22:58.688900Z","shell.execute_reply.started":"2023-08-05T01:22:56.266541Z","shell.execute_reply":"2023-08-05T01:22:58.685543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TRAINING AND CONFIGURATION","metadata":{}},{"cell_type":"code","source":"# get config and merge it with Faster R-CNN config frommap\ncfg = get_cfg()\ncfg.merge_from_file(model_zoo.get_config_file('COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml')) \n# cfg.merge_from_file(model_zoo.get_config_file('COCO-Detection/retinanet_R_101_FPN_3x.yaml'))     TODO TRY OUT: RETINA NET and others than 101\n \nimg_len = len(ds_train)\nbatchs = 4\nepochs = 10\n\ncfg.DATASETS.TRAIN = ('instances_train',) # always keep it a tuple of strings\ncfg.DATASETS.TEST = ('instances_test',) # TODO ADD TEST DATASET see if VALIDATION is possible\ncfg.DATALOADER.NUM_WORKERS = 2 # previously was set to 1, TODO CHECK P100\ncfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS = False # use all annotations\n# cfg.MODEL.WEIGHTS = \"../input/detectron2-faster-rcnn-x101/model_final_68b088.pkl\"  # get weigths from attached dataset\ncfg.MODEL.WEIGHTS = \"../input/faster-r-cnn-r50-fpn-x3/model_final_280758.pkl\"  \n\ncfg.SOLVER.IMS_PER_BATCH = batchs  # batch size\ncfg.SOLVER.BASE_LR = 0.001  # learning rate\ncfg.SOLVER.MAX_ITER = round(epochs * (img_len/batchs))#  \ncfg.SOLVER.STEPS = (round(cfg.SOLVER.MAX_ITER * 0.7) , round(cfg.SOLVER.MAX_ITER * 0.9))\ncfg.SOLVER.WEIGHT_DECAY = 0.0001\ncfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5   # set threshold for boxes to return (lower thresh ---> better recall (+worse acc))\ncfg.MODEL.ROI_HEADS.NUM_CLASSES = 2  # we 2 classes\ncfg.MODEL.RPN.NMS_THRESH = 0.35 # default is 0.5\ncfg.TEST.EVAL_PERIOD = 1000# set eval step intervals\ncfg.TEST.DETECTIONS_PER_IMAGE = 50\ncfg.TEST.AUG.FLIP = False\ncfg.TEST.AUG.FLIP = False\n\n\n\nprint(cfg)\n\n# Loss function is BBOX_REG_LOSS_TYPE: smooth_l1\n\n# REMOVE PANOPTIC FPN/ APPLY NON-MAX SUPRESSION ? check all possible configs in documentation\n# TODO CHECK\n# MODEL ANCHOR GENERATOR\n# BACKBONE FREEZE_AT?\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-08-05T01:46:25.832903Z","iopub.execute_input":"2023-08-05T01:46:25.833250Z","iopub.status.idle":"2023-08-05T01:46:25.865634Z","shell.execute_reply.started":"2023-08-05T01:46:25.833204Z","shell.execute_reply":"2023-08-05T01:46:25.864869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cfg)\n","metadata":{"execution":{"iopub.status.busy":"2023-08-05T01:31:42.379861Z","iopub.status.idle":"2023-08-05T01:31:42.380316Z","shell.execute_reply.started":"2023-08-05T01:31:42.380072Z","shell.execute_reply":"2023-08-05T01:31:42.380097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Clean output folder\n!rm output/*","metadata":{"execution":{"iopub.status.busy":"2023-08-05T01:33:34.564570Z","iopub.execute_input":"2023-08-05T01:33:34.564862Z","iopub.status.idle":"2023-08-05T01:33:35.578129Z","shell.execute_reply.started":"2023-08-05T01:33:34.564830Z","shell.execute_reply":"2023-08-05T01:33:35.577182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TRAIN EXECUTION :make output dir, created default trainer and go ahead)\nos.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\ntrainer = DefaultTrainer(cfg) \ntrainer.resume_or_load(resume=False)\ntrainer.train()","metadata":{"scrolled":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-08-05T01:46:42.455585Z","iopub.execute_input":"2023-08-05T01:46:42.456174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pickup weights from training and create predictor\ncfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final_10epoch_035nms_lr001.pth\")\n\npredictor = DefaultPredictor(cfg)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T15:37:22.668648Z","iopub.execute_input":"2023-07-30T15:37:22.669236Z","iopub.status.idle":"2023-07-30T15:37:27.996417Z","shell.execute_reply.started":"2023-07-30T15:37:22.669197Z","shell.execute_reply":"2023-07-30T15:37:27.995551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# simple function to format predictions\ndef format_predictions(out):\n    l = []\n    for i in range(len(out['instances'])):\n        box = out['instances'].pred_boxes[i].tensor.cpu().numpy().flatten()\n        score = out['instances'].scores[0].cpu().numpy()\n        pred_class = out['instances'].pred_classes[i].item()\n        # XYXY to XYWH\n        res={'bbox':[box[0],box[1],(box[2]-box[0]),(box[3]-box[1])], 'score':score, 'class':pred_class}\n        l.append(res) # X,Y,W,H,score\n    return l\n\n# # create prediction env and iterator\n# env = greatbarrierreef.make_env()   # initialize the environment\n# iter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\n\n# # iterate over test images and do prediction\n# for (pixel_array, sample_prediction_df) in iter_test:\n#     out = predictor(pixel_array)\n#     sample_prediction_df['annotations'] = format_predictions(out)\n#     env.predict(sample_prediction_df)\n\nprint('formated prediction: ',format_predictions(predictor(saved_sample_img[3])))\n\nprint('model prediction: ',predictor(saved_sample_img[3]))","metadata":{"execution":{"iopub.status.busy":"2023-08-03T22:30:04.100159Z","iopub.execute_input":"2023-08-03T22:30:04.100456Z","iopub.status.idle":"2023-08-03T22:30:04.127317Z","shell.execute_reply.started":"2023-08-03T22:30:04.100423Z","shell.execute_reply":"2023-08-03T22:30:04.126418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show predictions\nnrows, ncols = 5, 2\nfig, ax = plt.subplots(nrows, ncols, figsize=(20,31))\ndisplayed = 0\n#for ti in random.sample(range(23500), 10000):\nfor ti in range(len(saved_sample_img)): # |saved_sample_img| = 10\n    img = saved_sample_img[ti] # has no annotation\n    annotated_img = saved_sample_img_annotated[ti] # has true annotations\n    annotations = format_predictions(predictor(img))\n    for a in annotations:\n        # Blue-ish (green=dmg=1 + cian=dirt=0) ARE TRUE ANNOTATIONS, Red-ish (red=dmg,) ARE PREDICTED\n        color = (200, 100, 0) if a['class'] == 1 else (200, 200,0) # Orange DAMAGE = 1 , Yellow DIRT = 0\n        b = a['bbox'] \n        cv2.rectangle(annotated_img, (round(b[0]), round(b[1])) , (round(b[0]+b[2]), round(b[1]+b[3])) , color, 2) #XYWH -> XYXY\n    \n    ax[displayed // ncols, displayed % ncols].grid(False)\n    ax[displayed // ncols, displayed % ncols].axis('off')\n    ax[displayed // ncols, displayed % ncols].imshow(annotated_img)\n    \n    displayed += 1\n    if nrows * ncols <= displayed: break\n        \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:13:27.854727Z","iopub.execute_input":"2023-07-30T16:13:27.855205Z","iopub.status.idle":"2023-07-30T16:13:34.150503Z","shell.execute_reply.started":"2023-07-30T16:13:27.855167Z","shell.execute_reply":"2023-07-30T16:13:34.148146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from detectron2.evaluation import COCOEvaluator, inference_on_dataset\nfrom detectron2.data import build_detection_test_loader\nevaluator=None\nevaluator = COCOEvaluator('instances_test', output_dir=\"./output\")\nprint(evaluator)\n","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:13:37.978305Z","iopub.execute_input":"2023-07-30T16:13:37.978588Z","iopub.status.idle":"2023-07-30T16:13:38.005276Z","shell.execute_reply.started":"2023-07-30T16:13:37.978557Z","shell.execute_reply":"2023-07-30T16:13:38.004254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_loader = build_detection_test_loader(cfg, \"instances_test\") # USE instances_test / val\nprint(inference_on_dataset(predictor.model, val_loader,evaluator))\n","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:13:39.895337Z","iopub.execute_input":"2023-07-30T16:13:39.895893Z","iopub.status.idle":"2023-07-30T16:41:05.572420Z","shell.execute_reply.started":"2023-07-30T16:13:39.895855Z","shell.execute_reply":"2023-07-30T16:41:05.571585Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(os.path.exists('/kaggle/input/wind-turbine-dmg-coco/NordTank586x371/test/DJI_0031_03_04.png'))\n# !ls -l /kaggle/input/wind-turbine-dmg-coco/NordTank586x371/test | grep DJI_0031_03_04","metadata":{"execution":{"iopub.status.busy":"2023-07-30T16:10:18.391184Z","iopub.execute_input":"2023-07-30T16:10:18.391779Z","iopub.status.idle":"2023-07-30T16:10:18.398570Z","shell.execute_reply.started":"2023-07-30T16:10:18.391743Z","shell.execute_reply":"2023-07-30T16:10:18.397762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l output/*","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-07-30T19:01:05.038776Z","iopub.execute_input":"2023-07-30T19:01:05.039469Z","iopub.status.idle":"2023-07-30T19:01:06.243329Z","shell.execute_reply.started":"2023-07-30T19:01:05.039406Z","shell.execute_reply":"2023-07-30T19:01:06.242501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SAVE OUTPUT\nimport shutil\nshutil.make_archive('output', 'zip', '/kaggle/working/output')","metadata":{"execution":{"iopub.status.busy":"2023-07-30T19:13:26.308601Z","iopub.execute_input":"2023-07-30T19:13:26.308911Z","iopub.status.idle":"2023-07-30T19:15:18.156261Z","shell.execute_reply.started":"2023-07-30T19:13:26.308880Z","shell.execute_reply":"2023-07-30T19:15:18.155212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'output/coco_instances_results.json')","metadata":{"execution":{"iopub.status.busy":"2023-07-30T19:03:56.307126Z","iopub.execute_input":"2023-07-30T19:03:56.307508Z","iopub.status.idle":"2023-07-30T19:03:56.317657Z","shell.execute_reply.started":"2023-07-30T19:03:56.307470Z","shell.execute_reply":"2023-07-30T19:03:56.316800Z"},"trusted":true},"execution_count":null,"outputs":[]}]}