{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Getting Started\n\nTo get the best result, you can follow my preivous dicussion about [How to win object detection competetion](https://www.kaggle.com/c/global-wheat-detection/discussion/232550#1273363) where i got a gold medal.\n\nAnd [This notebook](https://www.kaggle.com/espsiyam/yolov5-ensemble-tta-transfer-learning-hpt)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-22T10:20:55.019063Z","iopub.execute_input":"2021-05-22T10:20:55.019397Z","iopub.status.idle":"2021-05-22T10:20:57.559012Z","shell.execute_reply.started":"2021-05-22T10:20:55.019362Z","shell.execute_reply":"2021-05-22T10:20:57.558201Z"}}},{"cell_type":"code","source":"# Importing Required packages\nimport os\nimport torch\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, clear_output\nprint('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:56:42.814521Z","iopub.execute_input":"2021-05-22T23:56:42.814942Z","iopub.status.idle":"2021-05-22T23:56:44.139574Z","shell.execute_reply.started":"2021-05-22T23:56:42.814855Z","shell.execute_reply":"2021-05-22T23:56:44.138787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#customize iPython writefile so we can write variables\nfrom IPython.core.magic import register_line_cell_magic\n\n@register_line_cell_magic\ndef writetemplate(line, cell):\n    with open(line, 'w') as f:\n        f.write(cell.format(**globals()))","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:56:44.141167Z","iopub.execute_input":"2021-05-22T23:56:44.141520Z","iopub.status.idle":"2021-05-22T23:56:44.149589Z","shell.execute_reply.started":"2021-05-22T23:56:44.141482Z","shell.execute_reply":"2021-05-22T23:56:44.148696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cloning the repo and installing requiements\n!git clone https://github.com/ultralytics/yolov5.git\n!mv ./yolov5/* ./\n!pip install -r requirements.txt\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:56:44.153240Z","iopub.execute_input":"2021-05-22T23:56:44.153525Z","iopub.status.idle":"2021-05-22T23:57:01.467100Z","shell.execute_reply.started":"2021-05-22T23:56:44.153489Z","shell.execute_reply":"2021-05-22T23:57:01.466159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing the dataset","metadata":{}},{"cell_type":"code","source":"# Copying the dataset to working directory\n!mkdir Dataset\n!cp ../input/covid19-detection-for-yolov5-siimfisabiorsna/Covid19 -r ./Dataset\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:01.468982Z","iopub.execute_input":"2021-05-22T23:57:01.469346Z","iopub.status.idle":"2021-05-22T23:57:28.870602Z","shell.execute_reply.started":"2021-05-22T23:57:01.469303Z","shell.execute_reply":"2021-05-22T23:57:28.869721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('./Dataset/Covid19/images'))","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:28.872254Z","iopub.execute_input":"2021-05-22T23:57:28.872624Z","iopub.status.idle":"2021-05-22T23:57:28.884533Z","shell.execute_reply.started":"2021-05-22T23:57:28.872577Z","shell.execute_reply":"2021-05-22T23:57:28.883328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writetemplate ./split_dataset.py\nfrom utils.datasets import * \nautosplit('./Dataset/Covid19', weights=(0.8, 0.2, 0.0))","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:28.886746Z","iopub.execute_input":"2021-05-22T23:57:28.887027Z","iopub.status.idle":"2021-05-22T23:57:28.896844Z","shell.execute_reply.started":"2021-05-22T23:57:28.886997Z","shell.execute_reply":"2021-05-22T23:57:28.896028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sptting the dataset for training and validation using datasets from yolov5 repo","metadata":{}},{"cell_type":"code","source":"!python split_dataset.py","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:28.899580Z","iopub.execute_input":"2021-05-22T23:57:28.900134Z","iopub.status.idle":"2021-05-22T23:57:32.098964Z","shell.execute_reply.started":"2021-05-22T23:57:28.900103Z","shell.execute_reply":"2021-05-22T23:57:32.097997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('./Dataset/Covid19')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:32.100800Z","iopub.execute_input":"2021-05-22T23:57:32.101178Z","iopub.status.idle":"2021-05-22T23:57:32.110521Z","shell.execute_reply.started":"2021-05-22T23:57:32.101135Z","shell.execute_reply":"2021-05-22T23:57:32.109775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir DataFile","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:32.113919Z","iopub.execute_input":"2021-05-22T23:57:32.114189Z","iopub.status.idle":"2021-05-22T23:57:32.750722Z","shell.execute_reply.started":"2021-05-22T23:57:32.114162Z","shell.execute_reply":"2021-05-22T23:57:32.749620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defining the dataset","metadata":{}},{"cell_type":"code","source":"%%writetemplate ./DataFile/data.yaml\n\ntrain: ./Dataset/Covid19/autosplit_train.txt\nval: ./Dataset/Covid19/autosplit_val.txt\n\nnc: 4\nnames: ['Negative for Pneumonia', 'Typical Appearance',\n        'Indeterminate Appearance', 'Atypical Appearance']","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:32.754362Z","iopub.execute_input":"2021-05-22T23:57:32.754638Z","iopub.status.idle":"2021-05-22T23:57:32.759095Z","shell.execute_reply.started":"2021-05-22T23:57:32.754604Z","shell.execute_reply":"2021-05-22T23:57:32.758164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Declaration\n\nyou can find all the models [here](https://github.com/ultralytics/yolov5/tree/master/models) and cutomize the classes accordingly.","metadata":{}},{"cell_type":"code","source":"%%writetemplate ./models/custom_yolov5x.yaml\n\n\n# parameters\nnc: 4  # number of classes\ndepth_multiple: 1.33  # model depth multiple\nwidth_multiple: 1.25  # layer channel multiple\n\n# anchors\nanchors:\n  - [10,13, 16,30, 33,23]  # P3/8\n  - [30,61, 62,45, 59,119]  # P4/16\n  - [116,90, 156,198, 373,326]  # P5/32\n\n# YOLOv5 backbone\nbackbone:\n  # [from, number, module, args]\n  [[-1, 1, Focus, [64, 3]],  # 0-P1/2\n   [-1, 1, Conv, [128, 3, 2]],  # 1-P2/4\n   [-1, 3, C3, [128]],\n   [-1, 1, Conv, [256, 3, 2]],  # 3-P3/8\n   [-1, 9, C3, [256]],\n   [-1, 1, Conv, [512, 3, 2]],  # 5-P4/16\n   [-1, 9, C3, [512]],\n   [-1, 1, Conv, [1024, 3, 2]],  # 7-P5/32\n   [-1, 1, SPP, [1024, [5, 9, 13]]],\n   [-1, 3, C3, [1024, False]],  # 9\n  ]\n\n# YOLOv5 head\nhead:\n  [[-1, 1, Conv, [512, 1, 1]],\n   [-1, 1, nn.Upsample, [None, 2, 'nearest']],\n   [[-1, 6], 1, Concat, [1]],  # cat backbone P4\n   [-1, 3, C3, [512, False]],  # 13\n\n   [-1, 1, Conv, [256, 1, 1]],\n   [-1, 1, nn.Upsample, [None, 2, 'nearest']],\n   [[-1, 4], 1, Concat, [1]],  # cat backbone P3\n   [-1, 3, C3, [256, False]],  # 17 (P3/8-small)\n\n   [-1, 1, Conv, [256, 3, 2]],\n   [[-1, 14], 1, Concat, [1]],  # cat head P4\n   [-1, 3, C3, [512, False]],  # 20 (P4/16-medium)\n\n   [-1, 1, Conv, [512, 3, 2]],\n   [[-1, 10], 1, Concat, [1]],  # cat head P5\n   [-1, 3, C3, [1024, False]],  # 23 (P5/32-large)\n\n   [[17, 20, 23], 1, Detect, [nc, anchors]],  # Detect(P3, P4, P5)\n  ]","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:32.760541Z","iopub.execute_input":"2021-05-22T23:57:32.760940Z","iopub.status.idle":"2021-05-22T23:57:32.775525Z","shell.execute_reply.started":"2021-05-22T23:57:32.760906Z","shell.execute_reply":"2021-05-22T23:57:32.774710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I've used wandb before, I don't want it now. So might not need to run this cell\n!wandb off","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:32.776666Z","iopub.execute_input":"2021-05-22T23:57:32.776994Z","iopub.status.idle":"2021-05-22T23:57:34.803131Z","shell.execute_reply.started":"2021-05-22T23:57:32.776968Z","shell.execute_reply":"2021-05-22T23:57:34.802065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"%%time\n\n!python train.py --img-size 602 --batch 10 --epochs 40 --data './DataFile/data.yaml' --cfg ./models/custom_yolov5x.yaml --weights yolov5x.pt --name experiment2  --cache","metadata":{"execution":{"iopub.status.busy":"2021-05-22T23:57:34.806499Z","iopub.execute_input":"2021-05-22T23:57:34.806796Z","iopub.status.idle":"2021-05-22T23:58:36.542246Z","shell.execute_reply.started":"2021-05-22T23:57:34.806754Z","shell.execute_reply":"2021-05-22T23:58:36.541343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext tensorboard\n%tensorboard --logdir runs","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:21:31.84943Z","iopub.status.idle":"2021-05-22T19:21:31.850202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyze the result","metadata":{}},{"cell_type":"code","source":"os.listdir('./runs/train/experiment2/')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:27:51.364131Z","iopub.execute_input":"2021-05-22T19:27:51.364481Z","iopub.status.idle":"2021-05-22T19:27:51.373766Z","shell.execute_reply.started":"2021-05-22T19:27:51.36445Z","shell.execute_reply":"2021-05-22T19:27:51.372267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image('./runs/train/experiment2/results.png')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:28:04.05145Z","iopub.execute_input":"2021-05-22T19:28:04.051923Z","iopub.status.idle":"2021-05-22T19:28:04.081048Z","shell.execute_reply.started":"2021-05-22T19:28:04.05189Z","shell.execute_reply":"2021-05-22T19:28:04.079952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image('./runs/train/experiment2/confusion_matrix.png',width=400)","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:28:17.508616Z","iopub.execute_input":"2021-05-22T19:28:17.509034Z","iopub.status.idle":"2021-05-22T19:28:17.525832Z","shell.execute_reply.started":"2021-05-22T19:28:17.508979Z","shell.execute_reply":"2021-05-22T19:28:17.524794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image('./runs/train/experiment2/test_batch1_labels.jpg')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:28:24.963627Z","iopub.execute_input":"2021-05-22T19:28:24.964142Z","iopub.status.idle":"2021-05-22T19:28:24.983821Z","shell.execute_reply.started":"2021-05-22T19:28:24.964113Z","shell.execute_reply":"2021-05-22T19:28:24.982565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image('./runs/train/experiment2/test_batch1_pred.jpg')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:28:35.043886Z","iopub.execute_input":"2021-05-22T19:28:35.044262Z","iopub.status.idle":"2021-05-22T19:28:35.059028Z","shell.execute_reply.started":"2021-05-22T19:28:35.044231Z","shell.execute_reply":"2021-05-22T19:28:35.057643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('./Dataset/Covid19')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:28:41.703632Z","iopub.execute_input":"2021-05-22T19:28:41.704065Z","iopub.status.idle":"2021-05-22T19:28:41.713323Z","shell.execute_reply.started":"2021-05-22T19:28:41.70402Z","shell.execute_reply":"2021-05-22T19:28:41.711923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detecting the classes ","metadata":{}},{"cell_type":"code","source":"!python detect.py --img-size 602  --conf 0.1 --source ../input/covid19-detection-for-yolov5-siimfisabiorsna/Covid19/images/1000_0.jpg --weights ./runs/train/experiment2/weights/best.pt","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:28:51.281345Z","iopub.execute_input":"2021-05-22T19:28:51.281777Z","iopub.status.idle":"2021-05-22T19:29:03.748022Z","shell.execute_reply.started":"2021-05-22T19:28:51.281744Z","shell.execute_reply":"2021-05-22T19:29:03.746687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image('./runs/detect/exp/1000_0.jpg')","metadata":{"execution":{"iopub.status.busy":"2021-05-22T19:29:03.752322Z","iopub.execute_input":"2021-05-22T19:29:03.752669Z","iopub.status.idle":"2021-05-22T19:29:03.767821Z","shell.execute_reply.started":"2021-05-22T19:29:03.75262Z","shell.execute_reply":"2021-05-22T19:29:03.766559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}