{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"thanks to https://www.kaggle.com/h053473666/siim-cov19-yolov5-train/data?select=image","metadata":{}},{"cell_type":"markdown","source":"Making YoloV5 format","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd\nfrom glob import glob\nimport os, shutil\nimport matplotlib as plt\nfrom sklearn.model_selection import GroupKFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-22T08:23:13.084256Z","iopub.execute_input":"2021-07-22T08:23:13.084678Z","iopub.status.idle":"2021-07-22T08:23:13.089291Z","shell.execute_reply.started":"2021-07-22T08:23:13.084642Z","shell.execute_reply":"2021-07-22T08:23:13.088362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.090675Z","iopub.execute_input":"2021-07-22T08:23:13.091159Z","iopub.status.idle":"2021-07-22T08:23:13.106333Z","shell.execute_reply.started":"2021-07-22T08:23:13.091117Z","shell.execute_reply":"2021-07-22T08:23:13.104857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.109017Z","iopub.execute_input":"2021-07-22T08:23:13.109428Z","iopub.status.idle":"2021-07-22T08:23:13.166792Z","shell.execute_reply.started":"2021-07-22T08:23:13.109391Z","shell.execute_reply":"2021-07-22T08:23:13.165564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = train_df\ngkf = GroupKFold(n_splits = 5)\ndf['fold'] = -1\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(df, groups = df.StudyInstanceUID.tolist())):\n    df.loc[val_idx, 'fold'] = fold\n","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.168854Z","iopub.execute_input":"2021-07-22T08:23:13.169206Z","iopub.status.idle":"2021-07-22T08:23:13.218517Z","shell.execute_reply.started":"2021-07-22T08:23:13.169174Z","shell.execute_reply":"2021-07-22T08:23:13.217317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.220394Z","iopub.execute_input":"2021-07-22T08:23:13.220852Z","iopub.status.idle":"2021-07-22T08:23:13.238441Z","shell.execute_reply.started":"2021-07-22T08:23:13.220805Z","shell.execute_reply":"2021-07-22T08:23:13.237052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = df","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.240427Z","iopub.execute_input":"2021-07-22T08:23:13.240895Z","iopub.status.idle":"2021-07-22T08:23:13.248952Z","shell.execute_reply.started":"2021-07-22T08:23:13.240844Z","shell.execute_reply":"2021-07-22T08:23:13.247515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['image_path'] = f'../input/second/image/' + train_df.id + '.png'\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.253566Z","iopub.execute_input":"2021-07-22T08:23:13.254067Z","iopub.status.idle":"2021-07-22T08:23:13.277481Z","shell.execute_reply.started":"2021-07-22T08:23:13.254029Z","shell.execute_reply":"2021-07-22T08:23:13.276186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nval_files = []\nval_files += list(train_df[train_df.fold==fold].image_path.unique())\ntrain_files += list(train_df[train_df.fold!=fold].image_path.unique())\nlen(train_files), len(val_files)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.279434Z","iopub.execute_input":"2021-07-22T08:23:13.27973Z","iopub.status.idle":"2021-07-22T08:23:13.296113Z","shell.execute_reply.started":"2021-07-22T08:23:13.279701Z","shell.execute_reply":"2021-07-22T08:23:13.294828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor _,_,files in os.walk('../input/thirdyolotxt'):\n    for file in files:\n        count += 1\nprint(count)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.297865Z","iopub.execute_input":"2021-07-22T08:23:13.298319Z","iopub.status.idle":"2021-07-22T08:23:13.31854Z","shell.execute_reply.started":"2021-07-22T08:23:13.298275Z","shell.execute_reply":"2021-07-22T08:23:13.317195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/siim-cov19/labels/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/labels/val', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/train', exist_ok = True)\nos.makedirs('/kaggle/working/siim-cov19/images/val', exist_ok = True)\nlabel_dir = '../input/thirdyolotxt'\n\n\nfor file in tqdm(val_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/val')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename + '.txt'),'/kaggle/working/siim-cov19/labels/val')\n    \nfor file in tqdm(train_files):\n    shutil.copy(file, '/kaggle/working/siim-cov19/images/train')\n    filename = file.split('/')[-1].split('.')[0]\n    shutil.copy(os.path.join(label_dir, filename + '.txt'), '/kaggle/working/siim-cov19/labels/train')\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.320166Z","iopub.execute_input":"2021-07-22T08:23:13.320635Z","iopub.status.idle":"2021-07-22T08:23:13.427177Z","shell.execute_reply.started":"2021-07-22T08:23:13.320599Z","shell.execute_reply":"2021-07-22T08:23:13.426137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from os import listdir\nfrom os.path import isfile, join\nimport yaml\n\ncwd ='/kaggle/working/'\n\nwith open(join(cwd, 'train.txt'), 'w') as f:\n    for path in glob('./siim-cov19/images/train/*'):\n        f.write(path+'\\n')\n        \nwith open(join(cwd, 'val.txt'), 'w') as f:\n    for path in glob('./siim-cov19/images/val/*'):\n        f.write(path+'\\n')","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.428711Z","iopub.execute_input":"2021-07-22T08:23:13.429318Z","iopub.status.idle":"2021-07-22T08:23:13.438448Z","shell.execute_reply.started":"2021-07-22T08:23:13.429275Z","shell.execute_reply":"2021-07-22T08:23:13.437008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = dict(\n    train = join(cwd, 'train.txt'),\n    val = join(cwd, 'val.txt'),\n    nc = 1,\n    names = ['0. opacity']\n)\n\nwith open(join(cwd, 'siim-cov19.yaml'), 'w') as outfile:\n    yaml.dump(data, outfile, default_flow_style = False)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.43985Z","iopub.execute_input":"2021-07-22T08:23:13.440333Z","iopub.status.idle":"2021-07-22T08:23:13.453466Z","shell.execute_reply.started":"2021-07-22T08:23:13.44029Z","shell.execute_reply":"2021-07-22T08:23:13.452177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f = open(join(cwd, 'siim-cov19.yaml'), 'r')\nprint('\\nyaml:')\nprint(f.read())","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.455023Z","iopub.execute_input":"2021-07-22T08:23:13.455439Z","iopub.status.idle":"2021-07-22T08:23:13.469882Z","shell.execute_reply.started":"2021-07-22T08:23:13.455397Z","shell.execute_reply":"2021-07-22T08:23:13.468318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/ultralytics/yolov5\n","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:13.471048Z","iopub.execute_input":"2021-07-22T08:23:13.471443Z","iopub.status.idle":"2021-07-22T08:23:14.242005Z","shell.execute_reply.started":"2021-07-22T08:23:13.471374Z","shell.execute_reply":"2021-07-22T08:23:14.240673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd yolov5\n%pip install -r requirements.txt","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:14.276997Z","iopub.status.idle":"2021-07-22T08:23:14.277543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom IPython.display import Image, clear_output  # to display images\n\nclear_output()\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:24:46.292196Z","iopub.execute_input":"2021-07-22T08:24:46.292609Z","iopub.status.idle":"2021-07-22T08:24:47.619701Z","shell.execute_reply.started":"2021-07-22T08:24:46.292575Z","shell.execute_reply":"2021-07-22T08:24:47.618465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shutil.copytree('/kaggle/input/yolov5-official-v31-dataset/yolov5', '/kaggle/working/yolov5')\n# os.chdir('/kaggle/working/yolov5')","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:24:33.046767Z","iopub.execute_input":"2021-07-22T08:24:33.047346Z","iopub.status.idle":"2021-07-22T08:24:33.731933Z","shell.execute_reply.started":"2021-07-22T08:24:33.047295Z","shell.execute_reply":"2021-07-22T08:24:33.730731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !WANDB_MODE=\"dryrun\" python train.py --img 512 --batch 24 --epochs 1 --data /kaggle/working/siim-cov19.yaml \\\n# --weights yolov5s.pt --project=/kaggle/working/results --nosave --exist-ok --cache","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:26:08.02356Z","iopub.execute_input":"2021-07-22T08:26:08.023995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/results', exist_ok = True)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:23:14.281303Z","iopub.status.idle":"2021-07-22T08:23:14.281885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!WANDB_MODE=\"dryrun\" python train.py --img 512 --batch 24 --epochs 30 --data /kaggle/working/siim-cov19.yaml --weights yolov5l.pt \\\n--project=/kaggle/working/results --nosave --exist-ok --cache\n\n# !python train.py --img 512 --batch 16 --epochs 30 --data /kaggle/working/siim-cov19.yaml --weights yolov5l.pt \\\n# --project=/kaggle/working/results --exist-ok --cache","metadata":{"execution":{"iopub.status.busy":"2021-07-22T08:24:50.3916Z","iopub.execute_input":"2021-07-22T08:24:50.391981Z","iopub.status.idle":"2021-07-22T08:25:28.156252Z","shell.execute_reply.started":"2021-07-22T08:24:50.391949Z","shell.execute_reply":"2021-07-22T08:25:28.154551Z"},"trusted":true},"execution_count":null,"outputs":[]}]}