{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-18T18:18:47.845495Z","iopub.execute_input":"2021-12-18T18:18:47.845854Z","iopub.status.idle":"2021-12-18T18:18:47.850699Z","shell.execute_reply.started":"2021-12-18T18:18:47.845823Z","shell.execute_reply":"2021-12-18T18:18:47.849190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nfrom tqdm import tqdm\nimport os\nimport numpy as np\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:18:48.242452Z","iopub.execute_input":"2021-12-18T18:18:48.242795Z","iopub.status.idle":"2021-12-18T18:18:48.247274Z","shell.execute_reply.started":"2021-12-18T18:18:48.242764Z","shell.execute_reply":"2021-12-18T18:18:48.246127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install W&B \n!pip install -q --upgrade wandb\n# Login \nimport wandb\nwandb.login()","metadata":{"execution":{"iopub.status.busy":"2021-12-14T09:30:20.10925Z","iopub.execute_input":"2021-12-14T09:30:20.109569Z","iopub.status.idle":"2021-12-14T09:31:45.901749Z","shell.execute_reply.started":"2021-12-14T09:30:20.109541Z","shell.execute_reply":"2021-12-14T09:31:45.900903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"FOR MALE FEMALE PA","metadata":{}},{"cell_type":"code","source":"male_pa_df = pd.read_csv('../input/male-adult-pa-pneumonea/male_adult_pa.csv')\nfemale_pa_df = pd.read_csv(\"../input/female-adult-pa-pneumonea/female_pa_adult.csv\")\nmale_pa_df['gender'] = 0\nfemale_pa_df['gender'] = 1\nmf_df = pd.concat([male_pa_df,female_pa_df], ignore_index = True)\nworking_df = mf_df.copy()\nworking_df = working_df.drop(working_df[working_df['class'] == 'No Lung Opacity / Not Normal'].index)\nworking_df['new_class'] = working_df['class'].map({ 'Normal':'Non-Pneumonia','Lung Opacity': 'Pneumonia' })\n\nworking_df.rename(columns = {'Unnamed: 0' :'unique_id'}, inplace = True)\nworking_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T08:48:15.414682Z","iopub.execute_input":"2021-12-10T08:48:15.415026Z","iopub.status.idle":"2021-12-10T08:48:15.561141Z","shell.execute_reply.started":"2021-12-10T08:48:15.414992Z","shell.execute_reply":"2021-12-10T08:48:15.560122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder, OneHotEncoder\nclass_enc = LabelEncoder()\nworking_df['class_idx'] = class_enc.fit_transform(working_df['new_class'])\noh_enc = OneHotEncoder(sparse=False)\nworking_df['class_vec'] = oh_enc.fit_transform(\n    working_df['class_idx'].values.reshape(-1, 1)).tolist() ","metadata":{"execution":{"iopub.status.busy":"2021-12-10T08:48:19.589757Z","iopub.execute_input":"2021-12-10T08:48:19.590074Z","iopub.status.idle":"2021-12-10T08:48:20.273708Z","shell.execute_reply.started":"2021-12-10T08:48:19.590045Z","shell.execute_reply":"2021-12-10T08:48:20.272851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimage_df = working_df.groupby('patientId').apply(lambda x: x.sample(1))\nraw_train_df, remain_df = train_test_split(image_df, test_size=0.20, random_state=2018,\n                                    stratify=image_df['new_class'])\n#valid_df, test_df = train_test_split(remain_df, test_size=0.33, random_state=2018,\n #                                   stratify=remain_df['new_class'])\nprint(raw_train_df.shape, 'training data')\nprint(remain_df.shape, 'remain data')\n#print(valid_df.shape, 'valid data')\n#print(test_df.shape, 'test data')","metadata":{"execution":{"iopub.status.busy":"2021-12-02T07:41:17.60566Z","iopub.execute_input":"2021-12-02T07:41:17.606027Z","iopub.status.idle":"2021-12-02T07:41:45.892055Z","shell.execute_reply.started":"2021-12-02T07:41:17.605995Z","shell.execute_reply":"2021-12-02T07:41:45.891128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nos.getcwd()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T07:15:01.293425Z","iopub.execute_input":"2021-12-12T07:15:01.293803Z","iopub.status.idle":"2021-12-12T07:15:01.299455Z","shell.execute_reply.started":"2021-12-12T07:15:01.293771Z","shell.execute_reply":"2021-12-12T07:15:01.298688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir Train_Valid_images #in kaggle/working","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:52:24.072653Z","iopub.execute_input":"2021-12-18T15:52:24.073039Z","iopub.status.idle":"2021-12-18T15:52:24.835392Z","shell.execute_reply.started":"2021-12-18T15:52:24.072984Z","shell.execute_reply":"2021-12-18T15:52:24.833824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir Test_images","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:18:53.618630Z","iopub.execute_input":"2021-12-18T18:18:53.618961Z","iopub.status.idle":"2021-12-18T18:18:54.308056Z","shell.execute_reply.started":"2021-12-18T18:18:53.618931Z","shell.execute_reply":"2021-12-18T18:18:54.307111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir Test2","metadata":{"execution":{"iopub.status.busy":"2021-11-25T10:17:23.948754Z","iopub.execute_input":"2021-11-25T10:17:23.9491Z","iopub.status.idle":"2021-11-25T10:17:24.6037Z","shell.execute_reply.started":"2021-11-25T10:17:23.949065Z","shell.execute_reply":"2021-11-25T10:17:24.602681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cd working","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:29:44.793718Z","iopub.execute_input":"2021-12-14T10:29:44.794069Z","iopub.status.idle":"2021-12-14T10:29:44.800495Z","shell.execute_reply.started":"2021-12-14T10:29:44.794036Z","shell.execute_reply":"2021-12-14T10:29:44.799591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cd Train_Valid_images","metadata":{"execution":{"iopub.status.busy":"2021-12-12T07:19:11.496488Z","iopub.execute_input":"2021-12-12T07:19:11.496871Z","iopub.status.idle":"2021-12-12T07:19:11.502684Z","shell.execute_reply.started":"2021-12-12T07:19:11.496836Z","shell.execute_reply":"2021-12-12T07:19:11.50169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -rf Train_Valid_images ","metadata":{"execution":{"iopub.status.busy":"2021-12-12T07:19:16.794022Z","iopub.execute_input":"2021-12-12T07:19:16.79435Z","iopub.status.idle":"2021-12-12T07:19:18.134702Z","shell.execute_reply.started":"2021-12-12T07:19:16.79432Z","shell.execute_reply":"2021-12-12T07:19:18.133663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../input/rsna-jpeg-stage-2/Train_Valid_images/Train_Valid_images/ \"Train_Valid_images\"","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:29:52.009547Z","iopub.execute_input":"2021-12-14T10:29:52.009896Z","iopub.status.idle":"2021-12-14T10:33:12.462911Z","shell.execute_reply.started":"2021-12-14T10:29:52.009858Z","shell.execute_reply":"2021-12-14T10:33:12.461872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r ../input/rsna-jpeg-stage-2/Test_images/Test_images/ \"Test_images\"","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:19:20.431186Z","iopub.execute_input":"2021-12-18T18:19:20.431542Z","iopub.status.idle":"2021-12-18T18:19:23.960737Z","shell.execute_reply.started":"2021-12-18T18:19:20.431510Z","shell.execute_reply":"2021-12-18T18:19:23.959492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting train images into jpg","metadata":{"execution":{"iopub.status.busy":"2021-07-24T17:54:58.977493Z","iopub.execute_input":"2021-07-24T17:54:58.977877Z","iopub.status.idle":"2021-07-24T17:54:58.983756Z","shell.execute_reply.started":"2021-07-24T17:54:58.977844Z","shell.execute_reply":"2021-07-24T17:54:58.982297Z"}}},{"cell_type":"code","source":"\nfor filename in sorted(tqdm(os.listdir(\"../input/rsna-pneumonia-detection-challenge/stage_2_train_images\"))):\n    array = pydicom.dcmread(os.path.join(\"../input/rsna-pneumonia-detection-challenge/stage_2_train_images\",filename)).pixel_array\n    image = np.repeat(array[:,:,np.newaxis],3,2)\n#     print(array[0][0])\n#     print(filename[:len(filename)-4])\n    cv2.imwrite(os.path.join(\"Train_Valid_images\",filename[:len(filename)-4]+\".jpg\"),image)\n#     break;\n  ","metadata":{"execution":{"iopub.status.busy":"2021-12-11T17:00:25.61434Z","iopub.execute_input":"2021-12-11T17:00:25.614696Z","iopub.status.idle":"2021-12-11T17:00:30.865221Z","shell.execute_reply.started":"2021-12-11T17:00:25.614661Z","shell.execute_reply":"2021-12-11T17:00:30.864384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for filename in sorted(tqdm(os.listdir(\"../input/rsna-pneumonia-detection-challenge/stage_2_train_images\"))):\n    pass","metadata":{"execution":{"iopub.status.busy":"2021-12-13T08:06:48.46536Z","iopub.execute_input":"2021-12-13T08:06:48.4657Z","iopub.status.idle":"2021-12-13T08:06:48.960074Z","shell.execute_reply.started":"2021-12-13T08:06:48.46567Z","shell.execute_reply":"2021-12-13T08:06:48.959035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for filename in sorted(tqdm(os.listdir(\"/kaggle/working/Train_Valid_images\"))):\n    pass","metadata":{"execution":{"iopub.status.busy":"2021-12-13T08:06:40.675068Z","iopub.execute_input":"2021-12-13T08:06:40.675461Z","iopub.status.idle":"2021-12-13T08:06:40.683724Z","shell.execute_reply.started":"2021-12-13T08:06:40.675427Z","shell.execute_reply":"2021-12-13T08:06:40.682731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Converting test images into jpg","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:10:29.855598Z","iopub.execute_input":"2021-12-13T10:10:29.85594Z","iopub.status.idle":"2021-12-13T10:10:30.553663Z","shell.execute_reply.started":"2021-12-13T10:10:29.855907Z","shell.execute_reply":"2021-12-13T10:10:30.552366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for filename in sorted(tqdm(os.listdir(\"../input/rsna-pneumonia-detection-challenge/stage_2_test_images\"))):\n    array = pydicom.dcmread(os.path.join(\"../input/rsna-pneumonia-detection-challenge/stage_2_test_images\",filename)).pixel_array\n    image = np.repeat(array[:,:,np.newaxis],3,2)\n#     print(array[0][0])\n#     print(filename[:len(filename)-4])\n    cv2.imwrite(os.path.join(\"Test_images\",filename[:len(filename)-4]+\".jpg\"),image)\n#     break;\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-11T15:39:01.347326Z","iopub.execute_input":"2021-12-11T15:39:01.347617Z","iopub.status.idle":"2021-12-11T15:41:28.825808Z","shell.execute_reply.started":"2021-12-11T15:39:01.347587Z","shell.execute_reply":"2021-12-11T15:41:28.824856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For sample test","metadata":{}},{"cell_type":"code","source":"i = 1\nfor filename in sorted(tqdm(os.listdir(\"../input/rsna-pneumonia-detection-challenge/stage_2_test_images\"))):\n    array = pydicom.dcmread(os.path.join(\"../input/rsna-pneumonia-detection-challenge/stage_2_test_images\",filename)).pixel_array\n    image = np.repeat(array[:,:,np.newaxis],3,2)\n#     print(array[0][0])\n#     print(filename[:len(filename)-4])\n    cv2.imwrite(os.path.join(\"Test2\",filename[:len(filename)-4]+\".jpg\"),image)\n    i = i + 1\n    if i > 100:\n        break\n#     break;","metadata":{"execution":{"iopub.status.busy":"2021-12-11T17:00:56.800666Z","iopub.execute_input":"2021-12-11T17:00:56.801005Z","iopub.status.idle":"2021-12-11T17:01:01.790919Z","shell.execute_reply.started":"2021-12-11T17:00:56.800973Z","shell.execute_reply":"2021-12-11T17:01:01.790034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for male female ","metadata":{}},{"cell_type":"markdown","source":"train data","metadata":{}},{"cell_type":"code","source":"for paths in raw_train_df['path']:\n    array = pydicom.dcmread(paths).pixel_array\n    image = np.repeat(array[:,:,np.newaxis],3,2)\n#     print(array[0][0])\n    filename = os.path.basename(paths)\n     #print(filename[:len(filename)-4])\n    cv2.imwrite(os.path.join(\"Train_Valid_images\",filename[:len(filename)-4]+\".jpg\"),image)\n#     break;","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test data","metadata":{}},{"cell_type":"code","source":"for paths in remain_df['path']:\n    array = pydicom.dcmread(paths).pixel_array\n    image = np.repeat(array[:,:,np.newaxis],3,2)\n#     print(array[0][0])\n    filename = os.path.basename(paths)\n     #print(filename[:len(filename)-4])\n    cv2.imwrite(os.path.join(\"Test_images\",filename[:len(filename)-4]+\".jpg\"),image)\n#     break;","metadata":{"execution":{"iopub.status.busy":"2021-12-02T07:49:41.752464Z","iopub.execute_input":"2021-12-02T07:49:41.752788Z","iopub.status.idle":"2021-12-02T07:51:00.73789Z","shell.execute_reply.started":"2021-12-02T07:49:41.752757Z","shell.execute_reply":"2021-12-02T07:51:00.736992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import yaml\nimport json\n\nPneumonia = {\n     \"train\" : \"pneumonia_data/images/train\",\n    \"val\" : \"pneumonia_data/images/validation\",\n    \"nc\" :  1,\n    \"names\" : [\"pneumonia\"]\n}\n\nwith open('Pneumonia.yaml', 'w') as outfile:\n    yaml.safe_dump( Pneumonia, outfile, allow_unicode=True, default_flow_style=False)\n#with open('file.yaml', 'w') as file:\n    #file.write(json.dumps(Pneumonia)) \n #    print(Pneumonia, file=file)\n        \n","metadata":{"execution":{"iopub.status.busy":"2021-07-22T20:37:48.780196Z","iopub.execute_input":"2021-07-22T20:37:48.780541Z","iopub.status.idle":"2021-07-22T20:37:48.823295Z","shell.execute_reply.started":"2021-07-22T20:37:48.780508Z","shell.execute_reply":"2021-07-22T20:37:48.822488Z"}}},{"cell_type":"markdown","source":"import yaml\n\nyolov5sp = {\n    # parameters\n\"nc\" : 1,  # number of classes\n\"depth_multiple\" : 0.33 , # model depth multiple\n\"width_multiple\" : 0.50 , # layer channel multiple\n\n# anchors\n\"anchors\" :\n  [[10,13, 16,30, 33,23] , # P3/8\n   [30,61, 62,45, 59,119] , # P4/16\n   [116,90, 156,198, 373,326]] , # P5/32\n\n# YOLOv5 backbone\n\"backbone\" :\n  # [from, number, module, args]\n  [[-1, 1, \"Focus\", [64, 3]],  # 0-P1/2\n   [-1, 1, \"Conv\", [128, 3, 2]],  # 1-P2/4\n   [-1, 3, \"BottleneckCSP\", [128]],\n   [-1, 1, \"Conv\", [256, 3, 2]],  # 3-P3/8\n   [-1, 9, \"BottleneckCSP\", [256]],\n   [-1, 1, \"Conv\", [512, 3, 2]],  # 5-P4/16\n   [-1, 9, \"BottleneckCSP\", [512]],\n   [-1, 1, \"Conv\", [1024, 3, 2]],  # 7-P5/32\n   [-1, 1, \"SPP\", [1024, [5, 9, 13]]],\n   [-1, 3, \"BottleneckCSP\", [1024, False]],  # 9\n  ],\n# YOLOv5 head\n\"head\" : \n  [[-1, 1, \"Conv\", [512, 1, 1]],\n   [-1, 1, \"nn.Upsample\", [None, 2, 'nearest']],\n   [[-1, 6], 1, \"Concat\", [1]],  # cat backbone P4\n   [-1, 3, \"BottleneckCSP\", [512, False]],  # 13\n\n   [-1, 1, \"Conv\", [256, 1, 1]],\n   [-1, 1, \"nn.Upsample\", [None, 2, 'nearest']],\n   [[-1, 4], 1, \"Concat\", [1]],  # cat backbone P3\n   [-1, 3, \"BottleneckCSP\", [256, False]],  # 17 (P3/8-small)\n\n   [-1, 1, \"Conv\", [256, 3, 2]],\n   [[-1, 14], 1, \"Concat\", [1]],  # cat head P4\n   [-1, 3, \"BottleneckCSP\", [512, False]],  # 20 (P4/16-medium)\n\n   [-1, 1, \"Conv\", [512, 3, 2]],\n   [[-1, 10], 1, \"Concat\", [1]],  # cat head P5\n   [-1, 3, \"BottleneckCSP\", [1024, False]],  # 23 (P5/32-large)\n\n   [[17, 20, 23], 1, \"Detect\", [\"nc\", \"anchors\"]],  # Detect(P3, P4, P5)\n  ]\n}\n\nwith open('yolov5sp.yaml', 'w') as outfile:\n    yaml.safe_dump( yolov5sp, outfile, indent=4,  allow_unicode=False, default_flow_style=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-22T20:37:51.738852Z","iopub.execute_input":"2021-07-22T20:37:51.739486Z","iopub.status.idle":"2021-07-22T20:37:51.772315Z","shell.execute_reply.started":"2021-07-22T20:37:51.73944Z","shell.execute_reply":"2021-07-22T20:37:51.771187Z"}}},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-12-13T08:25:16.640636Z","iopub.execute_input":"2021-12-13T08:25:16.641007Z","iopub.status.idle":"2021-12-13T08:25:16.64934Z","shell.execute_reply.started":"2021-12-13T08:25:16.640971Z","shell.execute_reply":"2021-12-13T08:25:16.648165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import yaml\n\nvgg16 = {\n    # parameters\n\"nc\" : 1,  # number of classes\n\"depth_multiple\" : 1.0 , # model depth multiple\n\"width_multiple\" : 1.0 , # layer channel multiple\n\n# anchors\n\"anchors\" :\n  [[10,13, 16,30, 33,23] , # P3/8\n   [30,61, 62,45, 59,119] , # P4/16\n   [116,90, 156,198, 373,326]] , # P5/32\n\n# YOLOv5 backbone\n\"backbone\" :\n  # [from, number, module, args]\n  [[-1, 1, \"Conv\", [64, 3, 1]],  # 0\n   [-1, 1, \"Conv\", [64, 3, 1]],\n   [-1, 1, \"Maxpool\", [2, 2]],   # 2-P1/2\n   [-1, 1, \"Conv\", [128, 3, 1]],\n   [-1, 1, \"Conv\", [128, 3, 1]],\n   [-1, 1, \"Maxpool\", [2, 2]],   # 5-P2/4\n   [-1, 1, \"Conv\", [256, 3, 1]],\n   [-1, 1, \"Conv\", [256, 3, 1]],\n   [-1, 1, \"Maxpool\", [2, 2]],   # 8-P3/8\n   [-1, 1, \"Conv\", [512, 3, 1]],\n   [-1, 1, \"Conv\", [512, 3, 1]],\n   [-1, 1, \"Conv\", [512, 3, 1]],\n   [-1, 1, \"Maxpool\", [2, 2]],   # 13-P4/16\n   [-1, 1, \"Conv\", [512, 3, 1]],\n   [-1, 1, \"Conv\", [512, 3, 1]],\n   [-1, 1, \"Conv\", [512, 3, 1]],\n   [-1, 1, \"Maxpool\", [2, 2]],   # 16-P5/32\n  ],\n\n# YOLOv5 head\n\"head\" : \n     [[-1, 1, \"Bottleneck\", [1024, False]],\n   [-1, 2, \"Bottleneck\", [1024, False]],  # 18\n\n   [-1, 1, \"Conv\", [256, 1, 1]],\n   [-1, 1, \"nn.Upsample\", [None, 2, \"nearest\"]],\n   [[-1, 14], 1, \"Concat\", [1]],  # concat backbone P4\n   [-1, 1, \"Bottleneck\", [512, False]],\n   [-1, 2, \"Bottleneck\", [512, False]],  # 23\n\n   [-1, 1, \"Conv\", [128, 1, 1]],\n   [-1, 1, \"nn.Upsample\", [None, 2, \"nearest\"]],\n   [[-1, 9], 1, \"Concat\", [1]],  # concat backbone P3\n   [-1, 1, \"Bottleneck\", [256, False]],\n   [-1, 2, \"Bottleneck\", [256, False]],  # 28\n\n   [[28, 23, 18], 1, \"Detect\", [\"nc\", \"anchors\"]],   # Detect(P3, P4, P5)\n  ]\n}\n\nwith open('vgg16_1.yaml', 'w') as outfile:\n    yaml.safe_dump(vgg16, outfile, indent=4,  allow_unicode=False, default_flow_style=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T08:31:01.07206Z","iopub.execute_input":"2021-12-13T08:31:01.072433Z","iopub.status.idle":"2021-12-13T08:31:01.097991Z","shell.execute_reply.started":"2021-12-13T08:31:01.072402Z","shell.execute_reply":"2021-12-13T08:31:01.096928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# parameters\nnc: 1  # number of classes\ndepth_multiple: 1.0  # model depth multiple\nwidth_multiple: 1.0  # layer channel multiple\n\n# anchors\nanchors:\n  - [10,13, 16,30, 33,23]  # P3/8\n  - [30,61, 62,45, 59,119]  # P4/16\n  - [116,90, 156,198, 373,326]  # P5/32\n\n# VGG16 backbone\nbackbone:\n  # [from, number, module, args]\n  [[-1, 1, Conv, [64, 3, 1]],  # 0\n   [-1, 1, Conv, [64, 3, 1]],\n   [-1, 1, Maxpool, [2, 2]],   # 2-P1/2\n   [-1, 1, Conv, [128, 3, 1]],\n   [-1, 1, Conv, [128, 3, 1]],\n   [-1, 1, Maxpool, [2, 2]],   # 5-P2/4\n   [-1, 1, Conv, [256, 3, 1]],\n   [-1, 1, Conv, [256, 3, 1]],\n   [-1, 1, Maxpool, [2, 2]],   # 8-P3/8\n   [-1, 1, Conv, [512, 3, 1]],\n   [-1, 1, Conv, [512, 3, 1]],\n   [-1, 1, Conv, [512, 3, 1]],\n   [-1, 1, Maxpool, [2, 2]],   # 13-P4/16\n   [-1, 1, Conv, [512, 3, 1]],\n   [-1, 1, Conv, [512, 3, 1]],\n   [-1, 1, Conv, [512, 3, 1]],\n   [-1, 1, Maxpool, [2, 2]],   # 16-P5/32\n  ]\n\n# YOLOv3 head\nhead:\n  [[-1, 1, Bottleneck, [1024, False]],\n   [-1, 2, Bottleneck, [1024, False]],  # 18\n\n   [-1, 1, Conv, [256, 1, 1]],\n   [-1, 1, nn.Upsample, [None, 2, \"nearest\"]],\n   [[-1, 14], 1, Concat, [1]],  # concat backbone P4\n   [-1, 1, Bottleneck, [512, False]],\n   [-1, 2, Bottleneck, [512, False]],  # 23\n\n   [-1, 1, Conv, [128, 1, 1]],\n   [-1, 1, nn.Upsample, [None, 2, \"nearest\"]],\n   [[-1, 9], 1, Concat, [1]],  # concat backbone P3\n   [-1, 1, Bottleneck, [256, False]],\n   [-1, 2, Bottleneck, [256, False]],  # 28\n\n   [[28, 23, 18], 1, Detect, [number_classes, anchors]],   # Detect(P3, P4, P5)\n  ]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd \"/kaggle/working\"","metadata":{"execution":{"iopub.status.busy":"2021-12-18T17:42:25.685953Z","iopub.execute_input":"2021-12-18T17:42:25.686315Z","iopub.status.idle":"2021-12-18T17:42:25.695234Z","shell.execute_reply.started":"2021-12-18T17:42:25.686276Z","shell.execute_reply":"2021-12-18T17:42:25.694426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! git clone https://github.com/ultralytics/yolov5.git","metadata":{"execution":{"iopub.status.busy":"2021-12-11T15:45:13.057823Z","iopub.execute_input":"2021-12-11T15:45:13.058147Z","iopub.status.idle":"2021-12-11T15:45:16.060344Z","shell.execute_reply.started":"2021-12-11T15:45:13.058116Z","shell.execute_reply":"2021-12-11T15:45:16.059422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp -r ../input/yolov5-mod/yolov5-master .","metadata":{"execution":{"iopub.status.busy":"2021-12-18T17:42:32.531251Z","iopub.execute_input":"2021-12-18T17:42:32.531571Z","iopub.status.idle":"2021-12-18T17:42:33.889329Z","shell.execute_reply.started":"2021-12-18T17:42:32.531542Z","shell.execute_reply":"2021-12-18T17:42:33.888198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp \"../input/yolov5-pneumonia-yaml/Pneumonia.yaml\" \"yolov5-master/\"\n!cp \"../input/flexible-yolov5-efficient/vgg16.yaml\" \"yolov5-master/models/\"  #../input/yolov5-pneumonia-yaml/yolov5sp.yaml\nos.chdir(\"yolov5-master\")\n!rm -rf pneumonia_data\n!mkdir pneumonia_data\nos.chdir(\"pneumonia_data\")\n!mkdir images\n!mkdir labels\nos.chdir(\"images\")\n!mkdir train\n!mkdir validation\n!mkdir test\nos.chdir(os.pardir)\nos.chdir(\"labels\")\n!mkdir train\n!mkdir validation\n!mkdir test\nos.chdir(os.pardir)\nos.chdir(os.pardir)","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:43:38.320757Z","iopub.execute_input":"2021-12-13T10:43:38.321082Z","iopub.status.idle":"2021-12-13T10:43:46.346813Z","shell.execute_reply.started":"2021-12-13T10:43:38.321047Z","shell.execute_reply":"2021-12-13T10:43:46.345706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tree","metadata":{"execution":{"iopub.status.busy":"2021-07-30T17:16:01.563507Z","iopub.execute_input":"2021-07-30T17:16:01.563951Z","iopub.status.idle":"2021-07-30T17:16:02.312109Z","shell.execute_reply.started":"2021-07-30T17:16:01.563914Z","shell.execute_reply":"2021-07-30T17:16:02.310628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport ast\nimport shutil\nfrom sklearn import model_selection\nfrom tqdm import tqdm\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:24:21.726519Z","iopub.execute_input":"2021-12-14T10:24:21.726932Z","iopub.status.idle":"2021-12-14T10:24:22.484898Z","shell.execute_reply.started":"2021-12-14T10:24:21.726895Z","shell.execute_reply":"2021-12-14T10:24:22.484062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!pip install -r requirements.txt","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:07:42.587558Z","iopub.execute_input":"2021-12-13T17:07:42.587908Z","iopub.status.idle":"2021-12-13T17:07:43.457556Z","shell.execute_reply.started":"2021-12-13T17:07:42.587869Z","shell.execute_reply":"2021-12-13T17:07:43.456725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install -U PyYAML","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:13:09.773439Z","iopub.execute_input":"2021-12-13T10:13:09.773827Z","iopub.status.idle":"2021-12-13T10:13:31.244872Z","shell.execute_reply.started":"2021-12-13T10:13:09.773786Z","shell.execute_reply":"2021-12-13T10:13:31.243334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"DATA_PATH = \"../input/rsna-pneumonia-detection-challenge/\"\nOUTPUT_PATH = \"/kaggle/working/yolov5-master/pneumonia_data\"","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:53:16.306681Z","iopub.execute_input":"2021-12-18T15:53:16.307092Z","iopub.status.idle":"2021-12-18T15:53:16.314129Z","shell.execute_reply.started":"2021-12-18T15:53:16.307059Z","shell.execute_reply":"2021-12-18T15:53:16.31069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_data(data,data_type=\"train\"):\n    for _,row in tqdm(data.iterrows(),total=len(data),position=0,leave=True):\n        image_name = row[\"patientId\"]\n        bounding_boxes = row[\"bboxes\"]\n        yolo_data = []\n        for bbox in bounding_boxes:\n            if(bbox[4]==\"0.0\"):\n              # print(\"hello\")\n              continue\n            x=float(bbox[0])\n            y=float(bbox[1])\n            w=float(bbox[2])\n            h=float(bbox[3])\n            x_center = x+w/2\n            y_center = y+h/2\n            x_center/=1024.0\n            y_center/=1024.0\n            w/=1024.0\n            h/=1024.0\n            yolo_data.append([0,x_center,y_center,w,h])\n        yolo_data = np.array(yolo_data)\n        np.savetxt(\n            os.path.join(OUTPUT_PATH,f\"labels/{data_type}/{image_name}.txt\"),\n            yolo_data,\n            fmt = [\"%d\",\"%f\",\"%f\",\"%f\",\"%f\"]\n        )\n        shutil.copyfile(\n            f\"/kaggle/working/Train_Valid_images/Train_Valid_images/{image_name}.jpg\",\n            os.path.join(OUTPUT_PATH,f\"images/{data_type}/{image_name}.jpg\")\n        )","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:53:16.458781Z","iopub.execute_input":"2021-12-18T15:53:16.4592Z","iopub.status.idle":"2021-12-18T15:53:16.469512Z","shell.execute_reply.started":"2021-12-18T15:53:16.459167Z","shell.execute_reply":"2021-12-18T15:53:16.467616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_from_txt(file):\n    prediction_string= \"\"\n    if(os.path.isfile('runs/detect/exp/labels/'+file+'.txt')):\n        words = convert_from_yolo_to_bbox('runs/detect/exp/labels/'+file+'.txt')\n        #return words\n        for i in range(len(words)):\n\n            clss = float(words[i][0])\n            x_center = float(words[i][1])\n            y_center = float(words[i][2])\n            w = float(words[i][3])\n            h = float(words[i][4])\n            conf = float(words[i][5])\n            w = w * 1024\n            h = h * 1024\n            x_center = x_center * 1024\n            y_center = y_center * 1024\n            x = x_center - w/2\n            y = y_center - h/2\n            if (prediction_string != ''):\n                prediction_string = prediction_string + ' '\n            prediction_string = prediction_string + str(round(conf,3))+ ' '+ str(int(x)) + ' ' + str(int(y)) + ' '+ str(int(w)) + ' '+ str(int(h))\n            \n      \n        \n    return prediction_string","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:53:16.631254Z","iopub.execute_input":"2021-12-18T15:53:16.631746Z","iopub.status.idle":"2021-12-18T15:53:16.642132Z","shell.execute_reply.started":"2021-12-18T15:53:16.6317Z","shell.execute_reply":"2021-12-18T15:53:16.640884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"str(int((conf * 100) + 0.5) / 100.0)\n\nstr(round(conf,3))","metadata":{}},{"cell_type":"code","source":"def convert_from_yolo_to_bbox(txt_file):\n    \n    f = open(txt_file, \"r\")\n    lines = f.readlines()\n    words = []\n    for line in lines:\n        word = line.split(\" \") \n        words.append(word)\n    return words","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:53:19.242762Z","iopub.execute_input":"2021-12-18T15:53:19.243289Z","iopub.status.idle":"2021-12-18T15:53:19.251078Z","shell.execute_reply.started":"2021-12-18T15:53:19.243243Z","shell.execute_reply":"2021-12-18T15:53:19.249567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ss = convert_from_yolo_to_bbox(\"runs/detect/exp3/labels/00991acc-85b3-41c7-a397-bdf925c3697a.txt\")","metadata":{"execution":{"iopub.status.busy":"2021-12-02T07:54:08.576523Z","iopub.execute_input":"2021-12-02T07:54:08.576869Z","iopub.status.idle":"2021-12-02T07:54:08.580566Z","shell.execute_reply.started":"2021-12-02T07:54:08.576837Z","shell.execute_reply":"2021-12-02T07:54:08.57975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = convert_from_txt(\"001031d9-f904-4a23-b3e5-2c088acd19c6\")","metadata":{"execution":{"iopub.status.busy":"2021-12-02T07:55:04.374701Z","iopub.execute_input":"2021-12-02T07:55:04.375072Z","iopub.status.idle":"2021-12-02T07:55:04.380237Z","shell.execute_reply.started":"2021-12-02T07:55:04.37504Z","shell.execute_reply":"2021-12-02T07:55:04.377922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss","metadata":{"execution":{"iopub.status.busy":"2021-12-02T07:55:12.568606Z","iopub.execute_input":"2021-12-02T07:55:12.568967Z","iopub.status.idle":"2021-12-02T07:55:12.57666Z","shell.execute_reply.started":"2021-12-02T07:55:12.568934Z","shell.execute_reply":"2021-12-02T07:55:12.575165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(ss)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T07:55:56.106766Z","iopub.execute_input":"2021-12-02T07:55:56.10717Z","iopub.status.idle":"2021-12-02T07:55:56.114891Z","shell.execute_reply.started":"2021-12-02T07:55:56.107137Z","shell.execute_reply":"2021-12-02T07:55:56.113888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss","metadata":{"execution":{"iopub.status.busy":"2021-07-31T08:32:22.493912Z","iopub.execute_input":"2021-07-31T08:32:22.494491Z","iopub.status.idle":"2021-07-31T08:32:22.501657Z","shell.execute_reply.started":"2021-07-31T08:32:22.494457Z","shell.execute_reply":"2021-07-31T08:32:22.50044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float(ss[0][4])","metadata":{"execution":{"iopub.status.busy":"2021-11-25T10:42:12.846091Z","iopub.execute_input":"2021-11-25T10:42:12.846683Z","iopub.status.idle":"2021-11-25T10:42:12.982216Z","shell.execute_reply.started":"2021-11-25T10:42:12.846646Z","shell.execute_reply":"2021-11-25T10:42:12.981054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir(os.pardir)","metadata":{"execution":{"iopub.status.busy":"2021-11-25T10:42:41.558854Z","iopub.execute_input":"2021-11-25T10:42:41.559245Z","iopub.status.idle":"2021-11-25T10:42:41.563588Z","shell.execute_reply.started":"2021-11-25T10:42:41.559211Z","shell.execute_reply":"2021-11-25T10:42:41.562111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.getcwd()","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:39:13.856845Z","iopub.execute_input":"2021-12-13T10:39:13.857193Z","iopub.status.idle":"2021-12-13T10:39:13.863251Z","shell.execute_reply.started":"2021-12-13T10:39:13.857157Z","shell.execute_reply":"2021-12-13T10:39:13.862417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ..","metadata":{"execution":{"iopub.status.busy":"2021-12-14T02:47:59.488296Z","iopub.execute_input":"2021-12-14T02:47:59.488644Z","iopub.status.idle":"2021-12-14T02:47:59.49462Z","shell.execute_reply.started":"2021-12-14T02:47:59.488612Z","shell.execute_reply":"2021-12-14T02:47:59.493659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -rf yolov5-master","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:43:23.838956Z","iopub.execute_input":"2021-12-13T10:43:23.839307Z","iopub.status.idle":"2021-12-13T10:43:25.010975Z","shell.execute_reply.started":"2021-12-13T10:43:23.839258Z","shell.execute_reply":"2021-12-13T10:43:25.009868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip uninstall -q -y wandb","metadata":{"execution":{"iopub.status.busy":"2021-07-29T08:56:25.448739Z","iopub.execute_input":"2021-07-29T08:56:25.449231Z","iopub.status.idle":"2021-07-29T08:56:25.453233Z","shell.execute_reply.started":"2021-07-29T08:56:25.449194Z","shell.execute_reply":"2021-07-29T08:56:25.452461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH","metadata":{"execution":{"iopub.status.busy":"2021-07-29T08:56:25.454649Z","iopub.execute_input":"2021-07-29T08:56:25.455328Z","iopub.status.idle":"2021-07-29T08:56:25.463244Z","shell.execute_reply.started":"2021-07-29T08:56:25.455286Z","shell.execute_reply":"2021-07-29T08:56:25.462298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:53:26.324343Z","iopub.execute_input":"2021-12-18T15:53:26.324743Z","iopub.status.idle":"2021-12-18T15:53:26.334556Z","shell.execute_reply.started":"2021-12-18T15:53:26.324709Z","shell.execute_reply":"2021-12-18T15:53:26.333156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"in kaggle/working dir","metadata":{}},{"cell_type":"code","source":"if __name__==\"__main__\":\n    full_path = str(os.path.join(DATA_PATH,\"stage_2_train_labels.csv\"))\n    print(full_path)\n    df = pd.read_csv(full_path)\n   # df_test = df_full.iloc[int(0.9*len(df_full)):]\n    #df = raw_train_df.reset_index(drop=True) #male female\n    df = df[df['Target']==1]\n    # df.bbox = df.bbox.apply(ast.literal_eval)\n    cols = ['x', 'y', 'width','height','Target']\n    df['combined'] = df[cols].apply(lambda row: ','.join(row.values.astype(str)), axis=1)\n    df['combined'] = df.combined.apply(lambda x: x.strip('()').split(','))\n    # print(type(df['combined'].iloc[0]))\n    # print(df)\n    df = df.groupby(\"patientId\")['combined'].apply(list).reset_index(name=\"bboxes\")\n    # print(df)\n    df_train,df_valid = model_selection.train_test_split(\n        df,\n        test_size=0.11,\n        random_state=True,\n        shuffle=True,\n    )\n    df_train = df_train.reset_index(drop=True)\n    df_valid = df_valid.reset_index(drop=True)\n    # print(df_train)\n    process_data(df_train,data_type=\"train\")\n    process_data(df_valid,data_type=\"validation\")","metadata":{"execution":{"iopub.status.busy":"2021-12-14T02:48:01.746138Z","iopub.execute_input":"2021-12-14T02:48:01.746468Z","iopub.status.idle":"2021-12-14T02:48:11.372892Z","shell.execute_reply.started":"2021-12-14T02:48:01.746436Z","shell.execute_reply":"2021-12-14T02:48:11.371959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:53:53.48325Z","iopub.execute_input":"2021-12-18T15:53:53.48361Z","iopub.status.idle":"2021-12-18T15:53:53.490643Z","shell.execute_reply.started":"2021-12-18T15:53:53.483581Z","shell.execute_reply":"2021-12-18T15:53:53.489429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T08:09:05.036598Z","iopub.execute_input":"2021-12-02T08:09:05.036975Z","iopub.status.idle":"2021-12-02T08:09:05.064753Z","shell.execute_reply.started":"2021-12-02T08:09:05.036941Z","shell.execute_reply":"2021-12-02T08:09:05.063852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_train_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-02T08:06:19.038361Z","iopub.execute_input":"2021-12-02T08:06:19.03868Z","iopub.status.idle":"2021-12-02T08:06:19.04613Z","shell.execute_reply.started":"2021-12-02T08:06:19.038649Z","shell.execute_reply":"2021-12-02T08:06:19.045124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_path = str(\"../input/rsna-pneumonia-detection-challenge/stage_2_train_labels.csv\")\nprint(full_path)\ndf = pd.read_csv(full_path)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:53:40.493678Z","iopub.execute_input":"2021-12-18T15:53:40.49414Z","iopub.status.idle":"2021-12-18T15:53:40.608343Z","shell.execute_reply.started":"2021-12-18T15:53:40.494077Z","shell.execute_reply":"2021-12-18T15:53:40.607031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir(\"yolov5-master\")\nos.getcwd()","metadata":{"execution":{"iopub.status.busy":"2021-12-18T15:56:29.209675Z","iopub.execute_input":"2021-12-18T15:56:29.210089Z","iopub.status.idle":"2021-12-18T15:56:29.21925Z","shell.execute_reply.started":"2021-12-18T15:56:29.210052Z","shell.execute_reply":"2021-12-18T15:56:29.217705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"may be yolov5 yaml model architecture will be same as pretrained weights","metadata":{}},{"cell_type":"code","source":"#!tree #see structure of directories","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!python3 train.py --resume \"../../input/yolov5-rsna/last.pt\" ","metadata":{"execution":{"iopub.status.busy":"2021-07-25T19:23:41.55674Z","iopub.execute_input":"2021-07-25T19:23:41.557096Z","iopub.status.idle":"2021-07-25T19:23:45.806875Z","shell.execute_reply.started":"2021-07-25T19:23:41.557063Z","shell.execute_reply":"2021-07-25T19:23:45.805972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 train.py --img 1024 --batch 8 --epochs 20 --data Pneumonia.yaml --cfg models/yolov5sp.yaml --name wm --weights \"https://github.com/ultralytics/yolov5/releases/download/v5.0/yolov5l6.pt\" --project yolov5-rsna   --single-cls #https://github.com/ultralytics/yolov5/releases/download/v5.0/yolov5l6.pt  ##\"../../input/d/sakib01/yolov5-rsna/last.pt\"","metadata":{"execution":{"iopub.status.busy":"2021-12-02T08:09:52.567626Z","iopub.execute_input":"2021-12-02T08:09:52.568081Z","iopub.status.idle":"2021-12-02T09:13:57.515733Z","shell.execute_reply.started":"2021-12-02T08:09:52.56803Z","shell.execute_reply":"2021-12-02T09:13:57.514603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train with vgg16","metadata":{}},{"cell_type":"code","source":"!wget 'https://download.pytorch.org/models/vgg16-397923af.pth'","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:45:33.644789Z","iopub.execute_input":"2021-12-13T10:45:33.645134Z","iopub.status.idle":"2021-12-13T10:46:07.946409Z","shell.execute_reply.started":"2021-12-13T10:45:33.645103Z","shell.execute_reply":"2021-12-13T10:46:07.945439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":" os.rename('vgg16-397923af.pth', \"vgg16-397923af\"+\".pt\")","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:46:25.736928Z","iopub.execute_input":"2021-12-13T10:46:25.737286Z","iopub.status.idle":"2021-12-13T10:46:25.742814Z","shell.execute_reply.started":"2021-12-13T10:46:25.737248Z","shell.execute_reply":"2021-12-13T10:46:25.741533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 train.py --img 1024 --batch 8 --epochs 2 --data Pneumonia.yaml --cfg models/vgg16.yaml --name wm_vgg16_test_1 --weights \"vgg16-397923af.pt\" --project yolov5-rsna    --single-cls  #https://github.com/ultralytics/yolov5/releases/download/v5.0/yolov5l6.pt  ##\"../../input/d/sakib01/yolov5-rsna/last.pt\"","metadata":{"execution":{"iopub.status.busy":"2021-12-13T10:47:13.615543Z","iopub.execute_input":"2021-12-13T10:47:13.615902Z","iopub.status.idle":"2021-12-13T10:47:38.830429Z","shell.execute_reply.started":"2021-12-13T10:47:13.615869Z","shell.execute_reply":"2021-12-13T10:47:38.829169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 train.py --img 1024 --batch 8 --epochs 10 --data Pneumonia.yaml --cfg models/yolov5sp.yaml --name wm --weights \"yolov5-rsna/wm/weights/last.pt\" --project yolov5-rsna   --single-cls #https://github.com/ultralytics/yolov5/releases/download/v5.0/yolov5l6.pt  ##\"../../input/d/sakib01/yolov5-rsna/last.pt\"","metadata":{"execution":{"iopub.status.busy":"2021-12-02T09:17:31.685636Z","iopub.execute_input":"2021-12-02T09:17:31.68609Z","iopub.status.idle":"2021-12-02T09:44:11.482408Z","shell.execute_reply.started":"2021-12-02T09:17:31.68604Z","shell.execute_reply":"2021-12-02T09:44:11.48146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --source \"/kaggle/working/Test2/\"  --weights \"../../input/d/sakib01/yolov5-rsna/last.pt\" --save-txt --conf-thres  0.3 --iou-thres 0.5 #--iou-thres 0.3 --conf-thres 0.6/ --conf 0","metadata":{"execution":{"iopub.status.busy":"2021-07-31T08:56:06.057313Z","iopub.execute_input":"2021-07-31T08:56:06.057786Z","iopub.status.idle":"2021-07-31T08:56:43.940027Z","shell.execute_reply.started":"2021-07-31T08:56:06.057749Z","shell.execute_reply":"2021-07-31T08:56:43.938918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --source \"/kaggle/working/Test2/\"  --weights \"../../input/d/sakib01/yolov5-rsna/last.pt\" --save-txt --save-conf --conf-thres  0.3 --iou-thres 0.5 #--iou-thres 0.3 --conf-thres 0.6/ --conf 0","metadata":{"execution":{"iopub.status.busy":"2021-07-31T08:55:14.972455Z","iopub.execute_input":"2021-07-31T08:55:14.97324Z","iopub.status.idle":"2021-07-31T08:55:53.480002Z","shell.execute_reply.started":"2021-07-31T08:55:14.973203Z","shell.execute_reply":"2021-07-31T08:55:53.479027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd working","metadata":{"execution":{"iopub.status.busy":"2021-12-18T17:50:47.233773Z","iopub.execute_input":"2021-12-18T17:50:47.23413Z","iopub.status.idle":"2021-12-18T17:50:47.239341Z","shell.execute_reply.started":"2021-12-18T17:50:47.234099Z","shell.execute_reply":"2021-12-18T17:50:47.238485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ./flexible-yolov5-main","metadata":{"execution":{"iopub.status.busy":"2021-12-18T17:52:08.127013Z","iopub.execute_input":"2021-12-18T17:52:08.127399Z","iopub.status.idle":"2021-12-18T17:52:08.135093Z","shell.execute_reply.started":"2021-12-18T17:52:08.127364Z","shell.execute_reply":"2021-12-18T17:52:08.134138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for flexible yolov5","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\nsys.path.append('.')\nfrom od.models.modules.experimental import *\nfrom od.data.datasets import letterbox\nfrom utils.general import *\nfrom utils.split_detector import SPLITINFERENCE\nfrom utils.torch_utils import *\n\n\nclass Detector(object):\n    def __init__(self, pt_path, img_size, conf_thres=0.4, iou_thres=0.3, classes=0, agnostic_nms=False,\n                 xcycwh=True, device=0):\n        self.pt_path = pt_path\n        self.img_size = img_size\n        self.device = torch.device('cuda:{}'.format(device))\n        self.model = self.load_model()\n        self.conf_thres = conf_thres\n        self.iou_thres = iou_thres\n        self.classes = classes\n        self.agnostic_nms = agnostic_nms\n        self.xcycwh = xcycwh\n       # self.class_names = self.load_class_names(namesfile)\n\n    def load_model(self):\n        model = attempt_load(self.pt_path, map_location=self.device)  # load FP32 model\n        return model\n\n    \n\n    def __call__(self, ori_img, split_width=1, split_height=1):\n        if split_width == 1 and split_height == 1:\n            bboxes, scores, ids = self.detect_image(ori_img)\n        else:\n            bboxes = []\n            scores = []\n            ids = []\n            output = self.detect_img_split(image=ori_img, split_width=split_width, split_height=split_height)['data']\n            for key in output.keys():\n                values = output[key]\n                for value in values:\n                    x_min = value[0]\n                    y_min = value[1]\n                    x_max = value[2]\n                    y_max = value[3]\n                    w = x_max - x_min\n                    h = y_max - y_min\n                    if self.xcycwh:\n                        bboxes.append([x_min + w / 2, y_min + h / 2, w, h])\n                    else:\n                        bboxes.append(value[:4])\n                    scores.append(value[4])\n                    ids.append(key)\n        return np.asarray(bboxes), np.asarray(scores), np.asarray(ids)\n\n    def detect_image(self, image):\n        bboxes = []\n        scores = []\n        ids = []\n        im0s = image\n        img = letterbox(im0s, new_shape=self.img_size)[0]\n        img = img[:, :, ::-1].transpose(2, 0, 1)\n        img = np.ascontiguousarray(img)\n        img = torch.from_numpy(img).to(self.device)\n        img = img.float()\n        img /= 255.0\n        if img.ndimension() == 3:\n            img = img.unsqueeze(0)\n        pred = self.model(img)[0]\n        pred = non_max_suppression(pred, self.conf_thres, self.iou_thres, classes=self.classes,\n                                   agnostic=self.agnostic_nms)\n        for i, det in enumerate(pred):\n            if det is not None and len(det):\n                det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0s.shape).round()\n                for *xyxy, conf, cls in det:\n                    x_min = xyxy[0].cpu()\n                    y_min = xyxy[1].cpu()\n                    x_max = xyxy[2].cpu()\n                    y_max = xyxy[3].cpu()\n                    score = conf.cpu()\n                    clas = cls.cpu()\n                    w = x_max - x_min\n                    h = y_max - y_min\n                    if self.xcycwh:\n                        # center coord, w, h\n                        bboxes.append([x_min + w / 2, y_min + h / 2, w, h])\n                    else:\n                        bboxes.append([x_min, y_min, x_max, y_max])\n                    scores.append(score)\n                    ids.append(clas)\n        return np.asarray(bboxes), np.asarray(scores), np.asarray(ids)\n\n    @SPLITINFERENCE(split_width=2, split_height=1)\n    def detect_img_split(self, image='', **kwargs):\n        outputs_json = {}\n        im0s = image\n        img = letterbox(im0s, new_shape=self.img_size)[0]\n        img = img[:, :, ::-1].transpose(2, 0, 1)\n        img = np.ascontiguousarray(img)\n        img = torch.from_numpy(img).to(self.device)\n        img = img.float()\n        img /= 255.0\n        if img.ndimension() == 3:\n            img = img.unsqueeze(0)\n        pred = self.model(img)[0]\n        pred = non_max_suppression(pred, self.conf_thres, self.iou_thres, classes=self.classes,\n                                   agnostic=self.agnostic_nms)\n        for i, det in enumerate(pred):\n            if det is not None and len(det):\n                det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0s.shape).round()\n                for *xyxy, conf, cls in det:\n                    x_min = xyxy[0].cpu()\n                    y_min = xyxy[1].cpu()\n                    x_max = xyxy[2].cpu()\n                    y_max = xyxy[3].cpu()\n                    score = conf.cpu()\n                    clas = cls.cpu()\n                    if clas in outputs_json:\n                        outputs_json[clas].append([x_min, y_min, x_max, y_max, score])\n                    else:\n                        outputs_json[clas] = [[x_min, y_min, x_max, y_max, score]]\n        return {'data': outputs_json}","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:20:29.960982Z","iopub.execute_input":"2021-12-18T18:20:29.961431Z","iopub.status.idle":"2021-12-18T18:20:30.002729Z","shell.execute_reply.started":"2021-12-18T18:20:29.961386Z","shell.execute_reply":"2021-12-18T18:20:30.001729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pt_path = '../../input/yolov5-efficientnetb7/best.pt'\nmodel = Detector(pt_path,  512, xcycwh=False)\nimgs_root = \"/kaggle/working/Test_images/Test_images/\"\nimgs = os.listdir(imgs_root)\nsave_dir = '/kaggle/working/output/'\nif not os.path.exists(save_dir):\n    os.mkdir(save_dir)\nfor img in imgs:\n    im = cv2.imread(os.path.join(imgs_root, img))\n \n    bboxes, scores, ids = model.detect_image(im)\n \n    for idx in range(bboxes.shape[0]):\n        bbox = bboxes[idx].astype(int)\n        score = scores[idx]\n        cv2.rectangle(im, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 0, 255), 2, 1)\n    cv2.imwrite(os.path.join(save_dir, img), im)","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:20:34.915714Z","iopub.execute_input":"2021-12-18T18:20:34.916121Z","iopub.status.idle":"2021-12-18T18:27:02.178909Z","shell.execute_reply.started":"2021-12-18T18:20:34.916084Z","shell.execute_reply":"2021-12-18T18:27:02.177985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python detect.py --source \"/kaggle/working/Test_images/\"  --weights \"../../input/yolov5-efficientnetb7/best.pt\" --save-txt --save-conf --conf-thres  0.3 --iou-thres 0.5 #--iou-thres 0.3 --conf-thres 0.6/ --conf 0","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:05:55.68229Z","iopub.execute_input":"2021-12-18T18:05:55.682657Z","iopub.status.idle":"2021-12-18T18:05:56.471896Z","shell.execute_reply.started":"2021-12-18T18:05:55.682627Z","shell.execute_reply":"2021-12-18T18:05:56.470936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv('../../input/rsna-pneumonia-detection-challenge/stage_2_sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:07:54.184958Z","iopub.execute_input":"2021-07-31T09:07:54.185521Z","iopub.status.idle":"2021-07-31T09:07:54.225403Z","shell.execute_reply.started":"2021-07-31T09:07:54.185484Z","shell.execute_reply":"2021-07-31T09:07:54.224089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:07:54.646952Z","iopub.execute_input":"2021-07-31T09:07:54.647313Z","iopub.status.idle":"2021-07-31T09:07:54.67491Z","shell.execute_reply.started":"2021-07-31T09:07:54.647283Z","shell.execute_reply":"2021-07-31T09:07:54.673864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = sample_df","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:07:58.124183Z","iopub.execute_input":"2021-07-31T09:07:58.124548Z","iopub.status.idle":"2021-07-31T09:07:58.129167Z","shell.execute_reply.started":"2021-07-31T09:07:58.124516Z","shell.execute_reply":"2021-07-31T09:07:58.127811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(test_df.shape[0]):\n    test_df['PredictionString'][i] = convert_from_txt(test_df[\"patientId\"][i])\n    ","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:07:58.637859Z","iopub.execute_input":"2021-07-31T09:07:58.638267Z","iopub.status.idle":"2021-07-31T09:07:59.060038Z","shell.execute_reply.started":"2021-07-31T09:07:58.638234Z","shell.execute_reply":"2021-07-31T09:07:59.058718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:07:59.061572Z","iopub.execute_input":"2021-07-31T09:07:59.061999Z","iopub.status.idle":"2021-07-31T09:07:59.078627Z","shell.execute_reply.started":"2021-07-31T09:07:59.061962Z","shell.execute_reply":"2021-07-31T09:07:59.076938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!dir","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:08:01.588807Z","iopub.execute_input":"2021-07-31T09:08:01.589322Z","iopub.status.idle":"2021-07-31T09:08:02.350836Z","shell.execute_reply.started":"2021-07-31T09:08:01.589274Z","shell.execute_reply":"2021-07-31T09:08:02.349598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ..","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:31:36.368847Z","iopub.execute_input":"2021-12-18T18:31:36.369300Z","iopub.status.idle":"2021-12-18T18:31:36.374678Z","shell.execute_reply.started":"2021-12-18T18:31:36.369253Z","shell.execute_reply":"2021-12-18T18:31:36.373633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('test_submission_final.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-31T09:08:06.478021Z","iopub.execute_input":"2021-07-31T09:08:06.478411Z","iopub.status.idle":"2021-07-31T09:08:06.50478Z","shell.execute_reply.started":"2021-07-31T09:08:06.478365Z","shell.execute_reply":"2021-07-31T09:08:06.503001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tarfile\nimport os.path\n\ndef make_tarfile(output_filename, source_dir):\n    with tarfile.open(output_filename, \"w:gz\") as tar:\n        tar.add(source_dir, arcname=os.path.basename(source_dir))","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:31:28.886425Z","iopub.execute_input":"2021-12-18T18:31:28.886760Z","iopub.status.idle":"2021-12-18T18:31:28.891582Z","shell.execute_reply.started":"2021-12-18T18:31:28.886731Z","shell.execute_reply":"2021-12-18T18:31:28.890676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd working","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:31:32.351795Z","iopub.execute_input":"2021-12-18T18:31:32.352142Z","iopub.status.idle":"2021-12-18T18:31:32.358067Z","shell.execute_reply.started":"2021-12-18T18:31:32.352095Z","shell.execute_reply":"2021-12-18T18:31:32.357134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_tarfile(\"output_yolov5_efficient.tar\",'/kaggle/working/output/')\n","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:33:02.269063Z","iopub.execute_input":"2021-12-18T18:33:02.269416Z","iopub.status.idle":"2021-12-18T18:33:29.331361Z","shell.execute_reply.started":"2021-12-18T18:33:02.269385Z","shell.execute_reply":"2021-12-18T18:33:29.330494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_tarfile(\"weights_25_epochs_25_11_12.tar\",'/kaggle/working/yolov5/runs')","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:44:51.910796Z","iopub.execute_input":"2021-11-25T16:44:51.911139Z","iopub.status.idle":"2021-11-25T16:44:51.951303Z","shell.execute_reply.started":"2021-11-25T16:44:51.911104Z","shell.execute_reply":"2021-11-25T16:44:51.949811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd 'yolov5'","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:46:53.72578Z","iopub.execute_input":"2021-11-25T16:46:53.726118Z","iopub.status.idle":"2021-11-25T16:46:53.73312Z","shell.execute_reply.started":"2021-11-25T16:46:53.726087Z","shell.execute_reply":"2021-11-25T16:46:53.732235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd 'Train_Valid_images'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!dir","metadata":{"execution":{"iopub.status.busy":"2021-11-25T16:47:18.892765Z","iopub.execute_input":"2021-11-25T16:47:18.893137Z","iopub.status.idle":"2021-11-25T16:47:19.616148Z","shell.execute_reply.started":"2021-11-25T16:47:18.893104Z","shell.execute_reply":"2021-11-25T16:47:19.61527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:09:13.147056Z","iopub.execute_input":"2021-12-13T17:09:13.147456Z","iopub.status.idle":"2021-12-13T17:09:13.157575Z","shell.execute_reply.started":"2021-12-13T17:09:13.147422Z","shell.execute_reply":"2021-12-13T17:09:13.156534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ..","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:27:41.945898Z","iopub.execute_input":"2021-12-14T10:27:41.946323Z","iopub.status.idle":"2021-12-14T10:27:41.955446Z","shell.execute_reply.started":"2021-12-14T10:27:41.946287Z","shell.execute_reply":"2021-12-14T10:27:41.954211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/yl305237731/flexible-yolov5.git","metadata":{"execution":{"iopub.status.busy":"2021-12-13T11:28:10.856681Z","iopub.execute_input":"2021-12-13T11:28:10.857066Z","iopub.status.idle":"2021-12-13T11:28:12.791187Z","shell.execute_reply.started":"2021-12-13T11:28:10.857023Z","shell.execute_reply":"2021-12-13T11:28:12.790126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf flexible-yolov5-main","metadata":{"execution":{"iopub.status.busy":"2021-12-18T17:51:25.78227Z","iopub.execute_input":"2021-12-18T17:51:25.782595Z","iopub.status.idle":"2021-12-18T17:51:26.470811Z","shell.execute_reply.started":"2021-12-18T17:51:25.782568Z","shell.execute_reply":"2021-12-18T17:51:26.469673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp -r ../input/flexible-yolov5/flexible-yolov5-main .","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:19:35.783842Z","iopub.execute_input":"2021-12-18T18:19:35.784199Z","iopub.status.idle":"2021-12-18T18:19:36.741211Z","shell.execute_reply.started":"2021-12-18T18:19:35.784164Z","shell.execute_reply":"2021-12-18T18:19:36.740131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -r flexible-yolov5-main/requirements.txt","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:19:38.462299Z","iopub.execute_input":"2021-12-18T18:19:38.462673Z","iopub.status.idle":"2021-12-18T18:19:44.782721Z","shell.execute_reply.started":"2021-12-18T18:19:38.462638Z","shell.execute_reply":"2021-12-18T18:19:44.781730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp \"../input/yolov5-pneumonia-yaml/Pneumonia.yaml\" \"flexible-yolov5-main\"","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:41:18.691346Z","iopub.execute_input":"2021-12-14T10:41:18.691685Z","iopub.status.idle":"2021-12-14T10:41:19.357951Z","shell.execute_reply.started":"2021-12-14T10:41:18.691653Z","shell.execute_reply":"2021-12-14T10:41:19.356925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp \"../input/flexible-yolov5-efficient/Flexible_Yolo_efficientnet.yaml\" \"flexible-yolov5-main\"","metadata":{"execution":{"iopub.status.busy":"2021-12-13T16:43:09.97819Z","iopub.execute_input":"2021-12-13T16:43:09.978553Z","iopub.status.idle":"2021-12-13T16:43:10.66662Z","shell.execute_reply.started":"2021-12-13T16:43:09.978518Z","shell.execute_reply":"2021-12-13T16:43:10.665478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ..","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:42:13.949608Z","iopub.execute_input":"2021-12-14T10:42:13.94993Z","iopub.status.idle":"2021-12-14T10:42:13.956096Z","shell.execute_reply.started":"2021-12-14T10:42:13.949899Z","shell.execute_reply":"2021-12-14T10:42:13.955013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"kaggle/working","metadata":{}},{"cell_type":"code","source":"def process_data(data,data_type=\"train\"):\n    for _,row in tqdm(data.iterrows(),total=len(data),position=0,leave=True):\n        image_name = row[\"patientId\"]\n        bounding_boxes = row[\"bboxes\"]\n        yolo_data = []\n        if bounding_boxes != 0:          \n            for bbox in bounding_boxes:\n                if(bbox[4]==\"0.0\"):\n                  # print(\"hello\")\n                  continue\n                x=float(bbox[0])\n                y=float(bbox[1])\n                w=float(bbox[2])\n                h=float(bbox[3])\n                x_center = x+w/2\n                y_center = y+h/2\n                x_center/=1024.0\n                y_center/=1024.0\n                w/=1024.0\n                h/=1024.0\n                yolo_data.append([0,x_center,y_center,w,h])\n            yolo_data = np.array(yolo_data)\n            np.savetxt(\n                os.path.join(OUTPUT_PATH,f\"labels/{data_type}/{image_name}.txt\"),\n                yolo_data,\n                fmt = [\"%d\",\"%f\",\"%f\",\"%f\",\"%f\"]\n            )\n            \n        shutil.copyfile(\n            f\"../input/rsna-jpeg-stage-2/Train_Valid_images/Train_Valid_images/{image_name}.jpg\",\n            os.path.join(OUTPUT_PATH,f\"images/{data_type}/{image_name}.jpg\")\n        )","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:42:19.242022Z","iopub.execute_input":"2021-12-14T10:42:19.242349Z","iopub.status.idle":"2021-12-14T10:42:19.25104Z","shell.execute_reply.started":"2021-12-14T10:42:19.242319Z","shell.execute_reply":"2021-12-14T10:42:19.250071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__==\"__main__\":\n    full_path = str(os.path.join(DATA_PATH,\"stage_2_train_labels.csv\"))\n    print(full_path)\n    df = pd.read_csv(full_path)\n    df1 = df[df['Target']==1]\n    # df.bbox = df.bbox.apply(ast.literal_eval)\n    cols = ['x', 'y', 'width','height','Target']\n    df1['combined'] = df1[cols].apply(lambda row: ','.join(row.values.astype(str)), axis=1)\n    df1['combined'] = df1.combined.apply(lambda x: x.strip('()').split(','))\n    df1 = df1.groupby(\"patientId\")['combined'].apply(list).reset_index(name=\"bboxes\")\n\n    df2 = df[df['Target']!=1]\n    df2.drop(cols, axis  = 1, inplace = True)\n    df2['bboxes'] = 0\n    df = pd.concat([df1,df2])\n    df = df.sample(df.shape[0]).reset_index()\n    # print(df)\n    df_train,df_valid = model_selection.train_test_split(\n        df,\n        test_size=0.11,\n        random_state=True,\n        shuffle=True,\n    )\n    df_train = df_train.reset_index(drop=True)\n    df_valid = df_valid.reset_index(drop=True)\n    # print(df_train)\n    process_data(df_train,data_type=\"train\")\n    process_data(df_valid,data_type=\"validation\")","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:42:19.699495Z","iopub.execute_input":"2021-12-14T10:42:19.699815Z","iopub.status.idle":"2021-12-14T10:44:41.413819Z","shell.execute_reply.started":"2021-12-14T10:42:19.699785Z","shell.execute_reply":"2021-12-14T10:44:41.412873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir(\"flexible-yolov5-main\")","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:20:18.925568Z","iopub.execute_input":"2021-12-18T18:20:18.925925Z","iopub.status.idle":"2021-12-18T18:20:18.932755Z","shell.execute_reply.started":"2021-12-18T18:20:18.925892Z","shell.execute_reply":"2021-12-18T18:20:18.931678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!rm -rf pneumonia_data\n!mkdir pneumonia_data\nos.chdir(\"pneumonia_data\")\n!mkdir images\n!mkdir labels\nos.chdir(\"images\")\n!mkdir train\n!mkdir validation\n!mkdir test\nos.chdir(os.pardir)\nos.chdir(\"labels\")\n!mkdir train\n!mkdir validation\n!mkdir test\nos.chdir(os.pardir)\nos.chdir(os.pardir)","metadata":{"execution":{"iopub.status.busy":"2021-12-18T18:20:19.107898Z","iopub.execute_input":"2021-12-18T18:20:19.108235Z","iopub.status.idle":"2021-12-18T18:20:25.982659Z","shell.execute_reply.started":"2021-12-18T18:20:19.108205Z","shell.execute_reply":"2021-12-18T18:20:25.981511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = \"../input/rsna-pneumonia-detection-challenge/\"\nOUTPUT_PATH = \"/kaggle/working/flexible-yolov5-main/pneumonia_data\"","metadata":{"execution":{"iopub.status.busy":"2021-12-18T17:55:37.219269Z","iopub.execute_input":"2021-12-18T17:55:37.219638Z","iopub.status.idle":"2021-12-18T17:55:37.225227Z","shell.execute_reply.started":"2021-12-18T17:55:37.219604Z","shell.execute_reply":"2021-12-18T17:55:37.224325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cp ../../input/flexible-yolov5-efficient/model_resnet.yaml .","metadata":{"execution":{"iopub.status.busy":"2021-12-13T12:25:17.947133Z","iopub.execute_input":"2021-12-13T12:25:17.947635Z","iopub.status.idle":"2021-12-13T12:25:18.630108Z","shell.execute_reply.started":"2021-12-13T12:25:17.947581Z","shell.execute_reply":"2021-12-13T12:25:18.628719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 scripts/train.py --img 512 --batch 8 --epochs 15 --data Pneumonia.yaml --cfg configs/model_efficientnet.yaml --name wm2_efficientnetb7_pretrained_with_negative_images --project yolov5-rsna   --single-cls #https://github.com/ultralytics/yolov5/releases/download/v5.0/yolov5l6.pt  ##\"../../input/d/sakib01/yolov5-rsna/last.pt\"","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:45:11.655509Z","iopub.execute_input":"2021-12-14T10:45:11.655837Z","iopub.status.idle":"2021-12-14T11:47:35.451242Z","shell.execute_reply.started":"2021-12-14T10:45:11.655804Z","shell.execute_reply":"2021-12-14T11:47:35.450254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget 'https://download.pytorch.org/models/resnet50-19c8e357.pth'","metadata":{"execution":{"iopub.status.busy":"2021-12-13T11:45:12.692868Z","iopub.execute_input":"2021-12-13T11:45:12.693269Z","iopub.status.idle":"2021-12-13T11:45:15.872318Z","shell.execute_reply.started":"2021-12-13T11:45:12.693221Z","shell.execute_reply":"2021-12-13T11:45:15.871069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" os.rename('resnet50-19c8e357.pth', \"resnet50-19c8e357\"+\".pt\")","metadata":{"execution":{"iopub.status.busy":"2021-12-14T06:24:08.957292Z","iopub.execute_input":"2021-12-14T06:24:08.957667Z","iopub.status.idle":"2021-12-14T06:24:08.976862Z","shell.execute_reply.started":"2021-12-14T06:24:08.957633Z","shell.execute_reply":"2021-12-14T06:24:08.975695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pwd","metadata":{"execution":{"iopub.status.busy":"2021-12-14T02:02:47.706845Z","iopub.execute_input":"2021-12-14T02:02:47.707222Z","iopub.status.idle":"2021-12-14T02:02:47.712597Z","shell.execute_reply.started":"2021-12-14T02:02:47.707191Z","shell.execute_reply":"2021-12-14T02:02:47.711569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 scripts/train.py --img 1024 --batch 8 --epochs 10 --data Pneumonia.yaml --cfg model_resnet.yaml  --name wm2_resnet50_flexible_yolov5 --project yolov5-rsna   --single-cls #https://github.com/ultralytics/yolov5/releases/download/v5.0/yolov5l6.pt  ##\"../../input/d/sakib01/yolov5-rsna/last.pt\"","metadata":{"execution":{"iopub.status.busy":"2021-12-13T12:29:57.996123Z","iopub.execute_input":"2021-12-13T12:29:57.996518Z","iopub.status.idle":"2021-12-13T14:04:42.127404Z","shell.execute_reply.started":"2021-12-13T12:29:57.996474Z","shell.execute_reply":"2021-12-13T14:04:42.126359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python3 scripts/train.py --img 1024 --batch 8 --epochs 10 --data Pneumonia.yaml --cfg configs/model_densenet.yaml   --name wm2_densenet121_flexible_yolov5 --project yolov5-rsna   --single-cls #https://github.com/ultralytics/yolov5/releases/download/v5.0/yolov5l6.pt  ##\"../../input/d/sakib01/yolov5-rsna/last.pt\"","metadata":{"execution":{"iopub.status.busy":"2021-12-13T17:11:00.722031Z","iopub.execute_input":"2021-12-13T17:11:00.722413Z","iopub.status.idle":"2021-12-13T17:11:12.87201Z","shell.execute_reply.started":"2021-12-13T17:11:00.722382Z","shell.execute_reply":"2021-12-13T17:11:12.870726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd ..","metadata":{"execution":{"iopub.status.busy":"2021-12-14T06:19:48.69862Z","iopub.execute_input":"2021-12-14T06:19:48.698972Z","iopub.status.idle":"2021-12-14T06:19:48.705659Z","shell.execute_reply.started":"2021-12-14T06:19:48.698941Z","shell.execute_reply":"2021-12-14T06:19:48.704609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_tarfile(\"efficientnetb7_weights_15_epochs_14_12_21.tar\",'/kaggle/working/flexible-yolov5-main/yolov5-rsna/wm2_efficientnet_pretrained2/weights')","metadata":{"execution":{"iopub.status.busy":"2021-12-14T06:22:43.228115Z","iopub.execute_input":"2021-12-14T06:22:43.228455Z","iopub.status.idle":"2021-12-14T06:22:56.56974Z","shell.execute_reply.started":"2021-12-14T06:22:43.228424Z","shell.execute_reply":"2021-12-14T06:22:56.568535Z"},"trusted":true},"execution_count":null,"outputs":[]}]}