{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Let's get all the relevant libraries\n\n# Data Managment  \nimport numpy as np\nimport pandas as pd\nimport os\nimport cv2 \nimport json \nfrom glob import glob\nfrom PIL import Image\n\n# Dicom readers \nimport pydicom \nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\n# Plotting and Vizualization \nimport seaborn as sns \nimport matplotlib.pyplot as plt\n\n# Miscellaneous \nfrom tqdm.auto import tqdm\n\n#Torch \nimport torch ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-24T13:34:51.500505Z","iopub.execute_input":"2023-03-24T13:34:51.501876Z","iopub.status.idle":"2023-03-24T13:34:56.753078Z","shell.execute_reply.started":"2023-03-24T13:34:51.501740Z","shell.execute_reply":"2023-03-24T13:34:56.751778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 512\nBATCH_SIZE = 16\nEPOCHS = 80\n","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.754831Z","iopub.execute_input":"2023-03-24T13:34:56.755345Z","iopub.status.idle":"2023-03-24T13:34:56.761363Z","shell.execute_reply.started":"2023-03-24T13:34:56.755311Z","shell.execute_reply":"2023-03-24T13:34:56.760294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir('/kaggle/input/siim-covid19-detection')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.762910Z","iopub.execute_input":"2023-03-24T13:34:56.763879Z","iopub.status.idle":"2023-03-24T13:34:56.783477Z","shell.execute_reply.started":"2023-03-24T13:34:56.763843Z","shell.execute_reply":"2023-03-24T13:34:56.782629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = '/kaggle/input/siim-covid19-detection'","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.786810Z","iopub.execute_input":"2023-03-24T13:34:56.787748Z","iopub.status.idle":"2023-03-24T13:34:56.791913Z","shell.execute_reply.started":"2023-03-24T13:34:56.787714Z","shell.execute_reply":"2023-03-24T13:34:56.790864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_study_df = pd.read_csv(dataset + '/train_study_level.csv')\ntrain_study_df ","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.793579Z","iopub.execute_input":"2023-03-24T13:34:56.794159Z","iopub.status.idle":"2023-03-24T13:34:56.850506Z","shell.execute_reply.started":"2023-03-24T13:34:56.794122Z","shell.execute_reply":"2023-03-24T13:34:56.849597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df = pd.read_csv(dataset + '/train_image_level.csv')\ntrain_image_df","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.851781Z","iopub.execute_input":"2023-03-24T13:34:56.852090Z","iopub.status.idle":"2023-03-24T13:34:56.910367Z","shell.execute_reply.started":"2023-03-24T13:34:56.852059Z","shell.execute_reply":"2023-03-24T13:34:56.909245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"There are {} images with no bounding boxes in the dataset\"\n                      .format(train_image_df[\"boxes\"].isna().sum()))","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.912071Z","iopub.execute_input":"2023-03-24T13:34:56.912429Z","iopub.status.idle":"2023-03-24T13:34:56.923109Z","shell.execute_reply.started":"2023-03-24T13:34:56.912392Z","shell.execute_reply":"2023-03-24T13:34:56.921994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df[\"label\"]\n\n# Let's have a look at the labels: the opacity or none class\n# opacity means that the image contains a bouding box, no means that there is no such box. \n# Then, the last 4 numbers correspond to the coordinates of the box, in the following format: \n# xmin ymin xmax ymax \n# and if the class is non, the values are 0 0 1 1 ","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.924967Z","iopub.execute_input":"2023-03-24T13:34:56.925325Z","iopub.status.idle":"2023-03-24T13:34:56.937796Z","shell.execute_reply.started":"2023-03-24T13:34:56.925290Z","shell.execute_reply":"2023-03-24T13:34:56.936676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's get an idea of what is asked in the submission file\n\nsubmission_df = pd.read_csv(dataset + '/sample_submission.csv')\nprint(submission_df.shape)\nfor i in range(10): \n    print(submission_df.loc[i,:])\n    \n# We need to return, for each study in the test dataset, and Predicition String that include\n# the opaque or none label (or, disease or no disease) and if opaque, the values of all coordinates ","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.939355Z","iopub.execute_input":"2023-03-24T13:34:56.940106Z","iopub.status.idle":"2023-03-24T13:34:56.962637Z","shell.execute_reply.started":"2023-03-24T13:34:56.940072Z","shell.execute_reply":"2023-03-24T13:34:56.961678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The train_study file also fives use, for each study, which kind of Pneumonia is \n# associated with the patients.\n\n# Let's plot each subtypes \nsubtypes = train_study_df.groupby(['Negative for Pneumonia', 'Typical Appearance',\n       'Indeterminate Appearance', 'Atypical Appearance']).count().reset_index()\nsubtypes[\"label\"] = ['Atypical Appearance', 'Indeterminate Appearance',\n               'Typical Appearance', 'Negative for Pneumonia']\n\nax = plt.subplots(figsize=(21,10))\nax = sns.barplot(x=subtypes.label, y=subtypes.id, palette=\"deep\", orient='v')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:56.966489Z","iopub.execute_input":"2023-03-24T13:34:56.967129Z","iopub.status.idle":"2023-03-24T13:34:57.267267Z","shell.execute_reply.started":"2023-03-24T13:34:56.967088Z","shell.execute_reply":"2023-03-24T13:34:57.266474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's see the distribution between opacity and none \nclass_df = train_image_df[\"label\"].apply(lambda x: x.split(\" \")[0]).value_counts().reset_index()\nclass_df\nsns.barplot(x=class_df.label, y=[\"opacity\",\"none\"], palette=\"deep\", orient='h')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:57.268877Z","iopub.execute_input":"2023-03-24T13:34:57.269650Z","iopub.status.idle":"2023-03-24T13:34:57.459600Z","shell.execute_reply.started":"2023-03-24T13:34:57.269595Z","shell.execute_reply":"2023-03-24T13:34:57.458647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now let's create a column with the study_ids, to make life a bit easier \ntrain_study_df[\"study_id\"] = train_study_df[\"id\"].apply(lambda x: x.split(\"_\")[0])\ntrain_study_df","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:57.461227Z","iopub.execute_input":"2023-03-24T13:34:57.461581Z","iopub.status.idle":"2023-03-24T13:34:57.479407Z","shell.execute_reply.started":"2023-03-24T13:34:57.461546Z","shell.execute_reply":"2023-03-24T13:34:57.478399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's create a final train dataframe with all the information \ntrain = pd.merge(train_image_df, train_study_df, \n                 left_on=\"StudyInstanceUID\", right_on=\"study_id\")\ntrain.drop([ \"StudyInstanceUID\", \"id_y\"], axis=1, inplace=True)\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:57.481100Z","iopub.execute_input":"2023-03-24T13:34:57.481792Z","iopub.status.idle":"2023-03-24T13:34:57.511394Z","shell.execute_reply.started":"2023-03-24T13:34:57.481756Z","shell.execute_reply":"2023-03-24T13:34:57.510396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.sort_values('study_id')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:57.512423Z","iopub.execute_input":"2023-03-24T13:34:57.512715Z","iopub.status.idle":"2023-03-24T13:34:57.535272Z","shell.execute_reply.started":"2023-03-24T13:34:57.512679Z","shell.execute_reply":"2023-03-24T13:34:57.534402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.rename(columns={\"id_x\":\"id\"})","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:57.536695Z","iopub.execute_input":"2023-03-24T13:34:57.537054Z","iopub.status.idle":"2023-03-24T13:34:57.542185Z","shell.execute_reply.started":"2023-03-24T13:34:57.537019Z","shell.execute_reply":"2023-03-24T13:34:57.541127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a list of all the paths for all the images \ndicom_paths = glob(f'{dataset}/train/*/*/*.dcm')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:34:57.543745Z","iopub.execute_input":"2023-03-24T13:34:57.544337Z","iopub.status.idle":"2023-03-24T13:36:01.421593Z","shell.execute_reply.started":"2023-03-24T13:34:57.544303Z","shell.execute_reply":"2023-03-24T13:36:01.420375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv(dataset + '/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:01.423091Z","iopub.execute_input":"2023-03-24T13:36:01.423455Z","iopub.status.idle":"2023-03-24T13:36:01.432811Z","shell.execute_reply.started":"2023-03-24T13:36:01.423416Z","shell.execute_reply":"2023-03-24T13:36:01.431897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:01.434206Z","iopub.execute_input":"2023-03-24T13:36:01.434656Z","iopub.status.idle":"2023-03-24T13:36:01.449635Z","shell.execute_reply.started":"2023-03-24T13:36:01.434591Z","shell.execute_reply":"2023-03-24T13:36:01.448675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = glob(f'{dataset}/test/*/*/*.dcm')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:01.450942Z","iopub.execute_input":"2023-03-24T13:36:01.451347Z","iopub.status.idle":"2023-03-24T13:36:14.223428Z","shell.execute_reply.started":"2023-03-24T13:36:01.451313Z","shell.execute_reply":"2023-03-24T13:36:14.222410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dcm = pd.DataFrame({'dcm_path':test_path})\ntest_dcm['id']  = test_dcm.dcm_path.map(lambda x: x.split('/')[-1].replace('.dcm','_image'))\ntest_dcm","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.224939Z","iopub.execute_input":"2023-03-24T13:36:14.225350Z","iopub.status.idle":"2023-03-24T13:36:14.243566Z","shell.execute_reply.started":"2023-03-24T13:36:14.225309Z","shell.execute_reply":"2023-03-24T13:36:14.242567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get a Dataframe that includes the path \ndcm_df = pd.DataFrame({'dcm_path':dicom_paths})\ndcm_df['id'] = dcm_df.dcm_path.map(lambda x: x.split('/')[-1].replace('.dcm','_image'))\ndcm_df","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.244980Z","iopub.execute_input":"2023-03-24T13:36:14.245738Z","iopub.status.idle":"2023-03-24T13:36:14.264466Z","shell.execute_reply.started":"2023-03-24T13:36:14.245704Z","shell.execute_reply":"2023-03-24T13:36:14.263521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge both dataframe to have the paths in the train DataFrame \ntrain = train.merge(dcm_df, on='id', how='left')\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.265704Z","iopub.execute_input":"2023-03-24T13:36:14.266548Z","iopub.status.idle":"2023-03-24T13:36:14.293030Z","shell.execute_reply.started":"2023-03-24T13:36:14.266512Z","shell.execute_reply":"2023-03-24T13:36:14.292034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge both dataframe to have the paths in the train DataFrame \ntest = test_df.merge(test_dcm, on='id', how='left')\ntest","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.294306Z","iopub.execute_input":"2023-03-24T13:36:14.295147Z","iopub.status.idle":"2023-03-24T13:36:14.312419Z","shell.execute_reply.started":"2023-03-24T13:36:14.295112Z","shell.execute_reply":"2023-03-24T13:36:14.310849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.dropna()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.314344Z","iopub.execute_input":"2023-03-24T13:36:14.317367Z","iopub.status.idle":"2023-03-24T13:36:14.324010Z","shell.execute_reply.started":"2023-03-24T13:36:14.317336Z","shell.execute_reply":"2023-03-24T13:36:14.323178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.325403Z","iopub.execute_input":"2023-03-24T13:36:14.325887Z","iopub.status.idle":"2023-03-24T13:36:14.344048Z","shell.execute_reply.started":"2023-03-24T13:36:14.325846Z","shell.execute_reply":"2023-03-24T13:36:14.342771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dev = train[:5067]\ntrain_dev","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.345664Z","iopub.execute_input":"2023-03-24T13:36:14.346122Z","iopub.status.idle":"2023-03-24T13:36:14.367676Z","shell.execute_reply.started":"2023-03-24T13:36:14.346088Z","shell.execute_reply":"2023-03-24T13:36:14.366653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dev = train[5067:]\nvalid_dev","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.369251Z","iopub.execute_input":"2023-03-24T13:36:14.369573Z","iopub.status.idle":"2023-03-24T13:36:14.392585Z","shell.execute_reply.started":"2023-03-24T13:36:14.369540Z","shell.execute_reply":"2023-03-24T13:36:14.391671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The dicom to array function simply reads the dicom image, and returns a numpy array\n# Then, the plot_img and plot_imgs functions can plot one or several images\n\n\ndef dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    if voi_lut: \n        array = apply_voi_lut(dicom.pixel_array, dicom)\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        array = np.amax(array) - array\n    array = array - np.min(array)\n    array = array / np.max(array)\n    array = (array * 255).astype(np.uint8)\n    return array\n\ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n\n\ndef plot_imgs(imgs, cols=4, size=7, is_rgb=True, title='',cmap='gray', img_size=(512,512)):\n    rows = len(imgs)//cols + 1 \n    print(rows)\n    fig = plt.figure(figsize=(cols*size, rows*size))\n    for i, img in enumerate(imgs):\n        if img_size is not None: \n            img = cv2.resize(img, img_size)\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.401433Z","iopub.execute_input":"2023-03-24T13:36:14.401714Z","iopub.status.idle":"2023-03-24T13:36:14.411046Z","shell.execute_reply.started":"2023-03-24T13:36:14.401689Z","shell.execute_reply":"2023-03-24T13:36:14.410107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's look at one image \nimg = dicom2array(dicom_paths[20])\nplot_img(img)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:14.412149Z","iopub.execute_input":"2023-03-24T13:36:14.412647Z","iopub.status.idle":"2023-03-24T13:36:15.770410Z","shell.execute_reply.started":"2023-03-24T13:36:14.412589Z","shell.execute_reply":"2023-03-24T13:36:15.769429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's look at several images \n\nimgs = [dicom2array(path) for path in dicom_paths[:4]]\nplot_imgs(imgs)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:15.771465Z","iopub.execute_input":"2023-03-24T13:36:15.771835Z","iopub.status.idle":"2023-03-24T13:36:18.062117Z","shell.execute_reply.started":"2023-03-24T13:36:15.771801Z","shell.execute_reply":"2023-03-24T13:36:18.060677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's make some bounding boxes, to visualize the task \n# The function plot_bboxes_with_label takes as imput a label, n images, and plots\n# n number of images from the corresping label with the boxes associated \n\n# while I know that in this project, the positive classes for COVID should be green, and every\n# thing else yellow. \n# I will keep it that was for development sake, and we will see later on\n\n# Credits to:  https://www.kaggle.com/piantic/siim-fisabio-rsna-covid-19-detection-basic-eda\n\nfrom colorama import Fore, Back, Style\n\nlabel2color = {\n    '[1, 0, 0]': [255,0,0], # Typical Appearance\n    '[0, 1, 0]': [0,255,0], # Indeterminate Appearance\n    '[0, 0, 1]': [0,0,255], # Atypical Appearance\n    '[0, 0, 0]': None, # negative\n}\n\nclass_names = ['Typical Appearance', 'Indeterminate Appearance', 'Atypical Appearance']\n\ndef plot_bboxes_with_label(label_name, n): \n    print('Typical Appearance: ' + Fore.RED + 'Red',Style.RESET_ALL)\n    print('Indeterminate Appearance: '  + Fore.GREEN + 'Green',Style.RESET_ALL)\n    print('Atypical Appearance: ' + Fore.BLUE + 'Blue',Style.RESET_ALL)\n    \n    imgs = []\n    \n    thickness = 2 \n    scale = 5 \n    \n    if label_name == \"Negative for Pneumonia\": \n        flag = 0\n    else: \n        flag = 1\n    \n    for _, row in train[train[label_name]==flag].iloc[:n].iterrows():\n        # _ is the index, row is well, the row \n        study_id=row['study_id'] # get the study ids \n        img_path = glob(f'{dataset}/train/{study_id}/*/*')[0] # get all the path, \n        img = dicom2array(img_path)\n        img = cv2.resize(img, None, fx=1/scale, fy=1/scale)\n        img = np.stack([img, img, img], axis=-1)\n        \n        claz = row[class_names].values\n        color = label2color[str(claz.tolist())]\n\n        bboxes = []\n        bbox = []\n        \n        for i, l in enumerate(row['label'].split(' ')): \n            # i is index, l the label\n            if (i % 6 == 0) | (i % 6 == 1):\n                continue\n            bbox.append(float(l)/scale)\n            if i % 6 == 5: \n                bboxes.append(bbox)\n                bbox = []\n        for box in bboxes: \n            img = cv2.rectangle(\n                img,\n                (int(box[0]), int(box[1])),\n                (int(box[2]), int(box[3])),\n                color, thickness\n            )\n        img = cv2.resize(img, (512,512))\n        imgs.append(img)\n    \n    plot_imgs(imgs, cmap=None)\n    \n    del img, imgs, bbox, bboxes","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:18.063956Z","iopub.execute_input":"2023-03-24T13:36:18.064660Z","iopub.status.idle":"2023-03-24T13:36:18.080899Z","shell.execute_reply.started":"2023-03-24T13:36:18.064606Z","shell.execute_reply":"2023-03-24T13:36:18.080085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This cell will print several images with bounding box\n# You can change the label to print different images from differnt categories \nplot_bboxes_with_label(\"Negative for Pneumonia\", 4)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:18.082280Z","iopub.execute_input":"2023-03-24T13:36:18.083340Z","iopub.status.idle":"2023-03-24T13:36:20.473285Z","shell.execute_reply.started":"2023-03-24T13:36:18.083300Z","shell.execute_reply":"2023-03-24T13:36:20.472385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now let's clone the model and save the images in a different directories for future use ","metadata":{}},{"cell_type":"code","source":"os.makedirs('/kaggle/working/tmp/', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:20.474501Z","iopub.execute_input":"2023-03-24T13:36:20.475553Z","iopub.status.idle":"2023-03-24T13:36:20.481392Z","shell.execute_reply.started":"2023-03-24T13:36:20.475514Z","shell.execute_reply":"2023-03-24T13:36:20.480022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/tmp","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:20.483729Z","iopub.execute_input":"2023-03-24T13:36:20.484074Z","iopub.status.idle":"2023-03-24T13:36:20.495654Z","shell.execute_reply.started":"2023-03-24T13:36:20.484041Z","shell.execute_reply":"2023-03-24T13:36:20.494537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !git clone https://github.com/ultralytics/yolov5\n!git clone https://github.com/WongKinYiu/yolov7 ","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:20.497383Z","iopub.execute_input":"2023-03-24T13:36:20.497767Z","iopub.status.idle":"2023-03-24T13:36:25.682359Z","shell.execute_reply.started":"2023-03-24T13:36:20.497733Z","shell.execute_reply":"2023-03-24T13:36:25.681005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd yolov5\n%cd yolov7\n!pip install -r requirements.txt","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:25.687861Z","iopub.execute_input":"2023-03-24T13:36:25.688213Z","iopub.status.idle":"2023-03-24T13:36:39.012213Z","shell.execute_reply.started":"2023-03-24T13:36:25.688176Z","shell.execute_reply":"2023-03-24T13:36:39.011056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%ls","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:39.014361Z","iopub.execute_input":"2023-03-24T13:36:39.014798Z","iopub.status.idle":"2023-03-24T13:36:39.969652Z","shell.execute_reply.started":"2023-03-24T13:36:39.014754Z","shell.execute_reply":"2023-03-24T13:36:39.968449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('data/images/train', exist_ok=True)\nos.makedirs('data/images/valid', exist_ok=True)\n\nos.makedirs('data/labels/train', exist_ok=True)\nos.makedirs('data/labels/valid', exist_ok=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:39.972016Z","iopub.execute_input":"2023-03-24T13:36:39.972419Z","iopub.status.idle":"2023-03-24T13:36:39.979080Z","shell.execute_reply.started":"2023-03-24T13:36:39.972376Z","shell.execute_reply":"2023-03-24T13:36:39.977828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd data","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:39.980939Z","iopub.execute_input":"2023-03-24T13:36:39.981349Z","iopub.status.idle":"2023-03-24T13:36:39.990124Z","shell.execute_reply.started":"2023-03-24T13:36:39.981313Z","shell.execute_reply":"2023-03-24T13:36:39.988842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create .yaml file \nimport yaml\n\ndata_yaml = dict(\n#     train = '/kaggle/working/tmp/yolov5/data/images/train',\n#     val = '/kaggle/working/tmp/yolov5/data/images/valid',\n    train = '/kaggle/working/tmp/yolov7/data/images/train',\n    val = '/kaggle/working/tmp/yolov7/data/images/valid',\n    nc = 2,\n    names = ['none', 'opacity']\n)\n\n# Note that I am creating the file in the yolov5/data/ directory.\n# with open('/kaggle/working/tmp/yolov5/data/data.yaml', 'w') as outfile:\n#     yaml.dump(data_yaml, outfile, default_flow_style=True)\n\nwith open('/kaggle/working/tmp/yolov7/data/data.yaml', 'w') as outfile:\n    yaml.dump(data_yaml, outfile, default_flow_style=True)\n    \n# %cat /kaggle/working/tmp/yolov5/data/data.yaml\n%cat /kaggle/working/tmp/yolov7/data/data.yaml","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:39.991872Z","iopub.execute_input":"2023-03-24T13:36:39.992274Z","iopub.status.idle":"2023-03-24T13:36:40.963224Z","shell.execute_reply.started":"2023-03-24T13:36:39.992241Z","shell.execute_reply":"2023-03-24T13:36:40.961951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\ndim0 = []\ndim1 = []\ndef resize_and_save(end_path, df):\n    dim0 = []\n    dim1 = []\n    filenames = []\n    for index, row in tqdm(df[['study_id', 'dcm_path']].iterrows(), total = df.shape[0]):\n        try: \n            array = dicom2array(row['dcm_path'])\n            dim0.append(array.shape[0])\n            dim1.append(array.shape[1])\n            img = cv2.resize(array, (IMG_SIZE,IMG_SIZE))\n            img = Image.fromarray(img)\n   \n            filename = row['dcm_path'].split('/')[-1].split('.')[0]\n            filenames.append(filename)\n            img.save(os.path.join(end_path, f'{filename}.png'))\n        except RuntimeError:\n            pass\n    return pd.DataFrame({'dim0':dim0, 'dim1': dim1, 'id': filenames})\n        #return filename.replace('dcm','') + '_image', array.shape[0], array.shape[1]","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:40.965093Z","iopub.execute_input":"2023-03-24T13:36:40.965933Z","iopub.status.idle":"2023-03-24T13:36:40.978117Z","shell.execute_reply.started":"2023-03-24T13:36:40.965881Z","shell.execute_reply":"2023-03-24T13:36:40.977011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's save the image in a new file for training \n\n# dims_train = resize_and_save('/kaggle/working/tmp/yolov5/data/images/train/', train_dev)\n# dims_valid = resize_and_save('/kaggle/working/tmp/yolov5/data/images/valid/', valid_dev)\ndims_train = resize_and_save('/kaggle/working/tmp/yolov7/data/images/train/', train_dev)\ndims_valid = resize_and_save('/kaggle/working/tmp/yolov7/data/images/valid/', valid_dev)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T13:36:40.981877Z","iopub.execute_input":"2023-03-24T13:36:40.982167Z","iopub.status.idle":"2023-03-24T14:13:45.435652Z","shell.execute_reply.started":"2023-03-24T13:36:40.982141Z","shell.execute_reply":"2023-03-24T14:13:45.434681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Let's change the train dataframe to include the name with png\ntrain['id'] = train['id'].apply(lambda x: x.replace('_image','.png'))\n\ntrain_dev['id'] = train_dev['id'].apply(lambda x: x.replace('_image','.png'))\n\nvalid_dev['id'] = valid_dev['id'].apply(lambda x: x.replace('_image','.png'))\nvalid_dev","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:45.437203Z","iopub.execute_input":"2023-03-24T14:13:45.437685Z","iopub.status.idle":"2023-03-24T14:13:45.468292Z","shell.execute_reply.started":"2023-03-24T14:13:45.437633Z","shell.execute_reply":"2023-03-24T14:13:45.467202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dims_valid['id'] = dims_valid['id'].astype(str) + '.png'\ndims_train['id'] = dims_train['id'].astype(str) + '.png'\ndims_train","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:45.469903Z","iopub.execute_input":"2023-03-24T14:13:45.470597Z","iopub.status.idle":"2023-03-24T14:13:45.493886Z","shell.execute_reply.started":"2023-03-24T14:13:45.470558Z","shell.execute_reply":"2023-03-24T14:13:45.492679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dev = train_dev.merge(dims_train, on='id', how='left')\nvalid_dev = valid_dev.merge(dims_valid, on='id', how='left')","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:45.495523Z","iopub.execute_input":"2023-03-24T14:13:45.495916Z","iopub.status.idle":"2023-03-24T14:13:45.515999Z","shell.execute_reply.started":"2023-03-24T14:13:45.495880Z","shell.execute_reply":"2023-03-24T14:13:45.514890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dev","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:45.517438Z","iopub.execute_input":"2023-03-24T14:13:45.517798Z","iopub.status.idle":"2023-03-24T14:13:45.541031Z","shell.execute_reply.started":"2023-03-24T14:13:45.517763Z","shell.execute_reply":"2023-03-24T14:13:45.539950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the raw bounding box by parsing the row value of the label column.\n# Ref: https://www.kaggle.com/yujiariyasu/plot-3positive-classes\ndef get_bbox(row):\n    bboxes = []\n    bbox = []\n    for i, l in enumerate(row.label.split(' ')):\n        if (i % 6 == 0) | (i % 6 == 1):\n            continue\n        bbox.append(float(l))\n        if i % 6 == 5:\n            bboxes.append(bbox)\n            bbox = []  \n            \n    return bboxes\n\n# Scale the bounding boxes according to the size of the resized image. \ndef scale_bbox(row, bboxes):\n    # Get scaling factor\n    scale_x = IMG_SIZE/row.dim1\n    scale_y = IMG_SIZE/row.dim0\n    \n    scaled_bboxes = []\n    for bbox in bboxes:\n        x = int(np.round(bbox[0]*scale_x, 4))\n        y = int(np.round(bbox[1]*scale_y, 4))\n        x1 = int(np.round(bbox[2]*(scale_x), 4))\n        y1= int(np.round(bbox[3]*scale_y, 4))\n\n        scaled_bboxes.append([x, y, x1, y1]) # xmin, ymin, xmax, ymax\n        \n    return scaled_bboxes\n\n# Convert the bounding boxes in YOLO format.\ndef get_yolo_format_bbox(img_w, img_h, bboxes):\n    yolo_boxes = []\n    for bbox in bboxes:\n        w = bbox[2] - bbox[0] # xmax - xmin\n        h = bbox[3] - bbox[1] # ymax - ymin\n        xc = bbox[0] + int(np.round(w/2)) # xmin + width/2\n        yc = bbox[1] + int(np.round(h/2)) # ymin + height/2\n        \n        yolo_boxes.append([xc/img_w, yc/img_h, w/img_w, h/img_h]) # x_center y_center width height\n    \n    return yolo_boxes","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:45.542562Z","iopub.execute_input":"2023-03-24T14:13:45.543006Z","iopub.status.idle":"2023-03-24T14:13:45.554235Z","shell.execute_reply.started":"2023-03-24T14:13:45.542971Z","shell.execute_reply":"2023-03-24T14:13:45.552761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dev['image_level'] = train_dev.apply(lambda x: x.label.split(' ')[0], axis=1)\ntrain_dev['id'] = train_dev['id'].apply(lambda x: x.replace('.png', '.txt'))\n\nvalid_dev['image_level'] = valid_dev.apply(lambda x: x.label.split(' ')[0], axis=1)\nvalid_dev['id'] = valid_dev['id'].apply(lambda x: x.replace('.png', '.txt'))","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:45.556177Z","iopub.execute_input":"2023-03-24T14:13:45.556855Z","iopub.status.idle":"2023-03-24T14:13:45.674538Z","shell.execute_reply.started":"2023-03-24T14:13:45.556820Z","shell.execute_reply":"2023-03-24T14:13:45.673643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare the txt files for bounding box\n\n\nfor i in tqdm(range(len(train_dev))):\n    row = train_dev.loc[i]\n    # Get image id\n    img_id = row.id\n    # Get image-level label\n    label = row.image_level\n    \n\n#     file_name = f'/kaggle/working/tmp/yolov5/data/labels/train/{row.id}'\n    file_name = f'/kaggle/working/tmp/yolov7/data/labels/train/{row.id}'\n        \n    try: \n        if label=='opacity':\n            # Get bboxes\n            bboxes = get_bbox(row)\n            # Scale bounding boxes\n            scale_bboxes = scale_bbox(row, bboxes)\n            # Format for YOLOv5\n            yolo_bboxes = get_yolo_format_bbox(IMG_SIZE, IMG_SIZE, scale_bboxes)\n        \n        \n            with open(file_name, 'w') as f:\n                for bbox in yolo_bboxes:\n                    \n                    bbox = [1]+bbox\n                    bbox = [str(i) for i in bbox]\n                    bbox = ' '.join(bbox)\n                    f.write(bbox)\n                    f.write('\\n')\n    except ValueError: \n        pass","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:45.675860Z","iopub.execute_input":"2023-03-24T14:13:45.676218Z","iopub.status.idle":"2023-03-24T14:13:47.712776Z","shell.execute_reply.started":"2023-03-24T14:13:45.676181Z","shell.execute_reply":"2023-03-24T14:13:47.711579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(len(valid_dev))):\n    row = valid_dev.loc[i]\n    # Get image id\n    img_id = row.id\n    # Get image-level label\n    label = row.image_level\n    \n\n#     file_name = f'/kaggle/working/tmp/yolov5/data/labels/valid/{row.id}'\n    file_name = f'/kaggle/working/tmp/yolov7/data/labels/valid/{row.id}'\n        \n    try: \n        if label=='opacity':\n            # Get bboxes\n            bboxes = get_bbox(row)\n            # Scale bounding boxes\n            scale_bboxes = scale_bbox(row, bboxes)\n            # Format for YOLOv5\n            yolo_bboxes = get_yolo_format_bbox(IMG_SIZE, IMG_SIZE, scale_bboxes)\n        \n        \n            with open(file_name, 'w') as f:\n                for bbox in yolo_bboxes:\n                    \n                    bbox = [1]+bbox\n                    bbox = [str(i) for i in bbox]\n                    bbox = ' '.join(bbox)\n                    f.write(bbox)\n                    f.write('\\n')\n    except ValueError: \n        pass","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:47.714248Z","iopub.execute_input":"2023-03-24T14:13:47.714731Z","iopub.status.idle":"2023-03-24T14:13:48.266564Z","shell.execute_reply.started":"2023-03-24T14:13:47.714693Z","shell.execute_reply":"2023-03-24T14:13:48.265508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/working/tmp/yolov5/data/labels/train\n%cd /kaggle/working/tmp/yolov7/data/labels/train\n%ls","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:48.268274Z","iopub.execute_input":"2023-03-24T14:13:48.268658Z","iopub.status.idle":"2023-03-24T14:13:49.287352Z","shell.execute_reply.started":"2023-03-24T14:13:48.268607Z","shell.execute_reply":"2023-03-24T14:13:49.286252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's verify that this is what we want \n\nf = open('000a312787f2.txt', 'r')\ncontent = f.read()\nf.close\nprint(content)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:49.289198Z","iopub.execute_input":"2023-03-24T14:13:49.291148Z","iopub.status.idle":"2023-03-24T14:13:49.298274Z","shell.execute_reply.started":"2023-03-24T14:13:49.291104Z","shell.execute_reply":"2023-03-24T14:13:49.297066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install W&B, login into your account and paste the API key \n\n# A note here: You can create and wandb account and login by uncommenting the last line of this \n# cell. This will save the run on your account, and allow you to vizualise the results very\n# easily, and give you access to several valuable options and tools \n# !pip install -q --upgrade wandb\n# Login \n# import wandb\n#wandb.login()","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:49.299862Z","iopub.execute_input":"2023-03-24T14:13:49.300174Z","iopub.status.idle":"2023-03-24T14:13:49.311504Z","shell.execute_reply.started":"2023-03-24T14:13:49.300148Z","shell.execute_reply":"2023-03-24T14:13:49.310698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nTODO\nDownload the parameters you want to use using the wget command.\nSpecify it in the --weight of the cell below.\n↓this example will help you\nhttps://www.kaggle.com/code/its7171/yolov7-finetune-with-soccernet\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:49.314530Z","iopub.execute_input":"2023-03-24T14:13:49.314824Z","iopub.status.idle":"2023-03-24T14:13:49.327250Z","shell.execute_reply.started":"2023-03-24T14:13:49.314799Z","shell.execute_reply":"2023-03-24T14:13:49.326313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-w6.pt","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:49.328502Z","iopub.execute_input":"2023-03-24T14:13:49.328851Z","iopub.status.idle":"2023-03-24T14:13:51.973219Z","shell.execute_reply.started":"2023-03-24T14:13:49.328815Z","shell.execute_reply":"2023-03-24T14:13:51.971909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:51.977194Z","iopub.execute_input":"2023-03-24T14:13:51.977550Z","iopub.status.idle":"2023-03-24T14:13:53.047954Z","shell.execute_reply.started":"2023-03-24T14:13:51.977515Z","shell.execute_reply":"2023-03-24T14:13:53.046678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pwd","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:53.049730Z","iopub.execute_input":"2023-03-24T14:13:53.050120Z","iopub.status.idle":"2023-03-24T14:13:54.004308Z","shell.execute_reply.started":"2023-03-24T14:13:53.050081Z","shell.execute_reply":"2023-03-24T14:13:54.002891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/tmp/yolov7","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:54.008310Z","iopub.execute_input":"2023-03-24T14:13:54.008655Z","iopub.status.idle":"2023-03-24T14:13:54.951317Z","shell.execute_reply.started":"2023-03-24T14:13:54.008599Z","shell.execute_reply":"2023-03-24T14:13:54.950111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip3 uninstall clearml\n!echo y | pip3 uninstall wandb","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:13:54.953344Z","iopub.execute_input":"2023-03-24T14:13:54.954096Z","iopub.status.idle":"2023-03-24T14:14:03.050441Z","shell.execute_reply.started":"2023-03-24T14:13:54.954054Z","shell.execute_reply":"2023-03-24T14:14:03.048592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### If you are running the model while being logged in a wandb account, remove the \n# calling \"WANDB_MODE=\"dryrun\" \n# %cd /kaggle/working/tmp/yolov5\n%cd /kaggle/working/tmp/yolov7\n\n#YOLOv7, YOLOV7-X\n# !WANDB_MODE=\"dryrun\" python train.py --img {IMG_SIZE} \\\n#                  --batch {BATCH_SIZE} \\\n#                  --epochs {EPOCHS} \\\n#                  --data data.yaml \\\n#                  --weights yolov5s.pt \\\n#                 # --save_period 1\\\n#                  --project kaggle-siim-covid\n\n# other YOLOv7\n# !WANDB_MODE=\"dryrun\" python train_aux.py --img {IMG_SIZE} \\\n#                  --batch {BATCH_SIZE} \\\n#                  --epochs {EPOCHS} \\\n#                  --data data.yaml \\\n#                  --weights /kaggle/working/tmp/yolov7/data/labels/train/yolov7-w6.pt \\\n#                  --project kaggle-siim-covid\n\n!python train_aux.py --img {IMG_SIZE} \\\n                 --batch {BATCH_SIZE} \\\n                 --epochs {EPOCHS} \\\n                 --data data.yaml \\\n                 --weights /kaggle/working/tmp/yolov7/data/labels/train/yolov7-w6.pt \\\n                 --project kaggle-siim-covid \\\n                 --cfg cfg/training/yolov7-w6.yaml \\\n                 --hyp data/hyp.scratch.p6.yaml","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:03.052210Z","iopub.execute_input":"2023-03-24T14:14:03.053077Z","iopub.status.idle":"2023-03-24T14:14:58.000401Z","shell.execute_reply.started":"2023-03-24T14:14:03.053026Z","shell.execute_reply":"2023-03-24T14:14:57.998732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/tmp/yolov7/kaggle-siim-covid/exp","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:58.008120Z","iopub.execute_input":"2023-03-24T14:14:58.008679Z","iopub.status.idle":"2023-03-24T14:14:59.178124Z","shell.execute_reply.started":"2023-03-24T14:14:58.008604Z","shell.execute_reply":"2023-03-24T14:14:59.176920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(30,15))\nplt.axis('off')\n# plt.imshow(plt.imread('/kaggle/working/tmp/yolov5/runs/train/exp/confusion_matrix.png'));\nplt.imshow(plt.imread('/kaggle/working/tmp/yolov7/kaggle-siim-covid/exp/confusion_matrix.png'));","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.180351Z","iopub.execute_input":"2023-03-24T14:14:59.180785Z","iopub.status.idle":"2023-03-24T14:14:59.504882Z","shell.execute_reply.started":"2023-03-24T14:14:59.180743Z","shell.execute_reply":"2023-03-24T14:14:59.503628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/working/tmp/yolov5/runs/train/exp\n%cd /kaggle/working/tmp/yolov7/kaggle-siim-covid/exp\n%ls","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.506040Z","iopub.status.idle":"2023-03-24T14:14:59.507914Z","shell.execute_reply.started":"2023-03-24T14:14:59.507646Z","shell.execute_reply":"2023-03-24T14:14:59.507674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This shows a batch of the validation data with the corresponding label \nplt.figure(figsize=(15,15))\n# plt.imshow(plt.imread('val_batch0_labels.jpg'))\nplt.imshow(plt.imread('test_batch0_labels.jpg'))","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.509138Z","iopub.status.idle":"2023-03-24T14:14:59.509814Z","shell.execute_reply.started":"2023-03-24T14:14:59.509543Z","shell.execute_reply":"2023-03-24T14:14:59.509568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(plt.imread('R_curve.png'))","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.511443Z","iopub.status.idle":"2023-03-24T14:14:59.512545Z","shell.execute_reply.started":"2023-03-24T14:14:59.512254Z","shell.execute_reply":"2023-03-24T14:14:59.512282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(plt.imread('P_curve.png'))","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.514055Z","iopub.status.idle":"2023-03-24T14:14:59.514945Z","shell.execute_reply.started":"2023-03-24T14:14:59.514683Z","shell.execute_reply":"2023-03-24T14:14:59.514708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This prints all the results curves \nplt.figure(figsize=(20,30))\nplt.imshow(plt.imread('results.png'))","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.516389Z","iopub.status.idle":"2023-03-24T14:14:59.522919Z","shell.execute_reply.started":"2023-03-24T14:14:59.522715Z","shell.execute_reply":"2023-03-24T14:14:59.522735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The weights are stored here, and could be used for inference \n# %cd /kaggle/working/tmp/yolov5/kaggle-siim-covid/exp/weights\n%cd /kaggle/working/tmp/yolov7/kaggle-siim-covid/exp/weights\n%ls","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.524251Z","iopub.status.idle":"2023-03-24T14:14:59.524857Z","shell.execute_reply.started":"2023-03-24T14:14:59.524569Z","shell.execute_reply":"2023-03-24T14:14:59.524594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# weights = '/kaggle/working/tmp/yolov5/kaggle-siim-covid/exp/weights/best.pt'\nweights = '/kaggle/working/tmp/yolov7/kaggle-siim-covid/exp/weights/best.pt'","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.526696Z","iopub.status.idle":"2023-03-24T14:14:59.527372Z","shell.execute_reply.started":"2023-03-24T14:14:59.527104Z","shell.execute_reply":"2023-03-24T14:14:59.527129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/working/tmp/yolov5/data/images\n%cd /kaggle/working/tmp/yolov7/data/images","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.529121Z","iopub.status.idle":"2023-03-24T14:14:59.529730Z","shell.execute_reply.started":"2023-03-24T14:14:59.529434Z","shell.execute_reply":"2023-03-24T14:14:59.529459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('test', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.531531Z","iopub.status.idle":"2023-03-24T14:14:59.532135Z","shell.execute_reply.started":"2023-03-24T14:14:59.531871Z","shell.execute_reply":"2023-03-24T14:14:59.531896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_test(end_path, df):\n\n    filenames = []\n    for index, row in tqdm(df[['id', 'dcm_path']].iterrows(), total = df.shape[0]):\n        try: \n            array = dicom2array(row['dcm_path'])\n            img = cv2.resize(array, (IMG_SIZE,IMG_SIZE))\n            img = Image.fromarray(img)\n   \n            filename = row['dcm_path'].split('/')[-1].split('.')[0]\n            filenames.append(filename)\n            img.save(os.path.join(end_path, f'{filename}.png'))\n        except RuntimeError:\n            pass\n        #return filename.replace('dcm','') + '_image', array.shape[0], array.shape[1]","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.534044Z","iopub.status.idle":"2023-03-24T14:14:59.534650Z","shell.execute_reply.started":"2023-03-24T14:14:59.534356Z","shell.execute_reply":"2023-03-24T14:14:59.534382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We save the test images in a new folder; not all the images are necessary, you can make this smaller by cutting the dataframe\nsave_test('test', test)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.536676Z","iopub.status.idle":"2023-03-24T14:14:59.537254Z","shell.execute_reply.started":"2023-03-24T14:14:59.536986Z","shell.execute_reply":"2023-03-24T14:14:59.537011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/working/tmp/yolov5\n%cd /kaggle/working/tmp/yolov7","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.539177Z","iopub.status.idle":"2023-03-24T14:14:59.540028Z","shell.execute_reply.started":"2023-03-24T14:14:59.539744Z","shell.execute_reply":"2023-03-24T14:14:59.539770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This makes all the necessary predicitions \n\n# !python detect.py --weights /kaggle/working/tmp/yolov5/kaggle-siim-covid/exp/weights/best.pt /kaggle/working/tmp/yolov5/kaggle-siim-covid/exp/weights/last.pt --img 512 --source data/images/test\n# !python detect.py --weights /kaggle/working/tmp/yolov7/kaggle-siim-covid/exp/weights/best.pt /kaggle/working/tmp/yolov7/kaggle-siim-covid/exp/weights/last.pt --img 512 --source data/images/test\n!python detect.py --weights /kaggle/working/tmp/yolov7/kaggle-siim-covid/exp/weights/best.pt --img 512 --source data/images/test","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.541566Z","iopub.status.idle":"2023-03-24T14:14:59.542442Z","shell.execute_reply.started":"2023-03-24T14:14:59.542170Z","shell.execute_reply":"2023-03-24T14:14:59.542197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/working/tmp/yolov5/runs/detect/\n%cd /kaggle/working/tmp/yolov7/runs/detect/\n%ls","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.543943Z","iopub.status.idle":"2023-03-24T14:14:59.544805Z","shell.execute_reply.started":"2023-03-24T14:14:59.544511Z","shell.execute_reply":"2023-03-24T14:14:59.544537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directory = os.listdir('exp')\nprint(directory)\nplt.figure(figsize=(15,15))\nfor i, file in enumerate((directory)[0:5]):\n    img = plt.imread('exp/' + file)\n#     img = plt.imread(file)\n    plt.subplot(3, 3, i+1)\n    plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2023-03-24T14:14:59.546356Z","iopub.status.idle":"2023-03-24T14:14:59.547458Z","shell.execute_reply.started":"2023-03-24T14:14:59.547189Z","shell.execute_reply":"2023-03-24T14:14:59.547216Z"},"trusted":true},"execution_count":null,"outputs":[]}]}