{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-13T14:59:55.192871Z","iopub.execute_input":"2021-07-13T14:59:55.193214Z","iopub.status.idle":"2021-07-13T15:01:04.410573Z","shell.execute_reply.started":"2021-07-13T14:59:55.193109Z","shell.execute_reply":"2021-07-13T15:01:04.409585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\nimport torch\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")\n\nimport os\nimport gc\nimport cv2\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm import tqdm\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:04.414065Z","iopub.execute_input":"2021-07-13T15:01:04.414348Z","iopub.status.idle":"2021-07-13T15:01:11.22657Z","shell.execute_reply.started":"2021-07-13T15:01:04.414316Z","shell.execute_reply":"2021-07-13T15:01:11.224683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n  try:\n    # Currently, memory growth needs to be the same across GPUs\n    for gpu in gpus:\n      tf.config.experimental.set_memory_growth(gpu, True)\n    logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n    print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n  except RuntimeError as e:\n    # Memory growth must be set before GPUs have been initialized\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:11.228633Z","iopub.execute_input":"2021-07-13T15:01:11.229266Z","iopub.status.idle":"2021-07-13T15:01:17.208547Z","shell.execute_reply.started":"2021-07-13T15:01:11.229223Z","shell.execute_reply":"2021-07-13T15:01:17.207568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = pd.read_csv('/kaggle/input/siim-covid19-detection/sample_submission.csv')\nprint(len(sub_df))\nsub_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:17.210349Z","iopub.execute_input":"2021-07-13T15:01:17.210697Z","iopub.status.idle":"2021-07-13T15:01:17.245579Z","shell.execute_reply.started":"2021-07-13T15:01:17.210658Z","shell.execute_reply":"2021-07-13T15:01:17.244659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = sub_df.loc[sub_df.id.str.contains('_study')]\nlen(study_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:17.246892Z","iopub.execute_input":"2021-07-13T15:01:17.247447Z","iopub.status.idle":"2021-07-13T15:01:17.258048Z","shell.execute_reply.started":"2021-07-13T15:01:17.247396Z","shell.execute_reply":"2021-07-13T15:01:17.257271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df = sub_df.loc[sub_df.id.str.contains('_image')]\nlen(image_df)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:17.259563Z","iopub.execute_input":"2021-07-13T15:01:17.259996Z","iopub.status.idle":"2021-07-13T15:01:17.269886Z","shell.execute_reply.started":"2021-07-13T15:01:17.259957Z","shell.execute_reply":"2021-07-13T15:01:17.268751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef resize_xray(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:17.271457Z","iopub.execute_input":"2021-07-13T15:01:17.271982Z","iopub.status.idle":"2021-07-13T15:01:17.281534Z","shell.execute_reply.started":"2021-07-13T15:01:17.271943Z","shell.execute_reply":"2021-07-13T15:01:17.280651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = f'/kaggle/tmp/test/'\nIMG_SIZE = 512\n\ndef prepare_test_images():\n    image_id = []\n    dim0 = []\n    dim1 = []\n\n    os.makedirs(TEST_PATH, exist_ok=True)\n\n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/test')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(os.path.join(dirname, file))\n            im = resize_xray(xray, IMG_SIZE)\n            im = np.array(im)\n            equ = cv2.equalizeHist(im)\n            clahe = cv2.createCLAHE(clipLimit=40.0, tileGridSize=(8,8))\n            clh = clahe.apply(im)\n            kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15, 15))\n            tophat = cv2.morphologyEx(equ, cv2.MORPH_TOPHAT, kernel)\n            bothat = cv2.morphologyEx(equ, cv2.MORPH_BLACKHAT, kernel)\n            morph = equ + tophat - bothat\n            output = np.dstack((im, clh, morph))\n            cv2.imwrite(os.path.join(TEST_PATH, file.replace('dcm', 'jpg')),output)\n\n            image_id.append(file.replace('.dcm', ''))\n            dim0.append(xray.shape[0])\n            dim1.append(xray.shape[1])\n\n    return image_id, dim0, dim1","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:17.284839Z","iopub.execute_input":"2021-07-13T15:01:17.28542Z","iopub.status.idle":"2021-07-13T15:01:17.293817Z","shell.execute_reply.started":"2021-07-13T15:01:17.28536Z","shell.execute_reply":"2021-07-13T15:01:17.2929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids, dim0, dim1 = prepare_test_images()\nprint(f'Number of test images: {len(os.listdir(TEST_PATH))}')","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:01:17.296316Z","iopub.execute_input":"2021-07-13T15:01:17.296827Z","iopub.status.idle":"2021-07-13T15:10:53.825019Z","shell.execute_reply.started":"2021-07-13T15:01:17.296788Z","shell.execute_reply":"2021-07-13T15:10:53.823514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df = pd.DataFrame.from_dict({'image_id': image_ids, 'dim0': dim0, 'dim1': dim1})\n\n# Associate image-level id with study-level ids.\n# Note that a study-level might have more than one image-level ids.\nfor study_dir in os.listdir('../input/siim-covid19-detection/test'):\n    for series in os.listdir(f'../input/siim-covid19-detection/test/{study_dir}'):\n        for image in os.listdir(f'../input/siim-covid19-detection/test/{study_dir}/{series}/'):\n            image_id = image[:-4]\n            meta_df.loc[meta_df['image_id'] == image_id, 'study_id'] = study_dir\n        \nmeta_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:10:53.827754Z","iopub.execute_input":"2021-07-13T15:10:53.828991Z","iopub.status.idle":"2021-07-13T15:10:56.432283Z","shell.execute_reply.started":"2021-07-13T15:10:53.828952Z","shell.execute_reply":"2021-07-13T15:10:56.431135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"YOLO_MODEL_PATH0 = '/kaggle/input/yolomodels1607/yolov5L512_1607.pt'\nYOLO_MODEL_PATH1 = '/kaggle/input/yolov5m6-512/yolov5m6_512.pt'\nYOLO_MODEL_PATH2 = '/kaggle/input/yolov5bestlarge1607/yolov5bestlarge.pt'\nYOLO_MODEL_PATH3 = ''\nYOLO_MODEL_PATH4 = ''\n","metadata":{"execution":{"iopub.status.busy":"2021-07-16T10:32:50.643678Z","iopub.execute_input":"2021-07-16T10:32:50.644006Z","iopub.status.idle":"2021-07-16T10:32:50.651855Z","shell.execute_reply.started":"2021-07-16T10:32:50.643932Z","shell.execute_reply":"2021-07-16T10:32:50.650894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n!python /kaggle/input/siimcovidyolov5l/yolov5/detect.py --weights /kaggle/input/yolov5l-3channels/yolov5l_fold0.pt /kaggle/input/yolov5l-3channels/yolov5l_fold1.pt \\\n                                      --source {TEST_PATH} \\\n                                      --img {IMG_SIZE} \\\n                                      --conf 0.22 \\\n                                      --iou-thres 0.5 \\\n                                      --max-det 10 \\\n                                      --save-txt \\\n                                      --save-conf","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:10:56.443786Z","iopub.execute_input":"2021-07-13T15:10:56.445341Z","iopub.status.idle":"2021-07-13T15:12:53.7835Z","shell.execute_reply.started":"2021-07-13T15:10:56.445302Z","shell.execute_reply":"2021-07-13T15:12:53.782512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PRED_PATH = 'runs/detect/exp/labels'\nprediction_files = os.listdir(PRED_PATH)\nprint(f'Number of opacity predicted by YOLOv5: {len(prediction_files)}')","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:53.785829Z","iopub.execute_input":"2021-07-13T15:12:53.786093Z","iopub.status.idle":"2021-07-13T15:12:53.798272Z","shell.execute_reply.started":"2021-07-13T15:12:53.786065Z","shell.execute_reply":"2021-07-13T15:12:53.797006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_bbox_format(bboxes):\n    correct_bboxes = []\n    for b in bboxes:\n        xc, yc = int(np.round(b[0]*IMG_SIZE)), int(np.round(b[1]*IMG_SIZE))\n        w, h = int(np.round(b[2]*IMG_SIZE)), int(np.round(b[3]*IMG_SIZE))\n\n        xmin = xc - int(np.round(w/2))\n        ymin = yc - int(np.round(h/2))\n        xmax = xc + int(np.round(w/2))\n        ymax = yc + int(np.round(h/2))\n        \n        correct_bboxes.append([xmin, ymin, xmax, ymax])\n        \n    return correct_bboxes\n\ndef scale_bboxes_to_original(row, bboxes):\n    # Get scaling factor\n    scale_x = IMG_SIZE/row.dim1\n    scale_y = IMG_SIZE/row.dim0\n    \n    scaled_bboxes = []\n    for bbox in bboxes:\n        xmin, ymin, xmax, ymax = bbox\n        \n        xmin = int(np.round(xmin/scale_x))\n        ymin = int(np.round(ymin/scale_y))\n        xmax = int(np.round(xmax/scale_x))\n        ymax = int(np.round(ymax/scale_y))\n        \n        scaled_bboxes.append([xmin, ymin, xmax, ymax])\n        \n    return scaled_bboxes\n\n# Read the txt file generated by YOLOv5 during inference and extract \n# confidence and bounding box coordinates.\ndef get_conf_bboxes(file_path):\n    confidence = []\n    bboxes = []\n    with open(file_path, 'r') as file:\n        for line in file:\n            preds = line.strip('\\n').split(' ')\n            preds = list(map(float, preds))\n            confidence.append(preds[-1])\n            bboxes.append(preds[1:-1])\n    return confidence, bboxes","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:53.799381Z","iopub.execute_input":"2021-07-13T15:12:53.799848Z","iopub.status.idle":"2021-07-13T15:12:53.816238Z","shell.execute_reply.started":"2021-07-13T15:12:53.799812Z","shell.execute_reply":"2021-07-13T15:12:53.815201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_pred_strings = []\nfor i in tqdm(range(len(image_df))):\n    row = meta_df.loc[i]\n    id_name = row.image_id\n    \n    if f'{id_name}.txt' in prediction_files:\n        # opacity label\n        confidence, bboxes = get_conf_bboxes(f'{PRED_PATH}/{id_name}.txt')\n        bboxes = correct_bbox_format(bboxes)\n        ori_bboxes = scale_bboxes_to_original(row, bboxes)\n        \n        pred_string = ''\n        for j, conf in enumerate(confidence):\n            pred_string += f'opacity {conf} ' + ' '.join(map(str, ori_bboxes[j])) + ' '\n        image_pred_strings.append(pred_string[:-1]) \n    else:\n        image_pred_strings.append(\"none 1 0 0 1 1\")","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:53.819341Z","iopub.execute_input":"2021-07-13T15:12:53.819892Z","iopub.status.idle":"2021-07-13T15:12:54.471092Z","shell.execute_reply.started":"2021-07-13T15:12:53.819856Z","shell.execute_reply":"2021-07-13T15:12:54.470273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df['PredictionString'] = image_pred_strings\nimage_df = meta_df[['study_id','image_id', 'PredictionString']]\n# image_df.insert(0, 'id', image_df.apply(lambda row: row.image_id+'_image', axis=1))\n# image_df = image_df.drop('image_id', axis=1)\nimage_df.head(20)\nimage_df.to_csv('object_yolo5x512_080721.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:54.473473Z","iopub.execute_input":"2021-07-13T15:12:54.47481Z","iopub.status.idle":"2021-07-13T15:12:54.850169Z","shell.execute_reply.started":"2021-07-13T15:12:54.474771Z","shell.execute_reply":"2021-07-13T15:12:54.84937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink(r'object_yolo5x512_080721.csv')\nimage_df","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:54.852807Z","iopub.execute_input":"2021-07-13T15:12:54.854621Z","iopub.status.idle":"2021-07-13T15:12:54.874599Z","shell.execute_reply.started":"2021-07-13T15:12:54.85458Z","shell.execute_reply":"2021-07-13T15:12:54.873732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imageDict = {}\n# for study in os.listdir('../input/siim-covid19-detection/test'):\n#     for _,__,files in os.walk('../input/siim-covid19-detection/test/'+study):\n#         for file in files:\n#             imageDict[file[:-4]]=study","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:54.877012Z","iopub.execute_input":"2021-07-13T15:12:54.878664Z","iopub.status.idle":"2021-07-13T15:12:54.883436Z","shell.execute_reply.started":"2021-07-13T15:12:54.878626Z","shell.execute_reply":"2021-07-13T15:12:54.882508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# arr=[]\n# imageList = image_df.id\n# for image in imageList:\n#     arr.append(imageDict[image.split(\"_\")[0]]+\"_study\")\n# image_df.insert(0,\"study_id\",arr)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:54.886129Z","iopub.execute_input":"2021-07-13T15:12:54.888146Z","iopub.status.idle":"2021-07-13T15:12:54.893514Z","shell.execute_reply.started":"2021-07-13T15:12:54.888107Z","shell.execute_reply":"2021-07-13T15:12:54.892701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import models\nimport tensorflow_addons as tfa\nfrom tensorflow.keras import mixed_precision\n\n\nimport tensorflow_probability as tfp\ntfd = tfp.distributions\n\nimport os\nimport gc\nimport json\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport cv2\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.utils.class_weight import compute_class_weight\n\n# Imports for augmentations. \nfrom albumentations import (Compose, RandomResizedCrop, Cutout, Rotate, HorizontalFlip, \n                            VerticalFlip, RandomBrightnessContrast, ShiftScaleRotate, \n                            CenterCrop, Resize)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:54.896867Z","iopub.execute_input":"2021-07-13T15:12:54.899352Z","iopub.status.idle":"2021-07-13T15:12:56.771118Z","shell.execute_reply.started":"2021-07-13T15:12:54.899313Z","shell.execute_reply":"2021-07-13T15:12:56.770309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PATH = f'/kaggle/tmp/test_study/'\nIMG_SIZE = 224\n\ndef prepare_test_images():\n    image_id = []\n    dim0 = []\n    dim1 = []\n\n    os.makedirs(TEST_PATH, exist_ok=True)\n\n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/test')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(os.path.join(dirname, file))\n            im = resize_xray(xray, size=IMG_SIZE)  \n            im.save(os.path.join(TEST_PATH, file.replace('dcm', 'png')))\n\n            image_id.append(file.replace('.dcm', ''))\n            \n    return image_id","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:56.77245Z","iopub.execute_input":"2021-07-13T15:12:56.77281Z","iopub.status.idle":"2021-07-13T15:12:56.78375Z","shell.execute_reply.started":"2021-07-13T15:12:56.772775Z","shell.execute_reply":"2021-07-13T15:12:56.782883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\nCONFIG = dict (\n    seed = 42,\n    num_labels = 4,\n    num_folds = 5,\n    img_width = 224, # If you change the resolution to 512 reduce batch size. \n    img_height = 224,\n    batch_size = 32,\n    epochs = 70,\n    learning_rate = 1e-3,\n    architecture = \"CNN\",\n    competition = 'siim-covid',\n    _wandb_kernel = 'aks',\n    infra = \"GCP\",\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:56.786863Z","iopub.execute_input":"2021-07-13T15:12:56.788808Z","iopub.status.idle":"2021-07-13T15:12:56.795546Z","shell.execute_reply.started":"2021-07-13T15:12:56.78877Z","shell.execute_reply":"2021-07-13T15:12:56.794732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    base_model = tf.keras.applications.EfficientNetB0(include_top=False, weights='../input/effnetweights/efficientnetb0_notop.h5')\n    base_model.trainabe = True\n\n    inputs = layers.Input((CONFIG['img_height'], CONFIG['img_width'], 3))\n    x = base_model(inputs, training=True)\n    x = layers.GlobalAveragePooling2D()(x)\n    x = layers.Dropout(0.5)(x)\n    \n    outputs = layers.Dense(CONFIG['num_labels'], kernel_regularizer=tf.keras.regularizers.l2(0.001))(x)\n    outputs = layers.Activation('softmax', dtype='float32', name='predictions')(outputs)\n    \n    return models.Model(inputs, outputs)\n\ntf.keras.backend.clear_session() \nmodel = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:12:56.806661Z","iopub.execute_input":"2021-07-13T15:12:56.808364Z","iopub.status.idle":"2021-07-13T15:12:59.685489Z","shell.execute_reply.started":"2021-07-13T15:12:56.808325Z","shell.execute_reply":"2021-07-13T15:12:59.684449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG['model_name'] = 'effnetb0_mixup'\nCONFIG['group'] = 'Effnetb0-Mixup-512'","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:13:00.008641Z","iopub.execute_input":"2021-07-13T15:13:00.009929Z","iopub.status.idle":"2021-07-13T15:13:00.023519Z","shell.execute_reply.started":"2021-07-13T15:13:00.009886Z","shell.execute_reply":"2021-07-13T15:13:00.020263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"modelpath = '/kaggle/input/effnetmodels/model-best.h5'\nmodel.load_weights(modelpath)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:13:00.02614Z","iopub.execute_input":"2021-07-13T15:13:00.030738Z","iopub.status.idle":"2021-07-13T15:13:00.742416Z","shell.execute_reply.started":"2021-07-13T15:13:00.030696Z","shell.execute_reply":"2021-07-13T15:13:00.741265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_list= prepare_test_images()","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:13:00.745537Z","iopub.execute_input":"2021-07-13T15:13:00.745915Z","iopub.status.idle":"2021-07-13T15:21:19.295897Z","shell.execute_reply.started":"2021-07-13T15:13:00.74586Z","shell.execute_reply":"2021-07-13T15:21:19.295045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef decode_image(image):\n    # convert the compressed string to a 3D uint8 tensor\n    image = tf.image.decode_png(image, channels=3)\n    print(image)\n    # Normalize image\n#     image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n    print(image)\n    return image\ndef load_image(df_dict):\n    # Load image\n    image = tf.io.read_file(df_dict)\n    image = decode_image(image)\n    \n#     # Parse label\n#     label = df_dict['study_level']\n#     label = tf.one_hot(indices=label, depth=CONFIG['num_labels'])\n    \n    return image","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:21:19.298493Z","iopub.execute_input":"2021-07-13T15:21:19.300169Z","iopub.status.idle":"2021-07-13T15:21:19.307683Z","shell.execute_reply.started":"2021-07-13T15:21:19.300115Z","shell.execute_reply":"2021-07-13T15:21:19.306857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imageDict = {}\nfor study in os.listdir('../input/siim-covid19-detection/test'):\n    for _,__,files in os.walk('../input/siim-covid19-detection/test/'+study):\n        for file in files:\n            imageDict[file[:-4]]=study","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:21:19.310522Z","iopub.execute_input":"2021-07-13T15:21:19.312372Z","iopub.status.idle":"2021-07-13T15:21:20.776778Z","shell.execute_reply.started":"2021-07-13T15:21:19.312335Z","shell.execute_reply":"2021-07-13T15:21:20.775913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_arr = ['negative','typical','indeterminate','atypical']","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:21:20.777984Z","iopub.execute_input":"2021-07-13T15:21:20.778336Z","iopub.status.idle":"2021-07-13T15:21:20.788177Z","shell.execute_reply.started":"2021-07-13T15:21:20.7783Z","shell.execute_reply":"2021-07-13T15:21:20.787032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_preds = []\nfor img in tqdm(os.listdir(TEST_PATH)):\n#     image = tf.io.read_file()\n    image = tf.keras.preprocessing.image.load_img(os.path.join(TEST_PATH,img))\n    x = tf.keras.preprocessing.image.img_to_array(image)\n    x = np.expand_dims(x, axis=0)\n    output = model.predict(x)\n#     print(output)\n    output_str = ''\n    for i,conf in enumerate(output[0]):\n        output_str +=output_arr[i]+' '+str(conf)+' 0 0 1 1 '\n        #output_arr[np.argmax(output)]+\" \"+str(output[0][np.argmax(output)])+\" 0 0 1 1\"]\n    study_preds.append([imageDict[img.split(\".\")[0]],img.split(\".\")[0],output_str])\n#     print(study_preds)\n#     break","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:54:13.693353Z","iopub.execute_input":"2021-07-13T15:54:13.693694Z","iopub.status.idle":"2021-07-13T15:55:23.765172Z","shell.execute_reply.started":"2021-07-13T15:54:13.693662Z","shell.execute_reply":"2021-07-13T15:55:23.764209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = pd.DataFrame(study_preds, columns =['study_id','image_id','PredictionString'])","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:44.776009Z","iopub.execute_input":"2021-07-13T15:55:44.776401Z","iopub.status.idle":"2021-07-13T15:55:44.782434Z","shell.execute_reply.started":"2021-07-13T15:55:44.776354Z","shell.execute_reply":"2021-07-13T15:55:44.781607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# arr=[]\n# study_df = \n# imageList = image_df.id\n# for image in imageList:\n#     arr.append(imageDict[image.split(\"_\")[0]]+\"_study\")\n# image_df.insert(0,\"study_id\",arr)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:45.655861Z","iopub.execute_input":"2021-07-13T15:55:45.656208Z","iopub.status.idle":"2021-07-13T15:55:45.66043Z","shell.execute_reply.started":"2021-07-13T15:55:45.656176Z","shell.execute_reply":"2021-07-13T15:55:45.659259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# study_df.to_csv('classification_EffNet_080721.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:46.261788Z","iopub.execute_input":"2021-07-13T15:55:46.262111Z","iopub.status.idle":"2021-07-13T15:55:46.26662Z","shell.execute_reply.started":"2021-07-13T15:55:46.26208Z","shell.execute_reply":"2021-07-13T15:55:46.264878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink(r'classification_EffNet_080721.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:46.890484Z","iopub.execute_input":"2021-07-13T15:55:46.890808Z","iopub.status.idle":"2021-07-13T15:55:46.894757Z","shell.execute_reply.started":"2021-07-13T15:55:46.890778Z","shell.execute_reply":"2021-07-13T15:55:46.893513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.insert(0, 'id', image_df.apply(lambda row: row.image_id+'_image', axis=1))","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:47.488477Z","iopub.execute_input":"2021-07-13T15:55:47.488881Z","iopub.status.idle":"2021-07-13T15:55:47.523756Z","shell.execute_reply.started":"2021-07-13T15:55:47.488848Z","shell.execute_reply":"2021-07-13T15:55:47.522762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_new = image_df[['id','PredictionString']]","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:48.042985Z","iopub.execute_input":"2021-07-13T15:55:48.043392Z","iopub.status.idle":"2021-07-13T15:55:48.054196Z","shell.execute_reply.started":"2021-07-13T15:55:48.043357Z","shell.execute_reply":"2021-07-13T15:55:48.05324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.insert(0, 'id', study_df.apply(lambda row: row.study_id+'_study', axis=1))","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:48.738473Z","iopub.execute_input":"2021-07-13T15:55:48.7388Z","iopub.status.idle":"2021-07-13T15:55:48.769859Z","shell.execute_reply.started":"2021-07-13T15:55:48.738769Z","shell.execute_reply":"2021-07-13T15:55:48.768923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_new = study_df[['id','PredictionString']]\nids = {}\nfor i in range(len(study_df_new)):\n    row = study_df_new.loc[i]\n    if row.id in ids:\n        ids[row.id] = ids[row.id] #+' '+row.PredictionString\n    else:\n        ids[row.id] = row.PredictionString\nstudylbls = []\nfor k,v in ids.items():\n    studylbls.append([k,v])\n    \nstudy_df_new = pd.DataFrame(studylbls, columns =['id','PredictionString'])\n","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:49.906066Z","iopub.execute_input":"2021-07-13T15:55:49.906437Z","iopub.status.idle":"2021-07-13T15:55:50.093827Z","shell.execute_reply.started":"2021-07-13T15:55:49.906405Z","shell.execute_reply":"2021-07-13T15:55:50.092911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_new = study_df_new.append(image_df_new).reset_index(drop=True)\nstudy_df_new.to_csv('/kaggle/working/submission.csv',index = False)","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:51.17757Z","iopub.execute_input":"2021-07-13T15:55:51.177903Z","iopub.status.idle":"2021-07-13T15:55:51.219011Z","shell.execute_reply.started":"2021-07-13T15:55:51.177871Z","shell.execute_reply":"2021-07-13T15:55:51.218085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df_new\n","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:52.941857Z","iopub.execute_input":"2021-07-13T15:55:52.942251Z","iopub.status.idle":"2021-07-13T15:55:52.957642Z","shell.execute_reply.started":"2021-07-13T15:55:52.942215Z","shell.execute_reply":"2021-07-13T15:55:52.956522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df_new","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:55:59.918854Z","iopub.execute_input":"2021-07-13T15:55:59.919303Z","iopub.status.idle":"2021-07-13T15:55:59.930551Z","shell.execute_reply.started":"2021-07-13T15:55:59.91926Z","shell.execute_reply":"2021-07-13T15:55:59.929356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%rm -rf runs","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:56:00.990086Z","iopub.execute_input":"2021-07-13T15:56:00.990476Z","iopub.status.idle":"2021-07-13T15:56:01.755351Z","shell.execute_reply.started":"2021-07-13T15:56:00.990439Z","shell.execute_reply":"2021-07-13T15:56:01.75429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%rm -rf yolov5","metadata":{"execution":{"iopub.status.busy":"2021-07-13T15:56:01.757431Z","iopub.execute_input":"2021-07-13T15:56:01.757797Z","iopub.status.idle":"2021-07-13T15:56:02.479927Z","shell.execute_reply.started":"2021-07-13T15:56:01.757765Z","shell.execute_reply":"2021-07-13T15:56:02.478823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}