{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! conda install gdcm -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-05-20T08:47:05.157902Z","iopub.execute_input":"2021-05-20T08:47:05.158471Z","iopub.status.idle":"2021-05-20T08:48:12.675041Z","shell.execute_reply.started":"2021-05-20T08:47:05.158370Z","shell.execute_reply":"2021-05-20T08:48:12.673920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install --no-deps '../input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null\n! pip install --no-deps '../input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:03:51.752595Z","iopub.execute_input":"2021-05-20T09:03:51.753657Z","iopub.status.idle":"2021-05-20T09:03:56.960729Z","shell.execute_reply.started":"2021-05-20T09:03:51.753601Z","shell.execute_reply":"2021-05-20T09:03:56.959642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Import Libraris**","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.insert(0, \"../input/timm-efficientdet-pytorch\")\nsys.path.insert(0, \"../input/omegaconf\")\n\nimport os\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport torch\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom glob import glob\n\nSEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-05-20T09:04:50.942127Z","iopub.execute_input":"2021-05-20T09:04:50.942542Z","iopub.status.idle":"2021-05-20T09:04:55.034641Z","shell.execute_reply.started":"2021-05-20T09:04:50.942503Z","shell.execute_reply":"2021-05-20T09:04:55.033268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Helper Functions**","metadata":{}},{"cell_type":"code","source":"dataset_path = \"../input/siim-covid19-detection\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im\n        \n    \ndef plot_img(img, size=(7, 7), is_rgb=True, title=\"\", cmap='gray'):\n    plt.figure(figsize=size)\n    plt.imshow(img, cmap=cmap)\n    plt.suptitle(title)\n    plt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimage_id = []\n\n\ncnt_img = 0 # for testing\n\nfor split in ['test', 'train']:\n    save_dir = f'/kaggle/tmp/{split}/'\n\n    os.makedirs(save_dir, exist_ok=True)\n    \n    for dirname, _, filenames in tqdm(os.walk(f'{dataset_path}/{split}')):\n        for file in filenames:\n            # set keep_ratio=True to have original aspect ratio\n            xray = read_xray(os.path.join(dirname, file))\n            im = resize(xray, size=256)  \n            im.save(os.path.join(save_dir, file.replace('dcm', 'jpg')))\n\n            image_id.append(file.replace('.dcm', ''))\n\n            splits.append(split)\n        \n        if cnt_img >= 10:\n            cnt_img = 0\n            break\n        \n        cnt_img += 1\n            ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df = pd.read_csv(f\"{dataset_path}/train_image_level.csv\")\ntrain_image_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df['class'] = train_image_df.label.apply(lambda x: x.split()[0])\ntrain_image_df['x_min'] = train_image_df.label.apply(lambda x: float(x.split()[2]))\ntrain_image_df['y_min'] = train_image_df.label.apply(lambda x: float(x.split()[3]))\ntrain_image_df['x_max'] = train_image_df.label.apply(lambda x: float(x.split()[4]))\ntrain_image_df['y_max'] = train_image_df.label.apply(lambda x: float(x.split()[5]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Loading Network**","metadata":{}},{"cell_type":"code","source":"from effdet import get_efficientdet_config, EfficientDet, DetBenchTrain\nfrom effdet.efficientdet import HeadNet\n\ndef get_net():\n    config = get_efficientdet_config('tf_efficientdet_d5')\n    net = EfficientDet(config, pretrained_backbone=False)\n    checkpoint = torch.load('../input/efficientdet/efficientdet_d5-ef44aea8.pth')\n    net.load_state_dict(checkpoint)\n    config.num_classes = 1\n    config.image_size = 512\n    net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))\n    return DetBenchTrain(net, config)\n\nnet = get_net()","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:10:20.541970Z","iopub.execute_input":"2021-05-20T09:10:20.542389Z","iopub.status.idle":"2021-05-20T09:10:25.608873Z","shell.execute_reply.started":"2021-05-20T09:10:20.542338Z","shell.execute_reply":"2021-05-20T09:10:25.607851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(net)","metadata":{"execution":{"iopub.status.busy":"2021-05-20T09:10:32.551636Z","iopub.execute_input":"2021-05-20T09:10:32.552382Z","iopub.status.idle":"2021-05-20T09:10:32.577850Z","shell.execute_reply.started":"2021-05-20T09:10:32.552312Z","shell.execute_reply":"2021-05-20T09:10:32.576524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}