{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## install pip wheels","metadata":{}},{"cell_type":"code","source":"#!conda install '../input/conda-packages/python-3.8.10-h49503c6_1_cpython.tar.bz2' -c conda-forge -y -q\n!conda install '../input/conda-packages/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y -q\n!conda install '../input/conda-packages/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y -q\n!conda install '../input/conda-packages/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y -q\n!conda install '../input/conda-packages/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y -q\n!conda install '../input/conda-packages/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y -q\n!conda install '../input/conda-packages/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y -q","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:19:37.401258Z","iopub.execute_input":"2021-08-22T20:19:37.401532Z","iopub.status.idle":"2021-08-22T20:20:42.865094Z","shell.execute_reply.started":"2021-08-22T20:19:37.401471Z","shell.execute_reply":"2021-08-22T20:20:42.86398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python --version","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:20:42.867916Z","iopub.execute_input":"2021-08-22T20:20:42.86828Z","iopub.status.idle":"2021-08-22T20:20:43.512324Z","shell.execute_reply.started":"2021-08-22T20:20:42.86824Z","shell.execute_reply":"2021-08-22T20:20:43.511375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#install pips\n# !pip install ../input/pip-wheels/pylibjpeg_openjpeg-1.1.1-cp37-cp37m-manylinux2010_x86_64.whl --no-index\n# !pip install ../input/pip-wheels/pylibjpeg_rle-1.1.0-cp37-cp37m-manylinux2010_x86_64.whl --no-index\n# !pip install ../input/pip-wheels/pylibjpeg-1.3.0-py3-none-any.whl --no-index\n\n!pip install ../input/pip-wheels/torch-1.9.0-cp37-cp37m-manylinux1_x86_64.whl --no-index --no-deps\n!pip install ../input/pip-wheels/torchvision-0.10.0-cp37-cp37m-manylinux1_x86_64.whl --no-index --no-deps\n\n!pip install ../input/pip-wheels/pycocotools-2.0.2.tar.gz --no-index\n!pip install ../input/pip-wheels/albumentations-1.0.0-py3-none-any.whl --no-index\n!pip install ../input/pip-wheels/pycocotools-2.0.2.tar.gz --no-index\n!pip install ../input/pip-wheels/antlr4-python3-runtime-4.8.tar.gz --no-index\n!pip install ../input/pip-wheels/omegaconf-2.1.0-py3-none-any.whl --no-index\n!pip install ../input/pip-wheels/timm-0.4.13.zip --no-index\n!pip install ../input/pip-wheels/effdet-0.2.4-py3-none-any.whl --no-index \n\n!pip install ../input/pip-wheels/pretrainedmodels-0.7.4.tar.gz --no-index --no-deps\n!pip install ../input/pip-wheels/efficientnet_pytorch-0.6.3.tar.gz --no-index --no-deps\n!pip install ../input/pip-wheels/segmentation_models_pytorch-0.2.0-py3-none-any.whl --no-index --no-deps\n\n!pip install ../input/pip-wheels/ensemble_boxes-1.0.6-py3-none-any.whl --no-index","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:20:43.52715Z","iopub.execute_input":"2021-08-22T20:20:43.52739Z","iopub.status.idle":"2021-08-22T20:22:49.765771Z","shell.execute_reply.started":"2021-08-22T20:20:43.527363Z","shell.execute_reply":"2021-08-22T20:22:49.764874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir configs\n!mkdir data\n!mkdir models\n\n!cp ../input/kaggle-siim-covid-detection-9th-place/* ./\n!tar -xvf ./configs.tar -C ./configs >> /dev/null\n!tar -xvf ./data.tar -C ./data >> /dev/null\n!tar -xvf ./models.tar -C ./models >> /dev/null\n!rm ./*.tar","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:22:49.767344Z","iopub.execute_input":"2021-08-22T20:22:49.767668Z","iopub.status.idle":"2021-08-22T20:22:54.919509Z","shell.execute_reply.started":"2021-08-22T20:22:49.76763Z","shell.execute_reply":"2021-08-22T20:22:54.918368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy as sp\nimport os\nimport timm\nimport sys\nimport os\nimport importlib\nimport multiprocessing as mp\nimport cv2\nfrom tqdm.notebook import tqdm\nimport numpy as np\nimport pandas as pd\nimport glob\nimport torch\nfrom copy import copy\nfrom torch.cuda.amp import GradScaler, autocast\n\nfrom torch.utils.data import DataLoader\n\ntorch.backends.cudnn.benchmark = True\ncv2.setNumThreads(0)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:22:54.925403Z","iopub.execute_input":"2021-08-22T20:22:54.925684Z","iopub.status.idle":"2021-08-22T20:22:56.377205Z","shell.execute_reply.started":"2021-08-22T20:22:54.925651Z","shell.execute_reply":"2021-08-22T20:22:56.376271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path.append('./configs')\nsys.path.append('./data')\nsys.path.append('./models')","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:22:56.379501Z","iopub.execute_input":"2021-08-22T20:22:56.379859Z","iopub.status.idle":"2021-08-22T20:22:56.385557Z","shell.execute_reply.started":"2021-08-22T20:22:56.379818Z","shell.execute_reply":"2021-08-22T20:22:56.384391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMP_FOLDER = '../input/siim-covid19-detection/'\n\n\nSAMPLE_SUBMISSION = pd.read_csv(COMP_FOLDER + 'sample_submission.csv').set_index('id')\nN_CORES = mp.cpu_count()\n\nRAM_CHECK = False\nMIXED_PRECISION = False\nDEVICE = \"cuda\"\n\nif len(SAMPLE_SUBMISSION) == 2477:\n    DEBUG = True\nelse:\n    DEBUG = False","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:22:56.387026Z","iopub.execute_input":"2021-08-22T20:22:56.388122Z","iopub.status.idle":"2021-08-22T20:22:56.421642Z","shell.execute_reply.started":"2021-08-22T20:22:56.388085Z","shell.execute_reply":"2021-08-22T20:22:56.420903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert all dicoms to png\n\nfrom pydicom import read_file\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data\n\ndef do_one(fp):\n    img = read_xray(fp)\n    h, w = img.shape\n    \n    study_id, series_id, image_id = fp.split('/')[4:]\n    \n    img_025 = cv2.resize(img, (0, 0), fx=0.25, fy = 0.25)\n    folder = f'test_025/{study_id}/{series_id}/'\n    if not os.path.exists(folder):\n        os.makedirs(folder)\n    cv2.imwrite(folder + image_id.replace('dcm','png'),img_025)\n    \n    img_05 = cv2.resize(img, (0, 0), fx=0.5, fy = 0.5)\n    folder = f'test_05/{study_id}/{series_id}/'\n    if not os.path.exists(folder):\n        os.makedirs(folder)\n    cv2.imwrite(folder + image_id.replace('dcm','png'),img_05)\n        \n    return image_id, study_id, series_id, w, h","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:22:56.422738Z","iopub.execute_input":"2021-08-22T20:22:56.423205Z","iopub.status.idle":"2021-08-22T20:22:56.578503Z","shell.execute_reply.started":"2021-08-22T20:22:56.423166Z","shell.execute_reply":"2021-08-22T20:22:56.577673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fps = []\nfor dirname, _, filenames in os.walk(f'{COMP_FOLDER}test'):\n    for file in filenames:\n        fps += [dirname + '/' + file]\n        \nif DEBUG:\n    fps = fps[:50]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:22:56.580484Z","iopub.execute_input":"2021-08-22T20:22:56.580728Z","iopub.status.idle":"2021-08-22T20:23:00.561562Z","shell.execute_reply.started":"2021-08-22T20:22:56.580703Z","shell.execute_reply":"2021-08-22T20:23:00.558253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with mp.Pool(N_CORES) as p:\n    res = list(tqdm(p.imap(do_one, fps),total=len(fps)))\ntest_info = np.array(res)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:23:00.576362Z","iopub.execute_input":"2021-08-22T20:23:00.57672Z","iopub.status.idle":"2021-08-22T20:23:17.585296Z","shell.execute_reply.started":"2021-08-22T20:23:00.57668Z","shell.execute_reply":"2021-08-22T20:23:17.584353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.DataFrame({'PredictionString':['none 1 0 0 1 1']* len(test_info)})\ntest_df[['id','StudyInstanceUID','series_id','width','height']] = test_info\ntest_df['id'] = test_df['id'].str.replace('.dcm','_image')\ntest_df[['width','height']] = test_df[['width','height']].astype(int)\ntest_df.to_csv('test_image_level_v2.csv',index=False)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:23:17.586583Z","iopub.execute_input":"2021-08-22T20:23:17.586863Z","iopub.status.idle":"2021-08-22T20:23:17.623161Z","shell.execute_reply.started":"2021-08-22T20:23:17.586817Z","shell.execute_reply":"2021-08-22T20:23:17.621949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#helper functions\n\ndef get_state_dict(sd_fp):\n    sd = torch.load(sd_fp, map_location=\"cpu\")['model']\n    #fix DDP if necessary\n    sd = {k.replace(\"module.\", \"\"):v for k,v in sd.items()}\n    return sd\n\ndef pp_transform(logits, pp_transformation):\n    if pp_transformation == \"softmax\":\n        pred = logits.softmax(1)\n    elif pp_transformation == \"sigmoid\":\n        pred = logits.sigmoid()\n    else:\n        pred = logits.detach()\n    return pred","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:23:17.624558Z","iopub.execute_input":"2021-08-22T20:23:17.624906Z","iopub.status.idle":"2021-08-22T20:23:17.631998Z","shell.execute_reply.started":"2021-08-22T20:23:17.624868Z","shell.execute_reply":"2021-08-22T20:23:17.630853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_study(config_file, state_dicts):\n    cfg = importlib.import_module('default_config')\n    importlib.reload(cfg)\n    cfg = importlib.import_module(config_file)\n    importlib.reload(cfg)\n    cfg = copy(cfg.cfg)\n    print(cfg.model, cfg.dataset, cfg.backbone, cfg.pretrained_weights)\n\n    cfg.test_df = 'test_image_level_v2.csv'\n    cfg.test_data_folder = f\"test_{str(cfg.scale).replace('.','')}/\"\n    print(config_file, cfg.test_data_folder)\n    cfg.pretrained = False\n\n    ds = importlib.import_module(cfg.dataset)\n    importlib.reload(ds)\n\n    CustomDataset = ds.CustomDataset\n    batch_to_device = ds.batch_to_device\n\n    cfg.batch_size = 8\n    test_ds = CustomDataset(test_df, cfg, cfg.val_aug, mode=\"test\")\n    test_dl = DataLoader(test_ds, shuffle=False, batch_size = cfg.batch_size, num_workers = N_CORES, pin_memory=True)\n\n    model = importlib.import_module(cfg.model)\n    importlib.reload(model)\n    Net = model.Net\n\n    nets = []\n\n    for i,state_dict in enumerate(state_dicts):\n        net = Net(cfg).eval().cuda()\n        sd = get_state_dict(state_dict)\n        print(\"loading dict\")\n        net.load_state_dict(sd, strict=True)\n        nets += [net]\n\n    #with torch.no_grad():    \n    with torch.inference_mode():\n\n        preds_0 = []\n        for batch in tqdm(test_dl):\n            batch = batch_to_device(batch, DEVICE)\n            preds_ = []\n            for net in nets:\n                with autocast():\n                    logits = net(batch)['class_logits'].float()\n                #pred = pp_transform(logits, cfg.pp_transformation)\n                pred = logits\n                preds_ += [pred.cpu().numpy()]\n\n            preds_0 += [np.array(preds_)]\n        preds_0 = np.concatenate(preds_0, axis=1)\n    preds_0.shape\n    return preds_0\n\npreds = []\n\nstate_dicts = []\nfor fold, seed in enumerate([487735,464165,633237,686759,203682]):\n    state_dicts.append(f'../input/weights-cfg-ch-study-xcit-2e15/fold{fold}/checkpoint_last_seed{seed}.pth')\nprint(state_dicts)\n\np = run_study(\"cfg_ch_study_xcit_2e15\", state_dicts)\npreds.append(p)\n\nstate_dicts = []\nfor fold, seed in enumerate([226896,259854,476990,577158,258586]):\n    state_dicts.append(f'../input/weights-cfg-ch-study-xcit-2d/fold{fold}/checkpoint_last_seed{seed}.pth')\nprint(state_dicts)\n\np = run_study(\"cfg_ch_study_xcit_2d\", state_dicts)\npreds.append(p)\n\nstate_dicts = []\nfor file in glob.glob(\"../input/cfg-ps-study-ben-7-weights/*.pth\"):\n    state_dicts.append(file)\nprint(state_dicts)\n\np = run_study(\"cfg_ps_study_ben_7\", state_dicts)\npreds.append(p)\n\nstate_dicts = []\nfor file in glob.glob(\"../input/cfg-ps-study-7-weights/*.pth\"):\n    state_dicts.append(file)\nprint(state_dicts)\n\np = run_study(\"cfg_ps_study_7\", state_dicts)\npreds.append(p)\n\nstate_dicts = []\nfor fold, seed in enumerate([322511,301949,777691,615690,207310]):\n    state_dicts.append(f'../input/weights-cfg-ch-study-12d/fold{fold}/checkpoint_last_seed{seed}.pth')\nprint(state_dicts)\n\np = run_study(\"cfg_ch_study_12d\", state_dicts)\npreds.append(p)\n\nstate_dicts = []\nfor fold, seed in enumerate([825690,473862,975082,699448,879221]):\n    state_dicts.append(f'../input/weights-cfg-ch-study-12d/fold{fold}/checkpoint_last_seed{seed}.pth')\nprint(state_dicts)\n\np = run_study(\"cfg_ch_study_12d\", state_dicts)\npreds.append(p)\n\nstate_dicts = []\nfor fold in range(5):\n    for file in glob.glob(f\"../input/weights-cfg-ch-study-7-1024b5/fold{fold}/*.pth\"):\n        state_dicts.append(file)\n    #state_dicts.append(f'../input/weights-cfg-ch-study-12d/fold{fold}/checkpoint_last_seed{seed}.pth')\nprint(state_dicts)\nprint(len(state_dicts))\n\n\np = run_study(\"cfg_ch_study_7_1024b5\", state_dicts)\npreds.append(p)\n\nstate_dicts = []\nfor fold, seed in enumerate([775009,776562,239357,624529,909642]):\n    state_dicts.append(f'../input/weights-cfg-ch-study-7-1024b7-2/fold{fold}/checkpoint_last_seed{seed}.pth')\nprint(state_dicts)\n\np = run_study(\"cfg_ch_study_7_1024b7_2\", state_dicts)\npreds.append(p)\n\n#preds = [sp.special.softmax(p, axis=2).mean(axis=0) for p in preds]\npreds = np.concatenate(preds, axis=0)\npreds = sp.special.softmax(preds, axis=2)\npreds = np.mean(preds, axis=0)\n\nprint(preds.shape)\n\nstudy_ids = test_df['StudyInstanceUID']","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:23:17.63343Z","iopub.execute_input":"2021-08-22T20:23:17.634044Z","iopub.status.idle":"2021-08-22T20:30:50.231987Z","shell.execute_reply.started":"2021-08-22T20:23:17.634004Z","shell.execute_reply":"2021-08-22T20:30:50.230881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_pred_str_study(row):\n    items = []\n    for label in [\"negative\", \"typical\", \"indeterminate\", \"atypical\"]:\n        items += [label,row[label],'0','0','1','1']\n    return ' '.join([str(i) for i in items])\n\ndef create_sub(test_preds):\n    df_study = pd.DataFrame({'id':study_ids})\n    df_study['id'] = df_study['id'].astype(str) + '_study'\n    df_study[[\"negative\", \"typical\", \"indeterminate\", \"atypical\"]] = test_preds\n    df_study = df_study.groupby('id').agg('mean').reset_index()\n    df_study_orig = df_study.copy()\n    df_study['PredictionString'] = df_study.apply(lambda row: create_pred_str_study(row), axis = 1)\n    df_study= df_study[['id','PredictionString']].copy()\n    df_study_orig[\"study_id\"] = df_study_orig[\"id\"].str.replace(\"_study\", \"\")\n    return df_study, df_study_orig\n\ntest_preds_study = preds\ntest_preds_study.shape\n\ndf_study, df_study_orig = create_sub(test_preds_study)\ndf_study = df_study.set_index('id')\ndf_study.head()\n\nSAMPLE_SUBMISSION.loc[df_study.index,'PredictionString'] = df_study['PredictionString'].values","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:30:50.233658Z","iopub.execute_input":"2021-08-22T20:30:50.234036Z","iopub.status.idle":"2021-08-22T20:30:50.32298Z","shell.execute_reply.started":"2021-08-22T20:30:50.233993Z","shell.execute_reply":"2021-08-22T20:30:50.322245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOPK_EFFDET = 75\n\ndef get_df_image_pred_effdet(cfg, detections, image_ids):\n\n    images = []\n    boxes = []\n    labels = []\n    confs = []\n    \n    for i,det in enumerate(detections):\n\n        detection = det#[det[:,4] > cfg.box_min_conf]\n        \n        images.append([image_ids[i]] * len(detection))\n        labels.append(['opacity'] * len(detection))\n        boxes.append((detection[:,:4] / cfg.image_size[0]).clip(0,1))\n        confs.append(detection[:,4])\n        images += [[image_ids[i]]]\n        labels += [['none']]\n        boxes += [np.array([0,0,1,1])[None,:]]\n        confs += [np.array([1-det[:,4].max()])]\n    \n    confs = np.concatenate(confs)\n    labels = np.concatenate(labels)\n    images = np.concatenate(images)\n    boxes = np.concatenate(boxes)\n    df_image_pred = pd.DataFrame({'ImageID':images,'LabelName':labels,'Conf':confs})\n    df_image_pred[['XMin','YMin','XMax','YMax']] = boxes\n\n    df_image_pred = df_image_pred[df_image_pred.XMin != df_image_pred.XMax]\n    df_image_pred = df_image_pred[df_image_pred.YMin != df_image_pred.YMax]\n\n    df_image_pred = df_image_pred[['ImageID','LabelName','Conf','XMin', 'XMax', 'YMin', 'YMax']].copy()\n    \n    return df_image_pred\n\ndef get_results_effdet(config_file, state_dicts):\n    cfg = importlib.import_module('default_config')\n    importlib.reload(cfg)\n    cfg = importlib.import_module(config_file)\n    importlib.reload(cfg)\n    cfg = copy(cfg.cfg)\n    print(cfg.model, cfg.dataset, cfg.backbone, cfg.pretrained_weights)\n\n    cfg.test_df = 'test_image_level_v2.csv'\n    cfg.test_data_folder = f\"test_{str(cfg.scale).replace('.','')}/\"\n    print(config_file, cfg.test_data_folder)\n    cfg.pretrained = False\n    cfg.pretrained_convhead = False\n\n    ds = importlib.import_module(cfg.dataset)\n    importlib.reload(ds)\n\n    CustomDataset = ds.CustomDataset\n    batch_to_device = ds.batch_to_device\n\n    cfg.batch_size = 8\n    test_ds = CustomDataset(test_df, cfg, cfg.val_aug, mode=\"test\")\n    test_dl = DataLoader(test_ds, shuffle=False, batch_size = cfg.batch_size, num_workers = N_CORES, pin_memory=True)\n\n    model = importlib.import_module(cfg.model)\n    importlib.reload(model)\n    Net = model.Net\n\n    nets = []\n\n    for i,state_dict in enumerate(state_dicts):\n        net = Net(cfg).eval().cuda()\n        sd = get_state_dict(state_dict)\n        print(\"loading dict\")\n        net.load_state_dict(sd, strict=True)\n        nets += [net]\n\n    #with torch.no_grad():    \n    with torch.inference_mode():\n\n        preds_0 = []\n        for batch in tqdm(test_dl):\n            batch = batch_to_device(batch, DEVICE)\n            preds_ = []\n            for net in nets:\n                with autocast():\n                    detections_ = net(batch)['detections'].float()\n                    detections_ = detections_[:,:TOPK_EFFDET,:]\n                #pred = pp_transform(logits, cfg.pp_transformation)\n                preds_ += [detections_.cpu().numpy()]\n                \n            preds_0 += [np.array(preds_)]\n        preds_0 = np.concatenate(preds_0, axis=1)\n        \n        \n    dfs_pred = []\n    for fold in tqdm([0,1,2,3,4]):\n        df_image_pred = get_df_image_pred_effdet(cfg, preds_0[fold], test_df['id'])\n        dfs_pred += [df_image_pred]\n    dfs_pred = pd.concat(dfs_pred)\n    \n    \n    \n    return dfs_pred","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:30:50.324343Z","iopub.execute_input":"2021-08-22T20:30:50.324682Z","iopub.status.idle":"2021-08-22T20:30:50.34386Z","shell.execute_reply.started":"2021-08-22T20:30:50.324645Z","shell.execute_reply":"2021-08-22T20:30:50.342878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_file = 'cfg_ps_35'\nstate_dicts = []\nfor file in glob.glob(\"../input/cfg-ps-35-weights/*.pth\"):\n    state_dicts.append(file)\nprint(state_dicts)\n\ndfs_pred_ps_35 = get_results_effdet(config_file, state_dicts)\ndfs_pred_ps_35.sort_values(['ImageID','Conf'])\n\ndfs_pred_ps_35.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:30:50.345435Z","iopub.execute_input":"2021-08-22T20:30:50.345785Z","iopub.status.idle":"2021-08-22T20:31:13.014624Z","shell.execute_reply.started":"2021-08-22T20:30:50.345749Z","shell.execute_reply":"2021-08-22T20:31:13.013537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_file = 'cfg_ps_27'\nstate_dicts = []\nfor file in glob.glob(\"../input/cfg-ps-27-weights/*.pth\"):\n    state_dicts.append(file)\nprint(state_dicts)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:13.016486Z","iopub.execute_input":"2021-08-22T20:31:13.016905Z","iopub.status.idle":"2021-08-22T20:31:13.035779Z","shell.execute_reply.started":"2021-08-22T20:31:13.016862Z","shell.execute_reply":"2021-08-22T20:31:13.034846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_ps_27 = get_results_effdet(config_file, state_dicts)\ndfs_pred_ps_27.sort_values(['ImageID','Conf'])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:13.037048Z","iopub.execute_input":"2021-08-22T20:31:13.037424Z","iopub.status.idle":"2021-08-22T20:31:32.965853Z","shell.execute_reply.started":"2021-08-22T20:31:13.037387Z","shell.execute_reply":"2021-08-22T20:31:32.964781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_ps_27.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:32.967403Z","iopub.execute_input":"2021-08-22T20:31:32.967755Z","iopub.status.idle":"2021-08-22T20:31:32.985819Z","shell.execute_reply.started":"2021-08-22T20:31:32.967715Z","shell.execute_reply":"2021-08-22T20:31:32.984886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_file = 'cfg_ps_13'\nstate_dicts = []\nfor file in glob.glob(\"../input/cfg-ps-13-weights/*.pth\"):\n    state_dicts.append(file)\nprint(state_dicts)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:32.991892Z","iopub.execute_input":"2021-08-22T20:31:32.992235Z","iopub.status.idle":"2021-08-22T20:31:33.007865Z","shell.execute_reply.started":"2021-08-22T20:31:32.992207Z","shell.execute_reply":"2021-08-22T20:31:33.007016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_ps_13 = get_results_effdet(config_file, state_dicts)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:33.010672Z","iopub.execute_input":"2021-08-22T20:31:33.011253Z","iopub.status.idle":"2021-08-22T20:31:43.666034Z","shell.execute_reply.started":"2021-08-22T20:31:33.011213Z","shell.execute_reply":"2021-08-22T20:31:43.665032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_ps_13.sort_values(['ImageID','Conf'])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:43.670963Z","iopub.execute_input":"2021-08-22T20:31:43.672958Z","iopub.status.idle":"2021-08-22T20:31:43.718424Z","shell.execute_reply.started":"2021-08-22T20:31:43.672911Z","shell.execute_reply":"2021-08-22T20:31:43.717664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config_file = 'cfg_ps_9'\nstate_dicts = []\nfor file in glob.glob(\"../input/cfg-ps-9-weights/*.pth\"):\n    state_dicts.append(file)\nprint(state_dicts)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:43.724043Z","iopub.execute_input":"2021-08-22T20:31:43.727362Z","iopub.status.idle":"2021-08-22T20:31:43.748277Z","shell.execute_reply.started":"2021-08-22T20:31:43.727313Z","shell.execute_reply":"2021-08-22T20:31:43.747537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_ps_9 = get_results_effdet(config_file, state_dicts)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:43.749648Z","iopub.execute_input":"2021-08-22T20:31:43.750244Z","iopub.status.idle":"2021-08-22T20:31:53.408162Z","shell.execute_reply.started":"2021-08-22T20:31:43.750208Z","shell.execute_reply":"2021-08-22T20:31:53.407326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_ps_9.sort_values(['ImageID','Conf'])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:53.410326Z","iopub.execute_input":"2021-08-22T20:31:53.41084Z","iopub.status.idle":"2021-08-22T20:31:53.441755Z","shell.execute_reply.started":"2021-08-22T20:31:53.410795Z","shell.execute_reply":"2021-08-22T20:31:53.441016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import yaml\n\nrun1 = dict(\n    path = '',\n    train = '/kaggle/working/test_05',\n    val = '/kaggle/working/test_05',\n    test = '/kaggle/working/test_05',\n    nc =  1,  # number of classes\n    names = [ '0. opacity' ],  # class names\n\n)\n\nrun2 = dict(\n    path = '',\n    train = '/kaggle/working/test_05',\n    val = '/kaggle/working/test_05',\n    test = '/kaggle/working/test_05',\n    nc =  2,  # number of classes\n    names = [ '0. opacity', '1. none' ],  # class names\n\n)\n\n\nwith open('data_1cls.yaml', 'w') as outfile:\n    yaml.dump(run1, outfile, default_flow_style=False)\n    \nwith open('data_2cls.yaml', 'w') as outfile:\n    yaml.dump(run2, outfile, default_flow_style=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:53.443019Z","iopub.execute_input":"2021-08-22T20:31:53.443581Z","iopub.status.idle":"2021-08-22T20:31:53.454585Z","shell.execute_reply.started":"2021-08-22T20:31:53.44354Z","shell.execute_reply":"2021-08-22T20:31:53.453436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:53.456203Z","iopub.execute_input":"2021-08-22T20:31:53.456549Z","iopub.status.idle":"2021-08-22T20:31:53.460328Z","shell.execute_reply.started":"2021-08-22T20:31:53.456512Z","shell.execute_reply":"2021-08-22T20:31:53.459358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_1b/fold0/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_1b/fold0/test_tta >>yolo_run_ch_1b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_1b/fold1/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_1b/fold1/test_tta >>yolo_run_ch_1b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_1b/fold2/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_1b/fold2/test_tta >>yolo_run_ch_1b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_1b/fold3/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_1b/fold3/test_tta >>yolo_run_ch_1b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_1b/fold4/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_1b/fold4/test_tta >>yolo_run_ch_1b.logs","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:31:53.461949Z","iopub.execute_input":"2021-08-22T20:31:53.462395Z","iopub.status.idle":"2021-08-22T20:33:07.659394Z","shell.execute_reply.started":"2021-08-22T20:31:53.462358Z","shell.execute_reply":"2021-08-22T20:33:07.658389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_2b/fold0/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_2b/fold0/test_tta >>yolo_run_ch_2b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_2b/fold1/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_2b/fold1/test_tta >>yolo_run_ch_2b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_2b/fold2/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_2b/fold2/test_tta >>yolo_run_ch_2b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_2b/fold3/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_2b/fold3/test_tta >>yolo_run_ch_2b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_2b/fold4/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_2b/fold4/test_tta >>yolo_run_ch_2b.logs","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:33:07.661016Z","iopub.execute_input":"2021-08-22T20:33:07.661388Z","iopub.status.idle":"2021-08-22T20:34:39.589895Z","shell.execute_reply.started":"2021-08-22T20:33:07.661337Z","shell.execute_reply":"2021-08-22T20:34:39.588934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_3b/fold0/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 640 --task test --save-txt --save-conf --project \"\" --name run_ch_3b/fold0/test_tta >>yolo_run_ch_3b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_3b/fold1/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 640 --task test --save-txt --save-conf --project \"\" --name run_ch_3b/fold1/test_tta >>yolo_run_ch_3b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_3b/fold2/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 640 --task test --save-txt --save-conf --project \"\" --name run_ch_3b/fold2/test_tta >>yolo_run_ch_3b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_3b/fold3/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 640 --task test --save-txt --save-conf --project \"\" --name run_ch_3b/fold3/test_tta >>yolo_run_ch_3b.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_3b/fold4/weights/last.pt --data /kaggle/working/data_1cls.yaml --augment --imgsz 640 --task test --save-txt --save-conf --project \"\" --name run_ch_3b/fold4/test_tta >>yolo_run_ch_3b.logs","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:34:39.591545Z","iopub.execute_input":"2021-08-22T20:34:39.591928Z","iopub.status.idle":"2021-08-22T20:35:57.971513Z","shell.execute_reply.started":"2021-08-22T20:34:39.591883Z","shell.execute_reply":"2021-08-22T20:35:57.970507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_7_2cls_2/fold0/weights/last.pt --data /kaggle/working/data_2cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_7_2cls_2/fold0/test_tta >>yolo_run_ch_7_2cls_2.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_7_2cls_2/fold1/weights/last.pt --data /kaggle/working/data_2cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_7_2cls_2/fold1/test_tta >>yolo_run_ch_7_2cls_2.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_7_2cls_2/fold2/weights/last.pt --data /kaggle/working/data_2cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_7_2cls_2/fold2/test_tta >>yolo_run_ch_7_2cls_2.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_7_2cls_2/fold3/weights/last.pt --data /kaggle/working/data_2cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_7_2cls_2/fold3/test_tta >>yolo_run_ch_7_2cls_2.logs\n!python ../input/yolov5-github/yolov5/test.py --conf-thres 0.0001 --weights ../input/weights-yolo/run_ch_7_2cls_2/fold4/weights/last.pt --data /kaggle/working/data_2cls.yaml --augment --imgsz 512 --task test --save-txt --save-conf --project \"\" --name run_ch_7_2cls_2/fold4/test_tta >>yolo_run_ch_7_2cls_2.logs","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:35:57.976162Z","iopub.execute_input":"2021-08-22T20:35:57.976447Z","iopub.status.idle":"2021-08-22T20:37:09.849085Z","shell.execute_reply.started":"2021-08-22T20:35:57.976415Z","shell.execute_reply":"2021-08-22T20:37:09.847881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TOPK = 75\n\ndef xywh2x1x2y1y2(x):\n    # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right\n    y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x)\n    y[:, 0] = x[:, 0] - x[:, 2] / 2  # top left x\n    y[:, 1] = x[:, 0] + x[:, 2] / 2  # bottom right x\n    y[:, 2] = x[:, 1] - x[:, 3] / 2  # top left y\n    y[:, 3] = x[:, 1] + x[:, 3] / 2  # bottom right y\n    return y\n\nLAB = np.array(['opacity','none'])\n\ndef get_results_yolo(val_df, pred_folder):\n\n\n    results = {}\n\n    images = []\n    boxes = []\n    labels = []\n    targets = []\n    confs = []\n    target_boxes = []\n    study_ids = []\n    image_ids = val_df['id'].values\n    detections = []\n    missing_preds = []\n    n_boxes = 0\n    for i,id_ in enumerate(image_ids):\n\n        w = val_df.iloc[i]['width']\n        h = val_df.iloc[i]['height']\n        if 'label' in val_df.columns:\n            label_box = np.array(val_df.iloc[i]['label'].split()).reshape(-1,6)\n            box_target = label_box[:,2:].astype(np.float32)\n            target = label_box[:,0]\n            mask = target == 'opacity'\n            if mask.sum() > 0:\n                box_target[mask,0] /= w\n                box_target[mask,2] /= w\n                box_target[mask,1] /= h\n                box_target[mask,3] /= h\n            target_boxes.append(box_target)\n            targets.append(target)\n        try:\n            det = pd.read_csv(f'{pred_folder}labels/{id_.replace(\"_image\",\"\")}.txt', sep=' ', header=None, names=['cls','x','y','w','h','Conf'])\n            det = det.values\n            det = det[det[:,0]==0]\n            det = det[det[:, 5].argsort()[::-1]]\n            det = det[:TOPK]\n        except:\n            det = np.empty((1,6))\n            missing_preds += [id_]\n\n        n_boxes += len(det)\n        images.append([image_ids[i]] * len(det))\n        labels.append(list(LAB[det[:,0].astype(int)]))\n        box = xywh2x1x2y1y2(det[:,1:-1])\n        box = box.clip(0,1)\n#         box[:,:2] *= w\n#         box[:,2:] *= h\n        boxes.append(box)\n        confs.append(det[:,5])\n\n        study_ids.append(val_df.iloc[i]['StudyInstanceUID'])\n#     print(len(no_preds))\n    #     images += [image_ids[i]]\n    #     labels += ['none']\n    #     boxes += [np.array([0,0,1,1])[None,:]]\n    #     confs += [1-det[:,5].max()]\n\n    results[\"images\"] = images\n    results[\"boxes\"] = boxes\n    results[\"box_target\"] = target_boxes\n    results[\"target\"] = targets\n    results[\"labels\"] = labels\n    results[\"confs\"] = confs\n    results[\"study_idx\"] = study_ids\n    print(n_boxes)\n    return results, missing_preds\n\ndef get_df_image_pred(results, single_class =True):\n    images = []\n    boxes = []\n    labels = []\n    confs = []\n\n\n    for i in range(len(results[\"boxes\"])):\n\n        images += results[\"images\"][i]\n        labels +=results[\"labels\"][i]\n        boxes += [results[\"boxes\"][i]]\n        confs += list(results[\"confs\"][i])\n        images += [results[\"images\"][i][0]]\n        labels += ['none']\n        boxes += [np.array([0,1,0,1])[None,:]]\n        confs += [1-results[\"confs\"][i].max()]\n\n\n    df_image_pred = pd.DataFrame({'ImageID':images,'LabelName':labels,'Conf':np.array(confs)})\n    df_image_pred[['XMin','XMax','YMin','YMax']] = np.concatenate(boxes)\n\n    df_image_pred = df_image_pred[df_image_pred.XMin != df_image_pred.XMax]\n    df_image_pred = df_image_pred[df_image_pred.YMin != df_image_pred.YMax]\n\n    df_image_pred = df_image_pred[['ImageID','LabelName','Conf','XMin', 'XMax', 'YMin', 'YMax']].copy()\n    return df_image_pred\n\n","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:09.854577Z","iopub.execute_input":"2021-08-22T20:37:09.855121Z","iopub.status.idle":"2021-08-22T20:37:09.888649Z","shell.execute_reply.started":"2021-08-22T20:37:09.855075Z","shell.execute_reply":"2021-08-22T20:37:09.887565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_1b = []\nmissing_preds = 0\nfor fold in tqdm([0,1,2,3,4]):\n    pred_folder = f'run_ch_1b/fold{fold}/test_tta/'\n#     print(len(os.listdir(pred_folder + 'labels/')))\n    results, m = get_results_yolo(test_df, pred_folder)\n    missing_preds += len(m)\n    df_image_pred = get_df_image_pred(results)\n    dfs_pred_1b += [df_image_pred]\ndfs_pred_1b = pd.concat(dfs_pred_1b)\nmissing_preds","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:09.889916Z","iopub.execute_input":"2021-08-22T20:37:09.890429Z","iopub.status.idle":"2021-08-22T20:37:11.027208Z","shell.execute_reply.started":"2021-08-22T20:37:09.890387Z","shell.execute_reply":"2021-08-22T20:37:11.026411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_1b.sort_values(['ImageID','Conf'])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:11.028574Z","iopub.execute_input":"2021-08-22T20:37:11.029149Z","iopub.status.idle":"2021-08-22T20:37:11.066275Z","shell.execute_reply.started":"2021-08-22T20:37:11.029096Z","shell.execute_reply":"2021-08-22T20:37:11.06534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_2b = []\nfor fold in tqdm([0,1,2,3,4]):\n    pred_folder = f'run_ch_2b/fold{fold}/test_tta/'\n#     print(len(os.listdir(pred_folder + 'labels/')))\n    results, m = get_results_yolo(test_df, pred_folder)\n    missing_preds += len(m)\n    df_image_pred = get_df_image_pred(results)\n    dfs_pred_2b += [df_image_pred]\ndfs_pred_2b = pd.concat(dfs_pred_2b)\nmissing_preds","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:11.067734Z","iopub.execute_input":"2021-08-22T20:37:11.068127Z","iopub.status.idle":"2021-08-22T20:37:11.815291Z","shell.execute_reply.started":"2021-08-22T20:37:11.068089Z","shell.execute_reply":"2021-08-22T20:37:11.814315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_3b = []\nfor fold in tqdm([0,1,2,3,4]):\n    pred_folder = f'run_ch_3b/fold{fold}/test_tta/'\n#     print(len(os.listdir(pred_folder + 'labels/')))\n    results, m = get_results_yolo(test_df, pred_folder)\n    missing_preds += len(m)\n    df_image_pred = get_df_image_pred(results)\n    dfs_pred_3b += [df_image_pred]\ndfs_pred_3b = pd.concat(dfs_pred_3b)\nmissing_preds","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:11.816787Z","iopub.execute_input":"2021-08-22T20:37:11.817157Z","iopub.status.idle":"2021-08-22T20:37:12.545449Z","shell.execute_reply.started":"2021-08-22T20:37:11.8171Z","shell.execute_reply":"2021-08-22T20:37:12.544509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_pred_7_2cls_2 = []\nfor fold in tqdm([0,1,2,3,4]):\n    pred_folder = f'run_ch_7_2cls_2/fold{fold}/test_tta/'\n#     print(len(os.listdir(pred_folder + 'labels/')))\n    results, m = get_results_yolo(test_df, pred_folder)\n    missing_preds += len(m)\n    df_image_pred = get_df_image_pred(results,single_class=False)\n    dfs_pred_7_2cls_2 += [df_image_pred]\ndfs_pred_7_2cls_2 = pd.concat(dfs_pred_7_2cls_2)\nmissing_preds","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:12.546851Z","iopub.execute_input":"2021-08-22T20:37:12.547224Z","iopub.status.idle":"2021-08-22T20:37:13.298696Z","shell.execute_reply.started":"2021-08-22T20:37:12.547184Z","shell.execute_reply":"2021-08-22T20:37:13.297681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image_pred_test = [dfs_pred_1b,dfs_pred_2b,dfs_pred_3b,dfs_pred_7_2cls_2,dfs_pred_ps_9,dfs_pred_ps_13,dfs_pred_ps_27,dfs_pred_ps_35]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.300185Z","iopub.execute_input":"2021-08-22T20:37:13.300545Z","iopub.status.idle":"2021-08-22T20:37:13.305371Z","shell.execute_reply.started":"2021-08-22T20:37:13.300503Z","shell.execute_reply":"2021-08-22T20:37:13.304409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_opacity = [df[df['LabelName'] == 'opacity'].copy() for df in df_image_pred_test]\ndf_test_none = [df[df['LabelName'] == 'none'].copy() for df in df_image_pred_test]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.306926Z","iopub.execute_input":"2021-08-22T20:37:13.307283Z","iopub.status.idle":"2021-08-22T20:37:13.390093Z","shell.execute_reply.started":"2021-08-22T20:37:13.307245Z","shell.execute_reply":"2021-08-22T20:37:13.389272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_opacity = pd.concat(df_test_opacity)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.391391Z","iopub.execute_input":"2021-08-22T20:37:13.391733Z","iopub.status.idle":"2021-08-22T20:37:13.409118Z","shell.execute_reply.started":"2021-08-22T20:37:13.391696Z","shell.execute_reply":"2021-08-22T20:37:13.408342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_opacity","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.410508Z","iopub.execute_input":"2021-08-22T20:37:13.410856Z","iopub.status.idle":"2021-08-22T20:37:13.431098Z","shell.execute_reply.started":"2021-08-22T20:37:13.41082Z","shell.execute_reply":"2021-08-22T20:37:13.430198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_test_none = []\nfor item in df_test_none:\n    df_image_pred_none = item[['ImageID','Conf','XMin','XMax','YMin','YMax']].groupby('ImageID').agg('max')\n    df_image_pred_none = df_image_pred_none.reset_index()\n    df_image_pred_none['LabelName'] = 'none'\n    df_image_pred_none[['XMin','YMin','XMax','YMax']] = [0,0,1,1]\n    dfs_test_none += [df_image_pred_none]\ndfs_test_none = pd.concat(dfs_test_none)","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.432484Z","iopub.execute_input":"2021-08-22T20:37:13.43282Z","iopub.status.idle":"2021-08-22T20:37:13.487469Z","shell.execute_reply.started":"2021-08-22T20:37:13.43278Z","shell.execute_reply":"2021-08-22T20:37:13.486727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_test_none2 = dfs_test_none[['ImageID','Conf','XMin','XMax','YMin','YMax']].groupby('ImageID').agg('mean')\ndfs_test_none2 = dfs_test_none2.reset_index()\ndfs_test_none2['LabelName'] = 'none'\ndfs_test_none2[['XMin','YMin','XMax','YMax']] = [0,0,1,1]\ndfs_test_none2","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.488715Z","iopub.execute_input":"2021-08-22T20:37:13.489073Z","iopub.status.idle":"2021-08-22T20:37:13.520614Z","shell.execute_reply.started":"2021-08-22T20:37:13.489036Z","shell.execute_reply":"2021-08-22T20:37:13.519661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.522107Z","iopub.execute_input":"2021-08-22T20:37:13.522521Z","iopub.status.idle":"2021-08-22T20:37:13.548713Z","shell.execute_reply.started":"2021-08-22T20:37:13.522482Z","shell.execute_reply":"2021-08-22T20:37:13.54776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_test_none2 = dfs_test_none2.merge(test_df[[\"id\", \"StudyInstanceUID\"]], left_on=\"ImageID\", right_on=\"id\")","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.550269Z","iopub.execute_input":"2021-08-22T20:37:13.550607Z","iopub.status.idle":"2021-08-22T20:37:13.56942Z","shell.execute_reply.started":"2021-08-22T20:37:13.550572Z","shell.execute_reply":"2021-08-22T20:37:13.568687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_test_none2 = dfs_test_none2.merge(df_study_orig[[\"study_id\", \"negative\", \"atypical\"]], left_on=\"StudyInstanceUID\", right_on=\"study_id\")","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.572816Z","iopub.execute_input":"2021-08-22T20:37:13.573062Z","iopub.status.idle":"2021-08-22T20:37:13.580566Z","shell.execute_reply.started":"2021-08-22T20:37:13.573038Z","shell.execute_reply":"2021-08-22T20:37:13.579688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PP 3\ndfs_test_none2[\"Conf\"] = 0.5*dfs_test_none2[\"Conf\"] + 0.5*dfs_test_none2[\"negative\"] + 0.2*dfs_test_none2[\"atypical\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.582045Z","iopub.execute_input":"2021-08-22T20:37:13.582662Z","iopub.status.idle":"2021-08-22T20:37:13.590501Z","shell.execute_reply.started":"2021-08-22T20:37:13.582617Z","shell.execute_reply":"2021-08-22T20:37:13.589573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def create_pred_str_study(row):\n#     items = []\n#     for label in [\"negative\", \"typical\", \"indeterminate\", \"atypical\"]:\n#         items += [label,row[label],'0','0','1','1']\n#     return ' '.join([str(i) for i in items])\n\n# def create_sub(test_preds, none_preds_image):\n#     df_study = pd.DataFrame({'id':study_ids})\n#     df_study['id'] = df_study['id'].astype(str) + '_study'\n#     df_study[[\"negative\", \"typical\", \"indeterminate\", \"atypical\"]] = test_preds\n#     df_study = df_study.groupby('id').agg('mean').reset_index()\n#     df_study[\"study_id\"] = df_study[\"id\"].str.replace(\"_study\", \"\")\n#     df_study = df_study.merge(none_preds_image, on=\"study_id\")\n    \n#     #PP 2\n#     df_study[\"negative\"] = df_study[\"Conf\"]\n#     df_study[\"id\"] = df_study[\"id_x\"]\n    \n#     df_study_orig = df_study.copy()\n#     df_study['PredictionString'] = df_study.apply(lambda row: create_pred_str_study(row), axis = 1)\n#     df_study= df_study[['id','PredictionString']].copy()\n#     df_study_orig[\"study_id\"] = df_study_orig[\"id\"].str.replace(\"_study\", \"\")\n#     return df_study, df_study_orig\n# df_study, df_study_orig = create_sub(test_preds_study, dfs_test_none2)\n# df_study = df_study.set_index('id')\n# df_study.head()\n\n# SAMPLE_SUBMISSION.loc[df_study.index,'PredictionString'] = df_study['PredictionString'].values","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.592209Z","iopub.execute_input":"2021-08-22T20:37:13.592765Z","iopub.status.idle":"2021-08-22T20:37:13.598821Z","shell.execute_reply.started":"2021-08-22T20:37:13.592729Z","shell.execute_reply":"2021-08-22T20:37:13.597891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del dfs_test_none2[\"negative\"]\ndel dfs_test_none2[\"atypical\"]\ndel dfs_test_none2[\"StudyInstanceUID\"]\ndel dfs_test_none2[\"study_id\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.600338Z","iopub.execute_input":"2021-08-22T20:37:13.601024Z","iopub.status.idle":"2021-08-22T20:37:13.608774Z","shell.execute_reply.started":"2021-08-22T20:37:13.600983Z","shell.execute_reply":"2021-08-22T20:37:13.607875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs_test_none2","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.610452Z","iopub.execute_input":"2021-08-22T20:37:13.611165Z","iopub.status.idle":"2021-08-22T20:37:13.646454Z","shell.execute_reply.started":"2021-08-22T20:37:13.611036Z","shell.execute_reply":"2021-08-22T20:37:13.64553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ensemble_boxes import weighted_boxes_fusion, non_maximum_weighted\nfrom joblib import Parallel, delayed\n# import warnings\n# import sys\n# if not sys.warnoptions:\n#     warnings.simplefilter(\"ignore\")\n\n#def run_blend_boxes():\ndef run_blend_boxes(df_image,image_id):\n    label = df_image['LabelName'].map({'opacity':0,'none':1}).values\n    box = df_image[['XMin','YMin','XMax','YMax']].values\n    conf = df_image['Conf'].values\n    box, conf, label  = weighted_boxes_fusion([box], [conf], [label], weights=None, iou_thr=0.5, conf_type=\"max\", skip_box_thr=0.0)\n    boxes = box\n    confs = conf\n    labels = label\n    image_ids = [image_id]*len(conf)\n    \n    return boxes, confs, labels, image_ids\n\ndef combine_boxes(df_image_pred):\n\n    image_ids_unique = df_image_pred['ImageID'].unique()\n    df_image_pred_gr = df_image_pred.groupby('ImageID')\n    boxes = []\n    confs = []\n    labels = []\n    image_ids = []\n    \n    res = Parallel(n_jobs=N_CORES)(delayed(run_blend_boxes)(df_image_pred_gr.get_group(image_id), image_id) for image_id in tqdm(image_ids_unique))\n\n    boxes = [x[0] for x in res]\n    confs = [x[1] for x in res]\n    labels = [x[2] for x in res]\n    image_ids = [x[3] for x in res]\n    \n    boxes = np.concatenate(boxes)\n    confs = np.concatenate(confs)\n    labels = np.concatenate(labels)\n    image_ids = np.concatenate(image_ids)\n\n    df = pd.DataFrame({'ImageID':image_ids,'LabelName':labels,'Conf':confs})\n    df[['XMin','YMin','XMax','YMax']] = boxes\n    df['LabelName'] = df['LabelName'].map({0:'opacity',1:'none'})\n    df = df[['ImageID','LabelName','Conf','XMin','XMax','YMin','YMax']]    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:13.647946Z","iopub.execute_input":"2021-08-22T20:37:13.648319Z","iopub.status.idle":"2021-08-22T20:37:14.678496Z","shell.execute_reply.started":"2021-08-22T20:37:13.648284Z","shell.execute_reply":"2021-08-22T20:37:14.677634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_opacity = combine_boxes(df_test_opacity)\ndf_test_opacity.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:37:14.679892Z","iopub.execute_input":"2021-08-22T20:37:14.680256Z","iopub.status.idle":"2021-08-22T20:38:37.263999Z","shell.execute_reply.started":"2021-08-22T20:37:14.680215Z","shell.execute_reply":"2021-08-22T20:38:37.262938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_opacity = df_test_opacity.merge(test_df[['id','width','height']],how='left',left_on='ImageID',right_on='id')\ndf_test_opacity.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.265944Z","iopub.execute_input":"2021-08-22T20:38:37.266635Z","iopub.status.idle":"2021-08-22T20:38:37.295749Z","shell.execute_reply.started":"2021-08-22T20:38:37.266589Z","shell.execute_reply":"2021-08-22T20:38:37.29462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = df_test_opacity['LabelName']=='opacity'\n\nw = df_test_opacity.loc[mask,'width'].values\nx1x2 = df_test_opacity.loc[mask,['XMin','XMax']].values\nx1x2 = x1x2 * w[:,None]\ndf_test_opacity.loc[mask,['XMin','XMax']] = x1x2\n\nh = df_test_opacity.loc[mask,'height'].values\ny1y2 = df_test_opacity.loc[mask,['YMin','YMax']].values\ny1y2 = y1y2 * h[:,None]\ndf_test_opacity.loc[mask,['YMin','YMax']] = y1y2\n\ndf_test_opacity.loc[~mask,['XMin','XMax','YMin','YMax']] = np.array([0,1,0,1])\n\ndf_test_opacity.shape, mask.sum()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.297432Z","iopub.execute_input":"2021-08-22T20:38:37.297982Z","iopub.status.idle":"2021-08-22T20:38:37.341421Z","shell.execute_reply.started":"2021-08-22T20:38:37.297937Z","shell.execute_reply":"2021-08-22T20:38:37.340416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_opacity=df_test_opacity[dfs_test_none2.columns].copy()","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.342819Z","iopub.execute_input":"2021-08-22T20:38:37.343221Z","iopub.status.idle":"2021-08-22T20:38:37.349508Z","shell.execute_reply.started":"2021-08-22T20:38:37.343183Z","shell.execute_reply":"2021-08-22T20:38:37.348477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_image_pred_test = pd.concat([df_test_opacity,dfs_test_none2])\ndf_image_pred_test","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.351256Z","iopub.execute_input":"2021-08-22T20:38:37.351874Z","iopub.status.idle":"2021-08-22T20:38:37.383262Z","shell.execute_reply.started":"2021-08-22T20:38:37.35183Z","shell.execute_reply":"2021-08-22T20:38:37.382084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAMPLE_SUBMISSION","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.38489Z","iopub.execute_input":"2021-08-22T20:38:37.385307Z","iopub.status.idle":"2021-08-22T20:38:37.399932Z","shell.execute_reply.started":"2021-08-22T20:38:37.385268Z","shell.execute_reply":"2021-08-22T20:38:37.398815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_image_ids = np.array([item for item in SAMPLE_SUBMISSION.index if item.endswith('_image')])\nsub_image_ids.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.402034Z","iopub.execute_input":"2021-08-22T20:38:37.402878Z","iopub.status.idle":"2021-08-22T20:38:37.413761Z","shell.execute_reply.started":"2021-08-22T20:38:37.402817Z","shell.execute_reply":"2021-08-22T20:38:37.412576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_str(id_):\n    df_tmp = df_image_pred_test[df_image_pred_test['ImageID'] == id_].copy()\n    preds = df_tmp[['LabelName','Conf','XMin','YMin','XMax','YMax']].values\n    preds = preds.reshape(-1)\n    return ' '.join([str(i) for i in preds])","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.415749Z","iopub.execute_input":"2021-08-22T20:38:37.416435Z","iopub.status.idle":"2021-08-22T20:38:37.423145Z","shell.execute_reply.started":"2021-08-22T20:38:37.416385Z","shell.execute_reply":"2021-08-22T20:38:37.422115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strs = [create_str(id_) for id_ in sub_image_ids]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:37.424997Z","iopub.execute_input":"2021-08-22T20:38:37.425841Z","iopub.status.idle":"2021-08-22T20:38:40.598907Z","shell.execute_reply.started":"2021-08-22T20:38:37.425656Z","shell.execute_reply":"2021-08-22T20:38:40.598064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAMPLE_SUBMISSION.loc[sub_image_ids,'PredictionString'] = strs","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:40.600207Z","iopub.execute_input":"2021-08-22T20:38:40.600548Z","iopub.status.idle":"2021-08-22T20:38:40.736199Z","shell.execute_reply.started":"2021-08-22T20:38:40.60051Z","shell.execute_reply":"2021-08-22T20:38:40.735273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAMPLE_SUBMISSION.to_csv('submission_image_ch_blend3.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:40.737526Z","iopub.execute_input":"2021-08-22T20:38:40.737996Z","iopub.status.idle":"2021-08-22T20:38:40.781029Z","shell.execute_reply.started":"2021-08-22T20:38:40.737955Z","shell.execute_reply":"2021-08-22T20:38:40.780284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAMPLE_SUBMISSION","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:40.782249Z","iopub.execute_input":"2021-08-22T20:38:40.782601Z","iopub.status.idle":"2021-08-22T20:38:40.795234Z","shell.execute_reply.started":"2021-08-22T20:38:40.782564Z","shell.execute_reply":"2021-08-22T20:38:40.794303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAMPLE_SUBMISSION[SAMPLE_SUBMISSION.PredictionString!=\"\"]","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:40.796818Z","iopub.execute_input":"2021-08-22T20:38:40.797676Z","iopub.status.idle":"2021-08-22T20:38:40.813665Z","shell.execute_reply.started":"2021-08-22T20:38:40.797633Z","shell.execute_reply":"2021-08-22T20:38:40.81287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAMPLE_SUBMISSION.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:40.820948Z","iopub.execute_input":"2021-08-22T20:38:40.821212Z","iopub.status.idle":"2021-08-22T20:38:40.862601Z","shell.execute_reply.started":"2021-08-22T20:38:40.821187Z","shell.execute_reply":"2021-08-22T20:38:40.861816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r test_025","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:40.864098Z","iopub.execute_input":"2021-08-22T20:38:40.864504Z","iopub.status.idle":"2021-08-22T20:38:41.576463Z","shell.execute_reply.started":"2021-08-22T20:38:40.864465Z","shell.execute_reply":"2021-08-22T20:38:41.575377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r test_05","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:41.581082Z","iopub.execute_input":"2021-08-22T20:38:41.581388Z","iopub.status.idle":"2021-08-22T20:38:42.286871Z","shell.execute_reply.started":"2021-08-22T20:38:41.581351Z","shell.execute_reply":"2021-08-22T20:38:42.285775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rd run_ch_1b","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:42.288507Z","iopub.execute_input":"2021-08-22T20:38:42.288849Z","iopub.status.idle":"2021-08-22T20:38:42.993683Z","shell.execute_reply.started":"2021-08-22T20:38:42.288808Z","shell.execute_reply":"2021-08-22T20:38:42.99248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rd run_ch_2b","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:42.998481Z","iopub.execute_input":"2021-08-22T20:38:42.998783Z","iopub.status.idle":"2021-08-22T20:38:43.707738Z","shell.execute_reply.started":"2021-08-22T20:38:42.998751Z","shell.execute_reply":"2021-08-22T20:38:43.706638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rd run_ch_3b","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:43.709492Z","iopub.execute_input":"2021-08-22T20:38:43.709895Z","iopub.status.idle":"2021-08-22T20:38:44.419491Z","shell.execute_reply.started":"2021-08-22T20:38:43.709853Z","shell.execute_reply":"2021-08-22T20:38:44.41831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rd run_ch_7_2cls_2","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:44.421232Z","iopub.execute_input":"2021-08-22T20:38:44.421657Z","iopub.status.idle":"2021-08-22T20:38:45.137003Z","shell.execute_reply.started":"2021-08-22T20:38:44.421614Z","shell.execute_reply":"2021-08-22T20:38:45.135722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rd configs","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:45.138783Z","iopub.execute_input":"2021-08-22T20:38:45.139246Z","iopub.status.idle":"2021-08-22T20:38:45.861071Z","shell.execute_reply.started":"2021-08-22T20:38:45.139202Z","shell.execute_reply":"2021-08-22T20:38:45.859974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rd models","metadata":{"execution":{"iopub.status.busy":"2021-08-22T20:38:45.862886Z","iopub.execute_input":"2021-08-22T20:38:45.863267Z","iopub.status.idle":"2021-08-22T20:38:46.705837Z","shell.execute_reply.started":"2021-08-22T20:38:45.863224Z","shell.execute_reply":"2021-08-22T20:38:46.704727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}