{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🛠 Install Libraries","metadata":{"papermill":{"duration":0.011749,"end_time":"2022-06-30T14:43:02.644399","exception":false,"start_time":"2022-06-30T14:43:02.63265","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# !pip install -q segmentation_models_pytorch\n!pip install -qU wandb\n!pip install -q scikit-learn==1.0","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:43:02.666875Z","iopub.status.busy":"2022-06-30T14:43:02.666395Z","iopub.status.idle":"2022-06-30T14:43:31.915242Z","shell.execute_reply":"2022-06-30T14:43:31.913912Z"},"papermill":{"duration":29.263876,"end_time":"2022-06-30T14:43:31.918726","exception":false,"start_time":"2022-06-30T14:43:02.65485","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📚 Import Libraries ","metadata":{"papermill":{"duration":0.016752,"end_time":"2022-06-30T14:43:31.95346","exception":false,"start_time":"2022-06-30T14:43:31.936708","status":"completed"},"tags":[]}},{"cell_type":"code","source":"! ls ../input/segmentation-pytorchmodel/segmentation_models_pytorch\n!cp -r ../input/segmentation-pytorchmodel/segmentation_models_pytorch/efficientnet_pytorch-0.6.3/ /tmp/st\n!cp -r ../input/segmentation-pytorchmodel/segmentation_models_pytorch/pretrainedmodels-0.7.4/ /tmp/st\n!pip install /tmp/st/efficientnet_pytorch-0.6.3/\n!pip install /tmp/st/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4\n!pip install segmentation_models_pytorch --no-index --find-links=../input/segmentation-pytorchmodel/segmentation_models_pytorch\n# !pip install resnet_pytorch --no-index --find-links=../input/resnet-pytorch020/resnet_pytorch-0.2.0","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:43:31.989043Z","iopub.status.busy":"2022-06-30T14:43:31.988567Z","iopub.status.idle":"2022-06-30T14:44:06.748307Z","shell.execute_reply":"2022-06-30T14:44:06.747174Z"},"papermill":{"duration":34.781027,"end_time":"2022-06-30T14:44:06.750727","exception":false,"start_time":"2022-06-30T14:43:31.9697","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%load_ext autoreload\n%autoreload 2","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:06.778641Z","iopub.status.busy":"2022-06-30T14:44:06.777172Z","iopub.status.idle":"2022-06-30T14:44:06.820113Z","shell.execute_reply":"2022-06-30T14:44:06.819267Z"},"papermill":{"duration":0.059029,"end_time":"2022-06-30T14:44:06.822191","exception":false,"start_time":"2022-06-30T14:44:06.763162","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\npd.options.plotting.backend = \"plotly\"\nimport random\nfrom glob import glob\nimport os, shutil\nfrom tqdm import tqdm\ntqdm.pandas()\nimport time\nimport copy\nimport joblib\nfrom collections import defaultdict\nimport gc\nfrom IPython import display as ipd\n\n# visualization\nimport cv2\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\n\n# Sklearn\nfrom sklearn.model_selection import StratifiedKFold, KFold, StratifiedGroupKFold\n\n# PyTorch \nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\n\nimport timm\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport rasterio\nfrom joblib import Parallel, delayed\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nc_  = Fore.GREEN\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"_kg_hide-input":false,"execution":{"iopub.execute_input":"2022-06-30T14:44:06.848551Z","iopub.status.busy":"2022-06-30T14:44:06.848284Z","iopub.status.idle":"2022-06-30T14:44:16.044424Z","shell.execute_reply":"2022-06-30T14:44:16.043413Z"},"papermill":{"duration":9.211905,"end_time":"2022-06-30T14:44:16.046983","exception":false,"start_time":"2022-06-30T14:44:06.835078","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⭐ WandB\n\n<img src=\"https://camo.githubusercontent.com/dd842f7b0be57140e68b2ab9cb007992acd131c48284eaf6b1aca758bfea358b/68747470733a2f2f692e696d6775722e636f6d2f52557469567a482e706e67\" width=\"400\" alt=\"Weights & Biases\" />\n\nWeights & Biases (W&B) is MLOps platform for tracking our experiemnts. We can use it to Build better models faster with experiment tracking, dataset versioning, and model management. Some of the cool features of **W&B**:\n\n* Track, compare, and visualize ML experiments\n* Get live metrics, terminal logs, and system stats streamed to the centralized dashboard.\n* Explain how your model works, show graphs of how model versions improved, discuss bugs, and demonstrate progress towards milestones.","metadata":{"papermill":{"duration":0.012525,"end_time":"2022-06-30T14:44:16.072079","exception":false,"start_time":"2022-06-30T14:44:16.059554","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\n\nsecret_label = \"WANDB\"\nsecret_value = UserSecretsClient().get_secret(secret_label)\nuser_secrets = UserSecretsClient()\napi_key = user_secrets.get_secret(\"WANDB\")\nwandb.login(key=api_key)\nanonymous = None","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:16.09818Z","iopub.status.busy":"2022-06-30T14:44:16.097666Z","iopub.status.idle":"2022-06-30T14:44:18.623055Z","shell.execute_reply":"2022-06-30T14:44:18.621955Z"},"papermill":{"duration":2.541372,"end_time":"2022-06-30T14:44:18.625703","exception":false,"start_time":"2022-06-30T14:44:16.084331","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    \n    secret_label = \"WANDB\"\n    secret_value = UserSecretsClient().get_secret(secret_label)\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret(\"WANDB\")\n    wandb.login(key=api_key)\n    anonymous = None\nexcept:\n    anonymous = \"must\"\n    print('To use your W&B account,\\nGo to Add-ons -> Secrets and provide your W&B access token. Use the Label name as WANDB. \\nGet your W&B access token from here: https://wandb.ai/authorize')","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:18.657754Z","iopub.status.busy":"2022-06-30T14:44:18.656826Z","iopub.status.idle":"2022-06-30T14:44:19.815892Z","shell.execute_reply":"2022-06-30T14:44:19.814941Z"},"papermill":{"duration":1.177422,"end_time":"2022-06-30T14:44:19.818164","exception":false,"start_time":"2022-06-30T14:44:18.640742","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.012708,"end_time":"2022-06-30T14:44:19.846156","exception":false,"start_time":"2022-06-30T14:44:19.833448","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⚙️ Configuration ","metadata":{"papermill":{"duration":0.012899,"end_time":"2022-06-30T14:44:19.871976","exception":false,"start_time":"2022-06-30T14:44:19.859077","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    seed          = 101\n    debug         = False # set debug=False for Full Training\n    exp_name      = '2.5D'\n    comment       = 'FPN_HFVF-efficientnet_b4-256x256-ep=12'\n    model_name    = 'FPN'\n    backbone      = 'efficientnet-b4'\n    train_bs      = 32\n    valid_bs      = train_bs*2\n    img_size      = [256, 256]\n    epochs        = 12\n    lr            = 2e-3\n    scheduler     = 'CosineAnnealingLR'\n    min_lr        = 1e-6\n    T_max         = int(30000/train_bs*epochs)+50\n    T_0           = 25\n    warmup_epochs = 0\n    wd            = 1e-6\n    n_accumulate  = max(1, 32//train_bs)\n    n_fold        = 5\n    folds         = [0]\n    num_classes   = 3\n    device        = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:19.899385Z","iopub.status.busy":"2022-06-30T14:44:19.899084Z","iopub.status.idle":"2022-06-30T14:44:20.018783Z","shell.execute_reply":"2022-06-30T14:44:20.017841Z"},"papermill":{"duration":0.136092,"end_time":"2022-06-30T14:44:20.021083","exception":false,"start_time":"2022-06-30T14:44:19.884991","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ❗ Reproducibility","metadata":{"papermill":{"duration":0.012746,"end_time":"2022-06-30T14:44:20.047812","exception":false,"start_time":"2022-06-30T14:44:20.035066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    print('> SEEDING DONE')\n    \nset_seed(CFG.seed)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:20.074979Z","iopub.status.busy":"2022-06-30T14:44:20.074335Z","iopub.status.idle":"2022-06-30T14:44:20.148091Z","shell.execute_reply":"2022-06-30T14:44:20.147065Z"},"papermill":{"duration":0.090249,"end_time":"2022-06-30T14:44:20.150768","exception":false,"start_time":"2022-06-30T14:44:20.060519","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📖 Meta Data\n\n* Files\n    * train.csv - IDs and masks for all training objects.\n    * sample_submission.csv - a sample submission file in the correct format\n    * train - a folder of case/day folders, each containing slice images for a particular case on a given day.\n\n> Note that the image filenames include 4 numbers (ex. `276_276_1.63_1.63.png`). These four numbers are slice height / width (integers in pixels) and heigh/width pixel spacing (floating points in mm). The first two defines the resolution of the slide. The last two record the physical size of each pixel.\n\n* Columns\n    * id - unique identifier for object\n    * class - the predicted class for the object\n    * EncodedPixels - RLE-encoded pixels for the identified object","metadata":{"papermill":{"duration":0.012488,"end_time":"2022-06-30T14:44:20.176831","exception":false,"start_time":"2022-06-30T14:44:20.164343","status":"completed"},"tags":[]}},{"cell_type":"code","source":"path_df = pd.DataFrame(glob('/kaggle/input/uwmgi-25d-stride2-dataset/images/images/*'), columns=['image_path'])\npath_df['mask_path'] = path_df.image_path.str.replace('image','mask')\npath_df['id'] = path_df.image_path.map(lambda x: x.split('/')[-1].replace('.npy',''))\npath_df.head()","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:20.203917Z","iopub.status.busy":"2022-06-30T14:44:20.20338Z","iopub.status.idle":"2022-06-30T14:44:21.168349Z","shell.execute_reply":"2022-06-30T14:44:21.167413Z"},"papermill":{"duration":0.980961,"end_time":"2022-06-30T14:44:21.170517","exception":false,"start_time":"2022-06-30T14:44:20.189556","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/uwmgi-mask-dataset/train.csv')\ndf['segmentation'] = df.segmentation.fillna('')\ndf['rle_len'] = df.segmentation.map(len) # length of each rle mask\n\ndf2 = df.groupby(['id'])['segmentation'].agg(list).to_frame().reset_index() # rle list of each id\ndf2 = df2.merge(df.groupby(['id'])['rle_len'].agg(sum).to_frame().reset_index()) # total length of all rles of each id\n\ndf = df.drop(columns=['segmentation', 'class', 'rle_len'])\ndf = df.groupby(['id']).head(1).reset_index(drop=True)\ndf = df.merge(df2, on=['id'])\ndf['empty'] = (df.rle_len==0) # empty masks\n\ndf = df.drop(columns=['image_path','mask_path'])\ndf = df.merge(path_df, on=['id'])\ndf.head()","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:21.200977Z","iopub.status.busy":"2022-06-30T14:44:21.199517Z","iopub.status.idle":"2022-06-30T14:44:23.192562Z","shell.execute_reply":"2022-06-30T14:44:23.191585Z"},"papermill":{"duration":2.010614,"end_time":"2022-06-30T14:44:23.19458","exception":false,"start_time":"2022-06-30T14:44:21.183966","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Remove Faulty","metadata":{"papermill":{"duration":0.013415,"end_time":"2022-06-30T14:44:23.22191","exception":false,"start_time":"2022-06-30T14:44:23.208495","status":"completed"},"tags":[]}},{"cell_type":"code","source":"fault1 = 'case7_day0'\nfault2 = 'case81_day30'\ndf = df[~df['id'].str.contains(fault1) & ~df['id'].str.contains(fault2)].reset_index(drop=True)\ndf.head()","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:23.251063Z","iopub.status.busy":"2022-06-30T14:44:23.250319Z","iopub.status.idle":"2022-06-30T14:44:23.377339Z","shell.execute_reply":"2022-06-30T14:44:23.376422Z"},"papermill":{"duration":0.143899,"end_time":"2022-06-30T14:44:23.379408","exception":false,"start_time":"2022-06-30T14:44:23.235509","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Class Distribution","metadata":{"papermill":{"duration":0.013841,"end_time":"2022-06-30T14:44:23.407266","exception":false,"start_time":"2022-06-30T14:44:23.393425","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df['empty'].value_counts().plot.bar()","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:23.435882Z","iopub.status.busy":"2022-06-30T14:44:23.435579Z","iopub.status.idle":"2022-06-30T14:44:26.067681Z","shell.execute_reply":"2022-06-30T14:44:26.066776Z"},"papermill":{"duration":2.648679,"end_time":"2022-06-30T14:44:26.069661","exception":false,"start_time":"2022-06-30T14:44:23.420982","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔨 Utility","metadata":{"papermill":{"duration":0.01399,"end_time":"2022-06-30T14:44:26.098056","exception":false,"start_time":"2022-06-30T14:44:26.084066","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Mask","metadata":{"papermill":{"duration":0.013869,"end_time":"2022-06-30T14:44:26.12603","exception":false,"start_time":"2022-06-30T14:44:26.112161","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def id2mask(id_):\n    idf = df[df['id']==id_]\n    wh = idf[['height','width']].iloc[0]\n    shape = (wh.height, wh.width, 3)\n    mask = np.zeros(shape, dtype=np.uint8)\n    for i, class_ in enumerate(['large_bowel', 'small_bowel', 'stomach']):\n        cdf = idf[idf['class']==class_]\n        rle = cdf.segmentation.squeeze()\n        if len(cdf) and not pd.isna(rle):\n            mask[..., i] = rle_decode(rle, shape[:2])\n    return mask\n\ndef rgb2gray(mask):\n    pad_mask = np.pad(mask, pad_width=[(0,0),(0,0),(1,0)])\n    gray_mask = pad_mask.argmax(-1)\n    return gray_mask\n\ndef gray2rgb(mask):\n    rgb_mask = tf.keras.utils.to_categorical(mask, num_classes=4)\n    return rgb_mask[..., 1:].astype(mask.dtype)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:26.155858Z","iopub.status.busy":"2022-06-30T14:44:26.15552Z","iopub.status.idle":"2022-06-30T14:44:26.239527Z","shell.execute_reply":"2022-06-30T14:44:26.238635Z"},"papermill":{"duration":0.101249,"end_time":"2022-06-30T14:44:26.241606","exception":false,"start_time":"2022-06-30T14:44:26.140357","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image","metadata":{"papermill":{"duration":0.015322,"end_time":"2022-06-30T14:44:26.271313","exception":false,"start_time":"2022-06-30T14:44:26.255991","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def load_img(path):\n    img = np.load(path)\n    img = img.astype('float32') # original is uint16\n    mx = np.max(img)\n    if mx:\n        img/=mx # scale image to [0, 1]\n    return img\n\ndef load_msk(path):\n    msk = np.load(path)\n    msk = msk.astype('float32')\n    msk/=255.0\n    return msk\n    \n\ndef show_img(img, mask=None):\n    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))\n    \n#     img = clahe.apply(img)\n#     plt.figure(figsize=(10,10))\n    plt.imshow(img, cmap='bone')\n    \n    if mask is not None:\n        # plt.imshow(np.ma.masked_where(mask!=1, mask), alpha=0.5, cmap='autumn')\n        plt.imshow(mask, alpha=0.5)\n        handles = [Rectangle((0,0),1,1, color=_c) for _c in [(0.667,0.0,0.0), (0.0,0.667,0.0), (0.0,0.0,0.667)]]\n        labels = [\"Large Bowel\", \"Small Bowel\", \"Stomach\"]\n        plt.legend(handles,labels)\n    plt.axis('off')","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:26.301356Z","iopub.status.busy":"2022-06-30T14:44:26.301064Z","iopub.status.idle":"2022-06-30T14:44:26.383552Z","shell.execute_reply":"2022-06-30T14:44:26.38267Z"},"papermill":{"duration":0.099471,"end_time":"2022-06-30T14:44:26.385623","exception":false,"start_time":"2022-06-30T14:44:26.286152","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## RLE","metadata":{"papermill":{"duration":0.014242,"end_time":"2022-06-30T14:44:26.414771","exception":false,"start_time":"2022-06-30T14:44:26.400529","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ref: https://www.kaggle.com/paulorzp/run-length-encode-and-decode\ndef rle_decode(mask_rle, shape):\n    '''\n    mask_rle: run-length as string formated (start length)\n    shape: (height,width) of array to return \n    Returns numpy array, 1 - mask, 0 - background\n\n    '''\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape)  # Needed to align to RLE direction\n\n\n# ref.: https://www.kaggle.com/stainsby/fast-tested-rle\ndef rle_encode(img):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    pixels = img.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:26.444827Z","iopub.status.busy":"2022-06-30T14:44:26.444041Z","iopub.status.idle":"2022-06-30T14:44:26.525932Z","shell.execute_reply":"2022-06-30T14:44:26.525084Z"},"papermill":{"duration":0.098935,"end_time":"2022-06-30T14:44:26.527939","exception":false,"start_time":"2022-06-30T14:44:26.429004","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📁 Create Folds","metadata":{"papermill":{"duration":0.014025,"end_time":"2022-06-30T14:44:26.556206","exception":false,"start_time":"2022-06-30T14:44:26.542181","status":"completed"},"tags":[]}},{"cell_type":"code","source":"skf = StratifiedGroupKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(df, df['empty'], groups = df[\"case\"])):\n    df.loc[val_idx, 'fold'] = fold\ndisplay(df.groupby(['fold','empty'])['id'].count())","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:26.586389Z","iopub.status.busy":"2022-06-30T14:44:26.585601Z","iopub.status.idle":"2022-06-30T14:44:26.815755Z","shell.execute_reply":"2022-06-30T14:44:26.814672Z"},"papermill":{"duration":0.247662,"end_time":"2022-06-30T14:44:26.818068","exception":false,"start_time":"2022-06-30T14:44:26.570406","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍚 Dataset","metadata":{"papermill":{"duration":0.014241,"end_time":"2022-06-30T14:44:26.847353","exception":false,"start_time":"2022-06-30T14:44:26.833112","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class BuildDataset(torch.utils.data.Dataset):\n    def __init__(self, df, label=True, transforms=None):\n        self.df         = df\n        self.label      = label\n        self.img_paths  = df['image_path'].tolist()\n        self.msk_paths  = df['mask_path'].tolist()\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path  = self.img_paths[index]\n        img = []\n        img = load_img(img_path)\n        \n        if self.label:\n            msk_path = self.msk_paths[index]\n            msk = load_msk(msk_path)\n            if self.transforms:\n                data = self.transforms(image=img, mask=msk)\n                img  = data['image']\n                msk  = data['mask']\n            img = np.transpose(img, (2, 0, 1))\n            msk = np.transpose(msk, (2, 0, 1))\n            return torch.tensor(img), torch.tensor(msk)\n        else:\n            if self.transforms:\n                data = self.transforms(image=img)\n                img  = data['image']\n            img = np.transpose(img, (2, 0, 1))\n            return torch.tensor(img)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:26.878245Z","iopub.status.busy":"2022-06-30T14:44:26.877948Z","iopub.status.idle":"2022-06-30T14:44:26.964322Z","shell.execute_reply":"2022-06-30T14:44:26.963406Z"},"papermill":{"duration":0.104747,"end_time":"2022-06-30T14:44:26.966935","exception":false,"start_time":"2022-06-30T14:44:26.862188","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🌈 Augmentations","metadata":{"papermill":{"duration":0.015276,"end_time":"2022-06-30T14:44:26.996992","exception":false,"start_time":"2022-06-30T14:44:26.981716","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_transforms = {\n    \"train\": A.Compose([\n#         A.Resize(*CFG.img_size, interpolation=cv2.INTER_NEAREST),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n#         A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.05, rotate_limit=10, p=0.5),\n#         A.OneOf([\n#             A.GridDistortion(num_steps=5, distort_limit=0.05, p=1.0),\n#             A.OpticalDistortion(distort_limit=0.05, shift_limit=0.05, p=1.0),\n#             A.ElasticTransform(alpha=1, sigma=50, alpha_affine=50, p=1.0)\n#         ], p=0.25),\n#         A.CoarseDropout(max_holes=8, max_height=CFG.img_size[0]//20, max_width=CFG.img_size[1]//20,\n#                          min_holes=5, fill_value=0, mask_fill_value=0, p=0.5),\n        ], p=1.0),\n    \n    \"valid\": A.Compose([\n#         A.Resize(*CFG.img_size, interpolation=cv2.INTER_NEAREST),\n        ], p=1.0)\n}","metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2022-06-30T14:44:27.027952Z","iopub.status.busy":"2022-06-30T14:44:27.027147Z","iopub.status.idle":"2022-06-30T14:44:27.106146Z","shell.execute_reply":"2022-06-30T14:44:27.105289Z"},"papermill":{"duration":0.096709,"end_time":"2022-06-30T14:44:27.108094","exception":false,"start_time":"2022-06-30T14:44:27.011385","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🍰 DataLoader","metadata":{"papermill":{"duration":0.014401,"end_time":"2022-06-30T14:44:27.13721","exception":false,"start_time":"2022-06-30T14:44:27.122809","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def prepare_loaders(fold, debug=False):\n    train_df = df.query(\"fold!=@fold\").reset_index(drop=True)\n    valid_df = df.query(\"fold==@fold\").reset_index(drop=True)\n    if debug:\n        train_df = train_df.head(32*5).query(\"empty==0\")\n        valid_df = valid_df.head(32*3).query(\"empty==0\")\n    train_dataset = BuildDataset(train_df, transforms=data_transforms['train'])\n    valid_dataset = BuildDataset(valid_df, transforms=data_transforms['valid'])\n\n    train_loader = DataLoader(train_dataset, batch_size=CFG.train_bs if not debug else 20, \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=False)\n    valid_loader = DataLoader(valid_dataset, batch_size=CFG.valid_bs if not debug else 20, \n                              num_workers=2, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader\n","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:27.169019Z","iopub.status.busy":"2022-06-30T14:44:27.167384Z","iopub.status.idle":"2022-06-30T14:44:27.25058Z","shell.execute_reply":"2022-06-30T14:44:27.249708Z"},"papermill":{"duration":0.101044,"end_time":"2022-06-30T14:44:27.252782","exception":false,"start_time":"2022-06-30T14:44:27.151738","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(fold=0, debug=True)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:27.284286Z","iopub.status.busy":"2022-06-30T14:44:27.284015Z","iopub.status.idle":"2022-06-30T14:44:27.386525Z","shell.execute_reply":"2022-06-30T14:44:27.385511Z"},"papermill":{"duration":0.12109,"end_time":"2022-06-30T14:44:27.388936","exception":false,"start_time":"2022-06-30T14:44:27.267846","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs, msks = next(iter(train_loader))\nimgs.size(), msks.size()","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:27.421351Z","iopub.status.busy":"2022-06-30T14:44:27.420547Z","iopub.status.idle":"2022-06-30T14:44:33.711656Z","shell.execute_reply":"2022-06-30T14:44:33.710353Z"},"papermill":{"duration":6.309766,"end_time":"2022-06-30T14:44:33.713999","exception":false,"start_time":"2022-06-30T14:44:27.404233","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📈 Visualization","metadata":{"papermill":{"duration":0.016707,"end_time":"2022-06-30T14:44:33.74765","exception":false,"start_time":"2022-06-30T14:44:33.730943","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def plot_batch(imgs, msks, size=3):\n    plt.figure(figsize=(5*5, 5))\n    for idx in range(size):\n        plt.subplot(1, 5, idx+1)\n        img = imgs[idx,].permute((1, 2, 0)).numpy()*255.0\n        img = img.astype('uint8')\n        msk = msks[idx,].permute((1, 2, 0)).numpy()*255.0\n        show_img(img, msk)\n    plt.tight_layout()\n    plt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2022-06-30T14:44:33.782461Z","iopub.status.busy":"2022-06-30T14:44:33.781366Z","iopub.status.idle":"2022-06-30T14:44:33.882251Z","shell.execute_reply":"2022-06-30T14:44:33.881101Z"},"papermill":{"duration":0.12107,"end_time":"2022-06-30T14:44:33.885026","exception":false,"start_time":"2022-06-30T14:44:33.763956","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_batch(imgs, msks, size=5)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:33.922093Z","iopub.status.busy":"2022-06-30T14:44:33.921731Z","iopub.status.idle":"2022-06-30T14:44:34.926685Z","shell.execute_reply":"2022-06-30T14:44:34.924833Z"},"papermill":{"duration":1.03196,"end_time":"2022-06-30T14:44:34.934596","exception":false,"start_time":"2022-06-30T14:44:33.902636","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:34.986305Z","iopub.status.busy":"2022-06-30T14:44:34.985976Z","iopub.status.idle":"2022-06-30T14:44:35.347408Z","shell.execute_reply":"2022-06-30T14:44:35.346318Z"},"papermill":{"duration":0.390196,"end_time":"2022-06-30T14:44:35.349961","exception":false,"start_time":"2022-06-30T14:44:34.959765","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📦 Model\n","metadata":{"papermill":{"duration":0.022808,"end_time":"2022-06-30T14:44:35.396532","exception":false,"start_time":"2022-06-30T14:44:35.373724","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import segmentation_models_pytorch as smp\n\ndef build_model():\n    model = smp.FPN(\n        encoder_name=CFG.backbone,      # choose encoder, e.g. mobilenet_v2 or efficientnet-b7\n        encoder_weights=\"imagenet\",     # use `imagenet` pre-trained weights for encoder initialization\n        in_channels=3,                  # model input channels (1 for gray-scale images, 3 for RGB, etc.)\n        classes=CFG.num_classes,        # model output channels (number of classes in your dataset)\n        activation=None,\n    )\n    model.to(CFG.device)\n    return model\n\ndef load_model(path):\n    model = build_model()\n    model.load_state_dict(torch.load(path))\n    model.eval()\n    return model","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:35.443865Z","iopub.status.busy":"2022-06-30T14:44:35.443332Z","iopub.status.idle":"2022-06-30T14:44:36.677929Z","shell.execute_reply":"2022-06-30T14:44:36.676905Z"},"papermill":{"duration":1.260968,"end_time":"2022-06-30T14:44:36.680531","exception":false,"start_time":"2022-06-30T14:44:35.419563","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔧 Loss Function","metadata":{"papermill":{"duration":0.023327,"end_time":"2022-06-30T14:44:36.7281","exception":false,"start_time":"2022-06-30T14:44:36.704773","status":"completed"},"tags":[]}},{"cell_type":"code","source":"JaccardLoss = smp.losses.JaccardLoss(mode='multilabel')\nDiceLoss    = smp.losses.DiceLoss(mode='multilabel')\nBCELoss     = smp.losses.SoftBCEWithLogitsLoss()\nLovaszLoss  = smp.losses.LovaszLoss(mode='multilabel', per_image=False)\nTverskyLoss = smp.losses.TverskyLoss(mode='multilabel', log_loss=False)\n\ndef dice_coef(y_true, y_pred, thr=0.5, dim=(2,3), epsilon=0.001):\n    y_true = y_true.to(torch.float32)\n    y_pred = (y_pred>thr).to(torch.float32)\n    inter = (y_true*y_pred).sum(dim=dim)\n    den = y_true.sum(dim=dim) + y_pred.sum(dim=dim)\n    dice = ((2*inter+epsilon)/(den+epsilon)).mean(dim=(1,0))\n    return dice\n\ndef iou_coef(y_true, y_pred, thr=0.5, dim=(2,3), epsilon=0.001):\n    y_true = y_true.to(torch.float32)\n    y_pred = (y_pred>thr).to(torch.float32)\n    inter = (y_true*y_pred).sum(dim=dim)\n    union = (y_true + y_pred - y_true*y_pred).sum(dim=dim)\n    iou = ((inter+epsilon)/(union+epsilon)).mean(dim=(1,0))\n    return iou\n\ndef criterion(y_pred, y_true):\n    return 0.5*BCELoss(y_pred, y_true) + 0.5*TverskyLoss(y_pred, y_true)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:36.776983Z","iopub.status.busy":"2022-06-30T14:44:36.776631Z","iopub.status.idle":"2022-06-30T14:44:36.867294Z","shell.execute_reply":"2022-06-30T14:44:36.866Z"},"papermill":{"duration":0.118019,"end_time":"2022-06-30T14:44:36.869862","exception":false,"start_time":"2022-06-30T14:44:36.751843","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚄 Training Function","metadata":{"papermill":{"duration":0.023846,"end_time":"2022-06-30T14:44:36.918632","exception":false,"start_time":"2022-06-30T14:44:36.894786","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n    model.train()\n    scaler = amp.GradScaler()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    pbar = tqdm(enumerate(dataloader), total=len(dataloader), desc='Train ')\n    for step, (images, masks) in pbar:         \n        images = images.to(device, dtype=torch.float)\n        masks  = masks.to(device, dtype=torch.float)\n        \n        batch_size = images.size(0)\n        \n        with amp.autocast(enabled=True):\n            y_pred = model(images)\n            loss   = criterion(y_pred, masks)\n            loss   = loss / CFG.n_accumulate\n            \n        scaler.scale(loss).backward()\n    \n        if (step + 1) % CFG.n_accumulate == 0:\n            scaler.step(optimizer)\n            scaler.update()\n\n            # zero the parameter gradients\n            optimizer.zero_grad()\n\n            if scheduler is not None:\n                scheduler.step()\n                \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0\n        current_lr = optimizer.param_groups[0]['lr']\n        pbar.set_postfix(train_loss=f'{epoch_loss:0.4f}',\n                        lr=f'{current_lr:0.5f}',\n                        gpu_mem=f'{mem:0.2f} GB')\n        torch.cuda.empty_cache()\n        gc.collect()\n    \n    return epoch_loss","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:36.969984Z","iopub.status.busy":"2022-06-30T14:44:36.968669Z","iopub.status.idle":"2022-06-30T14:44:37.05945Z","shell.execute_reply":"2022-06-30T14:44:37.058479Z"},"papermill":{"duration":0.119162,"end_time":"2022-06-30T14:44:37.061888","exception":false,"start_time":"2022-06-30T14:44:36.942726","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 👀 Validation Function","metadata":{"papermill":{"duration":0.023992,"end_time":"2022-06-30T14:44:37.110381","exception":false,"start_time":"2022-06-30T14:44:37.086389","status":"completed"},"tags":[]}},{"cell_type":"code","source":"@torch.no_grad()\ndef valid_one_epoch(model, dataloader, device, epoch):\n    model.eval()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    val_scores = []\n    \n    pbar = tqdm(enumerate(dataloader), total=len(dataloader), desc='Valid ')\n    for step, (images, masks) in pbar:        \n        images  = images.to(device, dtype=torch.float)\n        masks   = masks.to(device, dtype=torch.float)\n        \n        batch_size = images.size(0)\n        \n        y_pred  = model(images)\n        loss    = criterion(y_pred, masks)\n        \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        y_pred = nn.Sigmoid()(y_pred)\n        val_dice = dice_coef(masks, y_pred).cpu().detach().numpy()\n        val_jaccard = iou_coef(masks, y_pred).cpu().detach().numpy()\n        val_scores.append([val_dice, val_jaccard])\n        \n        mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0\n        current_lr = optimizer.param_groups[0]['lr']\n        pbar.set_postfix(valid_loss=f'{epoch_loss:0.4f}',\n                        lr=f'{current_lr:0.5f}',\n                        gpu_memory=f'{mem:0.2f} GB')\n    val_scores  = np.mean(val_scores, axis=0)\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    return epoch_loss, val_scores","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:37.158849Z","iopub.status.busy":"2022-06-30T14:44:37.157981Z","iopub.status.idle":"2022-06-30T14:44:37.247841Z","shell.execute_reply":"2022-06-30T14:44:37.246817Z"},"papermill":{"duration":0.117286,"end_time":"2022-06-30T14:44:37.250669","exception":false,"start_time":"2022-06-30T14:44:37.133383","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🏃 Run Training","metadata":{"papermill":{"duration":0.023871,"end_time":"2022-06-30T14:44:37.299204","exception":false,"start_time":"2022-06-30T14:44:37.275333","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n    # To automatically log gradients\n    wandb.watch(model, log_freq=100)\n    \n    if torch.cuda.is_available():\n        print(\"cuda: {}\\n\".format(torch.cuda.get_device_name()))\n    \n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_dice      = -np.inf\n    best_epoch     = -1\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1): \n        gc.collect()\n        print(f'Epoch {epoch}/{num_epochs}', end='')\n        train_loss = train_one_epoch(model, optimizer, scheduler, \n                                           dataloader=train_loader, \n                                           device=CFG.device, epoch=epoch)\n        \n        val_loss, val_scores = valid_one_epoch(model, valid_loader, \n                                                 device=CFG.device, \n                                                 epoch=epoch)\n        val_dice, val_jaccard = val_scores\n    \n        history['Train Loss'].append(train_loss)\n        history['Valid Loss'].append(val_loss)\n        history['Valid Dice'].append(val_dice)\n        history['Valid Jaccard'].append(val_jaccard)\n        \n        # Log the metrics\n        wandb.log({\"Train Loss\": train_loss, \n                   \"Valid Loss\": val_loss,\n                   \"Valid Dice\": val_dice,\n                   \"Valid Jaccard\": val_jaccard,\n                   \"LR\":scheduler.get_last_lr()[0]})\n        \n        print(f'Valid Dice: {val_dice:0.4f} | Valid Jaccard: {val_jaccard:0.4f}')\n        \n        # deep copy the model\n        if val_dice >= best_dice:\n            print(f\"{c_}Valid Score Improved ({best_dice:0.4f} ---> {val_dice:0.4f})\")\n            best_dice    = val_dice\n            best_jaccard = val_jaccard\n            best_epoch   = epoch\n            run.summary[\"Best Dice\"]    = best_dice\n            run.summary[\"Best Jaccard\"] = best_jaccard\n            run.summary[\"Best Epoch\"]   = best_epoch\n            best_model_wts = copy.deepcopy(model.state_dict())\n            PATH = f\"best_epoch-{fold:02d}.bin\"\n            torch.save(model.state_dict(), PATH)\n            # Save a model file from the current directory\n            wandb.save(PATH)\n            print(f\"Model Saved{sr_}\")\n            \n        last_model_wts = copy.deepcopy(model.state_dict())\n        PATH = f\"last_epoch-{fold:02d}.bin\"\n        torch.save(model.state_dict(), PATH)\n            \n        print(); print()\n    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best Score: {:.4f}\".format(best_jaccard))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    return model, history","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:37.348247Z","iopub.status.busy":"2022-06-30T14:44:37.347893Z","iopub.status.idle":"2022-06-30T14:44:37.442869Z","shell.execute_reply":"2022-06-30T14:44:37.441846Z"},"papermill":{"duration":0.122135,"end_time":"2022-06-30T14:44:37.445421","exception":false,"start_time":"2022-06-30T14:44:37.323286","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔍 Optimizer\n\n<img src=\"https://mlfromscratch.com/content/images/2019/12/saddle.gif\" width=500>","metadata":{"papermill":{"duration":0.023421,"end_time":"2022-06-30T14:44:37.492695","exception":false,"start_time":"2022-06-30T14:44:37.469274","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CFG.scheduler == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CFG.T_max, \n                                                   eta_min=CFG.min_lr)\n    elif CFG.scheduler == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CFG.T_0, \n                                                             eta_min=CFG.min_lr)\n    elif CFG.scheduler == 'ReduceLROnPlateau':\n        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer,\n                                                   mode='min',\n                                                   factor=0.1,\n                                                   patience=7,\n                                                   threshold=0.0001,\n                                                   min_lr=CFG.min_lr,)\n    elif CFG.scheduer == 'ExponentialLR':\n        scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.85)\n    elif CFG.scheduler == None:\n        return None\n        \n    return scheduler","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:37.541969Z","iopub.status.busy":"2022-06-30T14:44:37.541305Z","iopub.status.idle":"2022-06-30T14:44:37.626619Z","shell.execute_reply":"2022-06-30T14:44:37.625591Z"},"papermill":{"duration":0.11213,"end_time":"2022-06-30T14:44:37.628908","exception":false,"start_time":"2022-06-30T14:44:37.516778","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model()\noptimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.wd)\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T14:44:37.678591Z","iopub.status.busy":"2022-06-30T14:44:37.678159Z","iopub.status.idle":"2022-06-30T14:44:50.652477Z","shell.execute_reply":"2022-06-30T14:44:50.651487Z"},"papermill":{"duration":13.001953,"end_time":"2022-06-30T14:44:50.654923","exception":false,"start_time":"2022-06-30T14:44:37.65297","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🚅 Training","metadata":{"papermill":{"duration":0.023785,"end_time":"2022-06-30T14:44:50.703101","exception":false,"start_time":"2022-06-30T14:44:50.679316","status":"completed"},"tags":[]}},{"cell_type":"code","source":"for fold in CFG.folds:\n    print(f'#'*15)\n    print(f'### Fold: {fold}')\n    print(f'#'*15)\n    run = wandb.init(project='uwmadison_challenge', entity=\"patronuseeker\",\n                     config={k:v for k, v in dict(vars(CFG)).items() if '__' not in k},\n                     anonymous=anonymous,\n                     name=f\"fold-{fold}|dim-{CFG.img_size[0]}x{CFG.img_size[1]}|model-{CFG.model_name}\",\n                     group=CFG.comment,\n                    )\n    train_loader, valid_loader = prepare_loaders(fold=fold, debug=CFG.debug)\n    model     = build_model()\n    \n    \n\n    # Optional\n    optimizer = optim.Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.wd)\n    scheduler = fetch_scheduler(optimizer)\n    model, history = run_training(model, optimizer, scheduler,\n                                  device=CFG.device,\n                                  num_epochs=CFG.epochs)\n    run.finish()\n    display(ipd.IFrame(run.url, width=1000, height=720))","metadata":{"_kg_hide-output":false,"execution":{"iopub.execute_input":"2022-06-30T14:44:50.750935Z","iopub.status.busy":"2022-06-30T14:44:50.750584Z","iopub.status.idle":"2022-06-30T20:51:17.99401Z","shell.execute_reply":"2022-06-30T20:51:17.992894Z"},"papermill":{"duration":21987.269829,"end_time":"2022-06-30T20:51:17.996171","exception":false,"start_time":"2022-06-30T14:44:50.726342","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model,'FPN_hfvf_7thJuly')","metadata":{"execution":{"iopub.execute_input":"2022-06-30T20:51:20.704446Z","iopub.status.busy":"2022-06-30T20:51:20.704053Z","iopub.status.idle":"2022-06-30T20:51:21.782616Z","shell.execute_reply":"2022-06-30T20:51:21.781631Z"},"papermill":{"duration":2.380711,"end_time":"2022-06-30T20:51:21.785058","exception":false,"start_time":"2022-06-30T20:51:19.404347","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ","metadata":{"execution":{"iopub.execute_input":"2022-06-30T20:51:24.758371Z","iopub.status.busy":"2022-06-30T20:51:24.757996Z","iopub.status.idle":"2022-06-30T20:51:25.66601Z","shell.execute_reply":"2022-06-30T20:51:25.664542Z"},"papermill":{"duration":2.556299,"end_time":"2022-06-30T20:51:25.669106","exception":false,"start_time":"2022-06-30T20:51:23.112807","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔭 Prediction","metadata":{"papermill":{"duration":1.419877,"end_time":"2022-06-30T20:51:28.985407","exception":false,"start_time":"2022-06-30T20:51:27.56553","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test_dataset = BuildDataset(df.query(\"fold==0 & empty==0\").sample(frac=1.0), label=False, \n                            transforms=data_transforms['valid'])\ntest_loader  = DataLoader(test_dataset, batch_size=5, \n                          num_workers=2, shuffle=False, pin_memory=True)\nimgs = next(iter(test_loader))\nimgs = imgs.to(CFG.device, dtype=torch.float)\n\npreds = []\nfor fold in CFG.folds:\n    model = load_model(f\"best_epoch-{fold:02d}.bin\")\n    with torch.no_grad():\n        pred = model(imgs)\n        pred = (nn.Sigmoid()(pred)>0.5).double()\n    preds.append(pred)\n    \nimgs  = imgs.cpu().detach()\npreds = torch.mean(torch.stack(preds, dim=0), dim=0).cpu().detach()","metadata":{"execution":{"iopub.execute_input":"2022-06-30T20:51:31.720866Z","iopub.status.busy":"2022-06-30T20:51:31.720457Z","iopub.status.idle":"2022-06-30T20:51:33.22437Z","shell.execute_reply":"2022-06-30T20:51:33.223235Z"},"papermill":{"duration":2.928267,"end_time":"2022-06-30T20:51:33.227186","exception":false,"start_time":"2022-06-30T20:51:30.298919","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_batch(imgs, preds, size=5)","metadata":{"execution":{"iopub.execute_input":"2022-06-30T20:51:36.282397Z","iopub.status.busy":"2022-06-30T20:51:36.281834Z","iopub.status.idle":"2022-06-30T20:51:37.269168Z","shell.execute_reply":"2022-06-30T20:51:37.267484Z"},"papermill":{"duration":2.754743,"end_time":"2022-06-30T20:51:37.275502","exception":false,"start_time":"2022-06-30T20:51:34.520759","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ✂️ Remove Files","metadata":{"papermill":{"duration":1.400133,"end_time":"2022-06-30T20:51:39.979381","exception":false,"start_time":"2022-06-30T20:51:38.579248","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!rm -r ./wandb","metadata":{"execution":{"iopub.execute_input":"2022-06-30T20:51:42.698938Z","iopub.status.busy":"2022-06-30T20:51:42.698569Z","iopub.status.idle":"2022-06-30T20:51:43.515198Z","shell.execute_reply":"2022-06-30T20:51:43.513923Z"},"papermill":{"duration":2.234077,"end_time":"2022-06-30T20:51:43.518136","exception":false,"start_time":"2022-06-30T20:51:41.284059","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}