{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport csv\nimport torch\nimport numpy as np\nimport pandas as pd\nimport torch.nn as nn\nimport torchvision\nimport random\nimport matplotlib.pyplot as plt\nimport pandas as pd \nimport torch.optim as optim\nimport math\nimport cv2\nimport tensorflow as tf\nimport numpy.ma as ma\nfrom torch.nn import functional as F\n\nfrom sklearn.metrics import fbeta_score\nimport torch.utils.data as thd\nfrom PIL import Image\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import datasets\nimport gc\nimport glob\nimport json\nfrom collections import defaultdict\nimport multiprocessing as mp\nfrom pathlib import Path\nfrom types import SimpleNamespace\nfrom typing import Dict, List, Optional, Tuple\nimport warnings\n\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nimport numpy as np\nimport pandas as pd\nimport PIL.Image as Image\nfrom sklearn.exceptions import UndefinedMetricWarning\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as thd\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:38.261494Z","iopub.execute_input":"2023-06-15T13:00:38.262266Z","iopub.status.idle":"2023-06-15T13:00:43.031790Z","shell.execute_reply.started":"2023-06-15T13:00:38.262216Z","shell.execute_reply":"2023-06-15T13:00:43.030444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DoubleConv(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super(DoubleConv, self).__init__()\n        channels = int(out_channels / 2)\n        if in_channels > out_channels:\n            channels = int(in_channels / 2)\n\n        layers = [\n            nn.Conv2d(in_channels, channels, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(True),\n            nn.Conv2d(channels, out_channels, kernel_size=3, stride=1, padding=1),\n            nn.ReLU(True)\n        ]\n\n        self.double_conv = nn.Sequential(*layers)\n\n    def forward(self, x):\n        return self.double_conv(x)\n\nclass DownSampling(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super(DownSampling, self).__init__()\n        self.maxpool_to_conv = nn.Sequential(\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            DoubleConv(in_channels, out_channels)\n        )\n\n    def forward(self, x):\n        return self.maxpool_to_conv(x)\n\nclass UpSampling(nn.Module):\n    def __init__(self, in_channels, out_channels):\n        super(UpSampling, self).__init__()\n        self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n        self.conv = DoubleConv(in_channels + int(in_channels / 2), out_channels)\n\n    def forward(self, inputs1, inputs2):\n        \n        inputs1 = self.up(inputs1)\n        outputs = torch.cat([inputs1, inputs2], dim=1)\n        outputs = self.conv(outputs)\n        return outputs\n\nclass LastConv(nn.Module):\n    def __init__(self, in_channels, out_channels ):\n        super(LastConv, self).__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=1 )\n    def forward(self, x):\n        output = self.conv(x)\n        return output\n    \nclass InkDetector(nn.Module):\n    def __init__(self, in_channels=66):\n        super(InkDetector, self).__init__()\n        self.in_channels = in_channels\n        \n        self.inputs = DoubleConv(in_channels, 64,)\n        self.down_1 = DownSampling(64, 128)\n        self.down_2 = DownSampling(128, 256)\n        #self.down_3 = DownSampling(256, 512)\n\n        #self.up_1 = UpSampling(512, 256)\n        self.up_2 = UpSampling(256, 128)\n        self.up_3 = UpSampling(128, 64)\n        self.outputs = LastConv(64, 1)\n\n    def forward(self, x):\n        # down 部分\n        x1 = self.inputs(x)\n        x2 = self.down_1(x1)\n        x3 = self.down_2(x2)\n        #x4 = self.down_3(x3)\n        \n        # up部分\n        #x5 = self.up_1(x4, x3)\n        x6 = self.up_2(x3, x2)\n        x7 = self.up_3(x6, x1)\n        x = self.outputs(x7)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:43.036003Z","iopub.execute_input":"2023-06-15T13:00:43.037400Z","iopub.status.idle":"2023-06-15T13:00:43.059634Z","shell.execute_reply.started":"2023-06-15T13:00:43.037350Z","shell.execute_reply":"2023-06-15T13:00:43.058507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:43.061183Z","iopub.execute_input":"2023-06-15T13:00:43.062379Z","iopub.status.idle":"2023-06-15T13:00:43.100865Z","shell.execute_reply.started":"2023-06-15T13:00:43.062330Z","shell.execute_reply":"2023-06-15T13:00:43.099579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = InkDetector().to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:43.104714Z","iopub.execute_input":"2023-06-15T13:00:43.106139Z","iopub.status.idle":"2023-06-15T13:00:45.341369Z","shell.execute_reply.started":"2023-06-15T13:00:43.106097Z","shell.execute_reply":"2023-06-15T13:00:45.340216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LEARNING_RATE = 1e-5\nread = True # To avoid re-running when saving the notebook\nTRAIN_RUN = False\n\nWEIGHT_PATH = '/kaggle/input/kalen9099/32.pt'\nif read:\n    model_weights = torch.load(WEIGHT_PATH)\n    model.load_state_dict(model_weights) #TRAINING_STEPS","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:45.343056Z","iopub.execute_input":"2023-06-15T13:00:45.344249Z","iopub.status.idle":"2023-06-15T13:00:45.364936Z","shell.execute_reply.started":"2023-06-15T13:00:45.344200Z","shell.execute_reply":"2023-06-15T13:00:45.363798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BUFFER = 32\nif TRAIN_RUN:\n    index = 1\n    while(index <= 3):\n        print(\"Running dataset:\", index)\n        # training data location\n        \n        base_path = \"/kaggle/input/vesuvius-challenge/\"\n        testloader_PATH = base_path + \"train/\"\n        train_name = os.listdir(testloader_PATH+str(index)+'/surface_volume')\n        lable_name = os.listdir(testloader_PATH+str(index)) #讀取所有檔案名稱\n        print(testloader_PATH,str(index))\n        \n        train_PATH = []\n        lable_PATH = []\n        mask_PATH = []\n        for i in range(1):#所有檔案路徑 only dataset1\n            for j in range(65):\n                train_PATH.append(testloader_PATH+str(index)+'/surface_volume/'+train_name[j])\n            lable_PATH.append(testloader_PATH+str(index)+'/inklabels.png')\n            mask_PATH.append(testloader_PATH+str(index)+'/mask.png')\n          \n        train_dataset = []\n\n        transform = transforms.Compose([transforms.ToTensor()])\n        \n        for i in tqdm(range(65)): #讀取資料\n            mid = cv2.imread(train_PATH[i],cv2.IMREAD_GRAYSCALE)  # numpy數據\n            train_dataset.append(mid)\n            \n        lable_dataset = cv2.imread(lable_PATH[0],cv2.IMREAD_GRAYSCALE)\n        mask_dataset = cv2.imread(mask_PATH[0],cv2.IMREAD_GRAYSCALE)\n\n\n        del train_PATH\n        del lable_PATH\n        del mask_PATH\n\n      \n        for cut in range(20):\n            print(\"Running Cut:\", cut+1)\n\n            all_fragment = []\n            BUFFER = 32\n            epoch = 1000 #隨機切塊成n塊\n            transform = transforms.Compose([transforms.ToTensor()])\n            NP255 = np.ones((BUFFER*2,BUFFER*2))*255\n\n            for n in tqdm(range(epoch)):\n                x = random.randint(BUFFER, len(train_dataset[0]) - BUFFER)\n                y = random.randint(BUFFER, len(train_dataset[0][0]) - BUFFER)\n                while not (0.7*BUFFER*BUFFER > (lable_dataset[x-BUFFER:x+BUFFER,y-BUFFER:y+BUFFER] == NP255).sum() > 0.3*BUFFER*BUFFER) :\n                    x = random.randint(BUFFER, len(train_dataset[0]) - BUFFER)\n                    y = random.randint(BUFFER, len(train_dataset[0][0]) - BUFFER)\n\n                #存成[x,2,BATCH_SIZE,BUFFER,BUFFER],2指的是train的[66,BUFFER,BUFFER]和lable的[1,BUFFER,BUFFER]\n                temp = torch.zeros((len(train_dataset)+1,BUFFER*2,BUFFER*2))\n                mid = []\n                for a in range(len(train_dataset)):\n                    one = np.zeros((BUFFER*2,BUFFER*2))\n                    one[0:BUFFER*2,0:BUFFER*2] = train_dataset[a][x-BUFFER:x+BUFFER,y-BUFFER:y+BUFFER]\n                    if a == 0:\n                        temp[0] = transform(np.array(one.astype('uint8')))\n                    temp[a+1] = transform(np.array(one.astype('uint8')))\n                mid.append(temp)\n                temp = np.zeros((BUFFER*2,BUFFER*2))\n                temp[0:BUFFER*2,0:BUFFER*2] = lable_dataset[x-BUFFER:x+BUFFER,y-BUFFER:y+BUFFER]\n                mid.append(transform(np.array(temp.astype('uint8'))))\n                all_fragment.append(mid)\n\n            BATCH_SIZE = 10\n            train_loader = thd.DataLoader(all_fragment, batch_size=BATCH_SIZE, shuffle=False)\n            #print(\"Num batches:\", len(train_loader))\n\n            epoch = 10000\n            LEARNING_RATE = 1e-5\n            optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE, betas=(0.9, 0.999), eps=1e-09, weight_decay=0, amsgrad=False)\n            TRAINING_STEPS = len(train_loader)\n            print('TRAINING_STEPS:', TRAINING_STEPS)\n\n            fbeta_save = []\n            loss_save = []\n            for j in range(epoch):\n                criterion = nn.BCEWithLogitsLoss()\n                optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE)\n                scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=LEARNING_RATE, total_steps=TRAINING_STEPS)\n                model.train()\n                running_loss = 0.0\n                running_accuracy = 0.0\n                running_fbeta = 0.0\n                denom = 0\n                pbar = tqdm(enumerate(train_loader), total=TRAINING_STEPS)\n                for i, (subvolumes, inklabels) in pbar:\n                    if i >= TRAINING_STEPS:\n                        break\n                    optimizer.zero_grad()\n                    outputs = model(subvolumes.to(torch.float).to(DEVICE))\n                    loss = criterion(outputs.float(), inklabels.float().to(DEVICE))\n                    loss.backward()\n                    optimizer.step()\n                    scheduler.step()\n                    pred_ink = outputs.detach().sigmoid().gt(0.4).cpu().int()\n                    accuracy = (pred_ink == inklabels).sum().float()/(outputs.size(0)*outputs.size(1)*outputs.size(2)*outputs.size(3))\n                    running_fbeta += fbeta_score(inklabels.view(-1).numpy(), pred_ink.view(-1).numpy(), beta=0.5)\n                    running_accuracy += accuracy.item()\n                    running_loss += loss.item()\n                    denom += 1\n                    pbar.set_postfix({\"Loss\": running_loss / denom, \"Accuracy\": running_accuracy / denom, \"F0.5\": running_fbeta / denom, \"Epoch\": j})\n                    if (i + 1) % TRAINING_STEPS == 0:\n                        fbeta_save.append(running_fbeta / denom)\n                        loss_save.append(running_loss / denom)\n                        running_loss = 0.\n                        running_accuracy = 0.\n                        running_fbeta = 0.\n                        denom = 0\n                        torch.save(model.state_dict(), WEIGHT_PATH)\n\n        index = index + 1\n        del all_fragment\n        del lable_dataset\n        del mask_dataset\n        del subvolumes\n        del inklabels","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:45.367057Z","iopub.execute_input":"2023-06-15T13:00:45.367496Z","iopub.status.idle":"2023-06-15T13:00:45.407911Z","shell.execute_reply.started":"2023-06-15T13:00:45.367453Z","shell.execute_reply":"2023-06-15T13:00:45.406782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"warnings.simplefilter('ignore', UndefinedMetricWarning)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:45.409804Z","iopub.execute_input":"2023-06-15T13:00:45.410412Z","iopub.status.idle":"2023-06-15T13:00:45.423127Z","shell.execute_reply.started":"2023-06-15T13:00:45.410369Z","shell.execute_reply":"2023-06-15T13:00:45.421742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if TRAIN_RUN:\n    with torch.no_grad():\n        pbar = tqdm(enumerate(train_loader), total=TRAINING_STEPS)\n        for i,(subvolumes, inklabels) in pbar:\n            if i <20:\n                outputs = model(subvolumes.to(torch.float).to(DEVICE))\n                fig, (ax1, ax2, ax3, ax4) = plt.subplots(1,4)\n                #print(i)\n                #print(subvolumes.size())\n                #print(outputs.size())\n                #print(inklabels.size())\n                ax1.set_title(\"subvolumes\")\n                ax1.imshow(subvolumes[i][0], cmap = \"gray\")\n                ax2.set_title(\"outputs\")\n                ax2.imshow(outputs[i][0].cpu(), cmap = \"gray\")\n                ax4.set_title(\"inklabels\")\n                ax4.imshow(inklabels[i][0], cmap = \"gray\")\n                a = torch.empty((BUFFER*2,BUFFER*2))\n                for j in range(BUFFER*2):\n                    for k in range(BUFFER*2):\n                        a[j][k] = outputs[i][0][j][k].gt(0.4)\n                ax3.set_title(\"outputs to 0/1\")\n                ax3.imshow(a, cmap = \"gray\")\n                plt.show()\n\n    del train_dataset\n    del train_loader\n    del all_fragment\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:45.425127Z","iopub.execute_input":"2023-06-15T13:00:45.426171Z","iopub.status.idle":"2023-06-15T13:00:45.444197Z","shell.execute_reply.started":"2023-06-15T13:00:45.426098Z","shell.execute_reply":"2023-06-15T13:00:45.442850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Evaluate**","metadata":{}},{"cell_type":"code","source":"#del train_loader\n# test data location\nbase_path = \"/kaggle/input/vesuvius-challenge/\"\ntest_dir = \"a\"\ntestloader_PATH = base_path + \"test/\"\ntest_name = os.listdir(testloader_PATH+test_dir+'/surface_volume')\nlable_name = os.listdir(testloader_PATH+test_dir) #讀取所有檔案名稱\nprint(testloader_PATH+test_dir)\n\ntest_PATH = []\nmask_PATH = []\nfor i in range(1):#所有檔案路徑 only dataset1\n    for j in range(65):\n        test_PATH.append(testloader_PATH+test_dir+'/surface_volume/'+test_name[j])\n    mask_PATH.append(testloader_PATH+test_dir+'/mask.png')\n    \ntest_dataset = []\n\nnp.set_printoptions(precision=2) #控制小數精度\n\nfor i in tqdm(range(65)): #讀取資料\n    mid = cv2.imread(test_PATH[i],cv2.IMREAD_GRAYSCALE)  # numpy數據\n    test_dataset.append(mid)\n\nmask_dataset = cv2.imread(mask_PATH[0],cv2.IMREAD_GRAYSCALE)\n\ntest_PATH = None\nmask_PATH = None\ntest_name = None\nlable_name = None\nmid = None\ndel test_PATH\ndel mask_PATH\ndel test_name\ndel lable_name\ndel mid\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:45.446061Z","iopub.execute_input":"2023-06-15T13:00:45.446550Z","iopub.status.idle":"2023-06-15T13:00:53.765517Z","shell.execute_reply.started":"2023-06-15T13:00:45.446484Z","shell.execute_reply":"2023-06-15T13:00:53.763592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([transforms.ToTensor()])\nBUFFER = 32\nheight = len(mask_dataset)\nwidth = len(mask_dataset[0])\nprint(\"height:\", height, \" width:\", width)\n\niter_h = math.ceil(height / (BUFFER * 2))\niter_w = math.ceil(width / (BUFFER * 2))\nprint(\"iter_h:\", iter_h, \" iter_w:\", iter_w)\nprint(\"Buffer:\",BUFFER)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:53.770030Z","iopub.execute_input":"2023-06-15T13:00:53.770653Z","iopub.status.idle":"2023-06-15T13:00:53.781550Z","shell.execute_reply.started":"2023-06-15T13:00:53.770618Z","shell.execute_reply":"2023-06-15T13:00:53.778049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#存成[epoch,2,BATCH_SIZE,BUFFER,BUFFER]\ntemp_pad = torch.zeros(BUFFER*2,BUFFER*2)\nmid = []\nall_fragment = []\nx = 0\ny = 0\ntp = 0\nprint(len(test_dataset))\n\none = []\nfor i in tqdm(range(iter_h)):\n    for j in range(iter_w):\n        temp = torch.empty((len(test_dataset)+1,BUFFER*2,BUFFER*2))\n        for a in range(len(test_dataset)):\n            temp_pad = torch.zeros(BUFFER*2,BUFFER*2)\n            if y+(2*BUFFER)-1 > width and x+(2*BUFFER)-1 > height:\n                temp_pad[0:height-x, 0:width-y] = torch.from_numpy(test_dataset[a][x:height, y:width])\n                tp = 1\n            elif y+(2*BUFFER)-1 > width:\n                temp_pad[0:(2*BUFFER), 0:width-y] = transform(np.array(test_dataset[a][x:x+(2*BUFFER), y:width]))\n                tp = 2\n            elif x+(2*BUFFER)-1 > height:\n                temp_pad[0:height-x, 0:(2*BUFFER)] = transform(np.array(test_dataset[a][x:height, y:y+(2*BUFFER)]))\n                tp = 3\n            else:\n                temp_pad = transform(np.array(test_dataset[a][x:x+(2*BUFFER), y:y+(2*BUFFER)]))\n                tp = 4\n            if a == 0:\n                temp[a] = temp_pad\n            temp[a+1] = temp_pad\n        y = y + (BUFFER * 2)\n        if tp == 1:\n            y = 0\n            x = 0\n        elif tp == 2:\n            y = 0\n        all_fragment.append(temp)\n    x = x + BUFFER*2\n    temp = None\n    temp = None\n    del temp\n    del temp_pad\n    gc.collect()\n    \ntest_dataset = None\ndel test_dataset\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:00:53.783550Z","iopub.execute_input":"2023-06-15T13:00:53.783936Z","iopub.status.idle":"2023-06-15T13:01:22.209656Z","shell.execute_reply.started":"2023-06-15T13:00:53.783895Z","shell.execute_reply":"2023-06-15T13:01:22.208206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 1\ntest_loader = thd.DataLoader(all_fragment, batch_size=BATCH_SIZE, shuffle=False)\nTEST_STEPS = len(test_loader)\nprint('TEST_STEPS:', TEST_STEPS)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:01:22.211576Z","iopub.execute_input":"2023-06-15T13:01:22.212009Z","iopub.status.idle":"2023-06-15T13:01:22.219649Z","shell.execute_reply.started":"2023-06-15T13:01:22.211966Z","shell.execute_reply":"2023-06-15T13:01:22.218362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = 0\ny = 0\nm = 0\nn = 0\nall_a = torch.empty(BUFFER*2*iter_h,BUFFER*2*iter_w)\n\nwith torch.no_grad():\n    for i,subvolumes in enumerate(tqdm(test_loader)):\n        outputs = model(subvolumes.to(torch.float).to(DEVICE))\n        a = torch.empty(BUFFER*2,BUFFER*2)\n        a = outputs[0][0][0:BUFFER*2, 0:BUFFER*2].gt(0.4)*1\n        x =  m * BUFFER * 2 \n        y =  n * BUFFER * 2 \n        all_a[x:x+(BUFFER*2), y:y+(BUFFER*2)] = a[0:BUFFER*2, 0:BUFFER*2]\n        #next location\n        if n + 1 < iter_w:\n            n = n + 1\n        elif n + 1 == iter_w:\n            n = 0\n            m = m + 1\n        \n        if m  == iter_h:\n            print(\"break\")\n            break\n            \n\ntest_loader = None\nall_fragment = None\na =  None\ndel test_loader\ndel all_fragment\ndel a\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:01:22.221256Z","iopub.execute_input":"2023-06-15T13:01:22.222401Z","iopub.status.idle":"2023-06-15T13:02:33.697700Z","shell.execute_reply.started":"2023-06-15T13:01:22.222359Z","shell.execute_reply":"2023-06-15T13:02:33.696555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_a = all_a[0:height, 0:width]\nall_masked_a = all_a * mask_dataset","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:02:33.699548Z","iopub.execute_input":"2023-06-15T13:02:33.700340Z","iopub.status.idle":"2023-06-15T13:02:33.729805Z","shell.execute_reply.started":"2023-06-15T13:02:33.700293Z","shell.execute_reply":"2023-06-15T13:02:33.728744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1,ax2,ax3) = plt.subplots(1,3, figsize=(15, 15))\nax1.set_title(\"all_a\")\nax1.imshow(all_a, cmap = \"gray\")\nax2.set_title(\"mask\")\nax2.imshow(mask_dataset, cmap = \"gray\")\nax3.set_title(\"all_masked_a\")\nax3.imshow(all_masked_a, cmap = \"gray\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:02:33.731569Z","iopub.execute_input":"2023-06-15T13:02:33.731975Z","iopub.status.idle":"2023-06-15T13:02:35.973693Z","shell.execute_reply.started":"2023-06-15T13:02:33.731932Z","shell.execute_reply":"2023-06-15T13:02:35.972747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle(output):\n    output = output.view(-1)\n    flat_img = np.where(output > 0.4, 1, 0).astype(np.uint8)\n    temp = np.insert(flat_img,0,0)\n    temp = temp[0:len(temp)-1]\n    starts = np.array((flat_img[:-1] == 1) & (temp[:-1] == 0))\n    ends = np.array((flat_img[:-1] == 1) & (flat_img[1:] == 0))\n    starts_ix = np.where(starts)[0]\n    ends_ix = np.where(ends)[0]\n    lengths = ends_ix - starts_ix\n    output = None\n    flat_img = None\n    temp = None\n    starts = None\n    ends = None\n    ends_ix = None\n    del output\n    del flat_img\n    del temp\n    del starts\n    del ends\n    del ends_ix\n    return \" \".join(map(str, sum(zip(starts_ix, lengths), ())))","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:02:35.974960Z","iopub.execute_input":"2023-06-15T13:02:35.975654Z","iopub.status.idle":"2023-06-15T13:02:35.987772Z","shell.execute_reply.started":"2023-06-15T13:02:35.975612Z","shell.execute_reply":"2023-06-15T13:02:35.986372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = defaultdict(list)\nsubmission[\"Id\"].append(\"a\")\nsubmission[\"Predicted\"].append(rle(all_masked_a))\nsubmission[\"Id\"].append(\"b\")\nsubmission[\"Predicted\"].append(rle(all_masked_a))\nmask_dataset = None\nall_a = None\nall_masked_a = None\noutputs = None\ndel mask_dataset\ndel all_a\ndel all_masked_a\ndel outputs\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:02:35.989801Z","iopub.execute_input":"2023-06-15T13:02:35.990311Z","iopub.status.idle":"2023-06-15T13:03:33.849587Z","shell.execute_reply.started":"2023-06-15T13:02:35.990270Z","shell.execute_reply":"2023-06-15T13:03:33.848237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame.from_dict(submission).to_csv(\"/kaggle/working/submission.csv\", index=False)\npd.DataFrame.from_dict(submission)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T13:03:42.868586Z","iopub.execute_input":"2023-06-15T13:03:42.869106Z","iopub.status.idle":"2023-06-15T13:03:42.936857Z","shell.execute_reply.started":"2023-06-15T13:03:42.869066Z","shell.execute_reply":"2023-06-15T13:03:42.935623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}}]}