{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport glob\nimport torch\nimport numpy as np\nimport torch.nn as nn\nfrom tqdm import tqdm\nimport PIL.Image as Image\nimport matplotlib.pyplot as plt\nimport torch.utils.data as data\n\n# This is in size order to avoid OOM issues...\nPREFIX_1 = ['/kaggle/input/vesuvius-challenge-ink-detection/train/1/', '/kaggle/input/vevuvius-v5/train_1_v5_layer_1.npy','/kaggle/input/vevuvius-v5/train_1_v5_layer_1.npy', [16,18,20,22,24,26,28,30,32,34]]\nPREFIX_2 = ['/kaggle/input/vesuvius-challenge-ink-detection/train/2/', '/kaggle/input/vevuvius-v5/train_2_v5_layer_1.npy','/kaggle/input/vevuvius-v5/train_2_v5_layer_1.npy', [22,24,26,28,30,32,34,36,38,40]]\nPREFIX_3 = ['/kaggle/input/vesuvius-challenge-ink-detection/train/3/', '/kaggle/input/vevuvius-v5/train_3_v5_layer_1.npy','/kaggle/input/vevuvius-v5/train_3_v5_layer_1.npy', [20,22,24,26,28,30,32,34,36,38]]\nPREFIX_4 = ['/kaggle/input/vesuvius-challenge-ink-detection/test/a/', '/kaggle/input/vesuvius-v4-files/train_4_BCE_layer_1.npy']\nPREFIX_5 = ['/kaggle/input/vesuvius-challenge-ink-detection/test/b/', '/kaggle/input/vesuvius-v4-files/train_5_BCE_layer_1.npy']\n\nBUFFER_1 = 30  # Buffer size in x and y direction\nWINDOW_1 = 1  # Window = pixels in label in x and y direction \nBUFFER_2 = 60\nWINDOW_2 = 2\nBUFFER_3 = 120\nWINDOW_3 = 3\n\nBATCH_SIZE = 32\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","metadata":{"execution":{"iopub.status.busy":"2023-06-14T01:30:32.430721Z","iopub.execute_input":"2023-06-14T01:30:32.431481Z","iopub.status.idle":"2023-06-14T01:30:35.702591Z","shell.execute_reply.started":"2023-06-14T01:30:32.431450Z","shell.execute_reply":"2023-06-14T01:30:35.701795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SubvolumeDataset(data.Dataset):\n    def __init__(self, image_stack, pixels, BUFFER):\n        self.image_stack = image_stack\n        self.pixels = pixels\n        self.BUFFER = BUFFER\n    def __len__(self):\n        return len(self.pixels)\n    def __getitem__(self, index):\n        y, x = self.pixels[index]\n        subvolume = self.image_stack[:, y-self.BUFFER:y+self.BUFFER+1, x-self.BUFFER:x+self.BUFFER+1].view(1, len(self.image_stack), self.BUFFER*2+1, self.BUFFER*2+1)\n        return subvolume\n    ","metadata":{"execution":{"iopub.status.busy":"2023-06-14T01:30:39.579559Z","iopub.execute_input":"2023-06-14T01:30:39.580136Z","iopub.status.idle":"2023-06-14T01:30:39.589990Z","shell.execute_reply.started":"2023-06-14T01:30:39.580102Z","shell.execute_reply":"2023-06-14T01:30:39.587779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !!! This builds a dataloader to process layer 1 !!!\n\nPREFIX = PREFIX_3\n\nimage_selections = PREFIX[3]\nZ_DIM = len(image_selections) # We add x to this if we're including prediction layer(s).\nimg_layers = sorted(glob.glob(PREFIX[0]+\"surface_volume/*.tif\"))\n\nimages = [np.array(Image.open(img_layers[i-1]), dtype=np.float32)/65535.0 for i in tqdm(image_selections)]\nimage_stack = torch.stack([torch.from_numpy(image) for image in images], dim=0).to(DEVICE)\ndel images\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\ndel not_border, arr_mask, grid\n\n# eval_dataset = SubvolumeDataset(image_stack, label, all_pixels)\neval_dataset = SubvolumeDataset(image_stack, all_pixels, BUFFER_1)\n\ndel image_stack\ngc.collect()\n\neval_loader = data.DataLoader(eval_dataset, batch_size=BATCH_SIZE, shuffle=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !!! This builds a dataloader to process layer 2 !!!\n\nPREFIX = PREFIX_3\n\nimage_selections = PREFIX[3]\nZ_DIM = len(image_selections) + 1 # We add x to this if we're including prediction layer(s).\nimg_layers = sorted(glob.glob(PREFIX[0]+\"surface_volume/*.tif\"))\n\nimages = [np.array(Image.open(img_layers[i-1]), dtype=np.float32)/65535.0 for i in tqdm(image_selections)]\nimage_stack = torch.stack([torch.from_numpy(image) for image in images], dim=0).to(DEVICE)\ndel images\n\noutput_1 = np.float32(np.load(PREFIX[1]))\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nfor i in range(len(output_1)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output_1[i].reshape(3,3)\n\ndel output_1\n\nimage_stack = torch.concatenate((torch.tensor(full_rect, dtype=torch.float32).reshape(1,mask.shape[0],mask.shape[1]).to(DEVICE), image_stack))\ndel not_border, arr_mask, grid, full_rect, mask\n# The image stack is now complete.\n\n# This is remaking the pixel list with the correct window and buffer size.\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_2:mask.shape[0]-BUFFER_2, BUFFER_2:mask.shape[1]-BUFFER_2] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_2\nwhile i < grid.shape[0]:\n    j = WINDOW_2\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_2+1\n    i += 2*WINDOW_2+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\n# eval_dataset = SubvolumeDataset(image_stack, label, all_pixels)\neval_dataset = SubvolumeDataset(image_stack, all_pixels, BUFFER_2)\ndel image_stack, all_pixels\n\neval_loader = data.DataLoader(eval_dataset, batch_size=BATCH_SIZE, shuffle=False)\ndel eval_dataset\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T01:45:13.054886Z","iopub.execute_input":"2023-06-14T01:45:13.055664Z","iopub.status.idle":"2023-06-14T01:45:46.575786Z","shell.execute_reply.started":"2023-06-14T01:45:13.055621Z","shell.execute_reply":"2023-06-14T01:45:46.574698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n# !!! THIS SECTION RUNS THE MODEL AND SAVES THE OUTPUT !!!\n# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n# TO RUN ON A CPU, WHEN MODEL TRAINED ON GPU:\n#model = torch.load('/kaggle/input/uploads/bce_model_L1.pth', map_location=torch.device('cpu')).to(DEVICE)\n\n# TO RUN ON A GPU, WHEN MODEL TRAINED ON GPU:\nmodel = torch.load('/kaggle/input/vevuvius-v5/v5_model_L2.pth').to(DEVICE)\n\noutput=[]\nmodel.eval()\nwith torch.no_grad():\n    for i, (subvolumes) in enumerate(tqdm(eval_loader)):\n        for j, value in enumerate(model(subvolumes.to(DEVICE))):\n            output.append(value.tolist())\n\n# Saving the files.\nnp_output = np.array(output)\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.08:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(np_output > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \nnp_output = torch.tensor(np_output).gt(THRESHOLD).cpu()\n\nnp.save(\"train_3_BCE_layer_2.npy\", np_output)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T01:45:46.577425Z","iopub.execute_input":"2023-06-14T01:45:46.577720Z","iopub.status.idle":"2023-06-14T01:56:31.953702Z","shell.execute_reply.started":"2023-06-14T01:45:46.577694Z","shell.execute_reply":"2023-06-14T01:56:31.952739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n# !!! Here we're loading the saved data to review it. !!!\n# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_5\noutput = np.load('/kaggle/working/train_5_BCE_layer_2.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER:mask.shape[0]-BUFFER, BUFFER:mask.shape[1]-BUFFER] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW\nwhile i < grid.shape[0]:\n    j = WINDOW\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW+1\n    i += 2*WINDOW+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))","metadata":{"execution":{"iopub.status.busy":"2023-05-05T19:21:27.212879Z","iopub.execute_input":"2023-05-05T19:21:27.213249Z","iopub.status.idle":"2023-05-05T19:21:29.893806Z","shell.execute_reply.started":"2023-05-05T19:21:27.213217Z","shell.execute_reply":"2023-05-05T19:21:29.891886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW:pixel_y+WINDOW+1,pixel_x-WINDOW:pixel_x+WINDOW+1] = output[i].reshape(3,3)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T19:21:29.895408Z","iopub.execute_input":"2023-05-05T19:21:29.895678Z","iopub.status.idle":"2023-05-05T19:21:36.257347Z","shell.execute_reply.started":"2023-05-05T19:21:29.895654Z","shell.execute_reply":"2023-05-05T19:21:36.256505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_5\noutput = np.load('/kaggle/working/train_5_BCE_layer_1.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER:mask.shape[0]-BUFFER, BUFFER:mask.shape[1]-BUFFER] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW\nwhile i < grid.shape[0]:\n    j = WINDOW\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW+1\n    i += 2*WINDOW+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW:pixel_y+WINDOW+1,pixel_x-WINDOW:pixel_x+WINDOW+1] = output[i].reshape(3,3)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T19:23:09.162243Z","iopub.execute_input":"2023-05-05T19:23:09.162611Z","iopub.status.idle":"2023-05-05T19:23:17.964842Z","shell.execute_reply.started":"2023-05-05T19:23:09.162583Z","shell.execute_reply":"2023-05-05T19:23:17.96401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n# !!! Everything below here is experimental/test !!!\n# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!","metadata":{"execution":{"iopub.status.busy":"2023-05-03T22:02:48.144332Z","iopub.execute_input":"2023-05-03T22:02:48.14481Z","iopub.status.idle":"2023-05-03T22:02:50.073856Z","shell.execute_reply.started":"2023-05-03T22:02:48.144776Z","shell.execute_reply":"2023-05-03T22:02:50.072484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_1\noutput_1 = np.float32(np.load(PREFIX[1]))\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nfor i in range(len(output_1)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output_1[i].reshape(3,3)\n\ndel output_1\n\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T16:30:11.541907Z","iopub.execute_input":"2023-06-13T16:30:11.542836Z","iopub.status.idle":"2023-06-13T16:30:28.141349Z","shell.execute_reply.started":"2023-06-13T16:30:11.542798Z","shell.execute_reply":"2023-06-13T16:30:28.140559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_1\noutput_1 = np.float32(np.load(PREFIX[2]))\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nfor i in range(len(output_1)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output_1[i].reshape(3,3)\n\ndel output_1\n\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T16:30:31.648127Z","iopub.execute_input":"2023-06-13T16:30:31.648519Z","iopub.status.idle":"2023-06-13T16:30:46.348514Z","shell.execute_reply.started":"2023-06-13T16:30:31.648489Z","shell.execute_reply":"2023-06-13T16:30:46.347679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_3\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_3_BCE_layer_1.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.20:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(full_rect > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \nlayer_2_input = torch.tensor(full_rect).gt(THRESHOLD).cpu()\n\n# This is checking to make sure it looks normal.\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(layer_2_input, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:28:36.930682Z","iopub.execute_input":"2023-06-13T01:28:36.931208Z","iopub.status.idle":"2023-06-13T01:29:10.435854Z","shell.execute_reply.started":"2023-06-13T01:28:36.931169Z","shell.execute_reply":"2023-06-13T01:29:10.434639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_3\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_3_BCE_layer_1.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.18:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(full_rect > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \nlayer_2_input = torch.tensor(full_rect).gt(THRESHOLD).cpu()\n\n# This is checking to make sure it looks normal.\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(layer_2_input, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:29:10.438242Z","iopub.execute_input":"2023-06-13T01:29:10.438971Z","iopub.status.idle":"2023-06-13T01:29:42.422638Z","shell.execute_reply.started":"2023-06-13T01:29:10.438926Z","shell.execute_reply":"2023-06-13T01:29:42.421426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_3\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_3_BCE_layer_1.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.16:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(full_rect > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \nlayer_2_input = torch.tensor(full_rect).gt(THRESHOLD).cpu()\n\n# This is checking to make sure it looks normal.\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(layer_2_input, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:29:42.424161Z","iopub.execute_input":"2023-06-13T01:29:42.425357Z","iopub.status.idle":"2023-06-13T01:30:13.622677Z","shell.execute_reply.started":"2023-06-13T01:29:42.425313Z","shell.execute_reply":"2023-06-13T01:30:13.621511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_3\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_3_BCE_layer_1.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.14:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(full_rect > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \nlayer_2_input = torch.tensor(full_rect).gt(THRESHOLD).cpu()\n\n# This is checking to make sure it looks normal.\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(layer_2_input, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:30:13.625268Z","iopub.execute_input":"2023-06-13T01:30:13.625649Z","iopub.status.idle":"2023-06-13T01:30:44.255198Z","shell.execute_reply.started":"2023-06-13T01:30:13.625615Z","shell.execute_reply":"2023-06-13T01:30:44.254219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_3\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_3_BCE_layer_1.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.12:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(full_rect > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \nlayer_2_input = torch.tensor(full_rect).gt(THRESHOLD).cpu()\n\n# This is checking to make sure it looks normal.\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(layer_2_input, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:30:44.256394Z","iopub.execute_input":"2023-06-13T01:30:44.256696Z","iopub.status.idle":"2023-06-13T01:31:14.154093Z","shell.execute_reply.started":"2023-06-13T01:30:44.256670Z","shell.execute_reply":"2023-06-13T01:31:14.152890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_3\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_3_BCE_layer_1.npy')\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.10:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(full_rect > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \nlayer_2_input = torch.tensor(full_rect).gt(THRESHOLD).cpu()\n\n# This is checking to make sure it looks normal.\nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(layer_2_input, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:31:14.155333Z","iopub.execute_input":"2023-06-13T01:31:14.155651Z","iopub.status.idle":"2023-06-13T01:31:43.478298Z","shell.execute_reply.started":"2023-06-13T01:31:14.155625Z","shell.execute_reply":"2023-06-13T01:31:43.477080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_1\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_1_BCE_layer_1.npy')\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.08:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(output > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \noutput = torch.tensor(output).gt(THRESHOLD).cpu()\n\n# Saving the files.\nnp_output = np.array(output)\nnp.save(\"train_1_BCE_layer_1.npy\", np_output)\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:42:14.723980Z","iopub.execute_input":"2023-06-13T01:42:14.724450Z","iopub.status.idle":"2023-06-13T01:43:19.501654Z","shell.execute_reply.started":"2023-06-13T01:42:14.724413Z","shell.execute_reply":"2023-06-13T01:43:19.500150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_2\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_2_BCE_layer_1.npy')\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.08:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(output > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \noutput = torch.tensor(output).gt(THRESHOLD).cpu()\n\n# Saving the files.\nnp_output = np.array(output)\nnp.save(\"train_2_BCE_layer_1.npy\", np_output)\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:43:19.504772Z","iopub.execute_input":"2023-06-13T01:43:19.505608Z","iopub.status.idle":"2023-06-13T01:46:51.528932Z","shell.execute_reply.started":"2023-06-13T01:43:19.505560Z","shell.execute_reply":"2023-06-13T01:46:51.527427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREFIX = PREFIX_3\noutput = np.load('/kaggle/input/vesuvius-v4-files/train_3_BCE_layer_1.npy')\n\npositives = 0\nTHRESHOLD = 1.00\nwhile positives < 0.08:\n    THRESHOLD -= 0.01\n    threshold_test = np.where(output > THRESHOLD, 1, 0)\n    positives = np.mean(threshold_test)\n    \noutput = torch.tensor(output).gt(THRESHOLD).cpu()\n\n# Saving the files.\nnp_output = np.array(output)\nnp.save(\"train_3_BCE_layer_1.npy\", np_output)\n\nmask = np.array(Image.open(PREFIX[0]+\"mask.png\").convert('1'))\n\nnot_border = np.zeros(mask.shape, dtype=bool)\nnot_border[BUFFER_1:mask.shape[0]-BUFFER_1, BUFFER_1:mask.shape[1]-BUFFER_1] = True\n\ngrid = np.zeros(mask.shape)\ni = WINDOW_1\nwhile i < grid.shape[0]:\n    j = WINDOW_1\n    while j < grid.shape[1]:\n        grid[i,j] = 1\n        j += 2*WINDOW_1+1\n    i += 2*WINDOW_1+1\n\narr_mask = np.array(mask) * not_border * grid    \nfull_rect = np.ones(mask.shape, dtype=bool) * arr_mask\nall_pixels = np.argwhere(full_rect)\n\nprint(len(output))\nprint(len(all_pixels))\n\nfor i in range(len(output)):\n    pixel_y = all_pixels[i][0]\n    pixel_x = all_pixels[i][1]\n    full_rect[pixel_y-WINDOW_1:pixel_y+WINDOW_1+1,pixel_x-WINDOW_1:pixel_x+WINDOW_1+1] = output[i].reshape(WINDOW_1*2+1,WINDOW_1*2+1)\n    \nfig, (ax1) = plt.subplots(1, 1)\nax1.imshow(full_rect, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-13T01:46:51.530831Z","iopub.execute_input":"2023-06-13T01:46:51.531802Z","iopub.status.idle":"2023-06-13T01:47:47.138711Z","shell.execute_reply.started":"2023-06-13T01:46:51.531752Z","shell.execute_reply":"2023-06-13T01:47:47.137428Z"},"trusted":true},"execution_count":null,"outputs":[]}]}