{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-24T05:59:20.535030Z","iopub.execute_input":"2023-12-24T05:59:20.535807Z","iopub.status.idle":"2023-12-24T05:59:20.961666Z","shell.execute_reply.started":"2023-12-24T05:59:20.535745Z","shell.execute_reply":"2023-12-24T05:59:20.960647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport cv2\nfrom glob import glob\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom os import listdir\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_absolute_error\n\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nfrom tensorflow.keras import layers, models\nfrom keras.callbacks import EarlyStopping\n\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-12-24T05:59:20.963489Z","iopub.execute_input":"2023-12-24T05:59:20.963942Z","iopub.status.idle":"2023-12-24T05:59:38.402702Z","shell.execute_reply.started":"2023-12-24T05:59:20.963911Z","shell.execute_reply":"2023-12-24T05:59:38.401526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_img(path):\n    img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n    img = np.tile(img[...,None], [1, 1, 3])\n    img = img.astype('float32')\n    ma = np.max(img)\n    if ma:\n        img/=ma \n    return img\n\ndef load_msk(path):\n    msk = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n    msk = msk.astype('float32')\n    msk/=255.0\n    return msk","metadata":{"execution":{"iopub.status.busy":"2023-12-24T05:59:38.404213Z","iopub.execute_input":"2023-12-24T05:59:38.405074Z","iopub.status.idle":"2023-12-24T05:59:38.413589Z","shell.execute_reply.started":"2023-12-24T05:59:38.405034Z","shell.execute_reply":"2023-12-24T05:59:38.412148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = load_img('/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense/images/1200.tif')\nplt.imshow(img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-24T05:59:38.416938Z","iopub.execute_input":"2023-12-24T05:59:38.417643Z","iopub.status.idle":"2023-12-24T05:59:39.061136Z","shell.execute_reply.started":"2023-12-24T05:59:38.417599Z","shell.execute_reply":"2023-12-24T05:59:39.060334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = A.Compose([\n            A.Resize(height=256, width= 256, interpolation=cv2.INTER_NEAREST),\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.2),\n        ], p=1.0)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T05:59:39.062322Z","iopub.execute_input":"2023-12-24T05:59:39.062807Z","iopub.status.idle":"2023-12-24T05:59:39.067637Z","shell.execute_reply.started":"2023-12-24T05:59:39.062763Z","shell.execute_reply":"2023-12-24T05:59:39.066938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_path = '/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense/images/'\nmasks_path = '/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense/labels/'\nto_shuffle = []\nfor i in range(0, len(listdir(images_path))):\n    to_shuffle.append(listdir(images_path)[i] + '#$@%*(' + listdir(masks_path)[i])\n\nimport random\nrandom.shuffle(to_shuffle)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-24T05:59:39.069009Z","iopub.execute_input":"2023-12-24T05:59:39.069513Z","iopub.status.idle":"2023-12-24T05:59:45.771105Z","shell.execute_reply.started":"2023-12-24T05:59:39.069485Z","shell.execute_reply":"2023-12-24T05:59:45.769877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = []\nmasks = []\n\nfor paths in to_shuffle[:200]:\n    path1 = paths.split('#$@%*(')[0]\n    path2 = paths.split('#$@%*(')[1]\n    curimg = load_img(images_path + path1)\n    curmask = load_msk(masks_path + path2)\n    data = data_transforms(image=curimg, mask=curmask)\n    images.append(data['image'])\n    masks.append(data['mask'])\nimages = np.array(images)\nmasks = np.array(masks)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T05:59:45.773175Z","iopub.execute_input":"2023-12-24T05:59:45.773972Z","iopub.status.idle":"2023-12-24T06:00:04.179074Z","shell.execute_reply.started":"2023-12-24T05:59:45.773930Z","shell.execute_reply":"2023-12-24T06:00:04.177948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(images, masks, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:00:04.180400Z","iopub.execute_input":"2023-12-24T06:00:04.180744Z","iopub.status.idle":"2023-12-24T06:00:04.244491Z","shell.execute_reply.started":"2023-12-24T06:00:04.180708Z","shell.execute_reply":"2023-12-24T06:00:04.243053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential([\n    layers.Input(shape=images[0].shape),\n    layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),\n    layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.BatchNormalization(),\n    layers.MaxPool2D(pool_size=(2, 2), padding='same'),\n    layers.Dropout(0.5),\n    \n    layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),\n    layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.BatchNormalization(),\n    layers.MaxPool2D(pool_size=(2, 2), padding='same'),\n    layers.Dropout(0.5),\n    \n    layers.Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.BatchNormalization(),\n    layers.MaxPool2D(pool_size=(2, 2), padding='same'),\n    layers.Dropout(0.5),\n    \n    layers.Conv2D(filters=128, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2D(filters=128, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.BatchNormalization(),\n    layers.MaxPool2D(pool_size=(2, 2), padding='same'),\n    layers.Dropout(0.5),\n    \n    layers.Conv2D(filters=256, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2DTranspose(filters=1024, kernel_size=(3, 3), strides=(2, 2), padding=\"same\"),\n    layers.BatchNormalization(),\n    layers.Dropout(0.5),\n    \n    layers.Conv2D(filters=128, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2D(filters=128, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2DTranspose(filters=256, kernel_size=(3, 3), strides=(2, 2), padding=\"same\"),\n    layers.BatchNormalization(),\n    layers.Dropout(0.5),\n    \n    layers.Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2DTranspose(filters=128, kernel_size=(3, 3), strides=(2, 2), padding=\"same\"),\n    layers.BatchNormalization(),\n    layers.Dropout(0.5),\n    \n    layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding=\"same\"),\n    layers.Conv2DTranspose(filters=64, kernel_size=(3, 3), strides=(2, 2), padding=\"same\"),\n    layers.BatchNormalization(),\n    \n    layers.Conv2D(filters=1, kernel_size=(3, 3), activation='sigmoid', padding=\"same\")\n    \n])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:00:04.245923Z","iopub.execute_input":"2023-12-24T06:00:04.246254Z","iopub.status.idle":"2023-12-24T06:00:05.012270Z","shell.execute_reply.started":"2023-12-24T06:00:04.246225Z","shell.execute_reply":"2023-12-24T06:00:05.011244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer='adam',\n    loss='binary_crossentropy',\n    metrics=['accuracy']\n)\n\n\nstop_training = EarlyStopping(monitor='val_loss', patience = 3)\n\nhistory = model.fit(\n    X_train, y_train,\n    validation_data=[X_valid, y_valid],\n    batch_size=200,\n    epochs=50,\n    callbacks=stop_training\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:00:05.015388Z","iopub.execute_input":"2023-12-24T06:00:05.016033Z","iopub.status.idle":"2023-12-24T06:15:36.998427Z","shell.execute_reply.started":"2023-12-24T06:00:05.015999Z","shell.execute_reply":"2023-12-24T06:15:36.997372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df.loc[:, ['loss', 'val_loss']].plot(title=\"Cross-entropy\")","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:15:37.000017Z","iopub.execute_input":"2023-12-24T06:15:37.000307Z","iopub.status.idle":"2023-12-24T06:15:37.501569Z","shell.execute_reply.started":"2023-12-24T06:15:37.000281Z","shell.execute_reply":"2023-12-24T06:15:37.500373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_transforms = A.Compose([\n            A.Resize(height=256, width= 256, interpolation=cv2.INTER_NEAREST)\n        ], p=1.0)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:15:47.493194Z","iopub.execute_input":"2023-12-24T06:15:47.493653Z","iopub.status.idle":"2023-12-24T06:15:47.499727Z","shell.execute_reply.started":"2023-12-24T06:15:47.493617Z","shell.execute_reply":"2023-12-24T06:15:47.498446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATASET_FOLDER = \"/kaggle/input/blood-vessel-segmentation\"\nls_images = glob(os.path.join(DATASET_FOLDER, \"test\", \"*\", \"*\", \"*.tif\"))\nls_images.sort()\nls_images","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:15:49.251041Z","iopub.execute_input":"2023-12-24T06:15:49.251444Z","iopub.status.idle":"2023-12-24T06:15:49.262227Z","shell.execute_reply.started":"2023-12-24T06:15:49.251414Z","shell.execute_reply":"2023-12-24T06:15:49.261072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for path in ls_images:\n    curimg = load_img(path)\n    test_img_shape = curimg.shape\n    break","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:15:59.084079Z","iopub.execute_input":"2023-12-24T06:15:59.084511Z","iopub.status.idle":"2023-12-24T06:15:59.149176Z","shell.execute_reply.started":"2023-12-24T06:15:59.084480Z","shell.execute_reply":"2023-12-24T06:15:59.147941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unresize_transforms = A.Compose([\n            A.Resize(height=test_img_shape[0], width=test_img_shape[1], interpolation=cv2.INTER_NEAREST)\n        ], p=1.0)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:16:18.486252Z","iopub.execute_input":"2023-12-24T06:16:18.488638Z","iopub.status.idle":"2023-12-24T06:16:18.497802Z","shell.execute_reply.started":"2023-12-24T06:16:18.488585Z","shell.execute_reply":"2023-12-24T06:16:18.496567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rles = []\nsubmitnames = []\nfor path in ls_images:\n    curimg = load_img(path)\n    data = test_data_transforms(image=curimg)\n    importimage = np.array(data['image'])\n    arr = path.split('/')\n    submitstr = \"\"\n    for element in arr:\n        if 'kidney' in element:\n            submitstr += element + '_'\n        if '.tif' in element:\n            submitstr += element.replace('.tif', '')\n    submitnames.append(submitstr)\n    pred = model(np.array([importimage]))[0]\n    transformed = unresize_transforms(image=np.array(pred))\n    resized = transformed['image']\n    for i in range(0, len(resized)):\n        for j in range(0, len(resized[0])):\n            if resized[i][j] >= 0.5:\n                resized[i][j] = 1\n            else:\n                resized[i][j] = 0\n    rle = \"\"\n    flat = np.array(resized).flatten()\n    count = 1\n    for i in range(1, len(flat)):\n        if flat[i] != flat[i-1]:\n            rle += str(int(flat[i-1])) + ' ' + str(count) + ' '\n            count = 1\n        else:\n            count += 1\n            \n    rle += str(int(flat[i-1])) + ' ' + str(count) + ' '\n    rles.append(rle)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:18:07.470738Z","iopub.execute_input":"2023-12-24T06:18:07.471205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nimages = []\nsubmitnames = []\nfor path in ls_images:\n    curimg = load_img(path)\n    data = test_data_transforms(image=curimg)\n    images.append(data['image'])\n    arr = path.split('/')\n    submitstr = \"\"\n    for element in arr:\n        if 'kidney' in element:\n            submitstr += element + '_'\n        if '.tif' in element:\n            submitstr += element.replace('.tif', '')\n    submitnames.append(submitstr)\nimages = np.array(images)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-23T04:32:40.260502Z","iopub.execute_input":"2023-12-23T04:32:40.261411Z","iopub.status.idle":"2023-12-23T04:32:40.455775Z","shell.execute_reply.started":"2023-12-23T04:32:40.261337Z","shell.execute_reply":"2023-12-23T04:32:40.454667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\npredictions = model(images)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-23T04:35:46.703791Z","iopub.execute_input":"2023-12-23T04:35:46.704323Z","iopub.status.idle":"2023-12-23T04:35:47.878592Z","shell.execute_reply.started":"2023-12-23T04:35:46.704276Z","shell.execute_reply":"2023-12-23T04:35:47.877282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nfor path in ls_images:\n    curimg = load_img(path)\n    test_img_shape = curimg.shape\n    break\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-23T04:47:20.584830Z","iopub.execute_input":"2023-12-23T04:47:20.585360Z","iopub.status.idle":"2023-12-23T04:47:20.623259Z","shell.execute_reply.started":"2023-12-23T04:47:20.585321Z","shell.execute_reply":"2023-12-23T04:47:20.622101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nedited_pred = []\nfor pred in predictions:\n    transformed = unresize_transforms(image=np.array(pred))\n    resized = transformed['image']\n    for i in range(0, len(resized)):\n        for j in range(0, len(resized[0])):\n            if resized[i][j] >= 0.5:\n                resized[i][j] = 1\n            else:\n                resized[i][j] = 0\n    edited_pred.append(resized)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-23T04:51:47.353008Z","iopub.execute_input":"2023-12-23T04:51:47.353491Z","iopub.status.idle":"2023-12-23T04:52:23.928079Z","shell.execute_reply.started":"2023-12-23T04:51:47.353455Z","shell.execute_reply":"2023-12-23T04:52:23.926872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nrles = []\nfor pred in edited_pred:\n    rle = \"\"\n    flat = np.array(pred).flatten()\n    count = 1\n    for i in range(1, len(flat)):\n        if flat[i] != flat[i-1]:\n            rle += str(int(flat[i-1])) + ' ' + str(count) + ' '\n            count = 1\n        else:\n            count += 1\n            \n    rle += str(int(flat[i-1])) + ' ' + str(count) + ' '\n    rles.append(rle)\n'''","metadata":{"execution":{"iopub.status.busy":"2023-12-23T04:53:45.994741Z","iopub.execute_input":"2023-12-23T04:53:45.995198Z","iopub.status.idle":"2023-12-23T04:53:51.573384Z","shell.execute_reply.started":"2023-12-23T04:53:45.995166Z","shell.execute_reply":"2023-12-23T04:53:51.572056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    \"id\": submitnames,\n    \"rle\": rles\n})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:17:43.418783Z","iopub.execute_input":"2023-12-24T06:17:43.419261Z","iopub.status.idle":"2023-12-24T06:17:43.640050Z","shell.execute_reply.started":"2023-12-24T06:17:43.419225Z","shell.execute_reply":"2023-12-24T06:17:43.638807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-12-24T06:17:47.300329Z","iopub.execute_input":"2023-12-24T06:17:47.300757Z","iopub.status.idle":"2023-12-24T06:17:47.338187Z","shell.execute_reply.started":"2023-12-24T06:17:47.300724Z","shell.execute_reply":"2023-12-24T06:17:47.337115Z"},"trusted":true},"execution_count":null,"outputs":[]}]}