{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":61446,"databundleVersionId":6962461,"sourceType":"competition"},{"sourceId":7158841,"sourceType":"datasetVersion","datasetId":4134545},{"sourceId":7481880,"sourceType":"datasetVersion","datasetId":4355416},{"sourceId":7488455,"sourceType":"datasetVersion","datasetId":4359810},{"sourceId":7492326,"sourceType":"datasetVersion","datasetId":4362331},{"sourceId":7500578,"sourceType":"datasetVersion","datasetId":4367686},{"sourceId":7508292,"sourceType":"datasetVersion","datasetId":4371704},{"sourceId":7571712,"sourceType":"datasetVersion","datasetId":4407934},{"sourceId":132436161,"sourceType":"kernelVersion"},{"sourceId":158933141,"sourceType":"kernelVersion"},{"sourceId":159323821,"sourceType":"kernelVersion"},{"sourceId":160079581,"sourceType":"kernelVersion"}],"dockerImageVersionId":30636,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"debug = True","metadata":{"execution":{"iopub.status.busy":"2024-01-23T19:59:56.47413Z","iopub.execute_input":"2024-01-23T19:59:56.474376Z","iopub.status.idle":"2024-01-23T19:59:56.485467Z","shell.execute_reply.started":"2024-01-23T19:59:56.474353Z","shell.execute_reply":"2024-01-23T19:59:56.48452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## smp","metadata":{}},{"cell_type":"code","source":"try:\n    import segmentation_models_pytorch as smp\nexcept:   \n    !pip install --no-index --find-links=\"/kaggle/input/segmentation-models-pytorch-offline-installation/\" segmentation-models-pytorch\n    import segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2024-01-23T19:59:56.486833Z","iopub.execute_input":"2024-01-23T19:59:56.487151Z","iopub.status.idle":"2024-01-23T20:00:21.354518Z","shell.execute_reply.started":"2024-01-23T19:59:56.487121Z","shell.execute_reply":"2024-01-23T20:00:21.353474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastai.medical.imaging import * ## boyle omz cunku medical imaging import etmedin.... \n\nfrom tqdm.notebook import tqdm\nimport tifffile as tiff\nimport timm\n\nimport gc\nimport ctypes\nlibc = ctypes.CDLL(\"libc.so.6\")\nimport time","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:00:21.357669Z","iopub.execute_input":"2024-01-23T20:00:21.358302Z","iopub.status.idle":"2024-01-23T20:00:26.067531Z","shell.execute_reply.started":"2024-01-23T20:00:21.358274Z","shell.execute_reply":"2024-01-23T20:00:26.066523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CONFIG():\n#     stats = np.array([0.61561477, 0.5179343 , 0.64067212]), np.array([0.2915353 , 0.31549066, 0.28647661])\n    \n    stats = np.array([.4975]),np.array([.2825])\n    # pytorch model (segmentation_models_pytorch)\n    encoder_name = \"efficientnet-b3\"\n    encoder_weights = 'imagenet'\n    in_channels = 1\n    classes = 2 ## 6 classes in totl..\n    \n    # fastai Learner \n    mixed_precision_training = True\n    batch_size = 16\n    weight_decay = 0.01\n    loss_func = CrossEntropyLossFlat(axis=1)\n#     metrics = [Dice()] #Iou(), Dice_f1() these were for deepflash\n    metrics = [DiceMulti(), Dice()] # TumorDice(), SubclassOnlyDice()\n    optimizer = ranger  \n    max_learning_rate = 1e-4\n    epochs = 3\n    \n    ## training other params\n    min_mask_pixels_train = 350\n    min_mask_pixels_valid = 0\n    tile_size = 384\n    focus_size   = 224 # bu da ne kaedar icerden alacagim.z..\n    \n    deneme = False\n    partialdeneme = False\n    \ncfg = CONFIG()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:00:26.069086Z","iopub.execute_input":"2024-01-23T20:00:26.069716Z","iopub.status.idle":"2024-01-23T20:00:26.077009Z","shell.execute_reply.started":"2024-01-23T20:00:26.069681Z","shell.execute_reply":"2024-01-23T20:00:26.076076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## dogrudan isimlerini \n## kidney 5 ve 6 olarak koy\nkidney5_paths   = list(Path('/kaggle/input/blood-vessel-segmentation/test/kidney_5/images').glob('**/*.tif'))\n# label1_base = Path('/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense/labels')\n\nkidney6_paths   = list(Path('/kaggle/input/blood-vessel-segmentation/test/kidney_6/images').glob('**/*.tif'))\n# label2_base = Path('/kaggle/input/blood-vessel-segmentation/train/kidney_2/labels')\n\n\nif debug: \n    ## baska fonk istemiyorsan.. label3 icin biraz farkli bir husus...\n    ## cunku path olarak dogrudan yok label3 dense icinden cekeceksin\n    label3_dense_paths = list(Path('/kaggle/input/blood-vessel-segmentation/train/kidney_3_dense/labels').glob('**/*.tif'))\n    images_dense3_base = Path('/kaggle/input/blood-vessel-segmentation/train/kidney_3_sparse/images')\n    label3_base = Path('/kaggle/input/blood-vessel-segmentation/train/kidney_3_dense/labels')\n\n    ## labe3_base std olarak var.. tamam... kidney3_paths'i label3dense paths kullanarak cek.. \n    ### dikkat kidney 3 icin sparse olsun dense olsun hepsi sparse'in icinde... \n    #### ama labellar sparse vs. dense ayri folderlar icinde.. \n    kidney3_paths = [(images_dense3_base / path.name) for path in label3_dense_paths]\n    kidney3_paths.sort(key = lambda x: x.stem)\n\n\n    ## dikkat sadece dense olanlanlari aliyoruz....\n    len(kidney3_paths),kidney3_paths[:5]","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:00:26.078322Z","iopub.execute_input":"2024-01-23T20:00:26.079323Z","iopub.status.idle":"2024-01-23T20:00:26.240858Z","shell.execute_reply.started":"2024-01-23T20:00:26.079288Z","shell.execute_reply":"2024-01-23T20:00:26.239991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## siraya sok... ne path alirsan siraya sok\nkidney5_paths.sort(key = lambda x: x.stem)\nkidney6_paths.sort(key = lambda x: x.stem)\n\nlen(kidney5_paths), len(kidney5_paths), kidney5_paths[0:3], kidney6_paths[0:3]","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:00:26.249139Z","iopub.execute_input":"2024-01-23T20:00:26.249833Z","iopub.status.idle":"2024-01-23T20:00:26.259233Z","shell.execute_reply.started":"2024-01-23T20:00:26.249782Z","shell.execute_reply":"2024-01-23T20:00:26.258317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### data yuklemek icin dls. icin basit bir data yuklesek...\n- tek eksen yuklese..\n- 1024 x 1024 ve 100 slice olsun..\n- tek eksenden al... dogrudan al hatta. ","metadata":{}},{"cell_type":"code","source":"def find_bin(paths):\n    sample1 = random.sample(paths,333)\n#     arrays1 = [np.resize(tiff.imread(path),(384,384)) for path in sample1]\n    arrays1 = [tiff.imread(path) for path in sample1]\n    stacked_array1 = torch.stack([torch.tensor(x.astype(np.float32)) for x in arrays1])\n    bins1 = stacked_array1.freqhist_bins()\n    stacked_array_hist = torch.stack([x.hist_scaled(bins1) for x in stacked_array1])\n    mu,stdx = torch.mean(stacked_array_hist),torch.std(stacked_array_hist)\n    return mu,stdx,bins1","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:00:26.260347Z","iopub.execute_input":"2024-01-23T20:00:26.261152Z","iopub.status.idle":"2024-01-23T20:00:26.269342Z","shell.execute_reply.started":"2024-01-23T20:00:26.26112Z","shell.execute_reply":"2024-01-23T20:00:26.268473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## process image icinde 3te1 cikarma ekledim\ndef process_image(path, bins1):\n    img = tiff.imread(path).astype(np.float32)\n    img_tensor = torch.tensor(img)\n    img_tensor = img_tensor.hist_scaled(bins1)  # assuming hist_scaled is an in-place operation\n    return np.array(img_tensor * 255).astype(np.uint8)\n#     return (np.array(torch.clamp((img-.33)*1.5 ,0.,1.))*255).astype(np.uint8)\n#     return (np.array(torch.clamp((img-.50)*2.0 ,0.,1.))*255).astype(np.uint8)\n#     return (np.array(torch.clamp((img_tensor-.66)*3.0 ,0.,1.))*255).astype(np.uint8)\n#     return (np.array(torch.clamp((img-.75)*4.0 ,0.,1.))*255).astype(np.uint8)\n\n\ndef read_and_stack_images(paths, label_base):\n    mu, stdx, bins1 = find_bin(paths)\n    print('found the bins...')\n\n    # Process images using a generator expression\n    images = (process_image(path, bins1) for path in paths)\n    images_stacked = np.stack(images)\n\n    # Process labels\n    labels = np.stack((tiff.imread(label_base / path.name) > 0 for path in paths))\n\n    return images_stacked, labels, mu, stdx\n\nif debug:\n    ## azaltiyorsun... ama boyle yapinca sonuclari gorebilecek... kiyaslayabilecek misin????\n    kidney_3_images, kidney_3_labels ,mu3,sigma3 = read_and_stack_images(kidney3_paths, label3_base)\n    kidney_3_images = kidney_3_images[:64]\n\n    ## cok da buyuk olmasin diye boyle yaptim................\n    kidney_3_images = kidney_3_images[:,:1024,:1024] ## 64 x 1024  x 1024 oluyor... \n    kidney_3_labels = kidney_3_labels[:,:1024,:1024]\n    print(kidney_3_images.shape, kidney_3_labels.shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:00:26.27098Z","iopub.execute_input":"2024-01-23T20:00:26.271264Z","iopub.status.idle":"2024-01-23T20:02:42.110424Z","shell.execute_reply.started":"2024-01-23T20:00:26.27124Z","shell.execute_reply":"2024-01-23T20:02:42.109256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def read_and_stack_dense_images(image_base, label_paths):\n#     # Stacks images and labels directly into numpy arrays\n#     stacked_kidneys = np.stack((tiff.imread(image_base / path.name) for path in label_paths))\n#     stacked_labels = np.stack((tiff.imread(path)>0 for path in label_paths))\n\n#     return stacked_kidneys, stacked_labels\n\n# if debug:  ## dls olduktan sonra kidney3 images ve labels silebilirsin... \n#     label3_dense_paths = list(Path('/kaggle/input/blood-vessel-segmentation/train/kidney_3_dense/labels').glob('**/*.tif'))\n#     images_dense3_base = Path('/kaggle/input/14ocak-load-data-004-option1-final/kidney_3_sparse/images')\n\n    \n#     label3_dense_paths.sort(key = lambda x: x.stem)\n#     label3_dense_paths = label3_dense_paths[:64]\n    \n    \n#     kidney_3_images,kidney_3_labels = read_and_stack_dense_images(images_dense3_base, label3_dense_paths)\n#     kidney_3_images = kidney_3_images[:,:1024,:1024]\n#     kidney_3_labels = kidney_3_labels[:,:1024,:1024]\n#     print(kidney_3_images.shape, kidney_3_labels.shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.111918Z","iopub.execute_input":"2024-01-23T20:02:42.112299Z","iopub.status.idle":"2024-01-23T20:02:42.120245Z","shell.execute_reply.started":"2024-01-23T20:02:42.112265Z","shell.execute_reply":"2024-01-23T20:02:42.119169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## simdi burada slice alan dogrudan bu slicelari gonderen bir get_random_tile_and_mask\n- df icinde slice 0 to 64 olacak o kadar... ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## check #### HIP-CT-007_Generalized Tile and Mask\n## hangi slice is from 0 to 63\ndef get_random_tile_and_mask(image_array, slice_index, label_array, tile_size=512):\n    X, Y, Z = image_array.shape\n    \n    fark = (1024 - tile_size)//2 ## bunu boyle yaptim ki ortadan alalim...................en koseden degil\n    return (image_array[slice_index, fark:tile_size+fark, fark:tile_size+fark],\n            label_array[slice_index, fark:tile_size+fark, fark:tile_size+fark])\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.121303Z","iopub.execute_input":"2024-01-23T20:02:42.121545Z","iopub.status.idle":"2024-01-23T20:02:42.136074Z","shell.execute_reply.started":"2024-01-23T20:02:42.121523Z","shell.execute_reply":"2024-01-23T20:02:42.135118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 512yi 448 yaptim bakalim ne olacak... \ndef tiler(kidney,hangi_slice, mask,size = 448):\n  \n    random_image_tile, random_label_tile = get_random_tile_and_mask(kidney, hangi_slice, mask, tile_size=size)\n    \n    random_label_tile = random_label_tile.astype(np.uint8)\n#     print(random_image_tile.shape,random_label_tile.shape,random_image_tile[0],random_label_tile[0])\n#     print('one')\n    return random_image_tile,random_label_tile","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.137571Z","iopub.execute_input":"2024-01-23T20:02:42.13827Z","iopub.status.idle":"2024-01-23T20:02:42.146101Z","shell.execute_reply.started":"2024-01-23T20:02:42.138235Z","shell.execute_reply":"2024-01-23T20:02:42.145139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_slice, mask_slice = tiler(kidney_3_images,3,kidney_3_labels)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.147165Z","iopub.execute_input":"2024-01-23T20:02:42.147528Z","iopub.status.idle":"2024-01-23T20:02:42.154879Z","shell.execute_reply.started":"2024-01-23T20:02:42.147496Z","shell.execute_reply":"2024-01-23T20:02:42.153965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PILImageBW.create(image_slice)#.convert('L') ","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.156074Z","iopub.execute_input":"2024-01-23T20:02:42.156372Z","iopub.status.idle":"2024-01-23T20:02:42.196856Z","shell.execute_reply.started":"2024-01-23T20:02:42.156315Z","shell.execute_reply":"2024-01-23T20:02:42.196007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask_slice.shape,image_slice.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.197857Z","iopub.execute_input":"2024-01-23T20:02:42.198116Z","iopub.status.idle":"2024-01-23T20:02:42.204232Z","shell.execute_reply.started":"2024-01-23T20:02:42.198093Z","shell.execute_reply":"2024-01-23T20:02:42.203233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PILImage.create(image_slice)\nPILMask.create(mask_slice*255)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.205621Z","iopub.execute_input":"2024-01-23T20:02:42.205909Z","iopub.status.idle":"2024-01-23T20:02:42.217096Z","shell.execute_reply.started":"2024-01-23T20:02:42.205885Z","shell.execute_reply":"2024-01-23T20:02:42.216209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport cv2\nimport numpy as np\n\npng_image = image_slice\nmask = mask_slice\n\n# Assuming png_image and mask are already loaded and have shapes (1303, 912)\n\n# Convert png_image to a 3-channel image if it is single-channel\nif len(png_image.shape) == 2:  # Image is grayscale\n    png_image_color = cv2.cvtColor(png_image, cv2.COLOR_GRAY2BGR)\nelse:\n    png_image_color = png_image.copy()\n\n# Create a colored mask\ncolored_mask = np.zeros_like(png_image_color)  # Initialize with zeros\ncolored_mask[mask > 0] = [0, 0, 255]  # Red where the mask is positive\n\n# Overlay the mask - adjust alpha (transparency) to your preference\nalpha = 0.5  # Transparency factor\ncombined = cv2.addWeighted(png_image_color, 1, colored_mask, alpha, 0)\n\n\n# Optionally, save the result\n# Save the result instead of displaying\ncv2.imwrite('overlayed_image.jpg', combined)\n\n\n# import matplotlib.pyplot as plt\n\n# Convert BGR to RGB for Matplotlib display\ncombined_rgb = cv2.cvtColor(combined, cv2.COLOR_BGR2RGB)\n\n# Set the size of the figure\nplt.figure(figsize=(10, 8))  # You can adjust the size as needed\n\n\n# Display the image\nplt.imshow(combined_rgb)\nplt.axis('off')  # Turn off axis numbers\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.218253Z","iopub.execute_input":"2024-01-23T20:02:42.218513Z","iopub.status.idle":"2024-01-23T20:02:42.598598Z","shell.execute_reply.started":"2024-01-23T20:02:42.218482Z","shell.execute_reply":"2024-01-23T20:02:42.597608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## making of dataset","metadata":{}},{"cell_type":"code","source":"## infer... sadece train 3 cunku sadece onu yukleyecegim... model de zaten hazir..\n### eger normal training icinde yapmak istersen... ortaadn.. 448lik bolumleri algom...\n### as metric or cbs that is it... \ntrain_3 = 128\nslis = [random.randint(0,63) for x in range(128)]\n\n# train_set = ['1'] * train_1 + ['3'] * train_3 ## bunlara train height/width limitlerini koy\ntrain_set = ['3'] * train_3 ## bunlara train height/width limitlerini koy\n\n\ndf=pd.DataFrame({'kidney':train_set,'hangi_slice':slis})\ndf","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.599908Z","iopub.execute_input":"2024-01-23T20:02:42.600198Z","iopub.status.idle":"2024-01-23T20:02:42.625649Z","shell.execute_reply.started":"2024-01-23T20:02:42.600173Z","shell.execute_reply":"2024-01-23T20:02:42.62471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## dikkat img.size 2 yerde degisecek: item_tfms olarak da var..\n## sadece tiler degistirmen yetmez!!","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.626979Z","iopub.execute_input":"2024-01-23T20:02:42.627956Z","iopub.status.idle":"2024-01-23T20:02:42.63177Z","shell.execute_reply.started":"2024-01-23T20:02:42.62792Z","shell.execute_reply":"2024-01-23T20:02:42.630855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_and_mask(row):\n    # Extracting the kidney information from the row\n    kidney = row['kidney']\n    hangi_slice = row['hangi_slice']\n    \n    \n    \n    \n    # Call tiler function to get the image and mask\n    ##..............PARAMETRELERI FARKLI YAPARAK TRAININGE DEVAM... DIKKAT DENSE3: 501 SLICES\n    ## BU NEDENLE SIZE MAX 501, VE DIGER PARAMETRELERI DE KLLAN, FOR AXISALIGNED TILING: rotation_ration = -1\n    image, mask = tiler(kidney_3_images, hangi_slice, kidney_3_labels, size = cfg.tile_size)\n#     return PILImage.create(image), PILMask.create(mask) #to_tensor(image), to_tensor(mask)\n#     return PILImageBW(PILImageBW.create(image)), PILMask.create(mask) #to_tensor(image), to_tensor(mask)\n    return PILImageBW.create(image), PILMask.create(mask) #to_tensor(image), to_tensor(mask)\n    ## .convert('L').. bunu yapmadi .create().convert('L'): gerek kalmadi.... \n    ## kendisi yapiyor zaten \n\n\n# return two tuple series and not the zipped!!\ndef get_items(df):\n    im_label =  [get_image_and_mask(row) for _, row in df.iterrows()]\n    imic = [x for x,y in im_label]\n    lebil = [y for x,y in im_label]\n#     print(imic)\n    \n    return imic,lebil\n\n\n\ndblock = DataBlock.from_columns(blocks   = (ImageBlock(PILImageBW), MaskBlock(codes=np.array(['nothing','VASCO']))),\n    get_items=get_items,  # Use your custom function\n#     get_x=get_x,\n#     get_y=get_y,\n#     splitter=IndexSplitter(valid_idx), ## randomsplitter uygulayacak.. \n#     item_tfms =Resize(cfg.tile_size), ## bunu sonra ekledim... boylece ciktiyi 448 yapacak... \n                            ## diger bir ihtimal resimleri yukarida tiler icinde 384'e gore yapmak\n                            ## ve batch tfms aug_transform size. vs cikarmak\n    batch_tfms=[\n#                 *aug_transforms(size=448, min_scale=0.75),\n                Normalize.from_stats(*cfg.stats)] # imagenet_stats\n)  ## batch tfms: *aug_transforms(size=384, min_scale=0.75) ... henuz bu yok...\n\n# Create the DataLoaders.. ADJUST BSIZE AS REQUIRED FOR TRAIN/INFER\ndls = dblock.dataloaders(df, bs=16)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:42.63346Z","iopub.execute_input":"2024-01-23T20:02:42.634074Z","iopub.status.idle":"2024-01-23T20:02:43.127373Z","shell.execute_reply.started":"2024-01-23T20:02:42.634036Z","shell.execute_reply":"2024-01-23T20:02:43.126436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:43.129028Z","iopub.execute_input":"2024-01-23T20:02:43.12947Z","iopub.status.idle":"2024-01-23T20:02:43.993907Z","shell.execute_reply.started":"2024-01-23T20:02:43.129437Z","shell.execute_reply":"2024-01-23T20:02:43.992816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ## lets delete the kidney 3 data why not\n# del kidney_3_images, kidney_3_labels\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:43.995158Z","iopub.execute_input":"2024-01-23T20:02:43.995847Z","iopub.status.idle":"2024-01-23T20:02:44.000436Z","shell.execute_reply.started":"2024-01-23T20:02:43.995809Z","shell.execute_reply":"2024-01-23T20:02:43.999347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## this is new: set a path for the model to train and save things\n# Set the writable path for the model\nmodel_path = Path('/kaggle/working')\n# Ensure the model directory exists\nmodel_path.mkdir(exist_ok=True, parents=True)\n\nmodel = smp.Unet(encoder_name=cfg.encoder_name, \n                 encoder_weights=None,  # cfg.encoder_weights,, None is for inference\n                 in_channels=cfg.in_channels, \n                 classes=cfg.classes)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:44.001841Z","iopub.execute_input":"2024-01-23T20:02:44.002378Z","iopub.status.idle":"2024-01-23T20:02:44.17028Z","shell.execute_reply.started":"2024-01-23T20:02:44.00234Z","shell.execute_reply":"2024-01-23T20:02:44.169118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn1 = Learner(dls, model, metrics=cfg.metrics, wd=cfg.weight_decay, loss_func=cfg.loss_func,\n                opt_func=ranger) # , cbs=cbs: REMOVE FOR TESTING\n\nlearn2 = Learner(dls, model, metrics=cfg.metrics, wd=cfg.weight_decay, loss_func=cfg.loss_func,\n                opt_func=ranger)\n\nlearn3 = Learner(dls, model, metrics=cfg.metrics, wd=cfg.weight_decay, loss_func=cfg.loss_func,\n                opt_func=ranger)\n\nlearn4 = Learner(dls, model, metrics=cfg.metrics, wd=cfg.weight_decay, loss_func=cfg.loss_func,\n                opt_func=ranger)\n\nlearn5 = Learner(dls, model, metrics=cfg.metrics, wd=cfg.weight_decay, loss_func=cfg.loss_func,\n                opt_func=ranger)\n\nlearn6 = Learner(dls, model, metrics=cfg.metrics, wd=cfg.weight_decay, loss_func=cfg.loss_func,\n                opt_func=ranger)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:44.171627Z","iopub.execute_input":"2024-01-23T20:02:44.172514Z","iopub.status.idle":"2024-01-23T20:02:44.179106Z","shell.execute_reply.started":"2024-01-23T20:02:44.17248Z","shell.execute_reply":"2024-01-23T20:02:44.178081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learn.load('/kaggle/input/17ocak-009-training-bline-350-0/models/model_6')\n\n## 24ocak submission model\n# learn.load('/kaggle/input/23ocak-014-histscaled-traning-bline-19-11/models/model_16')\n\n##25ocak submission models subs 2 3 4 5 \n## 4te3trained aug x 3\n# learn.load('/kaggle/input/24ocak-models-locally-trained/24ocak-014-histscaled-traning-4te3_aug24_m15')\n# ## 3te2 trained aug x 3\n# learn.load('/kaggle/input/24ocak-models-locally-trained/24ocak-014-histscaled-traning-3te2_aug24_m22')\n# ## 3te2 trained low aug\n# learn.load('/kaggle/input/24ocak-models-locally-trained/24ocak-014-histscaled-traning-3te2_m21')\n\n\n# ## bline trained aug x 3\n# learn.load('/kaggle/input/24ocak-models-locally-trained/24ocak-014-histscaled-traning-bline_aug24_m24')\n\n## 26ocak sub1: finetune the pretrained\n# learn.load('/kaggle/input/26ocak-local-trained-fromscratch-and-finetuned/26ocak-014-histscaled-traning-bline_aug24_m24_7REFINETUNED')\n\n## 26ocak sub2: tune the nonpretrained\n# learn.load('/kaggle/input/26ocak-local-trained-fromscratch-and-finetuned/26ocak-nonpretrained_traning-bline_aug24_05randomtiles_7x')\n\n\n## 27ocak subs... dikkat birinci 384luk... ama digeri degil.. this is 80.2 veren !!!!\nlearn1.load('/kaggle/input/finfinal-models/06subat_384_local_voiFtune')\n## yukaridakinin raw 19x3 son hali. bu da 384luk: tek basina submit edilmemisti... \nlearn2.load('/kaggle/input/finfinal-models/06subat_384finetunes_01_ftune')\n\n\n## 28ocak ortak bins examples... you can use them in weighted ensemble why not... \n## asagiki ortak bin 741 verdi... \nlearn3.load('/kaggle/input/finfinal-models/06subat_384finetunes_02_ftune')\n## bunu sbumit etmemistim bilmiyorum...\nlearn4.load('/kaggle/input/finfinal-models/06subat_384finetunes_03_ftune_m0')\n\n## 29ocak subs.... 806 veren bu.....\nlearn5.load('/kaggle/input/finfinal-models/06subat_b3_384_VOIFtune_ep9')\n## 806nin geldigi 19 x 3lugun son 384lugu: tekbasina submit degil\nlearn6.load('/kaggle/input/finfinal-models/06subat_b3_480_33insteadOF41_17_VOIFTUNE')\n\n## ortak bin kotu gelmedi.... ama.... bakalim zaman yatecek mi.... \n# learn7.load('/kaggle/input/29ocak-models/29ocak_28ocak___ortakbin_m18_ftuned7_ftunedwithPerBin')\n\n\n\n# learn.load('/kaggle/input/29ocak-models/29ocak___randombin_including_cont2_ftuned_remove13e_ep7')\n# learn.load('/kaggle/input/29ocak-models/29ocak___randombin_including_cont2_ftuned_with_scale065_m10')\n\n##30ocak subs\n# learn.load('/kaggle/input/29ocak-models/29ocak____randombin_continue2_currentbest_lastM1')\n# learn.load('')\n# learn.load('')\n# learn.load('')\n# learn.load('')\n\n\n## add learners to learners\nlearners = [learn1,learn2,learn3,learn4,learn5,learn6]","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:44.189386Z","iopub.execute_input":"2024-01-23T20:02:44.18971Z","iopub.status.idle":"2024-01-23T20:02:44.84745Z","shell.execute_reply.started":"2024-01-23T20:02:44.18968Z","shell.execute_reply":"2024-01-23T20:02:44.846416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# learn.show_results()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:44.848847Z","iopub.execute_input":"2024-01-23T20:02:44.849643Z","iopub.status.idle":"2024-01-23T20:02:44.853997Z","shell.execute_reply.started":"2024-01-23T20:02:44.849607Z","shell.execute_reply":"2024-01-23T20:02:44.852979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.valid.after_batch","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:44.855309Z","iopub.execute_input":"2024-01-23T20:02:44.856017Z","iopub.status.idle":"2024-01-23T20:02:44.867659Z","shell.execute_reply.started":"2024-01-23T20:02:44.855984Z","shell.execute_reply":"2024-01-23T20:02:44.866819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_batch_tfms = [*aug_transforms(size=384, min_scale=0.75), Normalize.from_stats(*cfg.stats)]\n# dls.valid.after_batch = Pipeline(new_batch_tfms)\n## BU HATA VERDI...","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:44.868904Z","iopub.execute_input":"2024-01-23T20:02:44.869226Z","iopub.status.idle":"2024-01-23T20:02:44.876323Z","shell.execute_reply.started":"2024-01-23T20:02:44.869194Z","shell.execute_reply":"2024-01-23T20:02:44.875443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(3):\n    # Create the DataLoaders\n    dls = dblock.dataloaders(df, bs=16)\n    learn1.dls = dls\n    result = learn1.validate()\n    print(result)\n    \n## klasik ayni dls ile yani item_tfms yok iken\n## batch tfms icinde resize 384 varken\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.0042856899090111256, 0.831862823460708, 0.6643128254920999]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.008108304813504219, 0.8410498150544986, 0.683337949590472]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.003531221067532897, 0.871698948085167, 0.7439431307123169]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.005198875442147255, 0.8407163628293095, 0.6822777454968042]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.004640439059585333, 0.8649461652688102, 0.7306222551759539]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.0033839037641882896, 0.8471321671745814, 0.6947608200455581]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.005333773326128721, 0.8683360374224613, 0.7375594584705452]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.0034529222175478935, 0.879633552511897, 0.7597946120841247]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.003286712570115924, 0.8690230131438781, 0.7385373870172556]\n# Beginning training...\n# One epoch completed\n# Updating new dls.train\n# [0.006221315357834101, 0.871580952672194, 0.7440974866717441]","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:44.877435Z","iopub.execute_input":"2024-01-23T20:02:44.877762Z","iopub.status.idle":"2024-01-23T20:02:47.70431Z","shell.execute_reply.started":"2024-01-23T20:02:44.877731Z","shell.execute_reply":"2024-01-23T20:02:47.703061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn1.show_results()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:47.706066Z","iopub.execute_input":"2024-01-23T20:02:47.706389Z","iopub.status.idle":"2024-01-23T20:02:49.90628Z","shell.execute_reply.started":"2024-01-23T20:02:47.70636Z","shell.execute_reply":"2024-01-23T20:02:49.905254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## VALIDATE THE INFERENCE WAY... \n- sunu yapariz yine df olarak calisir....\n- ve x_limits, ve y_limits diye iki column eklersin...\n- bunlari istersen train icin kullanmana gerek yok..\n- ama istersen onun icin de doldurabilirsin...\n\n- size tiled,, with size /2 overlay, ve koselrede almayacak sekilde...\n- bu sekilde ","metadata":{}},{"cell_type":"code","source":"## o zaman once ilk eksen uzerinde kaydirarak baslayalim..\n### sonra eksenleri degisir olan uzerine koyarak tamamlariz... ","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:49.90767Z","iopub.execute_input":"2024-01-23T20:02:49.908046Z","iopub.status.idle":"2024-01-23T20:02:49.912226Z","shell.execute_reply.started":"2024-01-23T20:02:49.908011Z","shell.execute_reply":"2024-01-23T20:02:49.911274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_and_stack_images(paths):\n    # Using a generator expression to read and stack images efficiently\n    stacked_kidneys = np.stack((tiff.imread(path) for path in paths))\n    \n    return stacked_kidneys\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:49.913551Z","iopub.execute_input":"2024-01-23T20:02:49.91421Z","iopub.status.idle":"2024-01-23T20:02:49.922173Z","shell.execute_reply.started":"2024-01-23T20:02:49.914177Z","shell.execute_reply":"2024-01-23T20:02:49.921145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## lets make this iamge processing in 3d\n### this means the image to process_image_corrected_v2 will be the whole kidney\n### with the first axis as z axisk\npred_size = cfg.tile_size ## bu durumda tiler da 384 yaptim... ve 448 item_tfms, ya da resize448 kullanmadim\ndef extract_prediction_tile_corrected_v2(image, focus_x, focus_y, focus_size=cfg.focus_size, pred_size=pred_size):\n    \"\"\"Extract a prediction tile, adjusting for image boundaries and ensuring correct size.\"\"\"\n    half_focus_size = focus_size // 2\n    half_pred_size = pred_size // 2\n\n    # Determine start points, adjust to keep pred_tile within image boundaries\n    start_x = focus_x - half_pred_size\n    start_y = focus_y - half_pred_size\n\n    # Adjust start points to ensure the prediction tile is within image boundaries\n    if start_x < 0:\n        start_x = 0\n    elif start_x + pred_size > image.shape[2]: ## now widht is dimension final... \n        start_x = image.shape[2] - pred_size ##\n\n    if start_y < 0:\n        start_y = 0\n    elif start_y + pred_size > image.shape[1]: ## now height is dimension 1\n        start_y = image.shape[1] - pred_size\n\n    \n    return image[:,start_y:start_y + pred_size, start_x:start_x + pred_size], (start_x, start_y)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:49.923478Z","iopub.execute_input":"2024-01-23T20:02:49.923945Z","iopub.status.idle":"2024-01-23T20:02:50.264519Z","shell.execute_reply.started":"2024-01-23T20:02:49.923911Z","shell.execute_reply":"2024-01-23T20:02:50.263461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def model_infer_z(tile,learners): ##buraya weights gonderip, zipli olarka acip kullanabilrsin\n    tile = [PILImageBW.create(x) for x in tile]\n    dl = learners[0].dls.test_dl(tile) ## assuming identical models,,, single dl definition yeter\n    \n    for i,learn in enumerate(learners):\n        if i ==0:\n            preds, _ = learn.get_preds(dl=dl)\n            preds    = preds[:,1,:,:] ## now this has the shape and the segm. object pixel probs\n        else:\n            preds2, _ = learn.get_preds(dl=dl)\n            preds2    = preds2[:,1,:,:] ## now this has the shape and the segm. object pixel probs\n            preds+= preds2\n            \n    preds/=len(learners) ## 5learner varsa 5e boler bu kadar \n\n#     print(preds)\n    return preds\n\n\ndef process_image_corrected_v2(image, learners, focus_size=cfg.focus_size, pred_size=pred_size):\n    ## now image is 3d\n    h, w = image.shape[1:] ## bacause first dimension is the slice dimension, I dont tile it\n    \n    ## tek bir tane kullanacagim bu nedenle float olarak bunun icinde toplayacagim\n    result = np.zeros_like(image).astype(np.float32) ##.. uint8 kalsin bakalim\n    xtimes = np.zeros_like(image).astype(np.uint8) # image zaten uint8\n    \n    ## halihazirda hepsii birer kere yapiyorsun... ama belki ileride overlap daha farkli olacak\n    ## bu nedenle keep as is now...\n    \n    \n    ## image ile birlikte result ve xtime de donduruyorum..\n    ## en sonda tekrar ilk z-depth eksene donecekler\n    for rotation in tqdm(['z','y','x']):\n        if rotation   == 'z':\n            pass\n        elif rotation == 'y':\n            image = np.rot90(image, k=1, axes=(1, 0))\n            result = np.rot90(result, k=1, axes=(1, 0))\n            xtimes = np.rot90(xtimes, k=1, axes=(1, 0))\n\n        elif rotation == 'x':\n            image = np.rot90(image, k=1, axes=(2, 0))\n            result = np.rot90(result, k=1, axes=(2, 0))\n            xtimes = np.rot90(xtimes, k=1, axes=(2, 0))\n         \n        \n        for i in tqdm(range(0, h, focus_size)):\n            for j in range(0, w, focus_size):\n                # Calculate the actual size of the focus area (may be smaller at edges)\n                focus_tile_height = min(focus_size, h - i)\n                focus_tile_width = min(focus_size, w - j)\n#                 print(f'focus_tile_height: {focus_tile_height}')\n#                 print(f'focus_tile_width: {focus_tile_width}')\n\n\n                # The center of the focus area within the overall image\n                focus_center_x, focus_center_y = j + focus_tile_width // 2, i + focus_tile_height // 2\n#                 print(f'focus_center_x: {focus_center_x}')\n#                 print(f'focus_center_y: {focus_center_y}')\n\n                # Extract the appropriate prediction tile\n                pred_tile, (pred_start_x, pred_start_y) = extract_prediction_tile_corrected_v2(image, focus_center_x, focus_center_y)\n#                 print(f'pred_start_x: {pred_start_x}')\n#                 print(f'pred_start_y: {pred_start_y}')\n#                 print(f'pred_tile shape: {pred_tile.shape}')\n\n                ## asagida toplama cikarma yapacaksin focus area ile.. torch olarak toplama\n                pred_result = model_infer_z(pred_tile, learners).detach().cpu().numpy()\n#                 print(f'pred_result shape: {pred_result.shape}')\n\n                # Calculate the position of the focus area within the prediction tile\n                focus_start_x = focus_center_x - pred_start_x - (focus_tile_width // 2)\n                focus_start_y = focus_center_y - pred_start_y - (focus_tile_height // 2)\n#                 print(f'focus_start_x: {focus_start_x}')\n#                 print(f'focus_start_y: {focus_start_y}')\n                # Extract the focus area from the prediction result\n                focus_result = pred_result[:,focus_start_y:focus_start_y + focus_tile_height, focus_start_x:focus_start_x + focus_tile_width]\n#                 print(f'focus_result shape: {focus_result.shape}')\n\n\n                # Place the focus area in the result image at the correct position\n                ## mevcut olanin uzerine koyuyorum result dahil... \n                ##..........burda EGER FAZLA AGIRLIK VERMEK ISTEDIGIN VARSA.\n                ##......ONUN XTIMES: +1 YERINE +1.5 OLABILIR... SONUCTA ORADAKI DEGERE BOLUNECEK\n                \n                ## .... inplace yapmazsan concateneta-baskaislem yapmaya calisiyor... \n                ## .........FOR DIFFERENT XYZ YOU CAN PLACE DIFFEREN TWEIGHTS: 1 1.5 2 ETC. IN XTIMES\n#                 print(result[:,i:i + focus_tile_height, j:j + focus_tile_width].shape,focus_result.shape)\n                result[:,i:i + focus_tile_height, j:j + focus_tile_width] += focus_result\n                xtimes[:,i:i + focus_tile_height, j:j + focus_tile_width] = xtimes[:,i:i + focus_tile_height, j:j + focus_tile_width] + 1\n\n#                 print('......................................')\n#                 print('......................................')\n#                 print('......................................')\n#                 print('\\n')\n\n                  ##...........memory icin bunlari da silsek mi??????????????????...................\n                del pred_tile,pred_result,focus_result\n                gc.collect()\n                libc.malloc_trim(0)\n    \n        ## .............BU EKSENLE ISIN BITTI TEKRAR ORJINAL ZDEPTH EKSENE DON... \n        if rotation   == 'z': ## image ve result istedigim formatta z depth\n            pass\n        elif rotation == 'y': ## image'de geri al cunku buna ihtiyac var... \n            image  = np.rot90(image,  k=1, axes=(0, 1))\n            result = np.rot90(result, k=1, axes=(0, 1))\n            xtimes = np.rot90(xtimes, k=1, axes=(0, 1)) ## XTIMES da dondurecegiz.. orjinale\n        elif rotation == 'x':\n            image  = np.rot90(image,  k=1, axes=(0, 2))\n            result = np.rot90(result, k=1, axes=(0, 2))\n            xtimes = np.rot90(xtimes, k=1, axes=(0, 2)) \n            \n\n          \n    result /=xtimes ## result = result/xtimes\n    \n    del xtimes\n    gc.collect()\n    libc.malloc_trim(0)\n    \n    \n    return result","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:49.923478Z","iopub.execute_input":"2024-01-23T20:02:49.923945Z","iopub.status.idle":"2024-01-23T20:02:50.264519Z","shell.execute_reply.started":"2024-01-23T20:02:49.923911Z","shell.execute_reply":"2024-01-23T20:02:50.263461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deneme = cfg.deneme\npartialdeneme = cfg.partialdeneme ## kisa deneme.... tumdeneme only if you want to test time/memory etc.\n\nif deneme: \n    kidney1_paths   = list(Path('/kaggle/input/14ocak-load-data-004-option1-final/kidney_1_dense/images').glob('**/*.tif'))\n    # label1_base = Path('/kaggle/input/blood-vessel-segmentation/train/kidney_1_dense/labels')\n\n    kidney2_paths   = list(Path('/kaggle/input/14ocak-load-data-004-option1-final/kidney_2/images').glob('**/*.tif'))\n    # label2_base = Path('/kaggle/input/blood-vessel-segmentation/train/kidney_2/labels')\n    \n    kidney1_paths.sort(key = lambda x: x.stem)\n    kidney2_paths.sort(key = lambda x: x.stem)\n\n\n    if partialdeneme:\n        kidney1_paths = kidney1_paths[800:1200] ## ortadan alayim dedim\n        kidney2_paths = kidney2_paths[800:1200]\n\n    len(kidney1_paths),len(kidney2_paths)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:50.265759Z","iopub.execute_input":"2024-01-23T20:02:50.266607Z","iopub.status.idle":"2024-01-23T20:02:58.12707Z","shell.execute_reply.started":"2024-01-23T20:02:50.266578Z","shell.execute_reply":"2024-01-23T20:02:58.126044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## process image icinde 3te1 cikarma ekledim\ndef process_image(path, bins1):\n    img = tiff.imread(path).astype(np.float32)\n    img_tensor = torch.tensor(img)\n    img_tensor = img_tensor.hist_scaled(bins1)  # assuming hist_scaled is an in-place operation\n#     return np.array(img_tensor * 255).astype(np.uint8)\n#     return (np.array(torch.clamp((img_tensor-.33)*1.5 ,0.,1.))*255).astype(np.uint8)\n#     return (np.array(torch.clamp((img_tensor-.50)*2.0 ,0.,1.))*255).astype(np.uint8)\n    return (np.array(torch.clamp((img_tensor-.60)*2.5 ,0.,1.))*255).astype(np.uint8)\n#     return (np.array(torch.clamp((img_tensor-.75)*4.0 ,0.,1.))*255).astype(np.uint8)\n\n\ndef read_and_stack_images_only(paths): ## label bolumunu atiyorum\n    mu, stdx, bins1 = find_bin(paths)\n    print('found the bins...')\n\n    # Process images using a generator expression\n    images = (process_image(path, bins1) for path in paths)\n    images_stacked = np.stack(images)\n\n    # Process labels\n#     labels = np.stack((tiff.imread(label_base / path.name) > 0 for path in paths))\n\n    return images_stacked, mu, stdx","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:02:58.128693Z","iopub.execute_input":"2024-01-23T20:02:58.129093Z","iopub.status.idle":"2024-01-23T20:02:58.138448Z","shell.execute_reply.started":"2024-01-23T20:02:58.129058Z","shell.execute_reply":"2024-01-23T20:02:58.137504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df=[]\nthold = .45\n\ndef rle_encode(mask):\n    pixel = mask.flatten()\n    pixel = np.concatenate([[0], pixel, [0]])\n    run = np.where(pixel[1:] != pixel[:-1])[0] + 1\n    run[1::2] -= run[::2]\n    rle = ' '.join(str(r) for r in run)\n    if rle == '':\n        rle = '1 0'\n    return rle\n\n\n## mantik su...\n## deneme ise.... kidney 3 deneyecegim... ama pathleri az...\nif deneme and (len(kidney5_paths)==3): ##bir kere submit ise... zaten otomatik olarak kid5 ve 6 al!\n    print('deneme calisiyor')\n    infer_paths = [kidney3_paths] ## kidney 3 ile deneyeyim... sonuclari da kontrol ederim\nelse:\n    infer_paths = [kidney5_paths,kidney6_paths]\n    \n\nif (len(kidney5_paths)==3) and (not deneme):\n    print('kisa submit calisiyor')\n    kidney_3_images =kidney_3_images[:32,0:cfg.focus_size,0:cfg.focus_size]\n    results_all = process_image_corrected_v2(kidney_3_images,learners)\n    kidney3_paths = kidney3_paths[:32]\n    for pat,mask in zip(kidney3_paths,results_all):\n        subpath = pat.parent.parent.name\n        if subpath == 'kidney_3_sparse':\n            subpath = 'kidney_3_dense'\n        idx = subpath + '_'+ pat.stem\n        mask_pred=(mask>thold)\n\n        rle = rle_encode(mask_pred)\n\n        submission_df.append(\n            pd.DataFrame(data={\n                'id'  : idx,\n                'rle' : rle,\n            },index=[0])\n        )  \n    \nelse:\n    for image_paths in infer_paths:\n        # Reading and stacking images for kidney1 and labels1\n        ### su an icin mu ve sigma kullanmiorum.. stadandard orjinale boluyorum\n        #### degerlendirilebilir... \n        kidney_images,mu,sigma = read_and_stack_images_only(image_paths)\n        results_all = process_image_corrected_v2(kidney_images,learners)\n        for pat,mask in zip(image_paths,results_all):\n            subpath = pat.parent.parent.name\n            idx = subpath + '_'+ pat.stem\n            mask_pred=(mask>thold)\n\n            rle = rle_encode(mask_pred)\n\n            submission_df.append(\n                pd.DataFrame(data={\n                    'id'  : idx,\n                    'rle' : rle,\n                },index=[0])\n            )\n            \n        del results_all,kidney_images\n        gc.collect()\n        libc.malloc_trim(0)\n        \n    \n\nsubmission_df =pd.concat(submission_df).reset_index(drop=True)\nsubmission_df.to_csv('submission.csv', index=False)\n\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:51:16.41332Z","iopub.execute_input":"2024-01-23T20:51:16.413715Z","iopub.status.idle":"2024-01-23T20:52:40.97306Z","shell.execute_reply.started":"2024-01-23T20:51:16.413683Z","shell.execute_reply":"2024-01-23T20:52:40.971101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:52:40.973975Z","iopub.status.idle":"2024-01-23T20:52:40.974322Z","shell.execute_reply.started":"2024-01-23T20:52:40.974157Z","shell.execute_reply":"2024-01-23T20:52:40.974174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## erase all but submission","metadata":{}},{"cell_type":"code","source":"for f in glob.glob('*'):\n    if not f.startswith('subm'):\n        !rm -rf {f}","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:46:21.457222Z","iopub.execute_input":"2024-01-23T20:46:21.457758Z","iopub.status.idle":"2024-01-23T20:46:23.445015Z","shell.execute_reply.started":"2024-01-23T20:46:21.457724Z","shell.execute_reply":"2024-01-23T20:46:23.443628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## trial .. open in interactive mode","metadata":{}},{"cell_type":"code","source":"# sys.path.append('/kaggle/input/sennet-score/src')\n# from official_metric import create_table_neighbour_code_to_surface_area\n# device = 'cuda' if torch.cuda.is_available else 'cpu'\n# device","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:26.841952Z","iopub.execute_input":"2024-01-23T20:49:26.842394Z","iopub.status.idle":"2024-01-23T20:49:28.024Z","shell.execute_reply.started":"2024-01-23T20:49:26.842359Z","shell.execute_reply":"2024-01-23T20:49:28.023041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# di = '/kaggle/input/blood-vessel-segmentation'\n\n# # PyTorch version dependence on index data type\n# torch_ver_major = int(torch.__version__.split('.')[0])\n# dtype_index = torch.int32 if torch_ver_major >= 2 else torch.long","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:28.962197Z","iopub.execute_input":"2024-01-23T20:49:28.962587Z","iopub.status.idle":"2024-01-23T20:49:28.967725Z","shell.execute_reply.started":"2024-01-23T20:49:28.962553Z","shell.execute_reply":"2024-01-23T20:49:28.966693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def rle_decode(mask_rle: str, shape: tuple) -> np.array:\n#     \"\"\"\n#     Decode rle string\n#     https://www.kaggle.com/code/paulorzp/run-length-encode-and-decode/script\n#     https://www.kaggle.com/stainsby/fast-tested-rle\n\n#     Args:\n#       mask_rle: run length (rle) as string\n#       shape: (height, width) of the mask\n\n#     Returns:\n#       array[uint8], 1 - mask, 0 - background\n#     \"\"\"\n#     s = mask_rle.split()\n#     starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n#     starts -= 1\n#     ends = starts + lengths\n#     img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n#     for lo, hi in zip(starts, ends):\n#         img[lo:hi] = 1\n#     return img.reshape(shape)\n\n\n# def compute_area(y: list, unfold: nn.Unfold, area: torch.Tensor) -> torch.Tensor:\n#     \"\"\"\n#     Args:\n#       y (list[Tensor]): A pair of consecutive slices of mask\n#       unfold: nn.Unfold(kernel_size=(2, 2), padding=1)\n#       area (Tensor): surface area for 256 patterns (256, )\n\n#     Returns:\n#       Surface area of surface in 2x2x2 cube\n#     \"\"\"\n#     # Two layers of segmentation masks\n#     yy = torch.stack(y, dim=0).to(torch.float16).unsqueeze(0)\n#     # (batch_size=1, nch=2, H, W) \n#     # bit (0/1) but unfold requires float\n\n#     # unfold slides through the volume like a convolution\n#     # 2x2 kernel returns 8 values (2 channels * 2x2)\n#     cubes_float = unfold(yy).squeeze(0)  # (8, n_cubes)\n\n#     # Each of the 8 values are either 0 or 1\n#     # Convert those 8 bits to one uint8\n#     cubes_byte = torch.zeros(cubes_float.size(1), dtype=dtype_index, device=device)\n#     # indices are required to be int32 or long for area[cube_byte] below, not uint8\n#     # Can be int32 for torch 2.0.0, int32 raise IndexError in torch 1.13.1.\n    \n#     for k in range(8):\n#         cubes_byte += cubes_float[k, :].to(dtype_index) << k\n\n#     # Use area lookup table: pattern index -> area [float]\n#     cubes_area = area[cubes_byte]\n\n#     return cubes_area\n\n\n# def compute_surface_dice_score(submit: pd.DataFrame, label: pd.DataFrame) -> float:\n#     \"\"\"\n#     Compute surface Dice score for one 3D volume\n\n#     submit (pd.DataFrame): submission file with id and rle\n#     label (pd.DataFrame): ground truth id, rle, and also image height, width\n#     \"\"\"\n#     # submit and label must contain exact same id in same order\n# #     print(list(zip(submit.id,label.id)))\n#     assert (submit['id'] == label['id']).all()\n#     assert len(label) > 0\n\n#     # All height, width must be the same\n#     len(label['height'].unique()) == 1\n#     len(label['width'].unique()) == 1\n\n#     # Surface area lookup table: Tensor[float32] (256, )\n#     area = create_table_neighbour_code_to_surface_area((1, 1, 1))\n#     area = torch.from_numpy(area).to(device)  # torch.float32\n\n#     # Slide through the volume like a convolution\n#     unfold = torch.nn.Unfold(kernel_size=(2, 2), padding=1)\n\n#     r = label.iloc[0]\n#     h, w = r['height'], r['width']\n#     n_slices = len(label)\n\n#     # Padding before first slice\n#     y0 = y0_pred = torch.zeros((h, w), dtype=torch.uint8, device=device)\n\n#     num = 0     # numerator of surface Dice\n#     denom = 0   # denominator\n#     for i in range(n_slices + 1):\n#         # Load one slice\n#         if i < n_slices:\n#             r = label.iloc[i]\n#             y1 = rle_decode(r['rle'], (h, w))\n#             y1 = torch.from_numpy(y1).to(device)\n\n#             r = submit.iloc[i]\n#             y1_pred = rle_decode(r['rle'], (h, w))\n#             y1_pred = torch.from_numpy(y1_pred).to(device)\n#         else:\n#             # Padding after the last slice\n#             y1 = y1_pred = torch.zeros((h, w), dtype=torch.uint8, device=device)\n\n#         # Compute the surface area between two slices (n_cubes,)\n#         area_pred = compute_area([y0_pred, y1_pred], unfold, area)\n#         area_true = compute_area([y0, y1], unfold, area)\n\n#         # True positive cube indices\n#         idx = torch.logical_and(area_pred > 0, area_true > 0)\n\n#         # Surface dice numerator and denominator\n#         num += area_pred[idx].sum() + area_true[idx].sum()\n#         denom += area_pred.sum() + area_true.sum()\n\n#         # Next slice\n#         y0 = y1\n#         y0_pred = y1_pred\n\n#     dice = num / denom.clamp(min=1e-8)\n#     return dice.item()","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:31.288262Z","iopub.execute_input":"2024-01-23T20:49:31.289309Z","iopub.status.idle":"2024-01-23T20:49:31.309256Z","shell.execute_reply.started":"2024-01-23T20:49:31.289271Z","shell.execute_reply":"2024-01-23T20:49:31.308065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def add_size_columns(df: pd.DataFrame):\n#     \"\"\"\n#     df (DataFrame): including id column, e.g., kidney_1_dense_0000\n#     \"\"\"\n#     widths = []\n#     heights = []\n#     subdirs = []\n#     nums = []\n#     for i, r in df.iterrows():\n#         file_id = r['id']\n#         subdir = file_id[:-5]    # kidney_1_dense\n#         if subdir =='kidney_3_dense':\n#             subdir = 'kidney_3_sparse' # cunku images are there.. BU SADECE RESIM OKUMAK ICIN\n#         file_num = file_id[-4:]  # 0000\n\n#         filename = '%s/train/%s/images/%s.tif' % (di, subdir, file_num)\n#         img = Image.open(filename)\n#         w, h = img.size\n#         widths.append(w)\n#         heights.append(h)\n#         subdirs.append(subdir)\n#         nums.append(file_num)\n\n#     df['width'] = widths\n#     df['height'] = heights\n#     df['image_id'] = subdirs\n#     df['slice_id'] = nums","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:32.200101Z","iopub.execute_input":"2024-01-23T20:49:32.200486Z","iopub.status.idle":"2024-01-23T20:49:32.208929Z","shell.execute_reply.started":"2024-01-23T20:49:32.200456Z","shell.execute_reply":"2024-01-23T20:49:32.207724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Label for kidney_3_dense only\n\n# image_id = 'kidney_3_dense'\n\n# label = pd.read_csv(di + '/train_rles.csv')\n# idx = label['id'].str.startswith(image_id)\n# label = label[idx].reset_index(drop=True)\n# assert len(label) > 0\n\n# # Add height, width columns\n# add_size_columns(label)\n\n# label.head(n=2)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:33.61867Z","iopub.execute_input":"2024-01-23T20:49:33.619671Z","iopub.status.idle":"2024-01-23T20:49:36.057382Z","shell.execute_reply.started":"2024-01-23T20:49:33.619625Z","shell.execute_reply":"2024-01-23T20:49:36.056205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:36.059591Z","iopub.execute_input":"2024-01-23T20:49:36.060539Z","iopub.status.idle":"2024-01-23T20:49:36.072188Z","shell.execute_reply.started":"2024-01-23T20:49:36.060496Z","shell.execute_reply":"2024-01-23T20:49:36.070922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(submission_df), len(label)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:43.165367Z","iopub.execute_input":"2024-01-23T20:49:43.166224Z","iopub.status.idle":"2024-01-23T20:49:43.174444Z","shell.execute_reply.started":"2024-01-23T20:49:43.16618Z","shell.execute_reply":"2024-01-23T20:49:43.173344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# (submission_df.id==label.id).all() ## burayi gecemiyordu. rle scoring\n#                                    ## problem: indexleri farkli idi.. indexlerini ayniladim...","metadata":{"execution":{"iopub.status.busy":"2024-01-23T20:49:44.547333Z","iopub.execute_input":"2024-01-23T20:49:44.547784Z","iopub.status.idle":"2024-01-23T20:49:44.555782Z","shell.execute_reply.started":"2024-01-23T20:49:44.547749Z","shell.execute_reply":"2024-01-23T20:49:44.55454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# # Compute surface Dice score\n# score = compute_surface_dice_score(submission_df, label)\n# score","metadata":{},"execution_count":null,"outputs":[]}]}