{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\nimport glob\nimport SimpleITK as sitk\nfrom tqdm import tqdm\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-17T15:09:24.400833Z","iopub.execute_input":"2022-05-17T15:09:24.401097Z","iopub.status.idle":"2022-05-17T15:09:24.407341Z","shell.execute_reply.started":"2022-05-17T15:09:24.401067Z","shell.execute_reply":"2022-05-17T15:09:24.406622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = True\n\nif DEBUG:\n    TARGET_PATH = '/kaggle/input/uw-madison-gi-tract-image-segmentation/train/'\nelse:\n    TARGET_PATH = '/kaggle/input/uw-madison-gi-tract-image-segmentation/test/'","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:09:24.413001Z","iopub.execute_input":"2022-05-17T15:09:24.413754Z","iopub.status.idle":"2022-05-17T15:09:24.421543Z","shell.execute_reply.started":"2022-05-17T15:09:24.413726Z","shell.execute_reply":"2022-05-17T15:09:24.420392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/nnunetmaster712/nnUNet-master /kaggle/working/nnunet\n!cp -r /kaggle/input/nnunetdependencies /kaggle/working/nnunetdependencies","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:09:24.422769Z","iopub.execute_input":"2022-05-17T15:09:24.424583Z","iopub.status.idle":"2022-05-17T15:09:28.237551Z","shell.execute_reply.started":"2022-05-17T15:09:24.424545Z","shell.execute_reply":"2022-05-17T15:09:28.236551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile /kaggle/working/nnunet/nnunet/inference/predict.py\n#    Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nimport argparse\nfrom copy import deepcopy\nfrom typing import Tuple, Union, List\n\nimport numpy as np\nfrom batchgenerators.augmentations.utils import resize_segmentation\nfrom nnunet.inference.segmentation_export import save_segmentation_nifti_from_softmax, save_segmentation_nifti\nfrom batchgenerators.utilities.file_and_folder_operations import *\nfrom multiprocessing import Process, Queue\nimport torch\nimport SimpleITK as sitk\nimport shutil\nfrom multiprocessing import Pool\nfrom nnunet.postprocessing.connected_components import load_remove_save, load_postprocessing\nfrom nnunet.training.model_restore import load_model_and_checkpoint_files\nfrom nnunet.training.network_training.nnUNetTrainer import nnUNetTrainer\nfrom nnunet.utilities.one_hot_encoding import to_one_hot\n\n\nfrom memory_profiler import profile\nimport psutil\n\ndef preprocess_save_to_queue(preprocess_fn, q, list_of_lists, output_files, segs_from_prev_stage, classes,\n                             transpose_forward):\n    # suppress output\n    # sys.stdout = open(os.devnull, 'w')\n\n    errors_in = []\n    for i, l in enumerate(list_of_lists):\n        try:\n            output_file = output_files[i]\n            print(\"preprocessing\", output_file)\n            d, _, dct = preprocess_fn(l)\n            # print(output_file, dct)\n            if segs_from_prev_stage[i] is not None:\n                assert isfile(segs_from_prev_stage[i]) and segs_from_prev_stage[i].endswith(\n                    \".nii.gz\"), \"segs_from_prev_stage\" \\\n                                \" must point to a \" \\\n                                \"segmentation file\"\n                seg_prev = sitk.GetArrayFromImage(sitk.ReadImage(segs_from_prev_stage[i]))\n                # check to see if shapes match\n                img = sitk.GetArrayFromImage(sitk.ReadImage(l[0]))\n                assert all([i == j for i, j in zip(seg_prev.shape, img.shape)]), \"image and segmentation from previous \" \\\n                                                                                 \"stage don't have the same pixel array \" \\\n                                                                                 \"shape! image: %s, seg_prev: %s\" % \\\n                                                                                 (l[0], segs_from_prev_stage[i])\n                seg_prev = seg_prev.transpose(transpose_forward)\n                seg_reshaped = resize_segmentation(seg_prev, d.shape[1:], order=1)\n                seg_reshaped = to_one_hot(seg_reshaped, classes)\n                d = np.vstack((d, seg_reshaped)).astype(np.float32)\n            \"\"\"There is a problem with python process communication that prevents us from communicating objects \n            larger than 2 GB between processes (basically when the length of the pickle string that will be sent is \n            communicated by the multiprocessing.Pipe object then the placeholder (I think) does not allow for long \n            enough strings (lol). This could be fixed by changing i to l (for long) but that would require manually \n            patching system python code. We circumvent that problem here by saving softmax_pred to a npy file that will \n            then be read (and finally deleted) by the Process. save_segmentation_nifti_from_softmax can take either \n            filename or np.ndarray and will handle this automatically\"\"\"\n            print(d.shape)\n            if np.prod(d.shape) > (2e9 / 4 * 0.85):  # *0.85 just to be save, 4 because float32 is 4 bytes\n                print(\n                    \"This output is too large for python process-process communication. \"\n                    \"Saving output temporarily to disk\")\n                np.save(output_file[:-7] + \".npy\", d)\n                d = output_file[:-7] + \".npy\"\n            q.put((output_file, (d, dct)))\n        except KeyboardInterrupt:\n            raise KeyboardInterrupt\n        except Exception as e:\n            print(\"error in\", l)\n            print(e)\n    q.put(\"end\")\n    if len(errors_in) > 0:\n        print(\"There were some errors in the following cases:\", errors_in)\n        print(\"These cases were ignored.\")\n    else:\n        print(\"This worker has ended successfully, no errors to report\")\n    # restore output\n    # sys.stdout = sys.__stdout__\n\n\ndef preprocess_multithreaded(trainer, list_of_lists, output_files, num_processes=2, segs_from_prev_stage=None):\n    if segs_from_prev_stage is None:\n        segs_from_prev_stage = [None] * len(list_of_lists)\n\n    num_processes = min(len(list_of_lists), num_processes)\n\n    classes = list(range(1, trainer.num_classes))\n    assert isinstance(trainer, nnUNetTrainer)\n    q = Queue(1)\n    processes = []\n    for i in range(num_processes):\n        pr = Process(target=preprocess_save_to_queue, args=(trainer.preprocess_patient, q,\n                                                            list_of_lists[i::num_processes],\n                                                            output_files[i::num_processes],\n                                                            segs_from_prev_stage[i::num_processes],\n                                                            classes, trainer.plans['transpose_forward']))\n        pr.start()\n        processes.append(pr)\n\n    try:\n        end_ctr = 0\n        while end_ctr != num_processes:\n            item = q.get()\n            if item == \"end\":\n                end_ctr += 1\n                continue\n            else:\n                yield item\n\n    finally:\n        for p in processes:\n            if p.is_alive():\n                p.terminate()  # this should not happen but better safe than sorry right\n            p.join()\n\n        q.close()\n\ndef custom_postprocess(pred):\n\n    '''\n    cutoff_exp = 999\n    cutoff_ctrl = 999\n    \n    \n    \n    for i in range(pred.shape[1]):\n        if pred[1:,i].max()>=0.995:\n            cutoff_exp = i\n    \n    for n,j in enumerate(range(pred.shape[1])):\n        if n > cutoff_exp:\n            pred[2,n] = 0\n            pred[1,n] = 0\n            \n            #temp_tar = pred[:,n]\n            #temp[temp_tar[1]>0.4] = 1\n            #temp[temp_tar[2]>0.4] = 2\n            #temp[temp_tar[3]>0.4] = 3\n    '''\n    '''\n    softmax_max = np.argmax(softmax, 0)\n    \n    shape_ = softmax_max.shape[0]\n    flagg = 0\n    milestonesp = 999\n    milestone1 = 999\n    milestone2 = 999\n    for numb,ii in enumerate(softmax_max):\n        if numb < shape_ - 1:\n            if ((ii>0).sum().astype(float) / ((softmax_max[numb+1]>0).sum().astype(float) + 0.0001))>5 and flagg ==0:\n                milestonesp = numb\n                flagg = 1\n        \n        if 1 in ii:\n            milestone1 = numb\n        if 2 in ii:\n            milestone2 = numb\n    for numb,ii in enumerate(softmax_max):\n        if numb > milestone1:\n            softmax[2,numb] = 0.\n        \n        if numb > milestone2:\n            softmax[1,numb] = 0.\n        if numb > milestonesp - 1:\n            softmax[2,numb] = 0.\n            softmax[1,numb] = 0.\n        elif numb == milestonesp - 2:\n            softmax[0,numb] *= 0.9\n    '''\n    \n    #print('B:%.2f MB' % (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024))\n    \n    return pred\ndef predict_cases(model, list_of_lists, output_filenames, folds, save_npz, num_threads_preprocessing,\n                  num_threads_nifti_save, segs_from_prev_stage=None, do_tta=True, mixed_precision=True,\n                  overwrite_existing=False,\n                  all_in_gpu=False, step_size=0.5, checkpoint_name=\"model_final_checkpoint\",\n                  segmentation_export_kwargs: dict = None, disable_postprocessing: bool = False):\n    \"\"\"\n    :param segmentation_export_kwargs:\n    :param model: folder where the model is saved, must contain fold_x subfolders\n    :param list_of_lists: [[case0_0000.nii.gz, case0_0001.nii.gz], [case1_0000.nii.gz, case1_0001.nii.gz], ...]\n    :param output_filenames: [output_file_case0.nii.gz, output_file_case1.nii.gz, ...]\n    :param folds: default: (0, 1, 2, 3, 4) (but can also be 'all' or a subset of the five folds, for example use (0, )\n    for using only fold_0\n    :param save_npz: default: False\n    :param num_threads_preprocessing:\n    :param num_threads_nifti_save:\n    :param segs_from_prev_stage:\n    :param do_tta: default: True, can be set to False for a 8x speedup at the cost of a reduced segmentation quality\n    :param overwrite_existing: default: True\n    :param mixed_precision: if None then we take no action. If True/False we overwrite what the model has in its init\n    :return:\n    \"\"\"\n    assert len(list_of_lists) == len(output_filenames)\n    if segs_from_prev_stage is not None: assert len(segs_from_prev_stage) == len(output_filenames)\n\n    pool = Pool(num_threads_nifti_save)\n    results = []\n\n    cleaned_output_files = []\n    for o in output_filenames:\n        dr, f = os.path.split(o)\n        if len(dr) > 0:\n            maybe_mkdir_p(dr)\n        if not f.endswith(\".nii.gz\"):\n            f, _ = os.path.splitext(f)\n            f = f + \".nii.gz\"\n        cleaned_output_files.append(join(dr, f))\n\n    if not overwrite_existing:\n        print(\"number of cases:\", len(list_of_lists))\n        # if save_npz=True then we should also check for missing npz files\n        not_done_idx = [i for i, j in enumerate(cleaned_output_files) if (not isfile(j)) or (save_npz and not isfile(j[:-7] + '.npz'))]\n\n        cleaned_output_files = [cleaned_output_files[i] for i in not_done_idx]\n        list_of_lists = [list_of_lists[i] for i in not_done_idx]\n        if segs_from_prev_stage is not None:\n            segs_from_prev_stage = [segs_from_prev_stage[i] for i in not_done_idx]\n\n        print(\"number of cases that still need to be predicted:\", len(cleaned_output_files))\n\n    print(\"emptying cuda cache\")\n    torch.cuda.empty_cache()\n\n    print(\"loading parameters for folds,\", folds)\n    trainer, params = load_model_and_checkpoint_files(model, folds, mixed_precision=mixed_precision,\n                                                      checkpoint_name=checkpoint_name)\n\n    if segmentation_export_kwargs is None:\n        if 'segmentation_export_params' in trainer.plans.keys():\n            force_separate_z = trainer.plans['segmentation_export_params']['force_separate_z']\n            interpolation_order = trainer.plans['segmentation_export_params']['interpolation_order']\n            interpolation_order_z = trainer.plans['segmentation_export_params']['interpolation_order_z']\n        else:\n            force_separate_z = None\n            interpolation_order = 1\n            interpolation_order_z = 0\n    else:\n        force_separate_z = segmentation_export_kwargs['force_separate_z']\n        interpolation_order = segmentation_export_kwargs['interpolation_order']\n        interpolation_order_z = segmentation_export_kwargs['interpolation_order_z']\n\n    print(\"starting preprocessing generator\")\n    preprocessing = preprocess_multithreaded(trainer, list_of_lists, cleaned_output_files, num_threads_preprocessing,\n                                             segs_from_prev_stage)\n    print(\"starting prediction...\")\n    all_output_files = []\n    for preprocessed in preprocessing:\n        output_filename, (d, dct) = preprocessed\n        all_output_files.append(all_output_files)\n        if isinstance(d, str):\n            data = np.load(d)\n            os.remove(d)\n            d = data\n\n        print(\"predicting\", output_filename)\n        trainer.load_checkpoint_ram(params[0], False)\n        softmax = trainer.predict_preprocessed_data_return_seg_and_softmax(\n            d, do_mirroring=do_tta, mirror_axes=trainer.data_aug_params['mirror_axes'], use_sliding_window=True,\n            step_size=step_size, use_gaussian=True, all_in_gpu=all_in_gpu,\n            mixed_precision=mixed_precision)[1]\n        #print('A:%.2f MB' % (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024))\n        softmax = custom_postprocess(softmax)\n        #print('C:%.2f MB' % (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024))\n        print('softmax.shapenew')\n        print(softmax.shape)\n        for p in params[1:]:\n            print('doing folds ensemble...')\n            trainer.load_checkpoint_ram(p, False)\n            softmax_temp = trainer.predict_preprocessed_data_return_seg_and_softmax(\n                d, do_mirroring=do_tta, mirror_axes=trainer.data_aug_params['mirror_axes'], use_sliding_window=True,\n                step_size=step_size, use_gaussian=True, all_in_gpu=all_in_gpu,\n                mixed_precision=mixed_precision)[1]\n            \n            softmax_temp = custom_postprocess(softmax_temp)\n            # here is ensemble of folds\n            softmax += softmax_temp\n            del(softmax_temp)\n            \n        if len(params) > 1:\n            softmax /= len(params)\n\n        transpose_forward = trainer.plans.get('transpose_forward')\n        if transpose_forward is not None:\n            transpose_backward = trainer.plans.get('transpose_backward')\n            softmax = softmax.transpose([0] + [i + 1 for i in transpose_backward])\n\n        if save_npz:\n            npz_file = output_filename[:-7] + \".npz\"\n        else:\n            npz_file = None\n\n        if hasattr(trainer, 'regions_class_order'):\n            region_class_order = trainer.regions_class_order\n        else:\n            region_class_order = None\n\n        \"\"\"There is a problem with python process communication that prevents us from communicating objects \n        larger than 2 GB between processes (basically when the length of the pickle string that will be sent is \n        communicated by the multiprocessing.Pipe object then the placeholder (I think) does not allow for long \n        enough strings (lol). This could be fixed by changing i to l (for long) but that would require manually \n        patching system python code. We circumvent that problem here by saving softmax_pred to a npy file that will \n        then be read (and finally deleted) by the Process. save_segmentation_nifti_from_softmax can take either \n        filename or np.ndarray and will handle this automatically\"\"\"\n        bytes_per_voxel = 4\n        if all_in_gpu:\n            bytes_per_voxel = 2  # if all_in_gpu then the return value is half (float16)\n        if np.prod(softmax.shape) > (2e9 / bytes_per_voxel * 0.85):  # * 0.85 just to be save\n            print(\n                \"This output is too large for python process-process communication. Saving output temporarily to disk\")\n            np.save(output_filename[:-7] + \".npy\", softmax)\n            softmax = output_filename[:-7] + \".npy\"\n        \n        \n        results.append(pool.starmap_async(save_segmentation_nifti_from_softmax,\n                                          ((softmax, output_filename, dct, interpolation_order, region_class_order,\n                                            None, None,\n                                            npz_file, None, force_separate_z, interpolation_order_z),)\n                                          ))\n\n    print(\"inference done. Now waiting for the segmentation export to finish...\")\n    _ = [i.get() for i in results]\n    # now apply postprocessing\n    # first load the postprocessing properties if they are present. Else raise a well visible warning\n    if not disable_postprocessing:\n        results = []\n        pp_file = join(model, \"postprocessing.json\")\n        if isfile(pp_file):\n            print(\"postprocessing...\")\n            shutil.copy(pp_file, os.path.abspath(os.path.dirname(output_filenames[0])))\n            # for_which_classes stores for which of the classes everything but the largest connected component needs to be\n            # removed\n            for_which_classes, min_valid_obj_size = load_postprocessing(pp_file)\n            results.append(pool.starmap_async(load_remove_save,\n                                              zip(output_filenames, output_filenames,\n                                                  [for_which_classes] * len(output_filenames),\n                                                  [min_valid_obj_size] * len(output_filenames))))\n            _ = [i.get() for i in results]\n        else:\n            print(\"WARNING! Cannot run postprocessing because the postprocessing file is missing. Make sure to run \"\n                  \"consolidate_folds in the output folder of the model first!\\nThe folder you need to run this in is \"\n                  \"%s\" % model)\n\n    pool.close()\n    pool.join()\n\n\ndef predict_cases_fast(model, list_of_lists, output_filenames, folds, num_threads_preprocessing,\n                       num_threads_nifti_save, segs_from_prev_stage=None, do_tta=True, mixed_precision=True,\n                       overwrite_existing=False,\n                       all_in_gpu=False, step_size=0.5, checkpoint_name=\"model_final_checkpoint\",\n                       segmentation_export_kwargs: dict = None, disable_postprocessing: bool = False):\n    assert len(list_of_lists) == len(output_filenames)\n    if segs_from_prev_stage is not None: assert len(segs_from_prev_stage) == len(output_filenames)\n\n    pool = Pool(num_threads_nifti_save)\n    results = []\n\n    cleaned_output_files = []\n    for o in output_filenames:\n        dr, f = os.path.split(o)\n        if len(dr) > 0:\n            maybe_mkdir_p(dr)\n        if not f.endswith(\".nii.gz\"):\n            f, _ = os.path.splitext(f)\n            f = f + \".nii.gz\"\n        cleaned_output_files.append(join(dr, f))\n\n    if not overwrite_existing:\n        print(\"number of cases:\", len(list_of_lists))\n        not_done_idx = [i for i, j in enumerate(cleaned_output_files) if not isfile(j)]\n\n        cleaned_output_files = [cleaned_output_files[i] for i in not_done_idx]\n        list_of_lists = [list_of_lists[i] for i in not_done_idx]\n        if segs_from_prev_stage is not None:\n            segs_from_prev_stage = [segs_from_prev_stage[i] for i in not_done_idx]\n\n        print(\"number of cases that still need to be predicted:\", len(cleaned_output_files))\n\n    print(\"emptying cuda cache\")\n    torch.cuda.empty_cache()\n\n    print(\"loading parameters for folds,\", folds)\n    trainer, params = load_model_and_checkpoint_files(model, folds, mixed_precision=mixed_precision,\n                                                      checkpoint_name=checkpoint_name)\n\n    if segmentation_export_kwargs is None:\n        if 'segmentation_export_params' in trainer.plans.keys():\n            force_separate_z = trainer.plans['segmentation_export_params']['force_separate_z']\n            interpolation_order = trainer.plans['segmentation_export_params']['interpolation_order']\n            interpolation_order_z = trainer.plans['segmentation_export_params']['interpolation_order_z']\n        else:\n            force_separate_z = None\n            interpolation_order = 1\n            interpolation_order_z = 0\n    else:\n        force_separate_z = segmentation_export_kwargs['force_separate_z']\n        interpolation_order = segmentation_export_kwargs['interpolation_order']\n        interpolation_order_z = segmentation_export_kwargs['interpolation_order_z']\n\n    print(\"starting preprocessing generator\")\n    preprocessing = preprocess_multithreaded(trainer, list_of_lists, cleaned_output_files, num_threads_preprocessing,\n                                             segs_from_prev_stage)\n\n    print(\"starting prediction...\")\n    for preprocessed in preprocessing:\n        print(\"getting data from preprocessor\")\n        output_filename, (d, dct) = preprocessed\n        print(\"got something\")\n        if isinstance(d, str):\n            print(\"what I got is a string, so I need to load a file\")\n            data = np.load(d)\n            os.remove(d)\n            d = data\n\n        # preallocate the output arrays\n        # same dtype as the return value in predict_preprocessed_data_return_seg_and_softmax (saves time)\n        softmax_aggr = None  # np.zeros((trainer.num_classes, *d.shape[1:]), dtype=np.float16)\n        all_seg_outputs = np.zeros((len(params), *d.shape[1:]), dtype=int)\n        print(\"predicting\", output_filename)\n\n        for i, p in enumerate(params):\n            trainer.load_checkpoint_ram(p, False)\n\n            res = trainer.predict_preprocessed_data_return_seg_and_softmax(d, do_mirroring=do_tta,\n                                                                           mirror_axes=trainer.data_aug_params['mirror_axes'],\n                                                                           use_sliding_window=True,\n                                                                           step_size=step_size, use_gaussian=True,\n                                                                           all_in_gpu=all_in_gpu,\n                                                                           mixed_precision=mixed_precision)\n\n            if len(params) > 1:\n                # otherwise we dont need this and we can save ourselves the time it takes to copy that\n                print(\"aggregating softmax\")\n                if softmax_aggr is None:\n                    softmax_aggr = res[1]\n                else:\n                    softmax_aggr += res[1]\n            all_seg_outputs[i] = res[0]\n\n        print(\"obtaining segmentation map\")\n        if len(params) > 1:\n            # we dont need to normalize the softmax by 1 / len(params) because this would not change the outcome of the argmax\n            seg = softmax_aggr.argmax(0)\n        else:\n            seg = all_seg_outputs[0]\n\n        print(\"applying transpose_backward\")\n        transpose_forward = trainer.plans.get('transpose_forward')\n        if transpose_forward is not None:\n            transpose_backward = trainer.plans.get('transpose_backward')\n            seg = seg.transpose([i for i in transpose_backward])\n\n        if hasattr(trainer, 'regions_class_order'):\n            region_class_order = trainer.regions_class_order\n        else:\n            region_class_order = None\n        assert region_class_order is None, \"predict_cases_fast can only work with regular softmax predictions \" \\\n                                           \"and is therefore unable to handle trainer classes with region_class_order\"\n\n        print(\"initializing segmentation export\")\n        results.append(pool.starmap_async(save_segmentation_nifti,\n                                          ((seg, output_filename, dct, interpolation_order, force_separate_z,\n                                            interpolation_order_z),)\n                                          ))\n        print(\"done\")\n\n    print(\"inference done. Now waiting for the segmentation export to finish...\")\n    _ = [i.get() for i in results]\n    # now apply postprocessing\n    # first load the postprocessing properties if they are present. Else raise a well visible warning\n\n    if not disable_postprocessing:\n        results = []\n        pp_file = join(model, \"postprocessing.json\")\n        if isfile(pp_file):\n            print(\"postprocessing...\")\n            shutil.copy(pp_file, os.path.dirname(output_filenames[0]))\n            # for_which_classes stores for which of the classes everything but the largest connected component needs to be\n            # removed\n            for_which_classes, min_valid_obj_size = load_postprocessing(pp_file)\n            results.append(pool.starmap_async(load_remove_save,\n                                              zip(output_filenames, output_filenames,\n                                                  [for_which_classes] * len(output_filenames),\n                                                  [min_valid_obj_size] * len(output_filenames))))\n            _ = [i.get() for i in results]\n        else:\n            print(\"WARNING! Cannot run postprocessing because the postprocessing file is missing. Make sure to run \"\n                  \"consolidate_folds in the output folder of the model first!\\nThe folder you need to run this in is \"\n                  \"%s\" % model)\n\n    pool.close()\n    pool.join()\n\n\ndef predict_cases_fastest(model, list_of_lists, output_filenames, folds, num_threads_preprocessing,\n                          num_threads_nifti_save, segs_from_prev_stage=None, do_tta=True, mixed_precision=True,\n                          overwrite_existing=False, all_in_gpu=False, step_size=0.5,\n                          checkpoint_name=\"model_final_checkpoint\", disable_postprocessing: bool = False):\n    assert len(list_of_lists) == len(output_filenames)\n    if segs_from_prev_stage is not None: assert len(segs_from_prev_stage) == len(output_filenames)\n\n    pool = Pool(num_threads_nifti_save)\n    results = []\n\n    cleaned_output_files = []\n    for o in output_filenames:\n        dr, f = os.path.split(o)\n        if len(dr) > 0:\n            maybe_mkdir_p(dr)\n        if not f.endswith(\".nii.gz\"):\n            f, _ = os.path.splitext(f)\n            f = f + \".nii.gz\"\n        cleaned_output_files.append(join(dr, f))\n\n    if not overwrite_existing:\n        print(\"number of cases:\", len(list_of_lists))\n        not_done_idx = [i for i, j in enumerate(cleaned_output_files) if not isfile(j)]\n\n        cleaned_output_files = [cleaned_output_files[i] for i in not_done_idx]\n        list_of_lists = [list_of_lists[i] for i in not_done_idx]\n        if segs_from_prev_stage is not None:\n            segs_from_prev_stage = [segs_from_prev_stage[i] for i in not_done_idx]\n\n        print(\"number of cases that still need to be predicted:\", len(cleaned_output_files))\n\n    print(\"emptying cuda cache\")\n    torch.cuda.empty_cache()\n\n    print(\"loading parameters for folds,\", folds)\n    trainer, params = load_model_and_checkpoint_files(model, folds, mixed_precision=mixed_precision,\n                                                      checkpoint_name=checkpoint_name)\n\n    print(\"starting preprocessing generator\")\n    preprocessing = preprocess_multithreaded(trainer, list_of_lists, cleaned_output_files, num_threads_preprocessing,\n                                             segs_from_prev_stage)\n\n    print(\"starting prediction...\")\n    for preprocessed in preprocessing:\n        print(\"getting data from preprocessor\")\n        output_filename, (d, dct) = preprocessed\n        print(\"got something\")\n        if isinstance(d, str):\n            print(\"what I got is a string, so I need to load a file\")\n            data = np.load(d)\n            os.remove(d)\n            d = data\n\n        # preallocate the output arrays\n        # same dtype as the return value in predict_preprocessed_data_return_seg_and_softmax (saves time)\n        all_softmax_outputs = np.zeros((len(params), trainer.num_classes, *d.shape[1:]), dtype=np.float16)\n        all_seg_outputs = np.zeros((len(params), *d.shape[1:]), dtype=int)\n        print(\"predicting\", output_filename)\n\n        for i, p in enumerate(params):\n            trainer.load_checkpoint_ram(p, False)\n            res = trainer.predict_preprocessed_data_return_seg_and_softmax(d, do_mirroring=do_tta,\n                                                                           mirror_axes=trainer.data_aug_params['mirror_axes'],\n                                                                           use_sliding_window=True,\n                                                                           step_size=step_size, use_gaussian=True,\n                                                                           all_in_gpu=all_in_gpu,\n                                                                           mixed_precision=mixed_precision)\n            if len(params) > 1:\n                # otherwise we dont need this and we can save ourselves the time it takes to copy that\n                all_softmax_outputs[i] = res[1]\n            all_seg_outputs[i] = res[0]\n\n        if hasattr(trainer, 'regions_class_order'):\n            region_class_order = trainer.regions_class_order\n        else:\n            region_class_order = None\n        assert region_class_order is None, \"predict_cases_fastest can only work with regular softmax predictions \" \\\n                                           \"and is therefore unable to handle trainer classes with region_class_order\"\n\n        print(\"aggregating predictions\")\n        if len(params) > 1:\n            softmax_mean = np.mean(all_softmax_outputs, 0)\n            seg = softmax_mean.argmax(0)\n        else:\n            seg = all_seg_outputs[0]\n\n        print(\"applying transpose_backward\")\n        transpose_forward = trainer.plans.get('transpose_forward')\n        if transpose_forward is not None:\n            transpose_backward = trainer.plans.get('transpose_backward')\n            seg = seg.transpose([i for i in transpose_backward])\n\n        print(\"initializing segmentation export\")\n        results.append(pool.starmap_async(save_segmentation_nifti,\n                                          ((seg, output_filename, dct, 0, None),)\n                                          ))\n        print(\"done\")\n\n    print(\"inference done. Now waiting for the segmentation export to finish...\")\n    _ = [i.get() for i in results]\n    # now apply postprocessing\n    # first load the postprocessing properties if they are present. Else raise a well visible warning\n    if not disable_postprocessing:\n        results = []\n        pp_file = join(model, \"postprocessing.json\")\n        if isfile(pp_file):\n            print(\"postprocessing...\")\n            shutil.copy(pp_file, os.path.dirname(output_filenames[0]))\n            # for_which_classes stores for which of the classes everything but the largest connected component needs to be\n            # removed\n            for_which_classes, min_valid_obj_size = load_postprocessing(pp_file)\n            results.append(pool.starmap_async(load_remove_save,\n                                              zip(output_filenames, output_filenames,\n                                                  [for_which_classes] * len(output_filenames),\n                                                  [min_valid_obj_size] * len(output_filenames))))\n            _ = [i.get() for i in results]\n        else:\n            print(\"WARNING! Cannot run postprocessing because the postprocessing file is missing. Make sure to run \"\n                  \"consolidate_folds in the output folder of the model first!\\nThe folder you need to run this in is \"\n                  \"%s\" % model)\n\n    pool.close()\n    pool.join()\n\n\ndef check_input_folder_and_return_caseIDs(input_folder, expected_num_modalities):\n    print(\"This model expects %d input modalities for each image\" % expected_num_modalities)\n    files = subfiles(input_folder, suffix=\".nii.gz\", join=False, sort=True)\n\n    maybe_case_ids = np.unique([i[:-12] for i in files])\n\n    remaining = deepcopy(files)\n    missing = []\n\n    assert len(files) > 0, \"input folder did not contain any images (expected to find .nii.gz file endings)\"\n\n    # now check if all required files are present and that no unexpected files are remaining\n    for c in maybe_case_ids:\n        for n in range(expected_num_modalities):\n            expected_output_file = c + \"_%04.0d.nii.gz\" % n\n            if not isfile(join(input_folder, expected_output_file)):\n                missing.append(expected_output_file)\n            else:\n                remaining.remove(expected_output_file)\n\n    print(\"Found %d unique case ids, here are some examples:\" % len(maybe_case_ids),\n          np.random.choice(maybe_case_ids, min(len(maybe_case_ids), 10)))\n    print(\"If they don't look right, make sure to double check your filenames. They must end with _0000.nii.gz etc\")\n\n    if len(remaining) > 0:\n        print(\"found %d unexpected remaining files in the folder. Here are some examples:\" % len(remaining),\n              np.random.choice(remaining, min(len(remaining), 10)))\n\n    if len(missing) > 0:\n        print(\"Some files are missing:\")\n        print(missing)\n        raise RuntimeError(\"missing files in input_folder\")\n\n    return maybe_case_ids\n\n\ndef predict_from_folder(model: str, input_folder: str, output_folder: str, folds: Union[Tuple[int], List[int]],\n                        save_npz: bool, num_threads_preprocessing: int, num_threads_nifti_save: int,\n                        lowres_segmentations: Union[str, None],\n                        part_id: int, num_parts: int, tta: bool, mixed_precision: bool = True,\n                        overwrite_existing: bool = True, mode: str = 'normal', overwrite_all_in_gpu: bool = None,\n                        step_size: float = 0.5, checkpoint_name: str = \"model_final_checkpoint\",\n                        segmentation_export_kwargs: dict = None, disable_postprocessing: bool = False):\n    \"\"\"\n        here we use the standard naming scheme to generate list_of_lists and output_files needed by predict_cases\n\n    :param model:\n    :param input_folder:\n    :param output_folder:\n    :param folds:\n    :param save_npz:\n    :param num_threads_preprocessing:\n    :param num_threads_nifti_save:\n    :param lowres_segmentations:\n    :param part_id:\n    :param num_parts:\n    :param tta:\n    :param mixed_precision:\n    :param overwrite_existing: if not None then it will be overwritten with whatever is in there. None is default (no overwrite)\n    :return:\n    \"\"\"\n    maybe_mkdir_p(output_folder)\n    shutil.copy(join(model, 'plans.pkl'), output_folder)\n\n    assert isfile(join(model, \"plans.pkl\")), \"Folder with saved model weights must contain a plans.pkl file\"\n    expected_num_modalities = load_pickle(join(model, \"plans.pkl\"))['num_modalities']\n\n    # check input folder integrity\n    case_ids = check_input_folder_and_return_caseIDs(input_folder, expected_num_modalities)\n\n    output_files = [join(output_folder, i + \".nii.gz\") for i in case_ids]\n    all_files = subfiles(input_folder, suffix=\".nii.gz\", join=False, sort=True)\n    list_of_lists = [[join(input_folder, i) for i in all_files if i[:len(j)].startswith(j) and\n                      len(i) == (len(j) + 12)] for j in case_ids]\n\n    if lowres_segmentations is not None:\n        assert isdir(lowres_segmentations), \"if lowres_segmentations is not None then it must point to a directory\"\n        lowres_segmentations = [join(lowres_segmentations, i + \".nii.gz\") for i in case_ids]\n        assert all([isfile(i) for i in lowres_segmentations]), \"not all lowres_segmentations files are present. \" \\\n                                                               \"(I was searching for case_id.nii.gz in that folder)\"\n        lowres_segmentations = lowres_segmentations[part_id::num_parts]\n    else:\n        lowres_segmentations = None\n\n    if mode == \"normal\":\n        if overwrite_all_in_gpu is None:\n            all_in_gpu = False\n        else:\n            all_in_gpu = overwrite_all_in_gpu\n\n        return predict_cases(model, list_of_lists[part_id::num_parts], output_files[part_id::num_parts], folds,\n                             save_npz, num_threads_preprocessing, num_threads_nifti_save, lowres_segmentations, tta,\n                             mixed_precision=mixed_precision, overwrite_existing=overwrite_existing,\n                             all_in_gpu=all_in_gpu,\n                             step_size=step_size, checkpoint_name=checkpoint_name,\n                             segmentation_export_kwargs=segmentation_export_kwargs,\n                             disable_postprocessing=disable_postprocessing)\n    elif mode == \"fast\":\n        if overwrite_all_in_gpu is None:\n            all_in_gpu = False\n        else:\n            all_in_gpu = overwrite_all_in_gpu\n\n        assert save_npz is False\n        return predict_cases_fast(model, list_of_lists[part_id::num_parts], output_files[part_id::num_parts], folds,\n                                  num_threads_preprocessing, num_threads_nifti_save, lowres_segmentations,\n                                  tta, mixed_precision=mixed_precision, overwrite_existing=overwrite_existing,\n                                  all_in_gpu=all_in_gpu,\n                                  step_size=step_size, checkpoint_name=checkpoint_name,\n                                  segmentation_export_kwargs=segmentation_export_kwargs,\n                                  disable_postprocessing=disable_postprocessing)\n    elif mode == \"fastest\":\n        if overwrite_all_in_gpu is None:\n            all_in_gpu = False\n        else:\n            all_in_gpu = overwrite_all_in_gpu\n\n        assert save_npz is False\n        return predict_cases_fastest(model, list_of_lists[part_id::num_parts], output_files[part_id::num_parts], folds,\n                                     num_threads_preprocessing, num_threads_nifti_save, lowres_segmentations,\n                                     tta, mixed_precision=mixed_precision, overwrite_existing=overwrite_existing,\n                                     all_in_gpu=all_in_gpu,\n                                     step_size=step_size, checkpoint_name=checkpoint_name,\n                                     disable_postprocessing=disable_postprocessing)\n    else:\n        raise ValueError(\"unrecognized mode. Must be normal, fast or fastest\")\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"-i\", '--input_folder', help=\"Must contain all modalities for each patient in the correct\"\n                                                     \" order (same as training). Files must be named \"\n                                                     \"CASENAME_XXXX.nii.gz where XXXX is the modality \"\n                                                     \"identifier (0000, 0001, etc)\", required=True)\n    parser.add_argument('-o', \"--output_folder\", required=True, help=\"folder for saving predictions\")\n    parser.add_argument('-m', '--model_output_folder',\n                        help='model output folder. Will automatically discover the folds '\n                             'that were '\n                             'run and use those as an ensemble', required=True)\n    parser.add_argument('-f', '--folds', nargs='+', default='None', help=\"folds to use for prediction. Default is None \"\n                                                                         \"which means that folds will be detected \"\n                                                                         \"automatically in the model output folder\")\n    parser.add_argument('-z', '--save_npz', required=False, action='store_true', help=\"use this if you want to ensemble\"\n                                                                                      \" these predictions with those of\"\n                                                                                      \" other models. Softmax \"\n                                                                                      \"probabilities will be saved as \"\n                                                                                      \"compresed numpy arrays in \"\n                                                                                      \"output_folder and can be merged \"\n                                                                                      \"between output_folders with \"\n                                                                                      \"merge_predictions.py\")\n    parser.add_argument('-l', '--lowres_segmentations', required=False, default='None', help=\"if model is the highres \"\n                                                                                             \"stage of the cascade then you need to use -l to specify where the segmentations of the \"\n                                                                                             \"corresponding lowres unet are. Here they are required to do a prediction\")\n    parser.add_argument(\"--part_id\", type=int, required=False, default=0, help=\"Used to parallelize the prediction of \"\n                                                                               \"the folder over several GPUs. If you \"\n                                                                               \"want to use n GPUs to predict this \"\n                                                                               \"folder you need to run this command \"\n                                                                               \"n times with --part_id=0, ... n-1 and \"\n                                                                               \"--num_parts=n (each with a different \"\n                                                                               \"GPU (for example via \"\n                                                                               \"CUDA_VISIBLE_DEVICES=X)\")\n    parser.add_argument(\"--num_parts\", type=int, required=False, default=1,\n                        help=\"Used to parallelize the prediction of \"\n                             \"the folder over several GPUs. If you \"\n                             \"want to use n GPUs to predict this \"\n                             \"folder you need to run this command \"\n                             \"n times with --part_id=0, ... n-1 and \"\n                             \"--num_parts=n (each with a different \"\n                             \"GPU (via \"\n                             \"CUDA_VISIBLE_DEVICES=X)\")\n    parser.add_argument(\"--num_threads_preprocessing\", required=False, default=6, type=int, help=\n    \"Determines many background processes will be used for data preprocessing. Reduce this if you \"\n    \"run into out of memory (RAM) problems. Default: 6\")\n    parser.add_argument(\"--num_threads_nifti_save\", required=False, default=2, type=int, help=\n    \"Determines many background processes will be used for segmentation export. Reduce this if you \"\n    \"run into out of memory (RAM) problems. Default: 2\")\n    parser.add_argument(\"--tta\", required=False, type=int, default=1, help=\"Set to 0 to disable test time data \"\n                                                                           \"augmentation (speedup of factor \"\n                                                                           \"4(2D)/8(3D)), \"\n                                                                           \"lower quality segmentations\")\n    parser.add_argument(\"--overwrite_existing\", required=False, type=int, default=1, help=\"Set this to 0 if you need \"\n                                                                                          \"to resume a previous \"\n                                                                                          \"prediction. Default: 1 \"\n                                                                                          \"(=existing segmentations \"\n                                                                                          \"in output_folder will be \"\n                                                                                          \"overwritten)\")\n    parser.add_argument(\"--mode\", type=str, default=\"normal\", required=False)\n    parser.add_argument(\"--all_in_gpu\", type=str, default=\"None\", required=False, help=\"can be None, False or True\")\n    parser.add_argument(\"--step_size\", type=float, default=0.5, required=False, help=\"don't touch\")\n    # parser.add_argument(\"--interp_order\", required=False, default=3, type=int,\n    #                     help=\"order of interpolation for segmentations, has no effect if mode=fastest\")\n    # parser.add_argument(\"--interp_order_z\", required=False, default=0, type=int,\n    #                     help=\"order of interpolation along z is z is done differently\")\n    # parser.add_argument(\"--force_separate_z\", required=False, default=\"None\", type=str,\n    #                     help=\"force_separate_z resampling. Can be None, True or False, has no effect if mode=fastest\")\n    parser.add_argument('--disable_mixed_precision', default=False, action='store_true', required=False,\n                        help='Predictions are done with mixed precision by default. This improves speed and reduces '\n                             'the required vram. If you want to disable mixed precision you can set this flag. Note '\n                             'that this is not recommended (mixed precision is ~2x faster!)')\n\n    args = parser.parse_args()\n    input_folder = args.input_folder\n    output_folder = args.output_folder\n    part_id = args.part_id\n    num_parts = args.num_parts\n    model = args.model_output_folder\n    folds = args.folds\n    save_npz = args.save_npz\n    lowres_segmentations = args.lowres_segmentations\n    num_threads_preprocessing = args.num_threads_preprocessing\n    num_threads_nifti_save = args.num_threads_nifti_save\n    tta = args.tta\n    step_size = args.step_size\n\n    # interp_order = args.interp_order\n    # interp_order_z = args.interp_order_z\n    # force_separate_z = args.force_separate_z\n\n    # if force_separate_z == \"None\":\n    #     force_separate_z = None\n    # elif force_separate_z == \"False\":\n    #     force_separate_z = False\n    # elif force_separate_z == \"True\":\n    #     force_separate_z = True\n    # else:\n    #     raise ValueError(\"force_separate_z must be None, True or False. Given: %s\" % force_separate_z)\n\n    overwrite = args.overwrite_existing\n    mode = args.mode\n    all_in_gpu = args.all_in_gpu\n\n    if lowres_segmentations == \"None\":\n        lowres_segmentations = None\n\n    if isinstance(folds, list):\n        if folds[0] == 'all' and len(folds) == 1:\n            pass\n        else:\n            folds = [int(i) for i in folds]\n    elif folds == \"None\":\n        folds = None\n    else:\n        raise ValueError(\"Unexpected value for argument folds\")\n\n    if tta == 0:\n        tta = False\n    elif tta == 1:\n        tta = True\n    else:\n        raise ValueError(\"Unexpected value for tta, Use 1 or 0\")\n\n    if overwrite == 0:\n        overwrite = False\n    elif overwrite == 1:\n        overwrite = True\n    else:\n        raise ValueError(\"Unexpected value for overwrite, Use 1 or 0\")\n\n    assert all_in_gpu in ['None', 'False', 'True']\n    if all_in_gpu == \"None\":\n        all_in_gpu = None\n    elif all_in_gpu == \"True\":\n        all_in_gpu = True\n    elif all_in_gpu == \"False\":\n        all_in_gpu = False\n\n    predict_from_folder(model, input_folder, output_folder, folds, save_npz, num_threads_preprocessing,\n                        num_threads_nifti_save, lowres_segmentations, part_id, num_parts, tta,\n                        mixed_precision=not args.disable_mixed_precision,\n                        overwrite_existing=overwrite, mode=mode, overwrite_all_in_gpu=all_in_gpu, step_size=step_size)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile /kaggle/working/nnunet/nnunet/inference/ensemble_predictions.py\n#    Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany\n#\n#    Licensed under the Apache License, Version 2.0 (the \"License\");\n#    you may not use this file except in compliance with the License.\n#    You may obtain a copy of the License at\n#\n#        http://www.apache.org/licenses/LICENSE-2.0\n#\n#    Unless required by applicable law or agreed to in writing, software\n#    distributed under the License is distributed on an \"AS IS\" BASIS,\n#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#    See the License for the specific language governing permissions and\n#    limitations under the License.\n\n\nimport shutil\nfrom copy import deepcopy\n\nfrom nnunet.inference.segmentation_export import save_segmentation_nifti_from_softmax\nfrom batchgenerators.utilities.file_and_folder_operations import *\nimport numpy as np\nfrom multiprocessing import Pool\nfrom nnunet.postprocessing.connected_components import apply_postprocessing_to_folder, load_postprocessing\n\n\ndef custom_postprocess(pred):\n    '''\n    print('doing pp')\n    \n    cutoff_exp = 999\n    cutoff_ctrl = 999\n    \n    \n    \n    for i in range(pred.shape[1]):\n        if pred[1:,i].max()>=0.995:\n            cutoff_exp = i\n    \n    for n,j in enumerate(range(pred.shape[1])):\n        if n > cutoff_exp:\n            pred[2,n] = 0\n            pred[1,n] = 0\n            \n            #temp_tar = pred[:,n]\n            #temp[temp_tar[1]>0.4] = 1\n            #temp[temp_tar[2]>0.4] = 2\n            #temp[temp_tar[3]>0.4] = 3\n    '''\n    '''\n    softmax_max = np.argmax(softmax, 0)\n    \n    shape_ = softmax_max.shape[0]\n    flagg = 0\n    milestonesp = 999\n    milestone1 = 999\n    milestone2 = 999\n    for numb,ii in enumerate(softmax_max):\n        if numb < shape_ - 1:\n            if ((ii>0).sum().astype(float) / ((softmax_max[numb+1]>0).sum().astype(float) + 0.0001))>5 and flagg ==0:\n                milestonesp = numb\n                flagg = 1\n        \n        if 1 in ii:\n            milestone1 = numb\n        if 2 in ii:\n            milestone2 = numb\n    for numb,ii in enumerate(softmax_max):\n        if numb > milestone1:\n            softmax[2,numb] = 0.\n        \n        if numb > milestone2:\n            softmax[1,numb] = 0.\n        if numb > milestonesp - 1:\n            softmax[2,numb] = 0.\n            softmax[1,numb] = 0.\n        elif numb == milestonesp - 2:\n            softmax[0,numb] *= 0.9\n    '''\n    \n    #print('B:%.2f MB' % (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024))\n    \n    return pred\n\n\ndef merge_files(files, properties_files, out_file, override, store_npz):\n    if override or not isfile(out_file):\n        weights = [0.8,1.,0.5,0.5]\n\n        softmax = [np.load(f)['softmax'][None] for f in files]\n        softmax = np.vstack(softmax)\n\n        softmax = np.average(softmax, 0, weights)\n        \n        softmax = custom_postprocess(softmax)\n\n        props = [load_pickle(f) for f in properties_files]\n\n        reg_class_orders = [p['regions_class_order'] if 'regions_class_order' in p.keys() else None\n                            for p in props]\n\n        if not all([i is None for i in reg_class_orders]):\n            # if reg_class_orders are not None then they must be the same in all pkls\n            tmp = reg_class_orders[0]\n            for r in reg_class_orders[1:]:\n                assert tmp == r, 'If merging files with regions_class_order, the regions_class_orders of all ' \\\n                                 'files must be the same. regions_class_order: %s, \\n files: %s' % \\\n                                 (str(reg_class_orders), str(files))\n            regions_class_order = tmp\n        else:\n            regions_class_order = None\n\n        # Softmax probabilities are already at target spacing so this will not do any resampling (resampling parameters\n        # don't matter here)\n        save_segmentation_nifti_from_softmax(softmax, out_file, props[0], 3, regions_class_order, None, None,\n                                             force_separate_z=None)\n        if store_npz:\n            np.savez_compressed(out_file[:-7] + \".npz\", softmax=softmax)\n            save_pickle(props, out_file[:-7] + \".pkl\")\n\n\ndef merge(folders, output_folder, threads, override=True, postprocessing_file=None, store_npz=False):\n    maybe_mkdir_p(output_folder)\n\n    if postprocessing_file is not None:\n        output_folder_orig = deepcopy(output_folder)\n        output_folder = join(output_folder, 'not_postprocessed')\n        maybe_mkdir_p(output_folder)\n    else:\n        output_folder_orig = None\n\n    patient_ids = [subfiles(i, suffix=\".npz\", join=False) for i in folders]\n    patient_ids = [i for j in patient_ids for i in j]\n    patient_ids = [i[:-4] for i in patient_ids]\n    patient_ids = np.unique(patient_ids)\n\n    for f in folders:\n        assert all([isfile(join(f, i + \".npz\")) for i in patient_ids]), \"Not all patient npz are available in \" \\\n                                                                        \"all folders\"\n        assert all([isfile(join(f, i + \".pkl\")) for i in patient_ids]), \"Not all patient pkl are available in \" \\\n                                                                        \"all folders\"\n\n    files = []\n    property_files = []\n    out_files = []\n    for p in patient_ids:\n        files.append([join(f, p + \".npz\") for f in folders])\n        property_files.append([join(f, p + \".pkl\") for f in folders])\n        out_files.append(join(output_folder, p + \".nii.gz\"))\n\n    p = Pool(threads)\n    p.starmap(merge_files, zip(files, property_files, out_files, [override] * len(out_files), [store_npz] * len(out_files)))\n    p.close()\n    p.join()\n\n    if postprocessing_file is not None:\n        for_which_classes, min_valid_obj_size = load_postprocessing(postprocessing_file)\n        print('Postprocessing...')\n        apply_postprocessing_to_folder(output_folder, output_folder_orig,\n                                       for_which_classes, min_valid_obj_size, threads)\n        shutil.copy(postprocessing_file, output_folder_orig)\n\n\ndef main():\n    import argparse\n    parser = argparse.ArgumentParser(description=\"This script will merge predictions (that were prdicted with the \"\n                                                 \"-npz option!). You need to specify a postprocessing file so that \"\n                                                 \"we know here what postprocessing must be applied. Failing to do so \"\n                                                 \"will disable postprocessing\")\n    parser.add_argument('-f', '--folders', nargs='+', help=\"list of folders to merge. All folders must contain npz \"\n                                                           \"files\", required=True)\n    parser.add_argument('-o', '--output_folder', help=\"where to save the results\", required=True, type=str)\n    parser.add_argument('-t', '--threads', help=\"number of threads used to saving niftis\", required=False, default=2,\n                        type=int)\n    parser.add_argument('-pp', '--postprocessing_file', help=\"path to the file where the postprocessing configuration \"\n                                                             \"is stored. If this is not provided then no postprocessing \"\n                                                             \"will be made. It is strongly recommended to provide the \"\n                                                             \"postprocessing file!\",\n                        required=False, type=str, default=None)\n    parser.add_argument('--npz', action=\"store_true\", required=False, help=\"stores npz and pkl\")\n\n    args = parser.parse_args()\n\n    folders = args.folders\n    threads = args.threads\n    output_folder = args.output_folder\n    pp_file = args.postprocessing_file\n    npz = args.npz\n\n    merge(folders, output_folder, threads, override=True, postprocessing_file=pp_file, store_npz=npz)\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/nnunetdependencies')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:09:28.239332Z","iopub.execute_input":"2022-05-17T15:09:28.23968Z","iopub.status.idle":"2022-05-17T15:09:28.243168Z","shell.execute_reply.started":"2022-05-17T15:09:28.239646Z","shell.execute_reply":"2022-05-17T15:09:28.24251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install *.whl\n!pip install MedPy-0.4.0/MedPy-0.4.0/\n#!pip install argparse-1.4.0/argparse-1.4.0/\n!pip install dicom2nifti-2.3.0/dicom2nifti-2.3.0/\n#!pip install /kaggle/input/batchgenerators-whl/batchgenerators-0.23-py3-none-any.whl\n#!pip install linecache2-1.0.0/linecache2-1.0.0/\n#!pip install traceback2-1.4.0/traceback2-1.4.0\n#!pip install unittest2-1.1.0/unittest2-1.1.0\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:09:28.245593Z","iopub.execute_input":"2022-05-17T15:09:28.245998Z","iopub.status.idle":"2022-05-17T15:11:06.217314Z","shell.execute_reply.started":"2022-05-17T15:09:28.245963Z","shell.execute_reply":"2022-05-17T15:11:06.216473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r /kaggle/input/batchgenerators-whl /kaggle/working/batchgenerators-whl\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:11:06.219042Z","iopub.execute_input":"2022-05-17T15:11:06.219316Z","iopub.status.idle":"2022-05-17T15:11:07.191195Z","shell.execute_reply.started":"2022-05-17T15:11:06.219279Z","shell.execute_reply":"2022-05-17T15:11:07.190235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/working/batchgenerators-whl/","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:11:07.193632Z","iopub.execute_input":"2022-05-17T15:11:07.193927Z","iopub.status.idle":"2022-05-17T15:11:37.911751Z","shell.execute_reply.started":"2022-05-17T15:11:07.193889Z","shell.execute_reply":"2022-05-17T15:11:37.910777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/nnunet')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:11:37.913328Z","iopub.execute_input":"2022-05-17T15:11:37.91362Z","iopub.status.idle":"2022-05-17T15:11:37.918503Z","shell.execute_reply.started":"2022-05-17T15:11:37.913585Z","shell.execute_reply":"2022-05-17T15:11:37.91779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/pytorch-torchvision/torch-1.10.2cu111-cp37-cp37m-linux_x86_64.whl\n!pip install /kaggle/input/pytorch-torchvision/torchvision-0.11.3cu111-cp37-cp37m-linux_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:11:37.919994Z","iopub.execute_input":"2022-05-17T15:11:37.920655Z","iopub.status.idle":"2022-05-17T15:12:36.277495Z","shell.execute_reply.started":"2022-05-17T15:11:37.920613Z","shell.execute_reply":"2022-05-17T15:12:36.276656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:12:36.281032Z","iopub.execute_input":"2022-05-17T15:12:36.281532Z","iopub.status.idle":"2022-05-17T15:12:36.290689Z","shell.execute_reply.started":"2022-05-17T15:12:36.281489Z","shell.execute_reply":"2022-05-17T15:12:36.289843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:12:36.292847Z","iopub.execute_input":"2022-05-17T15:12:36.293819Z","iopub.status.idle":"2022-05-17T15:12:36.316443Z","shell.execute_reply.started":"2022-05-17T15:12:36.293781Z","shell.execute_reply":"2022-05-17T15:12:36.315632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -e .","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:12:36.317773Z","iopub.execute_input":"2022-05-17T15:12:36.318313Z","iopub.status.idle":"2022-05-17T15:13:08.507749Z","shell.execute_reply.started":"2022-05-17T15:12:36.318275Z","shell.execute_reply":"2022-05-17T15:13:08.506807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:08.509621Z","iopub.execute_input":"2022-05-17T15:13:08.509924Z","iopub.status.idle":"2022-05-17T15:13:19.88625Z","shell.execute_reply.started":"2022-05-17T15:13:08.509887Z","shell.execute_reply":"2022-05-17T15:13:19.885352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -rf /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_1","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:19.888069Z","iopub.execute_input":"2022-05-17T15:13:19.888346Z","iopub.status.idle":"2022-05-17T15:13:20.594804Z","shell.execute_reply.started":"2022-05-17T15:13:19.888311Z","shell.execute_reply":"2022-05-17T15:13:20.593832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs1\n!mkdir -p /kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs2","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:20.598626Z","iopub.execute_input":"2022-05-17T15:13:20.599532Z","iopub.status.idle":"2022-05-17T15:13:21.266702Z","shell.execute_reply.started":"2022-05-17T15:13:20.599481Z","shell.execute_reply":"2022-05-17T15:13:21.26572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['nnUNet_raw_data_base'] = \"/kaggle/working/nnUNet_raw\"\nos.environ['nnUNet_preprocessed'] = \"/kaggle/working/nnUNet_preprocessed\"\nos.environ['RESULTS_FOLDER'] = \"/kaggle/working/nnUNet_trained_models\"","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:21.268638Z","iopub.execute_input":"2022-05-17T15:13:21.269114Z","iopub.status.idle":"2022-05-17T15:13:21.274717Z","shell.execute_reply.started":"2022-05-17T15:13:21.269068Z","shell.execute_reply":"2022-05-17T15:13:21.274004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/nnUNet_preprocessed","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:21.275852Z","iopub.execute_input":"2022-05-17T15:13:21.276565Z","iopub.status.idle":"2022-05-17T15:13:21.945896Z","shell.execute_reply.started":"2022-05-17T15:13:21.276526Z","shell.execute_reply":"2022-05-17T15:13:21.944761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG == False:\n    sub_df = pd.read_csv('/kaggle/input/uw-madison-gi-tract-image-segmentation/sample_submission.csv')\nelse:\n    sub_df = pd.read_csv('/kaggle/input/uw-madison-gi-tract-image-segmentation/train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:21.94752Z","iopub.execute_input":"2022-05-17T15:13:21.947788Z","iopub.status.idle":"2022-05-17T15:13:22.225775Z","shell.execute_reply.started":"2022-05-17T15:13:21.947751Z","shell.execute_reply":"2022-05-17T15:13:22.225011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sub_df = sub_df.drop(['class'])","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:22.227059Z","iopub.execute_input":"2022-05-17T15:13:22.22731Z","iopub.status.idle":"2022-05-17T15:13:22.23113Z","shell.execute_reply.started":"2022-05-17T15:13:22.227276Z","shell.execute_reply":"2022-05-17T15:13:22.230139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for i in sub_df.index:\n#    id_ = sub_df.loc[i,'id']\n#    sub_df.loc[i,'case'] = id_.split('case')[-1].split('_')[0]\n#    sub_df.loc[i,'day'] = id_.split('day')[-1].split('_')[0]\n#    sub_df.loc[i,'slice'] = id_.split('day')[-1].split('_')[0]\n    ","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:22.232604Z","iopub.execute_input":"2022-05-17T15:13:22.233173Z","iopub.status.idle":"2022-05-17T15:13:22.243763Z","shell.execute_reply.started":"2022-05-17T15:13:22.233135Z","shell.execute_reply":"2022-05-17T15:13:22.243083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_list = glob.glob(TARGET_PATH+'*')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:22.245417Z","iopub.execute_input":"2022-05-17T15:13:22.246071Z","iopub.status.idle":"2022-05-17T15:13:22.252971Z","shell.execute_reply.started":"2022-05-17T15:13:22.246035Z","shell.execute_reply":"2022-05-17T15:13:22.252199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_list","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:22.254721Z","iopub.execute_input":"2022-05-17T15:13:22.255402Z","iopub.status.idle":"2022-05-17T15:13:22.266284Z","shell.execute_reply.started":"2022-05-17T15:13:22.255364Z","shell.execute_reply":"2022-05-17T15:13:22.265476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    case_list = case_list[:1]\n    sub_df = pd.read_csv('/kaggle/input/uw-madison-gi-tract-image-segmentation/sample_submission.csv')\n    train_df = pd.read_csv('/kaggle/input/uw-madison-gi-tract-image-segmentation/train.csv')\n    sub_df['id'] = train_df['id']\n    sub_df['class'] = train_df['class']\n    #sub_df.to_csv('/kaggle/input/uw-madison-gi-tract-image-segmentation/sample_submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:22.26789Z","iopub.execute_input":"2022-05-17T15:13:22.2682Z","iopub.status.idle":"2022-05-17T15:13:22.569078Z","shell.execute_reply.started":"2022-05-17T15:13:22.268162Z","shell.execute_reply":"2022-05-17T15:13:22.568314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"case_day_list = []\n\nfor c in case_list:\n    case_day_list.extend(glob.glob(f'{c}/*'))","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:22.570679Z","iopub.execute_input":"2022-05-17T15:13:22.570928Z","iopub.status.idle":"2022-05-17T15:13:22.576151Z","shell.execute_reply.started":"2022-05-17T15:13:22.570894Z","shell.execute_reply":"2022-05-17T15:13:22.575402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/nnUNet_raw/nnUNet_raw_data","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:22.581597Z","iopub.execute_input":"2022-05-17T15:13:22.582127Z","iopub.status.idle":"2022-05-17T15:13:23.251904Z","shell.execute_reply.started":"2022-05-17T15:13:22.582088Z","shell.execute_reply":"2022-05-17T15:13:23.251014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef _sort(list,b,a):\n    '''\n    list :待排列数组\n    b:数字前一个字符\n    a;数字后一个字符\n    '''\n    list.sort(key = lambda x:int(x.split(b)[-1].split(a)[0]))\n    return list\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:23.253344Z","iopub.execute_input":"2022-05-17T15:13:23.254132Z","iopub.status.idle":"2022-05-17T15:13:23.260192Z","shell.execute_reply.started":"2022-05-17T15:13:23.2541Z","shell.execute_reply":"2022-05-17T15:13:23.259024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n,i in enumerate(case_day_list):\n    slice_list = glob.glob(f'{i}/scans/*')\n    slice_list = _sort(slice_list,'slice_','_')\n    wid_get = float(slice_list[0].split('.png')[0].split('_')[-1])\n    \n    reader = sitk.ImageSeriesReader()\n    reader.SetFileNames(slice_list)\n    vol = reader.Execute()\n    vol.SetSpacing((wid_get,wid_get,4.5))\n    sitk.WriteImage(vol, f'/kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs1/gi_{n:03d}_0000.nii.gz')\n\nfor n,i in enumerate(case_day_list):\n    slice_list = glob.glob(f'{i}/scans/*')\n    slice_list = _sort(slice_list,'slice_','_')\n    wid_get = float(slice_list[0].split('.png')[0].split('_')[-1])\n    \n    reader = sitk.ImageSeriesReader()\n    reader.SetFileNames(slice_list)\n    vol = reader.Execute()\n    \n    sitk.WriteImage(vol, f'/kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs2/gi_{n:03d}_0000.nii.gz')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:23.261586Z","iopub.execute_input":"2022-05-17T15:13:23.261838Z","iopub.status.idle":"2022-05-17T15:13:37.37842Z","shell.execute_reply.started":"2022-05-17T15:13:23.261802Z","shell.execute_reply":"2022-05-17T15:13:37.377652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/nnunet_output1\n!mkdir /kaggle/working/nnunet_output2\n!mkdir /kaggle/working/nnunet_output3\n!mkdir /kaggle/working/nnunet_output4\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:37.379948Z","iopub.execute_input":"2022-05-17T15:13:37.380203Z","iopub.status.idle":"2022-05-17T15:13:38.063482Z","shell.execute_reply.started":"2022-05-17T15:13:37.380158Z","shell.execute_reply":"2022-05-17T15:13:38.062486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\ndef highres(path, value = 32):\n    file=open(path,\"rb\")\n    data=pickle.load(file)\n    data['plans']['plans_per_stage'][0]['patch_size'][0] +=value/2.\n    data['plans']['plans_per_stage'][0]['patch_size'][1] +=value\n    data['plans']['plans_per_stage'][0]['patch_size'][2] +=value\n    file.close()\n\n    f = open(path, 'wb')\n    pickle.dump(data, f)\n    f.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -rf /kaggle/working/nnUNet_trained_models\n!cp -r /kaggle/input/nnunet-3class/nn_UNet_trained_models_3class /kaggle/working/nnUNet_trained_models\n!rm -rf /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1\n!mkdir /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1\n!mkdir /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0\n!cp -r /kaggle/input/conv3-2nd/conv3/*  /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0/\n!cp -r /kaggle/input/conv3-2nd/plans.pkl  /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/\n!nnUNet_predict -i /kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs1 -o /kaggle/working/nnunet_output3 -t 505 -m 3d_fullres --save_npz --disable_tta\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/nnUNet_trained_models\n!cp -r /kaggle/input/nnunet-3class/nn_UNet_trained_models_3class /kaggle/working/nnUNet_trained_models\n!rm -rf /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1\n!mkdir /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1\n!mkdir /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0\n!cp -r /kaggle/input/nnunet-large-2nd/large/*  /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0/\n!cp -r /kaggle/input/nnunet-large-2nd/plans.pkl  /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/\n!nnUNet_predict -i /kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs1 -o /kaggle/working/nnunet_output4 -t 505 -m 3d_fullres --save_npz --disable_tta","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/nnUNet_trained_models\n!cp -r /kaggle/input/nnunet-224-allfold/nnUNet_trained_models_224/nnUNet_trained_models_224 /kaggle/working/nnUNet_trained_models\n!cp /kaggle/input/nnunet-224-allfold/plans.pkl /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1\n!rm -rf /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_3/*\n!rm -rf /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_4\n!cp -r /kaggle/input/nnunet-noval/nnUNet_trained_models_noval/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0/* /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_3/\n#!cp -r /kaggle/input/nnunet-noval/nnUNet_trained_models_noval/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_1/* /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_4/\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(4):\n    highres(f\"/kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_{i}/model_final_checkpoint.model.pkl\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nnUNet_predict -i /kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs2 -o /kaggle/working/nnunet_output1 -t 505 -m 3d_fullres --save_npz --disable_tta","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 1 is 224","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:13:38.065616Z","iopub.execute_input":"2022-05-17T15:13:38.065908Z","iopub.status.idle":"2022-05-17T15:15:10.587687Z","shell.execute_reply.started":"2022-05-17T15:13:38.065869Z","shell.execute_reply":"2022-05-17T15:15:10.586655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/nnUNet_trained_models\n!cp -r /kaggle/input/nnunet-3class/nn_UNet_trained_models_3class /kaggle/working/nnUNet_trained_models\n!rm -rf /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1\n!mkdir /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1\n!mkdir /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0\n!mkdir /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_1\n!cp -r /kaggle/input/nnunet-normalmodel-fix/*  /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0/\n!cp -r /kaggle/input/nnunettrainerv2-loss-novelloss-2-normalmodel/*  /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_1/\n!cp -r /kaggle/input/nnunet-nl-plans/* /kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(2):\n    highres(f\"/kaggle/working/nnUNet_trained_models/nnUNet/3d_fullres/Task505_Custom_Madison_GI/nnUNetTrainerV2__nnUNetPlansv2.1/fold_{i}/model_final_checkpoint.model.pkl\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nnUNet_predict -i /kaggle/working/nnUNet_raw/nnUNet_raw_data/Task505_Custom_Madison_GI/imagesTs1 -o /kaggle/working/nnunet_output2 -t 505 -m 3d_fullres --save_npz --disable_tta","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#4","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/nnunet_output","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nnUNet_ensemble -f /kaggle/working/nnunet_output1 /kaggle/working/nnunet_output2 /kaggle/working/nnunet_output3 /kaggle/working/nnunet_output4  -o /kaggle/working/nnunet_output","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_img(path):\n     img = sitk.ReadImage(path)\n     data = sitk.GetArrayFromImage(img)\n     return data","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.589434Z","iopub.execute_input":"2022-05-17T15:15:10.589756Z","iopub.status.idle":"2022-05-17T15:15:10.596876Z","shell.execute_reply.started":"2022-05-17T15:15:10.589716Z","shell.execute_reply":"2022-05-17T15:15:10.596165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ndef mask2rle(msk, thr=0.5):\n    '''\n    img: numpy array, 1 - mask, 0 - background\n    Returns run length as string formated\n    '''\n    msk    = np.array(msk)\n    pixels = msk.flatten()\n    pad    = np.array([0])\n    pixels = np.concatenate([pad, pixels, pad])\n    runs   = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.598561Z","iopub.execute_input":"2022-05-17T15:15:10.599162Z","iopub.status.idle":"2022-05-17T15:15:10.60768Z","shell.execute_reply.started":"2022-05-17T15:15:10.599119Z","shell.execute_reply":"2022-05-17T15:15:10.606797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_list = glob.glob('/kaggle/working/nnunet_output/*.nii.gz')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.609357Z","iopub.execute_input":"2022-05-17T15:15:10.609971Z","iopub.status.idle":"2022-05-17T15:15:10.617584Z","shell.execute_reply.started":"2022-05-17T15:15:10.609933Z","shell.execute_reply":"2022-05-17T15:15:10.616784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_list = _sort(output_list,'gi_','.nii.gz')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.619172Z","iopub.execute_input":"2022-05-17T15:15:10.619751Z","iopub.status.idle":"2022-05-17T15:15:10.626718Z","shell.execute_reply.started":"2022-05-17T15:15:10.619717Z","shell.execute_reply":"2022-05-17T15:15:10.62596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_list","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.628178Z","iopub.execute_input":"2022-05-17T15:15:10.628543Z","iopub.status.idle":"2022-05-17T15:15:10.637816Z","shell.execute_reply.started":"2022-05-17T15:15:10.6285Z","shell.execute_reply":"2022-05-17T15:15:10.63688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def masks2rles(msks, ids, heights, widths):\n    pred_strings = []; pred_ids = []; pred_classes = [];\n    for idx in range(msks.shape[0]):\n        msk = msks[idx]\n        height = heights[idx].item()\n        width = widths[idx].item()\n        shape0 = np.array([height, width])\n        resize = np.array([320, 384])\n        if np.any(shape0!=resize):\n            diff = resize - shape0\n            pad0 = diff[0]\n            pad1 = diff[1]\n            pady = [pad0//2, pad0//2 + pad0%2]\n            padx = [pad1//2, pad1//2 + pad1%2]\n            msk = msk[pady[0]:-pady[1], padx[0]:-padx[1], :]\n            msk = msk.reshape((*shape0, 3))\n        rle = [None]*3\n        for midx in [0, 1, 2]:\n            rle[midx] = mask2rle(msk[...,midx])\n        pred_strings.extend(rle)\n        pred_ids.extend([ids[idx]]*len(rle))\n        pred_classes.extend(['large_bowel', 'small_bowel', 'stomach'])\n    return pred_strings, pred_ids, pred_classes","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.6398Z","iopub.execute_input":"2022-05-17T15:15:10.640634Z","iopub.status.idle":"2022-05-17T15:15:10.65274Z","shell.execute_reply.started":"2022-05-17T15:15:10.640594Z","shell.execute_reply":"2022-05-17T15:15:10.651608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not DEBUG:\n    sub_df = pd.read_csv('/kaggle/input/uw-madison-gi-tract-image-segmentation/sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.654294Z","iopub.execute_input":"2022-05-17T15:15:10.654704Z","iopub.status.idle":"2022-05-17T15:15:10.665002Z","shell.execute_reply.started":"2022-05-17T15:15:10.654598Z","shell.execute_reply":"2022-05-17T15:15:10.664108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,j in tqdm(zip(case_day_list, output_list)):\n    img = read_img(j)\n    case = i.split('/case')[-1].split('_day')[0]\n    day = i.split('day')[-1]\n    \n    for u in range(img.shape[0]-1):\n        if ( (img[u]>0).sum().astype(float) / ( (img[u+1]>0).sum().astype(float)+0.0001) )>5:\n            milestonesp = u\n            break\n    \n    \n    \n    for n,i in enumerate(img):\n            if 1 in np.unique(i):\n                milestone = n\n    for n,i in enumerate(img):\n            if 2 in np.unique(i):\n                milestone2 = n\n    \n    \n    for n,j in enumerate(img):\n            \n            if n > milestone - 0:\n                temp = img[n]\n                temp[temp==2] = 0\n                #temp[temp==1] = 0\n                #temp_tar = pred[:,n]\n                #temp[temp_tar[1]>0.4] = 1\n                #temp[temp_tar[2]>0.4] = 2\n                #temp[temp_tar[3]>0.4] = 3\n                img[n] = temp\n            if n > milestone2 - 0:\n                temp = img[n]\n                temp[temp==1] = 0\n                #temp[temp==2] = 0\n                #temp_tar = pred[:,n]\n                #temp[temp_tar[1]>0.4] = 1\n                #temp[temp_tar[2]>0.4] = 2\n                #temp[temp_tar[3]>0.4] = 3\n                img[n] = temp\n            \n            if n > milestonesp - 1:\n                temp = img[n]\n                temp[temp==2] = 0\n                temp[temp==1] = 0\n                #temp_tar = pred[:,n]\n                #temp[temp_tar[1]>0.4] = 1\n                #temp[temp_tar[2]>0.4] = 2\n                #temp[temp_tar[3]>0.4] = 3\n                img[n] = temp\n\n    for n,slice_ in enumerate(img):\n        \n        slice_1 = (slice_==1) | (slice_==4) | (slice_==5) | (slice_==7)\n        \n        slice_2 = (slice_==2) | (slice_==4) | (slice_==6) | (slice_==7)\n        slice_3 = (slice_==3) | (slice_==5) | (slice_==6) | (slice_==7)\n        #print(np.unique(slice_))\n        rle1 = mask2rle(slice_1)\n        rle2 = mask2rle(slice_2)\n        rle3 = mask2rle(slice_3)\n        # 'large_bowel', 'small_bowel', 'stomach'\n        i1 = sub_df[(sub_df['id'] == f'case{case}_day{day}_slice_{n+1:04d}') & (sub_df['class'] == f'large_bowel')].index\n        #print(i1)\n        i2 = sub_df[(sub_df['id'] == f'case{case}_day{day}_slice_{n+1:04d}') & (sub_df['class'] == f'small_bowel')].index\n        i3 = sub_df[(sub_df['id'] == f'case{case}_day{day}_slice_{n+1:04d}') & (sub_df['class'] == f'stomach')].index\n        sub_df.loc[i1,'predicted'] = rle1\n        sub_df.loc[i2,'predicted'] = rle2\n        sub_df.loc[i3,'predicted'] = rle3","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:15:10.6666Z","iopub.execute_input":"2022-05-17T15:15:10.666875Z","iopub.status.idle":"2022-05-17T15:16:36.684902Z","shell.execute_reply.started":"2022-05-17T15:15:10.666845Z","shell.execute_reply":"2022-05-17T15:16:36.683514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.chdir('/kaggle/working/')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:16:36.686521Z","iopub.execute_input":"2022-05-17T15:16:36.687064Z","iopub.status.idle":"2022-05-17T15:16:36.691247Z","shell.execute_reply.started":"2022-05-17T15:16:36.687018Z","shell.execute_reply":"2022-05-17T15:16:36.690537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf *","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:16:36.692967Z","iopub.execute_input":"2022-05-17T15:16:36.693398Z","iopub.status.idle":"2022-05-17T15:16:37.507041Z","shell.execute_reply.started":"2022-05-17T15:16:36.693359Z","shell.execute_reply":"2022-05-17T15:16:37.505856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsub_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:16:37.511182Z","iopub.execute_input":"2022-05-17T15:16:37.511422Z","iopub.status.idle":"2022-05-17T15:16:37.817906Z","shell.execute_reply.started":"2022-05-17T15:16:37.511393Z","shell.execute_reply":"2022-05-17T15:16:37.817104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}