{"cells":[{"metadata":{},"cell_type":"markdown","source":"# References - \n### data preprocessing - https://www.kaggle.com/carlossouza/end-to-end-model-ct-scans-tabular","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport sys\nsys.path.append(\"../input/utils-model/\")\nimport os\nimport copy\nfrom datetime import timedelta, datetime\nimport imageio\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nimport multiprocessing\nfrom pathlib import Path\nimport pydicom\nimport pytest\nimport scipy.ndimage as ndimage\nfrom scipy.ndimage.interpolation import zoom\nfrom skimage import measure, morphology, segmentation\nfrom time import time, sleep\nfrom tqdm import trange\nfrom tqdm.auto import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, random_split, DistributedSampler, DataLoader\nfrom torch.utils.tensorboard import SummaryWriter\nfrom torchvision import transforms\nimport warnings\nimport glob\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from models import SegModel","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = SegModel(1,1)\nmodel = nn.DataParallel(model).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"basepath = \"../input/osic-pulmonary-fibrosis-progression\"\nroot_dir = f\"{basepath}/osic/\"\ntest_dir = f\"{basepath}/test/\"\ntrain_dir = f\"{basepath}/train/\"\n# model_file = f'/kaggle/working/diophantus.pt'\nresize_dims = (64, 256, 256)\nclip_bounds = (-1000, 200)\nwatershed_iterations = 1\npre_calculated_mean = 0.02865046213070556\nlatent_features = 10\nbatch_size = 16\nlearning_rate = 3e-5\nnum_epochs = 10\nval_size = 0.2\ntensorboard_dir = './runs'\ntr = pd.read_csv(f\"{basepath}/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy\nfrom datetime import timedelta, datetime\nimport imageio\nimport matplotlib.pyplot as plt\nfrom matplotlib import cm\nimport multiprocessing\nimport numpy as np\nimport os\nfrom pathlib import Path\nimport pydicom\nimport pytest\nimport scipy.ndimage as ndimage\nfrom scipy.ndimage.interpolation import zoom\nfrom skimage import measure, morphology, segmentation\nfrom time import time, sleep\nfrom tqdm import trange\nfrom tqdm.auto import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, random_split, DistributedSampler, DataLoader\nfrom torch.utils.tensorboard import SummaryWriter\nfrom torchvision import transforms\nimport warnings\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CTScansDataset(Dataset):\n    def __init__(self, root_dir, transform=None,transform2=None):\n        self.root_dir = Path(root_dir)\n        self.patients = [p for p in glob.glob(root_dir+'*')]\n        self.patients.sort()\n        self.transform = transform\n        self.transform2 = transform2\n\n    def __len__(self):\n        return len(self.patients)\n\n    def __getitem__(self, idx):\n        if torch.is_tensor(idx):\n            idx = idx.tolist()\n\n        image, metadata = self.load_scan(self.patients[idx])\n#         z,_,_ = image.shape\n#         if z>48:\n#             image = image[int(z/5):int(4*z/5)]\n#             margin = z//2\n#             image = image[margin-24:margin+24]\n#             print(image.shape)\n        \n        sample = {'image': image, 'metadata': metadata}\n#         if self.transform:\n#             sample_mask = self.transform(sample)\n        if self.transform2:\n            sample1_image = self.transform2(sample)    \n        sample = {'image': sample1_image['image'], \n                  'metadata': sample1_image['metadata']}\n        return sample\n\n    def save(self, path):\n        t0 = time()\n        Path(path).mkdir(exist_ok=True, parents=True)\n        print('Saving pre-processed dataset to disk')\n        sleep(1)\n        cum = 0\n\n        bar = trange(len(self))\n        for i in bar:\n            sample = self[i]\n            image, data = sample['image'], sample['metadata']\n            cum += torch.mean(image).item()\n\n            bar.set_description(f'Saving CT scan {data.PatientID}')\n            fname = Path(path) / f'{data.PatientID}.pt'\n            torch.save(image, fname)\n\n        sleep(1)\n        bar.close()\n        print(f'Done! Time {timedelta(seconds=time() - t0)}\\n'\n              f'Mean value: {cum / len(self)}')\n\n    def get_patient(self, patient_id):\n        patient_ids = [str(p.stem) for p in self.patients]\n        return self.__getitem__(patient_ids.index(patient_id))\n\n    @staticmethod\n    def load_scan(path):\n        slices = [pydicom.dcmread(path + \"/\" + file) for file in os.listdir(path)]\n        slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n        image = np.stack([s.pixel_array.astype(float) for s in slices])\n        return image, slices[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvertToHU:\n    def __call__(self, sample):\n        image, data = sample['image'], sample['metadata']\n#         img_type = data.ImageType\n#         is_hu = img_type[0] == 'ORIGINAL' and not (img_type[2] == 'LOCALIZER')\n        # if not is_hu:\n        #     warnings.warn(f'Patient {data.PatientID} CT Scan not cannot be'\n        #                   f'converted to Hounsfield Units (HU).')\n        intercept = data.RescaleIntercept if 'RescaleIntercept' in data else -1024\n        slope = data.RescaleSlope if 'RescaleSlope' in data else 1\n#         intercept = data.RescaleIntercept\n#         slope = data.RescaleSlope\n        image = (image * slope + intercept).astype(np.int16)\n        return {'image': image, 'metadata': data}\nclass Resize:\n    def __init__(self, output_size):\n        assert isinstance(output_size, tuple)\n        self.output_size = output_size\n\n    def __call__(self, sample):\n        image, data = sample['image'], sample['metadata']\n        resize_factor = np.array(self.output_size) / np.array(image.shape)\n        image = zoom(image, resize_factor, mode='nearest')\n        return {'image': image, 'metadata': data}\nclass Clip:\n    def __init__(self, bounds=(-1000, 500)):\n        self.min = min(bounds)\n        self.max = max(bounds)\n\n    def __call__(self, sample):\n        image, data = sample['image'], sample['metadata']\n        image[image < self.min] = self.min\n        image[image > self.max] = self.max\n        return {'image': image, 'metadata': data}\n    \n    \nclass Normalize:\n    def __init__(self, bounds=(-1000, 500)):\n        self.min = min(bounds)\n        self.max = max(bounds)\n\n    def __call__(self, sample):\n        image, data = sample['image'], sample['metadata']\n        image = image.astype(np.float)\n        image = (image - self.min) / (self.max - self.min)\n        return {'image': image, 'metadata': data}\n    \n\nclass ToTensor:\n    def __init__(self, add_channel=True):\n        self.add_channel = add_channel\n\n    def __call__(self, sample):\n        image, data = sample['image'], sample['metadata']\n        if self.add_channel:\n            image = np.expand_dims(image, axis=0)\n\n        return {'image': torch.from_numpy(image), 'metadata': data}\n    \n    \nclass ZeroCenter:\n    def __init__(self, pre_calculated_mean):\n        self.pre_calculated_mean = pre_calculated_mean\n\n    def __call__(self, tensor):\n        return tensor - self.pre_calculated_mean\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = CTScansDataset(\n    root_dir=train_dir,\n    transform2=transforms.Compose([\n        ConvertToHU(),\n        Resize(resize_dims),\n        Clip(bounds=clip_bounds),\n        Normalize(bounds=clip_bounds)\n    ])\n\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = glob.glob('../input/*/best.pt')[0]\nmodel.load_state_dict(torch.load(model_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_imgs = test[2]\nimage = list_imgs['image']\nimage_tensor = torch.tensor(image, dtype=torch.float32).cuda().unsqueeze(0).unsqueeze(0)\nwith torch.no_grad():\n    z = model(image_tensor)\n    z = torch.sigmoid(z)[0,0].cpu().numpy()\nplt.figure(figsize=[20,20])\nfor row in range(23,32):\n    plt.subplot(3,3,row-22)\n    plt.imshow(image[row],cmap='gray')\n    plt.imshow(z[row],alpha=0.5,cmap='hot')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_imgs = test[80]\nimage = list_imgs['image']\nimage_tensor = torch.tensor(image, dtype=torch.float32).cuda().unsqueeze(0).unsqueeze(0)\nwith torch.no_grad():\n    z = model(image_tensor)\n    z = torch.sigmoid(z)[0,0].cpu().numpy()\nplt.figure(figsize=[20,20])\nfor row in range(23,32):\n    plt.subplot(3,3,row-22)\n    plt.imshow(image[row],cmap='gray')\n    plt.imshow(z[row],alpha=0.5,cmap='hot')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_imgs = test[100]\nimage = list_imgs['image']\nimage_tensor = torch.tensor(image, dtype=torch.float32).cuda().unsqueeze(0).unsqueeze(0)\nwith torch.no_grad():\n    z = model(image_tensor)\n    z = torch.sigmoid(z)[0,0].cpu().numpy()\nplt.figure(figsize=[20,20])\nfor row in range(23,32):\n    plt.subplot(3,3,row-22)\n    plt.imshow(image[row],cmap='gray')\n    plt.imshow(z[row],alpha=0.5,cmap='hot')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}