{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Party!","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport torch ,torchvision,glob,os,time,pydicom,pdb\nimport pytorch_lightning as pl\nimport pydicom \nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport warnings\nwarnings.filterwarnings('ignore')\nimport pdb","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-05T08:56:52.1914Z","iopub.execute_input":"2022-05-05T08:56:52.191667Z","iopub.status.idle":"2022-05-05T08:56:59.436516Z","shell.execute_reply.started":"2022-05-05T08:56:52.191639Z","shell.execute_reply":"2022-05-05T08:56:59.435602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dcmtag2table(folder, list_of_tags):\n    \"\"\"\n    # Create a Pandas DataFrame with the <list_of_tags> DICOM tags\n    # from the DICOM files in <folder>\n    # Parameters:\n    #    folder (str): folder to be recursively walked looking for DICOM files.\n    #    list_of_tags (list of strings): list of DICOM tags with no whitespaces.\n    # Returns:\n    #    df (DataFrame): table of DICOM tags from the files in folder.\n    \"\"\"\n    list_of_tags = list_of_tags.copy()\n    items = []\n    table = []\n    filelist = []\n    print(\"Listing all files...\")\n    start = time.time()\n    for root, dirs, files in os.walk(folder, topdown=False):\n        for name in files:\n            filelist.append(os.path.join(root, name))\n    print(\"Time: \" + str(time.time() - start))\n    print(\"Reading files...\")\n    time.sleep(2)\n    for _f in tqdm(filelist):\n            ds = pydicom.dcmread(_f, stop_before_pixels=True)\n            items = []\n            items.append(_f)\n\n            for _tag in list_of_tags:\n                if _tag in ds:\n                    items.append(ds.data_element(_tag).value)\n                else:\n                    items.append(\"Not found\")\n\n            table.append((items))\n    \n    list_of_tags.insert(0, \"Filename\")\n    test = list(map(list, zip(*table)))\n    dictone = {}\n\n    for i, _tag in enumerate (list_of_tags):\n        dictone[_tag] = test[i]\n\n    df = pd.DataFrame(dictone)\n    time.sleep(2)\n    print(\"Finished.\")\n    return df\n","metadata":{"execution":{"iopub.status.busy":"2022-05-05T08:56:59.438622Z","iopub.execute_input":"2022-05-05T08:56:59.438911Z","iopub.status.idle":"2022-05-05T08:56:59.450958Z","shell.execute_reply.started":"2022-05-05T08:56:59.438872Z","shell.execute_reply":"2022-05-05T08:56:59.45004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic File Visualition","metadata":{}},{"cell_type":"code","source":"def convert_to_list(l):\n        l=l.split()\n        for n in range(len(l)):\n            l[n]=int(l[n])\n        return l\n\n    \ndef display_(df_path,train_dir):\n    tags = ['PhotometricInterpretation','BitsAllocated', 'SOPInstanceUID' ]\n    dicom_tags_train =  dcmtag2table(train_dir, tags)\n    df=pd.read_csv(df_path)\n    df = dicom_tags_train.merge(df, on =  'SOPInstanceUID')\n    df.Target=df.Target.map(lambda x: convert_to_list(x))\n    display(df)\n    print(df.info())\n    return df\n\n\ndef plot_images(paths,r=8,c=8,figsize=(20,20)):\n    \n    _,axs=plt.subplots(r,c,figsize=figsize)\n    axs=axs.flatten()\n    \n    for n, ax in enumerate(axs):\n        img=read_img(paths[n])\n        ax.imshow(img,cmap='gray')\n        ax.axis('off')\n        \n    plt.tight_layout()\n    plt.show()\n    \ndef read_img(path):\n    dcm=pydicom.dcmread(path)\n    img=dcm.pixel_array\n    return img\n    \ndef plot_imgs(imgs,r=8,c=8,figsize=(20,20)):\n    _,axs=plt.subplots(8,c,figsize=figsize)\n    axs=axs.flatten()\n    for n, ax in enumerate(axs):\n        ax.imshow(torchvision.transforms.functional.to_pil_image(imgs[n]),cmap='gray')\n        ax.axis('off')\n        \n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T08:56:59.454143Z","iopub.execute_input":"2022-05-05T08:56:59.454438Z","iopub.status.idle":"2022-05-05T08:56:59.467168Z","shell.execute_reply.started":"2022-05-05T08:56:59.454393Z","shell.execute_reply":"2022-05-05T08:56:59.466337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n    print('#'*100)\n    print('training images')\n    \n    train_df_path='../input/unifesp-x-ray-body-part-classifier/train.csv'\n    test_df_path='../input/unifesp-x-ray-body-part-classifier/sample_submission.csv'\n    train_dir='../input/unifesp-x-ray-body-part-classifier/train/train'\n    test_dir='../input/unifesp-x-ray-body-part-classifier/test/test'\n    \n    train_df=display_(train_df_path,train_dir)\n    print(train_df.Filename[1])\n    plot_images(train_df.Filename)\n    \n    print('#'*100)\n    print('test images')\n    \n    test_df=display_(test_df_path,test_dir)\n    plot_images(test_df.Filename)\n    \n\n\n\n\nmain()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T09:13:04.830739Z","iopub.execute_input":"2022-05-05T09:13:04.831119Z","iopub.status.idle":"2022-05-05T09:21:35.709176Z","shell.execute_reply.started":"2022-05-05T09:13:04.831075Z","shell.execute_reply":"2022-05-05T09:21:35.708323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Iput Pipeline","metadata":{}},{"cell_type":"code","source":"class base_pipe(torch.utils.data.Dataset):\n    \n    def __init__(\n        self,\n        df\n    ):\n        super().__init__()\n        self.df=df.reset_index()\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def get_label(self,label):\n        label=torch.nn.functional.one_hot(torch.tensor(label),num_classes=22).sum(axis=0)\n        label=torch.tensor(label,dtype=torch.float32)\n        return label\n    \n    def read_img(self,path):\n        t=torchvision.transforms.Resize(size=(512,512))\n        dcm=pydicom.dcmread(path)\n        img=dcm.pixel_array\n        img=img.astype(np.float32)\n        img=torch.tensor([img],dtype=torch.float32)\n        img=img.repeat(3,1,1)\n        img=t(img)\n        return img/img.max()\n    \n    \n    def __getitem__(self, idx):\n        img=self.read_img(self.df.Filename[idx])\n        label=self.get_label(self.df.Target[idx])\n        return img,label","metadata":{"execution":{"iopub.status.busy":"2022-05-05T09:22:23.314081Z","iopub.execute_input":"2022-05-05T09:22:23.31468Z","iopub.status.idle":"2022-05-05T09:22:23.326505Z","shell.execute_reply.started":"2022-05-05T09:22:23.314636Z","shell.execute_reply":"2022-05-05T09:22:23.325522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class pipeline(pl.LightningDataModule):\n    def __init__(\n        self,\n        ds,\n        df,\n        test_df,\n        bs=5\n    ):\n        super().__init__()\n        self.ds=ds\n        self.train_df, self.val_df = train_test_split(df, test_size=0.2)\n        self.test_df=test_df\n        self.bs=bs\n        \n    def train_dataloader(self):\n        ds=self.ds(self.train_df)\n        dataloader=torch.utils.data.DataLoader(ds,batch_size=self.bs)\n        return dataloader\n    \n    def val_dataloader(self):\n        ds=self.ds(self.val_df)\n        dataloader=torch.utils.data.DataLoader(ds,batch_size=self.bs)\n        return dataloader\n    \n    def test_dataloader(self):\n        ds=self.ds(self.test_df)\n        dataloader=torch.utils.data.DataLoader(ds,batch_size=self.bs)\n        return dataloader","metadata":{"execution":{"iopub.status.busy":"2022-05-05T09:22:23.872824Z","iopub.execute_input":"2022-05-05T09:22:23.873095Z","iopub.status.idle":"2022-05-05T09:22:23.882311Z","shell.execute_reply.started":"2022-05-05T09:22:23.873064Z","shell.execute_reply":"2022-05-05T09:22:23.881219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model and Classifier","metadata":{}},{"cell_type":"code","source":"   \nclass Base_Model(pl.LightningModule):\n    \n    def __init__(self,\n                 model\n                ):\n        super().__init__()\n        self.model=model\n        self.model.classifier=torch.nn.Sequential(\n            \n            torch.nn.Linear(in_features=self.model.classifier.in_features,out_features=512),\n            torch.nn.Dropout(0.3),\n            torch.nn.ReLU(),\n            torch.nn.Linear(512,128),\n            torch.nn.Dropout(0.3),\n            torch.nn.ReLU(),\n            torch.nn.Linear(128,22),\n            torch.nn.Softmax()\n        )\n        \n        \n    def forward(self,x):\n        out=self.model(x)\n        return out\n    \nclass Classifier(pl.LightningModule):\n    def __init__(\n        self,\n        model,\n        loss,\n        lr\n    ):\n        super().__init__()\n        self.model=model\n        self.loss=loss\n        self.lr=lr\n\n        \n    def training_step(self,batch,batch_idx):\n        imgs,targets=batch\n        preds=self.model(imgs)\n        loss=self.loss(preds,targets)\n        self.log('train_loss',loss)\n        return loss\n        \n    def valdation_step(self,batch,batch_idx):\n        imgs,targets=batch\n        preds=self.model(imgs)\n        val_loss=self.loss(preds,targets)\n        self.log('val_loss',val_loss)\n        return val_loss\n    \n    def test_step(self,batch,batch_idx):\n        imgs,targets=batch\n        preds=self.model(imgs)\n        test_loss=self.loss(preds,targets)\n        self.log('test_loss',test_loss)\n        return test_loss\n        \n    def configure_optimizers(self):\n        opt=torch.optim.Adam(self.model.parameters(),lr=self.lr)\n        return opt","metadata":{"execution":{"iopub.status.busy":"2022-05-05T09:24:32.011923Z","iopub.execute_input":"2022-05-05T09:24:32.012395Z","iopub.status.idle":"2022-05-05T09:24:32.028125Z","shell.execute_reply.started":"2022-05-05T09:24:32.012343Z","shell.execute_reply":"2022-05-05T09:24:32.026862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let the fireworks begin!!","metadata":{}},{"cell_type":"code","source":"def main():\n    \n    print('#'*100)\n    print('training images')\n\n    train_df_path='../input/unifesp-x-ray-body-part-classifier/train.csv'\n    test_df_path='../input/unifesp-x-ray-body-part-classifier/sample_submission.csv'\n    train_dir='../input/unifesp-x-ray-body-part-classifier/train/train'\n    test_dir='../input/unifesp-x-ray-body-part-classifier/test/test'\n\n    train_df=display_(train_df_path,train_dir)\n    test_df=display_(test_df_path,test_dir)\n    \n    #WandB Logger\n#     wandb_logger=pl.loggers.WandbLogger(name='Exp_1_1',project='X-Ray Body part',job_type='EXP_1')\n\n    #Input Pipeline\n    dataloader=pipeline(base_pipe,train_df,test_df,8)\n    \n    #Model\n    Model_=torchvision.models.densenet121(pretrained=True,progress=False)\n    model=Base_Model(Model_)\n    \n    #Classifer Logic\n    loss=torch.nn.BCELoss()\n    lr=1e-4\n    classifier=Classifier(model,loss,lr)\n    \n    \n    #Trainer\n    trainer=pl.Trainer(max_epochs=1,\n                       accelerator='gpu',\n#                        logger=wandb_logger,\n                       log_every_n_steps=1\n                      )\n    \n    trainer.fit(classifier,dataloader)\n\n\nmain()\n\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-05-05T09:26:04.788091Z","iopub.execute_input":"2022-05-05T09:26:04.788841Z"},"trusted":true},"execution_count":null,"outputs":[]}]}