{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n#!pip install pylibjpeg pylibjpeg-libjpeg pydicom\n#!pip install python-gdcm\n#import pylibjpeg\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\nos.mkdir('/kaggle/working/train/')\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T10:44:50.493611Z","iopub.execute_input":"2021-06-18T10:44:50.493965Z","iopub.status.idle":"2021-06-18T10:44:50.552814Z","shell.execute_reply.started":"2021-06-18T10:44:50.493936Z","shell.execute_reply":"2021-06-18T10:44:50.551362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain = pd.read_csv('/kaggle/input/siim-covid19-detection/train_image_level.csv')\ntrain_study = pd.read_csv('/kaggle/input/siim-covid19-detection/train_study_level.csv')\ntrain_study['StudyInstanceUID'] = train_study['id'].apply(lambda x: x.replace('_study', ''))\ndel train_study['id']\ntrain = train.merge(train_study, on='StudyInstanceUID')\nclass_names = ['Negative for Pneumonia','Typical Appearance', 'Indeterminate Appearance','Atypical Appearance']\nclassMap={0:[1, 0, 0, 0],\n         1:[0, 1, 0, 0],\n         2:[0, 0, 1, 0],\n         3:[0, 0, 0, 1]}\n\ndef findClass(value):\n    value=value.tolist()\n    for key,val in classMap.items():\n        if value==val:\n            return key\ntrain[\"class\"] =[findClass(li) for li in train[class_names].values]\ntrain.to_csv('/kaggle/working/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T10:44:55.573266Z","iopub.execute_input":"2021-06-18T10:44:55.573577Z","iopub.status.idle":"2021-06-18T10:44:55.945560Z","shell.execute_reply.started":"2021-06-18T10:44:55.573548Z","shell.execute_reply":"2021-06-18T10:44:55.944525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filepath=[]\nimgfile=[]\nfor dirname, _, filenames in os.walk('/kaggle/input/siim-covid19-detection/train'):\n    for filename in filenames:\n        imgfile.append(os.path.join(dirname, filename))\n        #print(dirname)\n        img_path = os.path.join('/kaggle/working/train/',filename)\n        #img_path = img_path.replace('.dcm', '.png')\n        filepath.append(img_path)\n\n#import shutil\n#for i in range(0,1200):\n        #os.system('med2image -i '+imgfile[i]+' -d '+filepath[i])\n        #shutil.copy(imgfile[i], filepath[i])\n        \n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2021-06-18T10:44:59.667369Z","iopub.execute_input":"2021-06-18T10:44:59.667871Z","iopub.status.idle":"2021-06-18T10:45:22.285611Z","shell.execute_reply.started":"2021-06-18T10:44:59.667826Z","shell.execute_reply":"2021-06-18T10:45:22.284805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport torch\nfrom torch import nn\nfrom torch import optim\nimport torch.nn.functional as F\nimport os\nimport pandas as pd\nfrom torchvision.io import read_image\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils, models\nfrom torchvision.transforms import ToTensor,Lambda\nfrom torch.utils.data import DataLoader\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport cv2\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T10:46:19.445324Z","iopub.execute_input":"2021-06-18T10:46:19.445661Z","iopub.status.idle":"2021-06-18T10:46:20.944094Z","shell.execute_reply.started":"2021-06-18T10:46:19.445630Z","shell.execute_reply":"2021-06-18T10:46:20.943247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() \n                                  else \"cpu\")\nmodel = models.resnet50(pretrained=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T10:48:12.977248Z","iopub.execute_input":"2021-06-18T10:48:12.977577Z","iopub.status.idle":"2021-06-18T10:48:13.397908Z","shell.execute_reply.started":"2021-06-18T10:48:12.977545Z","shell.execute_reply":"2021-06-18T10:48:13.397028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in model.parameters():\n    param.requires_grad = True\n    \nmodel.fc = nn.Sequential(nn.Linear(2048, 512),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(512, 4),\n                                 nn.Softmax(dim=1))\ncriterion =nn.MultiLabelSoftMarginLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.0001)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T11:01:37.509337Z","iopub.execute_input":"2021-06-18T11:01:37.509701Z","iopub.status.idle":"2021-06-18T11:01:37.542708Z","shell.execute_reply.started":"2021-06-18T11:01:37.509666Z","shell.execute_reply":"2021-06-18T11:01:37.541791Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.train()\nfor epoch in range(0,3):\n    for i in range(0,1000):\n        img_path=imgfile[i]\n        dicom= pydicom.read_file(img_path)\n        try:\n            image = apply_voi_lut(dicom.pixel_array, dicom)\n        except:\n            print('exception')\n            continue\n        image = image - np.min(image)\n        image = image / np.max(image)\n        image = (image*255).astype(np.uint8)\n        image=cv2.resize(image, dsize=(406, 406), interpolation=cv2.INTER_CUBIC)\n        image=np.stack((image,)*3, axis=2)\n        test_transforms = transforms.Compose([transforms.ToTensor(),])\n        image_tensor = test_transforms(image)\n        #image_tensor=torch.stack((image_tensor,)*64, axis=0)\n        image_tensor = image_tensor.unsqueeze(0)\n        inputData = image_tensor.to(device)\n        logps = model.forward(inputData.type(torch.cuda.FloatTensor))\n        img_id=img_path.split(\"/\")\n        img_id=img_id[len(img_id)-1].replace('.dcm','_image')\n        labels=train[train['id']==img_id]['class']\n        #print(labels)\n        labels=torch.Tensor(list(labels)).to(device).type(torch.cuda.LongTensor)\n        #print(logps)\n        #print(labels)\n        loss = criterion(logps, labels)\n        loss.backward()\n        optimizer.step()\n        loss.item()\n    \n        \n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-18T11:40:30.908452Z","iopub.execute_input":"2021-06-18T11:40:30.908796Z","iopub.status.idle":"2021-06-18T11:40:47.579287Z","shell.execute_reply.started":"2021-06-18T11:40:30.908765Z","shell.execute_reply":"2021-06-18T11:40:47.577192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model, 'model.pth')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T10:21:34.917789Z","iopub.execute_input":"2021-06-18T10:21:34.918112Z","iopub.status.idle":"2021-06-18T10:21:35.113992Z","shell.execute_reply.started":"2021-06-18T10:21:34.918084Z","shell.execute_reply":"2021-06-18T10:21:35.113087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T11:40:55.615441Z","iopub.execute_input":"2021-06-18T11:40:55.615785Z","iopub.status.idle":"2021-06-18T11:40:55.624501Z","shell.execute_reply.started":"2021-06-18T11:40:55.615753Z","shell.execute_reply":"2021-06-18T11:40:55.623546Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col=['id','PredictionString']\nsubmission=pd.DataFrame(columns = col) \nfor dirname, _, filenames in os.walk('/kaggle/input/siim-covid19-detection/test'):\n    for filename in filenames:\n        img_path= os.path.join(dirname, filename)\n        dicom= pydicom.read_file(img_path)\n        try:\n            image = apply_voi_lut(dicom.pixel_array, dicom)\n        except:\n            PredictionString= f'negative 0 0 0 1 1 typical 0 0 0 1 1 indeterminate 1 0 0 1 1 atypical 0 0 0 1 1'\n            id_=img_path.split('/')[5]+\"_study\"\n            submission=submission.append({\"id\":id_,\"PredictionString\":PredictionString},ignore_index=True)\n            continue\n        image = image - np.min(image)\n        image = image / np.max(image)\n        image = (image*255).astype(np.uint8)\n        image=cv2.resize(image, dsize=(1984, 1984), interpolation=cv2.INTER_CUBIC)\n        image=np.stack((image,)*3, axis=2)\n        test_transforms = transforms.Compose([transforms.ToTensor(),])\n        image_tensor = test_transforms(image)\n        #image_tensor=torch.stack((image_tensor,)*64, axis=0)\n        image_tensor = image_tensor.unsqueeze(0)\n        inputData = image_tensor.to(device)\n\n        output = model(inputData.type(torch.cuda.FloatTensor))\n        index = output.data.cpu().numpy().argmax()\n        predProb=output.data.cpu().numpy()[0]\n        id_=img_path.split('/')[5]+\"_study\"\n        #try:\n            #PredictionString=submission[submission['id']==id_]['PredictionString'].iloc[0]+' '#.to_string()\n        #except:\n            #PredictionString=''\n        PredictionString=f'negative {format(predProb[0],\".8f\")} 0 0 1 1 typical {format(predProb[1],\".5f\")} 0 0 1 1 indeterminate {format(predProb[2],\".5f\")} 0 0 1 1 atypical {format(predProb[3],\".5f\")} 0 0 1 1'           #PredictionString+class_names[index].split(' ')[0]+str(predProb[0])+' 0 0 1 1'\n        #df2 = pd.DataFrame([[id_, PredictionString]], columns=col)\n        submission=submission.append({\"id\":id_,\"PredictionString\":PredictionString},ignore_index=True)\n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-18T11:41:01.687285Z","iopub.execute_input":"2021-06-18T11:41:01.687614Z","iopub.status.idle":"2021-06-18T11:42:31.597007Z","shell.execute_reply.started":"2021-06-18T11:41:01.687584Z","shell.execute_reply":"2021-06-18T11:42:31.594602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Bounding box predictions using pretrained weights","metadata":{}},{"cell_type":"code","source":"!cp ../input/yolov3/darknet/backup/darknet-yolov3_final.weights .\n!cp ../input/yolov3/darknet/cfg/darknet-yolov3.cfg .\n!cp ../input/yolov3/darknet/obj.names .","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:45:18.624214Z","iopub.execute_input":"2021-06-16T13:45:18.624579Z","iopub.status.idle":"2021-06-16T13:45:21.180338Z","shell.execute_reply.started":"2021-06-16T13:45:18.624533Z","shell.execute_reply":"2021-06-16T13:45:21.17914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport numpy as np \nimport argparse\nimport time\nimport os\n\nnet = cv2.dnn.readNet(\"/kaggle/working/darknet-yolov3_final.weights\",\"/kaggle/working/darknet-yolov3.cfg\")\nclasses = []\nwith open(\"/kaggle/working/obj.names\", \"r\") as f:\n    classes = [line.strip() for line in f.readlines()]\nlayers_names = net.getLayerNames()\noutput_layers = [layers_names[i[0]-1] for i in net.getUnconnectedOutLayers()]\ncolors = np.random.uniform(0, 255, size=(len(classes), 3))","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:46:37.697461Z","iopub.execute_input":"2021-06-16T13:46:37.697817Z","iopub.status.idle":"2021-06-16T13:46:37.856258Z","shell.execute_reply.started":"2021-06-16T13:46:37.697789Z","shell.execute_reply":"2021-06-16T13:46:37.855375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_box_dimensions(outputs, height, width):\n\tboxes = []\n\tconfs = []\n\tclass_ids = []\n\tfor output in outputs:\n\t\tfor detect in output:\n\t\t\tscores = detect[5:]\n\t\t\t\n\t\t\tclass_id = np.argmax(scores)\n\t\t\tconf = scores[class_id]\n\t\t\tif conf > 0.3:\n\t\t\t\tcenter_x = int(detect[0] * width)\n\t\t\t\tcenter_y = int(detect[1] * height)\n\t\t\t\tw = int(detect[2] * width)\n\t\t\t\th = int(detect[3] * height)\n\t\t\t\tx = int(center_x - w/2)\n\t\t\t\ty = int(center_y - h / 2)\n\t\t\t\tboxes.append([x, y, w, h])\n\t\t\t\tconfs.append(float(conf))\n\t\t\t\tclass_ids.append(class_id)\n\treturn boxes, confs, class_ids","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:46:42.897255Z","iopub.execute_input":"2021-06-16T13:46:42.8976Z","iopub.status.idle":"2021-06-16T13:46:42.91299Z","shell.execute_reply.started":"2021-06-16T13:46:42.897544Z","shell.execute_reply":"2021-06-16T13:46:42.911507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input/siim-covid19-detection/test'):\n    for filename in filenames:\n        img_path= os.path.join(dirname, filename)\n        dicom= pydicom.read_file(img_path)\n        id_=img_path.split('/')[7].replace('.dcm','_image')\n        try:\n            image = apply_voi_lut(dicom.pixel_array, dicom)\n        except:\n            PredictionString='none 1 0 0 1 1'\n            \n            submission=submission.append({\"id\":id_,\"PredictionString\":PredictionString},ignore_index=True)\n            continue\n        image = image - np.min(image)\n        image = image / np.max(image)\n        image = (image*255).astype(np.uint8)\n        img=cv2.resize(image, dsize=(224, 224), interpolation=cv2.INTER_CUBIC)\n    \n        height, width = img.shape\n        blob = cv2.dnn.blobFromImage(img, scalefactor=0.00392, size=(224, 224), mean=(0, 0, 0), swapRB=True, crop=False)\n        net.setInput(blob)\n        outputs = net.forward(output_layers)\n        boxes, confs, class_ids=get_box_dimensions(outputs,224,224)\n        PredictionString='none 1 0 0 1 1'\n        submission=submission.append({\"id\":id_,\"PredictionString\":PredictionString},ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:48:56.967934Z","iopub.execute_input":"2021-06-16T13:48:56.968306Z","iopub.status.idle":"2021-06-16T13:52:12.384432Z","shell.execute_reply.started":"2021-06-16T13:48:56.968277Z","shell.execute_reply":"2021-06-16T13:52:12.382416Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('/kaggle/working/submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:52:21.772125Z","iopub.execute_input":"2021-06-16T13:52:21.772441Z","iopub.status.idle":"2021-06-16T13:52:21.904818Z","shell.execute_reply.started":"2021-06-16T13:52:21.772408Z","shell.execute_reply":"2021-06-16T13:52:21.904051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}