{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"! conda install -c conda-forge gdcm -y;","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-18T07:27:07.069586Z","iopub.execute_input":"2021-07-18T07:27:07.069903Z","iopub.status.idle":"2021-07-18T07:27:40.390633Z","shell.execute_reply.started":"2021-07-18T07:27:07.069869Z","shell.execute_reply":"2021-07-18T07:27:40.389622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/timmeffnetv2\")\n\nimport platform\nimport numpy as np\nimport pandas as pd\nimport os\nfrom tqdm.notebook import tqdm\nimport cv2\nimport pydicom\nimport gdcm\nimport glob\nimport gc\nfrom math import ceil\nimport matplotlib.pyplot as plt\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:07.973966Z","iopub.execute_input":"2021-07-18T07:28:07.974352Z","iopub.status.idle":"2021-07-18T07:28:07.982047Z","shell.execute_reply.started":"2021-07-18T07:28:07.974317Z","shell.execute_reply":"2021-07-18T07:28:07.981329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image = pd.read_csv(\"../input/siim-covid19-detection/train_image_level.csv\")\ntrain_study = pd.read_csv(\"../input/siim-covid19-detection/train_study_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:10.07007Z","iopub.execute_input":"2021-07-18T07:28:10.070399Z","iopub.status.idle":"2021-07-18T07:28:10.128242Z","shell.execute_reply.started":"2021-07-18T07:28:10.07037Z","shell.execute_reply":"2021-07-18T07:28:10.127503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_DIR = \"../input/siim-covid19-detection/train/\"\ntrain_study['StudyInstanceUID'] = train_study['id'].apply(lambda x: x.replace('_study', ''))\ntrain = train_image.merge(train_study, on='StudyInstanceUID')\n\n# Make a path folder\npaths = []\nfor instance_id in tqdm(train['StudyInstanceUID']):\n    paths.append(glob.glob(os.path.join(TRAIN_DIR, instance_id +\"/*/*\"))[0])\n\ntrain['path'] = paths\n\ntrain = train.drop(['id_x', 'id_y'], axis=1)\n\ntrain = train.sample(frac=1).reset_index(drop=True)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:10.395069Z","iopub.execute_input":"2021-07-18T07:28:10.395408Z","iopub.status.idle":"2021-07-18T07:28:31.668416Z","shell.execute_reply.started":"2021-07-18T07:28:10.395378Z","shell.execute_reply":"2021-07-18T07:28:31.667695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:31.669612Z","iopub.execute_input":"2021-07-18T07:28:31.669878Z","iopub.status.idle":"2021-07-18T07:28:31.676204Z","shell.execute_reply.started":"2021-07-18T07:28:31.669852Z","shell.execute_reply":"2021-07-18T07:28:31.675437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dicom2array(path, voi_lut=True, fix_monochrome=True):\n    dicom = pydicom.read_file(path)\n    # VOI LUT (if available by DICOM device) is used to\n    # transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:31.677657Z","iopub.execute_input":"2021-07-18T07:28:31.677989Z","iopub.status.idle":"2021-07-18T07:28:31.686477Z","shell.execute_reply.started":"2021-07-18T07:28:31.677959Z","shell.execute_reply":"2021-07-18T07:28:31.685848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('/kaggle/working/output/', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:31.687593Z","iopub.execute_input":"2021-07-18T07:28:31.688021Z","iopub.status.idle":"2021-07-18T07:28:31.696044Z","shell.execute_reply.started":"2021-07-18T07:28:31.687993Z","shell.execute_reply":"2021-07-18T07:28:31.695262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import h5py","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:31.697194Z","iopub.execute_input":"2021-07-18T07:28:31.697774Z","iopub.status.idle":"2021-07-18T07:28:31.833983Z","shell.execute_reply.started":"2021-07-18T07:28:31.697745Z","shell.execute_reply":"2021-07-18T07:28:31.833181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nidx = 0\nimage_id = train['StudyInstanceUID'].values[idx]\nimage_path = train['path'].values[idx]\nimage = dicom2array(image_path)\nimage = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\nimage = cv2.resize(image, (512, 512))\nlabel = train[train['StudyInstanceUID'] == image_id].values.tolist()[0][3:7]\nexample = h5py.File('/kaggle/working/output/'+'test.hdf5', 'w')\nexample.create_dataset(\"img\", data=image)\nexample.create_dataset(\"label\", data=label)\nexample.close()\nexample = h5py.File('/kaggle/working/output/'+'test.hdf5', 'r')\ndisplay(np.mean(image == example['img'][:]))\ndisplay(label == example['label'][:])\n'''\n","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:05:44.176986Z","iopub.execute_input":"2021-07-18T07:05:44.177477Z","iopub.status.idle":"2021-07-18T07:05:45.556079Z","shell.execute_reply.started":"2021-07-18T07:05:44.177424Z","shell.execute_reply":"2021-07-18T07:05:45.55536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:05:47.489718Z","iopub.execute_input":"2021-07-18T07:05:47.490357Z","iopub.status.idle":"2021-07-18T07:05:47.503187Z","shell.execute_reply.started":"2021-07-18T07:05:47.49032Z","shell.execute_reply":"2021-07-18T07:05:47.502131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx in range(6334):\n    print(idx)\n    image_id = train['StudyInstanceUID'].values[idx]\n    image_path = train['path'].values[idx]\n    image = dicom2array(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n    image = cv2.resize(image, (512, 512))\n    label = train[train['StudyInstanceUID'] == image_id].values.tolist()[0][3:7]\n    example = h5py.File('/kaggle/working/output/'+'{x}.hdf5'.format(x=image_id), 'w')\n    example.create_dataset(\"img\", data=image)\n    example.create_dataset(\"label\", data=label)\n    example.close()","metadata":{"execution":{"iopub.status.busy":"2021-07-18T07:28:31.836044Z","iopub.execute_input":"2021-07-18T07:28:31.836313Z","iopub.status.idle":"2021-07-18T07:29:12.490372Z","shell.execute_reply.started":"2021-07-18T07:28:31.836287Z","shell.execute_reply":"2021-07-18T07:29:12.488597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}