{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# SIIIM-COVID Stratified sampling\n\n1. Split training data into Stratified 5 folds by human label (4) and OPACITIES PER IMAGE/PATIENT (1 2 3 4 5 8)\n2. Resize , scale and nomalize dcm files\n3. create mask\n4. find duplicate \n\n# Reffernces \n1. EDA https://www.kaggle.com/dschettler8845/visual-in-depth-eda-siim-covid19-detection\n2. incorrect anotation ? https://www.kaggle.com/dschettler8845/covid-detection-studies-with-multiple-images-viz\n3. duplication count https://www.kaggle.com/kwk100/siim-covid-19-duplicate-training-images?select=pairs.csv\n4. convert 2 png https://www.kaggle.com/xhlulu/siim-covid-19-convert-to-jpg-256px/notebook?scriptVersionId=63196459&select=meta.csv","metadata":{}},{"cell_type":"markdown","source":"# Problem explain\n\n\n- NEGATIVE FOR PNEUMONIA\n\n    No lung opacities\n\n- TYPICAL APPEARANCE\n\n    Multifocal bilateral, peripheral opacities with rounded morphology, lower lung–predominant distribution\n\n- INDETERMINATE APPEARANCE\n\n    Absence of typical findings AND unilateral, central or upper lung predominant distribution\n\n- ATYPICAL APPEARANCE\n\n    Pneumothorax, pleural effusion, pulmonary edema, lobar consolidation, solitary lung nodule or mass, diffuse tiny nodules, cavity\n    \n  \n#  Suspect\n\n1. Maybe \"INDETERMINATE APPEARANCE\" very simillar to \"ATYPICAL APPEARANCE\" or \"NEGATIVE FOR PNEUMONIA\"\n2. some images has no bouding box but has human label non-neg\n3. how to Deal with duplicate label ？？？https://www.kaggle.com/c/siim-covid19-detection/discussion/246597","metadata":{}},{"cell_type":"code","source":"!conda install gdcm -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:05:14.149021Z","iopub.execute_input":"2021-07-10T05:05:14.149463Z","iopub.status.idle":"2021-07-10T05:06:24.833552Z","shell.execute_reply.started":"2021-07-10T05:05:14.149380Z","shell.execute_reply":"2021-07-10T05:06:24.832392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport ast\n\nimport math\n\nimport pandas as pd\nimport numpy as np\n\nfrom tqdm.auto import tqdm\n\nfrom sklearn.model_selection import StratifiedKFold\n\nimport cv2\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\n\n# read dicom\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport csv","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:07:43.641520Z","iopub.execute_input":"2021-07-10T05:07:43.641910Z","iopub.status.idle":"2021-07-10T05:07:45.034781Z","shell.execute_reply.started":"2021-07-10T05:07:43.641859Z","shell.execute_reply":"2021-07-10T05:07:45.033654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_study_level.csv\")\nimage_df = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_image_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:07:48.928312Z","iopub.execute_input":"2021-07-10T05:07:48.928792Z","iopub.status.idle":"2021-07-10T05:07:49.001432Z","shell.execute_reply.started":"2021-07-10T05:07:48.928761Z","shell.execute_reply":"2021-07-10T05:07:49.000313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:07:57.613109Z","iopub.execute_input":"2021-07-10T05:07:57.613469Z","iopub.status.idle":"2021-07-10T05:07:57.640687Z","shell.execute_reply.started":"2021-07-10T05:07:57.613439Z","shell.execute_reply":"2021-07-10T05:07:57.639535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_study_total = len(study_df)\nprint(\"total data length\" , _study_total)\n\n# Stduy data for stratify\n_study_neg_row = study_df.loc[study_df[\"Negative for Pneumonia\"] == 1]\n_study_tp_row = study_df.loc[study_df[\"Typical Appearance\"] == 1]\n_study_nd_row = study_df.loc[study_df[\"Indeterminate Appearance\"] == 1]\n_study_ap_row = study_df.loc[study_df[\"Atypical Appearance\"] == 1]\n\nprint(\"Negative for Pneumonia count\" , len(_study_neg_row))\nprint(\"Typical Appearance count\" , len(_study_tp_row))\nprint(\"Indeterminate Appearance\tcount\" , len(_study_nd_row))\nprint(\"Atypical Appearance count\" , len(_study_ap_row))\n\nprint(\"one hot embedding ?\", len(_study_neg_row) + len(_study_tp_row) + len(_study_nd_row) + len(_study_ap_row) == _study_total)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:00.002202Z","iopub.execute_input":"2021-07-10T05:08:00.002570Z","iopub.status.idle":"2021-07-10T05:08:00.034532Z","shell.execute_reply.started":"2021-07-10T05:08:00.002539Z","shell.execute_reply":"2021-07-10T05:08:00.033215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:03.883794Z","iopub.execute_input":"2021-07-10T05:08:03.884193Z","iopub.status.idle":"2021-07-10T05:08:03.896898Z","shell.execute_reply.started":"2021-07-10T05:08:03.884158Z","shell.execute_reply":"2021-07-10T05:08:03.895659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Image data for stratify\n_img_nan_row = image_df.loc[image_df[\"label\"] == \"none 1 0 0 1 1\"]\nprint(\"image total count\", len(image_df))\nprint(\"image none row count\", len(_img_nan_row))","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:06.003987Z","iopub.execute_input":"2021-07-10T05:08:06.004337Z","iopub.status.idle":"2021-07-10T05:08:06.012252Z","shell.execute_reply.started":"2021-07-10T05:08:06.004307Z","shell.execute_reply":"2021-07-10T05:08:06.011097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm\"\n_one = image_df.loc[image_df[\"StudyInstanceUID\"] == \"a7335b2f9815\"]\n_one","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:09.394838Z","iopub.execute_input":"2021-07-10T05:08:09.395243Z","iopub.status.idle":"2021-07-10T05:08:09.410755Z","shell.execute_reply.started":"2021-07-10T05:08:09.395203Z","shell.execute_reply":"2021-07-10T05:08:09.409595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/siim-covid19-detection/train/a7335b2f9815","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:32.444621Z","iopub.execute_input":"2021-07-10T05:08:32.444995Z","iopub.status.idle":"2021-07-10T05:08:33.189009Z","shell.execute_reply.started":"2021-07-10T05:08:32.444961Z","shell.execute_reply":"2021-07-10T05:08:33.187772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# \"../input/siim-covid19-detection/train/00086460a852/9e8302230c91/65761e66de9f.dcm\"\n_one = image_df.loc[image_df[\"StudyInstanceUID\"] == \"00086460a852\"]\n_one","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:34.451898Z","iopub.execute_input":"2021-07-10T05:08:34.452267Z","iopub.status.idle":"2021-07-10T05:08:34.466913Z","shell.execute_reply.started":"2021-07-10T05:08:34.452228Z","shell.execute_reply":"2021-07-10T05:08:34.465792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:36.388186Z","iopub.execute_input":"2021-07-10T05:08:36.388615Z","iopub.status.idle":"2021-07-10T05:08:36.399346Z","shell.execute_reply.started":"2021-07-10T05:08:36.388578Z","shell.execute_reply":"2021-07-10T05:08:36.398583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Fold","metadata":{}},{"cell_type":"code","source":"colnames = {\n    \"Negative for Pneumonia\": 0,\n    \"Typical Appearance\": 1,\n    \"Indeterminate Appearance\": 2,\n    \"Atypical Appearance\": 3\n}","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:38.469787Z","iopub.execute_input":"2021-07-10T05:08:38.470837Z","iopub.status.idle":"2021-07-10T05:08:38.479153Z","shell.execute_reply.started":"2021-07-10T05:08:38.470760Z","shell.execute_reply":"2021-07-10T05:08:38.477550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, row in study_df.iterrows():    \n    for x in colnames:\n        if row[x] == 1:\n            study_df.loc[index, \"bin\"] = colnames[x]","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:39.947764Z","iopub.execute_input":"2021-07-10T05:08:39.948161Z","iopub.status.idle":"2021-07-10T05:08:42.275233Z","shell.execute_reply.started":"2021-07-10T05:08:39.948130Z","shell.execute_reply":"2021-07-10T05:08:42.274185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:42.277082Z","iopub.execute_input":"2021-07-10T05:08:42.277515Z","iopub.status.idle":"2021-07-10T05:08:42.292637Z","shell.execute_reply.started":"2021-07-10T05:08:42.277469Z","shell.execute_reply":"2021-07-10T05:08:42.291207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for index, row in image_df.iterrows():       \n    image_df.loc[index, \"bin\"] = study_df.loc[study_df[\"id\"] == row[\"StudyInstanceUID\"] + \"_study\"][\"bin\"].item()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:08:51.527951Z","iopub.execute_input":"2021-07-10T05:08:51.528460Z","iopub.status.idle":"2021-07-10T05:09:04.249656Z","shell.execute_reply.started":"2021-07-10T05:08:51.528426Z","shell.execute_reply":"2021-07-10T05:09:04.248672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:13.776384Z","iopub.execute_input":"2021-07-10T05:09:13.776786Z","iopub.status.idle":"2021-07-10T05:09:13.797643Z","shell.execute_reply.started":"2021-07-10T05:09:13.776748Z","shell.execute_reply":"2021-07-10T05:09:13.796300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.loc[study_df[\"id\"] == \"ff0879eb20ed_study\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:04.251386Z","iopub.execute_input":"2021-07-10T05:09:04.251786Z","iopub.status.idle":"2021-07-10T05:09:04.265037Z","shell.execute_reply.started":"2021-07-10T05:09:04.251743Z","shell.execute_reply":"2021-07-10T05:09:04.263973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fold = 0\n\nX = image_df\ny = image_df['bin']\nskf = StratifiedKFold(n_splits=5, random_state=None, shuffle=True)\nskf.get_n_splits(X, y)\nfor train_index, test_index in skf.split(X, y):\n    print(\"TRAIN:\", len(train_index), \"TEST:\", len(test_index))\n    #X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n    #y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n    image_df.loc[test_index, 'fold'] = int(fold)\n    fold +=1 ","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:04.267060Z","iopub.execute_input":"2021-07-10T05:09:04.267509Z","iopub.status.idle":"2021-07-10T05:09:04.296036Z","shell.execute_reply.started":"2021-07-10T05:09:04.267462Z","shell.execute_reply":"2021-07-10T05:09:04.295003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:17.899542Z","iopub.execute_input":"2021-07-10T05:09:17.899921Z","iopub.status.idle":"2021-07-10T05:09:17.925588Z","shell.execute_reply.started":"2021-07-10T05:09:17.899855Z","shell.execute_reply":"2021-07-10T05:09:17.923986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_df.loc[study_df[\"id\"] == \"5776db0cec75_study\"]","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:20.771895Z","iopub.execute_input":"2021-07-10T05:09:20.772299Z","iopub.status.idle":"2021-07-10T05:09:20.785719Z","shell.execute_reply.started":"2021-07-10T05:09:20.772248Z","shell.execute_reply":"2021-07-10T05:09:20.785017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df.to_csv(\"./updated_iamge_level.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:23.673029Z","iopub.execute_input":"2021-07-10T05:09:23.673579Z","iopub.status.idle":"2021-07-10T05:09:23.765742Z","shell.execute_reply.started":"2021-07-10T05:09:23.673537Z","shell.execute_reply":"2021-07-10T05:09:23.764658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find duplicate ","metadata":{}},{"cell_type":"code","source":"# hengk's duplicate list, we shall verify it \nduplicateList = [\n'87adda1908f9','35e398a5a431','1b92142f4362','ef6e312ca719','866e3622cb24','3577ee4f26c4','d74ef8961bff','6f54e9cbd180','b97c6b32105e','8093df07a5d0','efc93a3917b6','b3ffe59e37c7','9939f63af4ff','00c1515729a8','267a250932bc','3e7b2ffc97db','b121806162c3','2f973757eaa4','a39667fe9a81','a5bbd30ed109','07c6b234a94b','089bd77c8c10','0575769456b9','149356f04849','ace7a9702770','0acec7010081','611348a721f7','deea6d6f81a5','b0866caa201a','ab55abb953d1','7626c521cad7','bee62c601ae9','582c442e440b','869476b0763a','9b1de1c45491','32222cc776a2','66dabc6f972d','1dc7459cb081','a2ee4b862182','3b6ad60071d4','39d52f244db3','ce51b397b1a6','2a7a456d367e','a94171e98807','21518ca15050','e96133d06736','ea2688741043','7b7275d670e7','59bc532be971','684230477525','8d4b3609ed92','608d574388ba','76c66ee8e58d','6ff218d10741','7966f780f27f','7f1924880cf8','0102b5cac730','99f3642f50f5','6e9fad584bff','830063223a31','72cf260ddf4c','156cb1f5c689','6d36ffbc7864','9fcbe25a88e0','6534a837497d','04cc2f7f4c4b','30b18db28900','1c1069c57757','f208dc529d16','df2bb22fa871','05c063f5cef5','eea3a910fa9e','2c130ee08736','c6e92e59a0ae','1ea01196514a','4c414b793562','00e3a7e91a34','6e5946091b8a','c3a09e8a600d','9872a8a48f23','12e97ed89297','66712e2fc6a4','c843d08b49b8','df4f1240317e','a5a364383f34','0dd32589bdc3','93a881fb3292','550f057ee0b0','ea2117b53323','6728e11290af','be65d1a22de5','08acae0bf785','0d4d6acc9ed3','779f0040d1b2','caa7fd25ee9d','e73f2cc047ad','75b52bec817f','ddb051c1233b','9108cdfd43dc','a57bf6dd6119','c4b68b29a072','717ea5155b46','49664f078f0e','e738c549fe8e','4cbc17936e7d','e3e2f20e0264','b5a415f70aa9'\n]","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:26.232632Z","iopub.execute_input":"2021-07-10T05:09:26.233302Z","iopub.status.idle":"2021-07-10T05:09:26.243593Z","shell.execute_reply.started":"2021-07-10T05:09:26.233257Z","shell.execute_reply":"2021-07-10T05:09:26.242649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(duplicateList)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:09:27.976772Z","iopub.execute_input":"2021-07-10T05:09:27.977611Z","iopub.status.idle":"2021-07-10T05:09:27.986461Z","shell.execute_reply.started":"2021-07-10T05:09:27.977541Z","shell.execute_reply":"2021-07-10T05:09:27.985090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicate_df_path = '../input/siim-covid-19-duplicate-training-images/duplicates.csv'","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:00.027453Z","iopub.execute_input":"2021-07-10T05:10:00.027805Z","iopub.status.idle":"2021-07-10T05:10:00.032232Z","shell.execute_reply.started":"2021-07-10T05:10:00.027775Z","shell.execute_reply":"2021-07-10T05:10:00.031034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_result = []\nwith open(duplicate_df_path, newline='') as csvfile:\n    spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n    for row in spamreader:\n        rows = row[0].split(',')\n        _count = 0\n        _cand = []\n        for item in rows:\n            _org = image_df.loc[image_df['id'] == item]\n            if str(_org['boxes'].item()) != 'nan':\n                _count +=1\n            else:\n                _cand.append(item)\n        if _count == 0:\n            _result += _cand[1:]\n        if _count > 0: \n            _result += _cand\n        if _count > 1:\n            print(row)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:01.927578Z","iopub.execute_input":"2021-07-10T05:10:01.927973Z","iopub.status.idle":"2021-07-10T05:10:02.232488Z","shell.execute_reply.started":"2021-07-10T05:10:01.927933Z","shell.execute_reply":"2021-07-10T05:10:02.231513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_result = [x.replace('_image','') for x in _result]\n#_result\nlen(_result)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:04.019246Z","iopub.execute_input":"2021-07-10T05:10:04.019577Z","iopub.status.idle":"2021-07-10T05:10:04.025885Z","shell.execute_reply.started":"2021-07-10T05:10:04.019549Z","shell.execute_reply":"2021-07-10T05:10:04.024773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# duplicate wrong anotation\n_result.append('7b7275d670e7')","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:05.571499Z","iopub.execute_input":"2021-07-10T05:10:05.571869Z","iopub.status.idle":"2021-07-10T05:10:05.575855Z","shell.execute_reply.started":"2021-07-10T05:10:05.571838Z","shell.execute_reply":"2021-07-10T05:10:05.574812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\nfor x in _result:\n    if x in duplicateList:\n        count += 1\nprint(count)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:07.019487Z","iopub.execute_input":"2021-07-10T05:10:07.019845Z","iopub.status.idle":"2021-07-10T05:10:07.025667Z","shell.execute_reply.started":"2021-07-10T05:10:07.019811Z","shell.execute_reply":"2021-07-10T05:10:07.024509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('dublicate.txt', 'w') as text_file:    \n    # using csv.writer method from CSV package\n    text_file.write('\\n'.join(_result))","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:10.807517Z","iopub.execute_input":"2021-07-10T05:10:10.808090Z","iopub.status.idle":"2021-07-10T05:10:10.813602Z","shell.execute_reply.started":"2021-07-10T05:10:10.808052Z","shell.execute_reply":"2021-07-10T05:10:10.812577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create PNG files","metadata":{}},{"cell_type":"code","source":"_SIZE_ = 512 \n_Type_ = \"png\"","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:12.527346Z","iopub.execute_input":"2021-07-10T05:10:12.527853Z","iopub.status.idle":"2021-07-10T05:10:12.532475Z","shell.execute_reply.started":"2021-07-10T05:10:12.527819Z","shell.execute_reply":"2021-07-10T05:10:12.531480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:13.591095Z","iopub.execute_input":"2021-07-10T05:10:13.591579Z","iopub.status.idle":"2021-07-10T05:10:13.598865Z","shell.execute_reply.started":"2021-07-10T05:10:13.591539Z","shell.execute_reply":"2021-07-10T05:10:13.598149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:15.659296Z","iopub.execute_input":"2021-07-10T05:10:15.659669Z","iopub.status.idle":"2021-07-10T05:10:15.664780Z","shell.execute_reply.started":"2021-07-10T05:10:15.659637Z","shell.execute_reply":"2021-07-10T05:10:15.664030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/siim-covid19-detection/train/ae3e63d94c13/288554eb6182/e00f9fe0cce5.dcm'\ndicom = pydicom.read_file(path)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:10:18.047032Z","iopub.execute_input":"2021-07-10T05:10:18.047630Z","iopub.status.idle":"2021-07-10T05:10:18.359289Z","shell.execute_reply.started":"2021-07-10T05:10:18.047596Z","shell.execute_reply":"2021-07-10T05:10:18.358354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dicom","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:51.033814Z","iopub.execute_input":"2021-07-10T05:02:51.034314Z","iopub.status.idle":"2021-07-10T05:02:51.042305Z","shell.execute_reply.started":"2021-07-10T05:02:51.034280Z","shell.execute_reply":"2021-07-10T05:02:51.041184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example = read_xray(path)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:51.043710Z","iopub.execute_input":"2021-07-10T05:02:51.044106Z","iopub.status.idle":"2021-07-10T05:02:51.927959Z","shell.execute_reply.started":"2021-07-10T05:02:51.044049Z","shell.execute_reply":"2021-07-10T05:02:51.926859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(example, cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:51.929190Z","iopub.execute_input":"2021-07-10T05:02:51.929472Z","iopub.status.idle":"2021-07-10T05:02:52.770363Z","shell.execute_reply.started":"2021-07-10T05:02:51.929444Z","shell.execute_reply":"2021-07-10T05:02:52.769347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create mask","metadata":{}},{"cell_type":"code","source":"example","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:52.771755Z","iopub.execute_input":"2021-07-10T05:02:52.772097Z","iopub.status.idle":"2021-07-10T05:02:52.778466Z","shell.execute_reply.started":"2021-07-10T05:02:52.772045Z","shell.execute_reply":"2021-07-10T05:02:52.777342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:52.779899Z","iopub.execute_input":"2021-07-10T05:02:52.780308Z","iopub.status.idle":"2021-07-10T05:02:52.803913Z","shell.execute_reply.started":"2021-07-10T05:02:52.780266Z","shell.execute_reply":"2021-07-10T05:02:52.802641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/siim-covid19-detection/train/e28208b7c58e/4b7cd369719c","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:52.805290Z","iopub.execute_input":"2021-07-10T05:02:52.805578Z","iopub.status.idle":"2021-07-10T05:02:53.598969Z","shell.execute_reply.started":"2021-07-10T05:02:52.805551Z","shell.execute_reply":"2021-07-10T05:02:53.597946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path2 = '../input/siim-covid19-detection/train/e28208b7c58e/4b7cd369719c/0a0bb7af0cab.dcm'\ndicom2 = pydicom.read_file(path)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:53.600501Z","iopub.execute_input":"2021-07-10T05:02:53.600786Z","iopub.status.idle":"2021-07-10T05:02:53.610430Z","shell.execute_reply.started":"2021-07-10T05:02:53.600749Z","shell.execute_reply":"2021-07-10T05:02:53.609552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example2_meta = image_df.loc[image_df['id'] == \"0a0bb7af0cab_image\"]\nexample2_meta","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:53.611547Z","iopub.execute_input":"2021-07-10T05:02:53.611809Z","iopub.status.idle":"2021-07-10T05:02:53.627618Z","shell.execute_reply.started":"2021-07-10T05:02:53.611784Z","shell.execute_reply":"2021-07-10T05:02:53.626500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_df","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:53.629050Z","iopub.execute_input":"2021-07-10T05:02:53.629346Z","iopub.status.idle":"2021-07-10T05:02:53.651725Z","shell.execute_reply.started":"2021-07-10T05:02:53.629318Z","shell.execute_reply":"2021-07-10T05:02:53.650603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bbox = example2_meta['boxes'].item()\nbbox","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:53.653274Z","iopub.execute_input":"2021-07-10T05:02:53.653662Z","iopub.status.idle":"2021-07-10T05:02:53.665521Z","shell.execute_reply.started":"2021-07-10T05:02:53.653621Z","shell.execute_reply":"2021-07-10T05:02:53.664413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example2 = read_xray(path2)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:53.666666Z","iopub.execute_input":"2021-07-10T05:02:53.666927Z","iopub.status.idle":"2021-07-10T05:02:54.339610Z","shell.execute_reply.started":"2021-07-10T05:02:53.666901Z","shell.execute_reply":"2021-07-10T05:02:54.338538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example2.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:54.341015Z","iopub.execute_input":"2021-07-10T05:02:54.341472Z","iopub.status.idle":"2021-07-10T05:02:54.347031Z","shell.execute_reply.started":"2021-07-10T05:02:54.341427Z","shell.execute_reply":"2021-07-10T05:02:54.346042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for box in ast.literal_eval(bbox):\n    x1, y1, x2, y2 = box[\"x\"], box[\"y\"], box[\"x\"] + box[\"width\"], box[\"y\"] + box[\"height\"]\n    x1 = int(x1) \n    y1 = int(y1) \n    x2 = int(x2)  \n    y2 = int(y2) \n    cv2.rectangle(example2, (x1, y1), (x2, y2), (0, 80, 255), 15)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:54.348506Z","iopub.execute_input":"2021-07-10T05:02:54.349100Z","iopub.status.idle":"2021-07-10T05:02:54.368706Z","shell.execute_reply.started":"2021-07-10T05:02:54.349035Z","shell.execute_reply":"2021-07-10T05:02:54.367408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(example2, cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:54.370337Z","iopub.execute_input":"2021-07-10T05:02:54.370638Z","iopub.status.idle":"2021-07-10T05:02:55.291908Z","shell.execute_reply.started":"2021-07-10T05:02:54.370608Z","shell.execute_reply":"2021-07-10T05:02:55.290921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"blankimg = np.zeros((example2.shape[0],example2.shape[1],3), dtype=np.uint8)\nfor box in ast.literal_eval(bbox):\n    x1, y1, x2, y2 = box[\"x\"], box[\"y\"], box[\"x\"] + box[\"width\"], box[\"y\"] + box[\"height\"]\n    x1 = int(x1) \n    y1 = int(y1) \n    x2 = int(x2)  \n    y2 = int(y2) \n    cv2.rectangle(blankimg, (x1, y1), (x2, y2), (255, 255, 255), -1)","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:55.293208Z","iopub.execute_input":"2021-07-10T05:02:55.293473Z","iopub.status.idle":"2021-07-10T05:02:55.304247Z","shell.execute_reply.started":"2021-07-10T05:02:55.293447Z","shell.execute_reply":"2021-07-10T05:02:55.303089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(blankimg, cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:02:55.305606Z","iopub.execute_input":"2021-07-10T05:02:55.305994Z","iopub.status.idle":"2021-07-10T05:02:56.318283Z","shell.execute_reply.started":"2021-07-10T05:02:55.305960Z","shell.execute_reply":"2021-07-10T05:02:56.317373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = []\ndim0 = []\ndim1 = []\nsplits = []\nstudy = []\n\nmaskdir = \"/kaggle/tmp/mask/\"\nos.makedirs(maskdir, exist_ok=True)\n    \n#for split in ['test', 'train']:\nfor split in ['train', 'test']:\n    save_dir = f'/kaggle/tmp/{split}/'\n    \n    os.makedirs(save_dir, exist_ok=True)\n    \n    for study_id in os.listdir(f'../input/siim-covid19-detection/{split}'):\n        for dirname, _, filenames in os.walk(f'../input/siim-covid19-detection/{split}/{study_id}'):\n            for file in filenames:\n                # set keep_ratio=True to have original aspect ratio\n                xray = read_xray(os.path.join(dirname, file))\n\n                im = resize(xray, size=_SIZE_)\n                _name = file.replace('dcm', _Type_)\n                im.save(os.path.join(save_dir, _name))\n\n                _img_id = file.replace('.dcm', '')\n                image_id.append(_img_id)\n\n                _w = xray.shape[0]\n                _h = xray.shape[1]\n                dim0.append(_w)\n                dim1.append(_h)\n                splits.append(split)\n                study.append(study_id)\n\n                if split != 'test':\n                    _mate = image_df.loc[image_df['id'] == _img_id + \"_image\"]\n                    _bbox = _mate['boxes'].item()\n                    if str(_bbox) != \"nan\":\n                        blankimg = np.zeros((_w, _h,3), dtype=np.uint8)\n\n                        for box in ast.literal_eval(_bbox):\n                            x1, y1, x2, y2 = box[\"x\"], box[\"y\"], box[\"x\"] + box[\"width\"], box[\"y\"] + box[\"height\"]\n                            x1 = int(x1) \n                            y1 = int(y1) \n                            x2 = int(x2)  \n                            y2 = int(y2) \n                            cv2.rectangle(blankimg, (x1, y1), (x2, y2), (255, 255, 255), -1)\n\n                        blankimg = resize(blankimg, size=_SIZE_)\n                        blankimg.save(os.path.join(maskdir, _name))","metadata":{"execution":{"iopub.status.busy":"2021-07-10T05:47:13.759405Z","iopub.execute_input":"2021-07-10T05:47:13.759760Z","iopub.status.idle":"2021-07-10T05:50:43.753787Z","shell.execute_reply.started":"2021-07-10T05:47:13.759729Z","shell.execute_reply":"2021-07-10T05:50:43.752037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!tar -zcf train.tar.gz -C \"/kaggle/tmp/train/\" .\n!tar -zcf test.tar.gz -C \"/kaggle/tmp/test/\" .\n!tar -zcf mask.tar.gz -C \"/kaggle/tmp/mask/\" .","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_df = pd.DataFrame.from_dict({'image_id': image_id, 'dim0': dim0, 'dim1': dim1, 'split': splits, 'study': study})\nmeta_df.to_csv('meta.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}