{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!conda install gdcm -c conda-forge -y","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-07-16T06:30:54.043663Z","iopub.execute_input":"2021-07-16T06:30:54.044052Z","iopub.status.idle":"2021-07-16T06:32:00.354999Z","shell.execute_reply.started":"2021-07-16T06:30:54.043973Z","shell.execute_reply":"2021-07-16T06:32:00.353895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfrom PIL import Image\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport cv2 as cv\nfrom glob import glob\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-16T06:32:00.356920Z","iopub.execute_input":"2021-07-16T06:32:00.357268Z","iopub.status.idle":"2021-07-16T06:32:00.501742Z","shell.execute_reply.started":"2021-07-16T06:32:00.357226Z","shell.execute_reply":"2021-07-16T06:32:00.500944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:00.504349Z","iopub.execute_input":"2021-07-16T06:32:00.504698Z","iopub.status.idle":"2021-07-16T06:32:00.732596Z","shell.execute_reply.started":"2021-07-16T06:32:00.504662Z","shell.execute_reply":"2021-07-16T06:32:00.731772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n\n    im = array \n    hist_im = cv.equalizeHist(im)\n    im = Image.fromarray(hist_im)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:00.734164Z","iopub.execute_input":"2021-07-16T06:32:00.734490Z","iopub.status.idle":"2021-07-16T06:32:00.748497Z","shell.execute_reply.started":"2021-07-16T06:32:00.734455Z","shell.execute_reply":"2021-07-16T06:32:00.745459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/siim-covid19-detection/train_image_level.csv')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:00.749904Z","iopub.execute_input":"2021-07-16T06:32:00.750469Z","iopub.status.idle":"2021-07-16T06:32:00.814264Z","shell.execute_reply.started":"2021-07-16T06:32:00.750427Z","shell.execute_reply":"2021-07-16T06:32:00.813345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/siim-covid19-detection/train/ae3e63d94c13/288554eb6182/e00f9fe0cce5.dcm'\ndicom = pydicom.read_file(path)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:00.818850Z","iopub.execute_input":"2021-07-16T06:32:00.819411Z","iopub.status.idle":"2021-07-16T06:32:01.145231Z","shell.execute_reply.started":"2021-07-16T06:32:00.819368Z","shell.execute_reply":"2021-07-16T06:32:01.142941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv\nfile = open('../input/siimdup/duplicates.csv' , \"r\")\ncsv_reader = csv.reader(file)\n\nlists_from_csv = []\nfor row in csv_reader:\n    lists_from_csv.append(row)\n\nprint(lists_from_csv)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:01.146636Z","iopub.execute_input":"2021-07-16T06:32:01.146979Z","iopub.status.idle":"2021-07-16T06:32:01.165544Z","shell.execute_reply.started":"2021-07-16T06:32:01.146941Z","shell.execute_reply":"2021-07-16T06:32:01.164614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keep = ['74077a8e3b7c_image', '9e4824fcee2e_image', '19fc87ff0612_image', '961e67cadcdc_image', '70fbcd6dcc53_image', '6494a03199e5_image', '089bd77c8c10_image', 'b61f3493c551_image', '0842f032a217_image', '830063223a31_image', 'efc93a3917b6_image', '26f643772090_image', 'c8156ae4d6e8_image', '93979c3e3177_image', '9e844dea386a_image', 'b4b931e5ad31_image', 'bb3076795a01_image', 'b6b74c8a97e4_image', '12e97ed89297_image', 'd180fed57716_image', 'c05a1da5efe0_image', '06f6423be3f9_image', '2da1eb17b0b7_image', '9fa8318fb7f6_image', '66712e2fc6a4_image', 'ea516e218fe6_image', '7966f780f27f_image', '93301812b0e7_image', '1a0a148c030f_image', 'd93b8a8335cf_image', 'caa7fd25ee9d_image', 'd342f75ccb55_image', '1dc7459cb081_image', '63c6e1324ac3_image', '04f41a8958f7_image', '41e9a794b342_image', '0d4d6acc9ed3_image', 'fa447a409bd6_image', 'f7edf5c476c4_image', '893fde8abd42_image', 'ea2688741043_image', 'b43de320e7d9_image', 'ac212043ee3c_image', '6f5e70a99d77_image', 'c7925ab50eb0_image', '42f55c3da74f_image', 'abb96ea8a826_image', '7be323e5f816_image', '0c6b440ba98e_image', '19701de5ae89_image', '40e1d57e03be_image', '077c85f226d8_image', 'c64ddad4795a_image', 'f7c7683d6ed9_image', '32ebc75d961e_image', 'df565bcf3504_image', 'a0178e3a0d8f_image', '3e7b2ffc97db_image', '4bb94cd7f2f4_image', '61f3ac249c50_image', '847f48f57169_image', '172a7ed6d7e2_image', '84135cf828e7_image', '7e8966bfec61_image', '7b3e9f13d4e7_image', 'f208dc529d16_image', 'd74ef8961bff_image', 'a2ee4b862182_image', 'd787c9bd4fa8_image', '68ad4b624a6d_image', '2f6019c75d6d_image', 'b0866caa201a_image', '52f97cad2b63_image', 'cbf0a27f993e_image', '3566e20a178e_image', '9108cdfd43dc_image', '33c026e51b02_image', '1c1069c57757_image', 'a414f67f5735_image', '173c23887f9b_image' ]","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:01.167799Z","iopub.execute_input":"2021-07-16T06:32:01.168151Z","iopub.status.idle":"2021-07-16T06:32:01.175046Z","shell.execute_reply.started":"2021-07-16T06:32:01.168116Z","shell.execute_reply":"2021-07-16T06:32:01.173784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remove = []\nfor i in lists_from_csv:\n    for j in i:\n        if j not in keep:\n            remove.append(j)\n#remove","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:01.176953Z","iopub.execute_input":"2021-07-16T06:32:01.177353Z","iopub.status.idle":"2021-07-16T06:32:01.186775Z","shell.execute_reply.started":"2021-07-16T06:32:01.177314Z","shell.execute_reply":"2021-07-16T06:32:01.185990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(keep), len(remove))","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:01.188099Z","iopub.execute_input":"2021-07-16T06:32:01.188463Z","iopub.status.idle":"2021-07-16T06:32:01.196519Z","shell.execute_reply.started":"2021-07-16T06:32:01.188426Z","shell.execute_reply":"2021-07-16T06:32:01.195596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 640\nimage_id = []\ndim0 = []\ndim1 = []\nsplits = []\n\ntuples = []\nfor split in ['train', 'test']:\n    save_dir = f'/kaggle/working/image/{split}/'\n\n    os.makedirs(save_dir, exist_ok=True)\n\n    for dirname, _, filenames in tqdm(os.walk(f'../input/siim-covid19-detection/{split}')):\n        for file in filenames:\n            tuples.append((file, dirname, save_dir, split))\nimport multiprocessing\n\ndef resize_and_save(t):\n    file, dirname, save_dir, split = t\n    # set keep_ratio=True to have original aspect ratio\n    xray = read_xray(os.path.join(dirname, file))\n    im = resize(xray, size=IMG_SIZE) \n#     print(os.path.join(dirname, file), os.path.join(save_dir, file.replace('dcm', 'png')))\n    if (file[:-4] + '_image') in remove:\n        pass\n    else:\n        im.save(os.path.join(save_dir, file[:-4] + '_image' + '.png'))\n\n    image_id.append(file.replace('.dcm', '') + '_image')\n    dim0.append(xray.shape[0])\n    dim1.append(xray.shape[1])\n    splits.append(split)\n\n\npool = multiprocessing.Pool(processes=multiprocessing.cpu_count())\n# for t in tqdm(tuples):\nfor _ in tqdm(pool.imap_unordered(resize_and_save, tuples), total=len(tuples)):\n    pass\n","metadata":{"execution":{"iopub.status.busy":"2021-07-16T06:32:01.197853Z","iopub.execute_input":"2021-07-16T06:32:01.198375Z","iopub.status.idle":"2021-07-16T07:08:14.684297Z","shell.execute_reply.started":"2021-07-16T06:32:01.198337Z","shell.execute_reply":"2021-07-16T07:08:14.682995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tuples","metadata":{"execution":{"iopub.status.busy":"2021-07-12T02:20:44.007411Z","iopub.execute_input":"2021-07-12T02:20:44.007894Z","iopub.status.idle":"2021-07-12T02:20:44.0139Z","shell.execute_reply.started":"2021-07-12T02:20:44.007855Z","shell.execute_reply":"2021-07-12T02:20:44.011203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_id = []\ndim0 = []\ndim1 = []\nsplits = []\nfor file, dirname, save_dir, split in tqdm(tuples):\n    image_id.append(file.replace('.dcm', '') + '_image')\n    xray = read_xray(os.path.join(dirname, file))\n    dim0.append(xray.shape[0])\n    dim1.append(xray.shape[1])\n    splits.append(split)","metadata":{"execution":{"iopub.status.busy":"2021-07-12T02:22:57.827101Z","iopub.execute_input":"2021-07-12T02:22:57.827381Z","iopub.status.idle":"2021-07-12T02:23:36.076955Z","shell.execute_reply.started":"2021-07-12T02:22:57.827358Z","shell.execute_reply":"2021-07-12T02:23:36.075317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!tar -zcf train.tar.gz -C \"/kaggle/working/image/train/\" .\n!tar -zcf test.tar.gz -C \"/kaggle/working/image/test/\" .","metadata":{"execution":{"iopub.status.busy":"2021-07-16T07:08:57.791259Z","iopub.execute_input":"2021-07-16T07:08:57.791880Z","iopub.status.idle":"2021-07-16T07:10:00.531252Z","shell.execute_reply.started":"2021-07-16T07:08:57.791806Z","shell.execute_reply":"2021-07-16T07:10:00.530178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame.from_dict({'image_id': image_id, 'dim0': dim0, 'dim1': dim1, 'split': splits})\ndf.to_csv('meta.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T07:10:00.533009Z","iopub.execute_input":"2021-07-16T07:10:00.533324Z","iopub.status.idle":"2021-07-16T07:10:00.849158Z","shell.execute_reply.started":"2021-07-16T07:10:00.533288Z","shell.execute_reply":"2021-07-16T07:10:00.848352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'./train.tar.gz')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T07:10:00.850848Z","iopub.execute_input":"2021-07-16T07:10:00.851180Z","iopub.status.idle":"2021-07-16T07:10:00.859820Z","shell.execute_reply.started":"2021-07-16T07:10:00.851142Z","shell.execute_reply":"2021-07-16T07:10:00.858798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'./test.tar.gz')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T07:10:00.861568Z","iopub.execute_input":"2021-07-16T07:10:00.862058Z","iopub.status.idle":"2021-07-16T07:10:00.870067Z","shell.execute_reply.started":"2021-07-16T07:10:00.862021Z","shell.execute_reply":"2021-07-16T07:10:00.868966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(glob('../input/siimcovid19-512-img-png-600-study-png/image/*')):\n    if i[-22:-4] == '0a990c89256a_image':\n        im = cv.imread(i, 0)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:29:42.481563Z","iopub.execute_input":"2021-07-16T04:29:42.482142Z","iopub.status.idle":"2021-07-16T04:29:42.741414Z","shell.execute_reply.started":"2021-07-16T04:29:42.482094Z","shell.execute_reply":"2021-07-16T04:29:42.740358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"im \nhist_im = cv.equalizeHist(im)\nkernel = cv.getStructuringElement(cv.MORPH_RECT, (15, 15)) # MORPH_ELLIPSE\n\ntophat_img = cv.morphologyEx(hist_im, cv.MORPH_TOPHAT, kernel)\nbothat_img = cv.morphologyEx(hist_im, cv.MORPH_BLACKHAT, kernel) # Black --> Bottom\n\nim = hist_im + tophat_img - bothat_img\nim = Image.fromarray(im)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:21:19.498094Z","iopub.execute_input":"2021-07-16T04:21:19.498462Z","iopub.status.idle":"2021-07-16T04:21:19.505340Z","shell.execute_reply.started":"2021-07-16T04:21:19.498428Z","shell.execute_reply":"2021-07-16T04:21:19.504302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist_test = cv.equalizeHist(im)","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:30:08.353695Z","iopub.execute_input":"2021-07-16T04:30:08.354212Z","iopub.status.idle":"2021-07-16T04:30:08.360789Z","shell.execute_reply.started":"2021-07-16T04:30:08.354176Z","shell.execute_reply":"2021-07-16T04:30:08.359833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(im, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T05:23:11.849664Z","iopub.execute_input":"2021-07-16T05:23:11.850009Z","iopub.status.idle":"2021-07-16T05:23:12.034565Z","shell.execute_reply.started":"2021-07-16T05:23:11.849980Z","shell.execute_reply":"2021-07-16T05:23:12.033506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(hist_test, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T04:33:00.336557Z","iopub.execute_input":"2021-07-16T04:33:00.336977Z","iopub.status.idle":"2021-07-16T04:33:00.517730Z","shell.execute_reply.started":"2021-07-16T04:33:00.336944Z","shell.execute_reply":"2021-07-16T04:33:00.516785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kernel = cv.getStructuringElement(cv.MORPH_RECT, (15, 15)) # MORPH_ELLIPSE\n\ntophat_img = cv.morphologyEx(im, cv.MORPH_TOPHAT, kernel)\nbothat_img = cv.morphologyEx(im, cv.MORPH_BLACKHAT, kernel) # Black --> Bottom\n\nimg = im + tophat_img - bothat_img","metadata":{"execution":{"iopub.status.busy":"2021-07-16T05:25:58.666143Z","iopub.execute_input":"2021-07-16T05:25:58.666631Z","iopub.status.idle":"2021-07-16T05:25:58.675652Z","shell.execute_reply.started":"2021-07-16T05:25:58.666576Z","shell.execute_reply":"2021-07-16T05:25:58.674637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(img, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2021-07-16T05:26:00.591776Z","iopub.execute_input":"2021-07-16T05:26:00.592345Z","iopub.status.idle":"2021-07-16T05:26:00.776760Z","shell.execute_reply.started":"2021-07-16T05:26:00.592312Z","shell.execute_reply":"2021-07-16T05:26:00.775679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}