{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport shutil\nimport os\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:19:48.823749Z","iopub.execute_input":"2021-07-20T12:19:48.824068Z","iopub.status.idle":"2021-07-20T12:19:48.828452Z","shell.execute_reply.started":"2021-07-20T12:19:48.824038Z","shell.execute_reply":"2021-07-20T12:19:48.827595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom glob import glob\nimport random\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom shutil import copyfile\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n\n#customize iPython writefile so we can write variables\nfrom IPython.core.magic import register_line_cell_magic\n\n@register_line_cell_magic\ndef writetemplate(line, cell):\n    with open(line, 'w') as f:\n        f.write(cell.format(**globals()))\n        \nimport torch\nprint(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport tensorflow.keras.preprocessing.image as Image","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:19:51.192462Z","iopub.execute_input":"2021-07-20T12:19:51.192989Z","iopub.status.idle":"2021-07-20T12:19:51.209085Z","shell.execute_reply.started":"2021-07-20T12:19:51.192947Z","shell.execute_reply":"2021-07-20T12:19:51.208213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:19:53.447124Z","iopub.execute_input":"2021-07-20T12:19:53.447446Z","iopub.status.idle":"2021-07-20T12:21:03.161863Z","shell.execute_reply.started":"2021-07-20T12:19:53.447416Z","shell.execute_reply":"2021-07-20T12:21:03.160941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv('/kaggle/input/siim-covid19-detection/sample_submission.csv')\nfast_df = False\nIMG_SIZE = 600","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.163662Z","iopub.execute_input":"2021-07-20T12:21:03.164010Z","iopub.status.idle":"2021-07-20T12:21:03.173174Z","shell.execute_reply.started":"2021-07-20T12:21:03.163971Z","shell.execute_reply":"2021-07-20T12:21:03.172420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study = pd.read_csv(\"/kaggle/input/siim-covid19-detection/train_study_level.csv\")\nstudy['id'] = study['id'].str.split('_',expand=True)[0]\n","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.174918Z","iopub.execute_input":"2021-07-20T12:21:03.175270Z","iopub.status.idle":"2021-07-20T12:21:03.263221Z","shell.execute_reply.started":"2021-07-20T12:21:03.175226Z","shell.execute_reply":"2021-07-20T12:21:03.262400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.265414Z","iopub.execute_input":"2021-07-20T12:21:03.265676Z","iopub.status.idle":"2021-07-20T12:21:03.288455Z","shell.execute_reply.started":"2021-07-20T12:21:03.265651Z","shell.execute_reply":"2021-07-20T12:21:03.287703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(study[\"Negative for Pneumonia\"].sum())\nprint(study[\"Typical Appearance\"].sum())\nprint(study[\"Indeterminate Appearance\"].sum())\nprint(study[\"Atypical Appearance\"].sum())","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.289723Z","iopub.execute_input":"2021-07-20T12:21:03.290069Z","iopub.status.idle":"2021-07-20T12:21:03.296631Z","shell.execute_reply.started":"2021-07-20T12:21:03.290034Z","shell.execute_reply":"2021-07-20T12:21:03.295762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#absolute difference\nprint(study[\"Typical Appearance\"].sum()-study[\"Negative for Pneumonia\"].sum())\nprint(study[\"Typical Appearance\"].sum()-study[\"Indeterminate Appearance\"].sum())\nprint(study[\"Typical Appearance\"].sum()-study[\"Atypical Appearance\"].sum())","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.298129Z","iopub.execute_input":"2021-07-20T12:21:03.298749Z","iopub.status.idle":"2021-07-20T12:21:03.306330Z","shell.execute_reply.started":"2021-07-20T12:21:03.298709Z","shell.execute_reply":"2021-07-20T12:21:03.305393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/real')\nos.mkdir('/kaggle/working/augmented')\nos.mkdir('/kaggle/working/real/train')\nos.mkdir('/kaggle/working/augmented/train')\nos.mkdir('/kaggle/working/real/train/Negative')\nos.mkdir('/kaggle/working/real/train/Typical')\nos.mkdir('/kaggle/working/real/train/Indeterminate')\nos.mkdir('/kaggle/working/real/train/Atypical')\nos.mkdir('/kaggle/working/augmented/train/Negative')\nos.mkdir('/kaggle/working/augmented/train/Typical')\nos.mkdir('/kaggle/working/augmented/train/Indeterminate')\nos.mkdir('/kaggle/working/augmented/train/Atypical')\n","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.307708Z","iopub.execute_input":"2021-07-20T12:21:03.308271Z","iopub.status.idle":"2021-07-20T12:21:03.315121Z","shell.execute_reply.started":"2021-07-20T12:21:03.308235Z","shell.execute_reply":"2021-07-20T12:21:03.314304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    # Original from: https://www.kaggle.com/raddar/convert-dicom-to-np-array-the-correct-way\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \n    # \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.317467Z","iopub.execute_input":"2021-07-20T12:21:03.317844Z","iopub.status.idle":"2021-07-20T12:21:03.581696Z","shell.execute_reply.started":"2021-07-20T12:21:03.317807Z","shell.execute_reply":"2021-07-20T12:21:03.580845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize(array, size, keep_ratio=False, resample=Image.LANCZOS):\n    # Original from: https://www.kaggle.com/xhlulu/vinbigdata-process-and-resize-to-image\n    im = Image.fromarray(array)\n    \n    if keep_ratio:\n        im.thumbnail((size, size), resample)\n    else:\n        im = im.resize((size, size), resample)\n    \n    return im","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:21:03.583161Z","iopub.execute_input":"2021-07-20T12:21:03.583502Z","iopub.status.idle":"2021-07-20T12:21:03.590670Z","shell.execute_reply.started":"2021-07-20T12:21:03.583465Z","shell.execute_reply":"2021-07-20T12:21:03.589883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_dir = f'/kaggle/working/real/train/'\n\nfor dirname, _, filenames in tqdm(os.walk(f'/kaggle/input/siim-covid19-detection/train')):\n    for file in filenames:\n        # set keep_ratio=True to have original aspect ratio\n        xray = read_xray(os.path.join(dirname, file))\n        im = resize(xray, size=600)\n        entry = study.loc[study['id']==dirname.split(\"/\")[-2]]\n        if entry[\"Negative for Pneumonia\"].item()==1:  \n            im.save(os.path.join(save_dir+\"Negative\",entry['id'].item()+\"_study.jpeg\"))\n        elif entry[\"Typical Appearance\"].item()==1:  \n            im.save(os.path.join(save_dir+\"Typical\",entry['id'].item()+\"_study.jpeg\"))\n        elif entry[\"Indeterminate Appearance\"].item()==1:  \n            im.save(os.path.join(save_dir+\"Indeterminate\",entry['id'].item()+\"_study.jpeg\"))\n        elif entry[\"Atypical Appearance\"].item()==1:  \n            im.save(os.path.join(save_dir+\"Atypical\",entry['id'].item()+\"_study.jpeg\"))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-20T12:22:10.646115Z","iopub.execute_input":"2021-07-20T12:22:10.646451Z","iopub.status.idle":"2021-07-20T13:14:29.978064Z","shell.execute_reply.started":"2021-07-20T12:22:10.646420Z","shell.execute_reply":"2021-07-20T13:14:29.976913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = len(os.listdir(\"/kaggle/working/real/train/Typical\"))","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:14:29.979564Z","iopub.execute_input":"2021-07-20T13:14:29.979912Z","iopub.status.idle":"2021-07-20T13:14:29.986303Z","shell.execute_reply.started":"2021-07-20T13:14:29.979876Z","shell.execute_reply":"2021-07-20T13:14:29.985388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:14:48.217523Z","iopub.execute_input":"2021-07-20T13:14:48.217894Z","iopub.status.idle":"2021-07-20T13:14:48.222246Z","shell.execute_reply.started":"2021-07-20T13:14:48.217862Z","shell.execute_reply":"2021-07-20T13:14:48.221151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_paths = sorted([os.path.abspath(os.path.join('/kaggle/working/real/train/Indeterminate/', p)) for p in os.listdir('/kaggle/working/real/train/Indeterminate/')])\n\nli = l- len(imgs_paths)-200\nprint(len(imgs_paths))\nprint(li)\ni = 0\nfinal_path = \"/kaggle/working/augmented/train/Indeterminate/\"\nwhile i<li:\n    img_path = random.choice(imgs_paths)\n    number = random.randint(1,3)\n    image = tf.keras.preprocessing.image.load_img(img_path)\n    if number == 1:\n        image = tf.keras.preprocessing.image.random_rotation(Image.img_to_array(image),rg=30)\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_rot_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n            \n    if number == 2:\n        image = tf.image.flip_left_right(Image.img_to_array(image))\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_flip_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n                \n    if number ==3:\n        image= tf.keras.preprocessing.image.random_shift(Image.img_to_array(image),0.2,0.1)\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_trans_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n    i += 1\n              ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:14:49.852021Z","iopub.execute_input":"2021-07-20T13:14:49.852348Z","iopub.status.idle":"2021-07-20T13:16:58.190889Z","shell.execute_reply.started":"2021-07-20T13:14:49.852318Z","shell.execute_reply":"2021-07-20T13:16:58.189910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_paths = sorted([os.path.abspath(os.path.join('/kaggle/working/real/train/Negative', p)) for p in os.listdir('/kaggle/working/real/train/Negative')])\n\nli = l- len(imgs_paths)-50\nprint(len(imgs_paths))\nprint(li)\ni = 0\nfinal_path = \"/kaggle/working/augmented/train/Negative/\"\nwhile i<li:\n    img_path = random.choice(imgs_paths)\n    number = random.randint(1,3)\n    image = tf.keras.preprocessing.image.load_img(img_path)\n    if number == 1:\n        image = tf.keras.preprocessing.image.random_rotation(Image.img_to_array(image),rg=30)\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_rot_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n            \n    if number == 2:\n        image = tf.image.flip_left_right(Image.img_to_array(image))\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_flip_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n                \n    if number ==3:\n        image= tf.keras.preprocessing.image.random_shift(Image.img_to_array(image),0.2,0.1)\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_trans_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n    i += 1\n              ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:16:58.192477Z","iopub.execute_input":"2021-07-20T13:16:58.192819Z","iopub.status.idle":"2021-07-20T13:18:20.674849Z","shell.execute_reply.started":"2021-07-20T13:16:58.192784Z","shell.execute_reply":"2021-07-20T13:18:20.673892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_paths = sorted([os.path.abspath(os.path.join('/kaggle/working/real/train/Atypical', p)) for p in os.listdir('/kaggle/working/real/train/Atypical')])\n\nli = l- len(imgs_paths)-500\nprint(len(imgs_paths))\nprint(li)\ni = 0\nfinal_path = \"/kaggle/working/augmented/train/Atypical/\"\nwhile i<li:\n    img_path = random.choice(imgs_paths)\n    number = random.randint(1,3)\n    image = tf.keras.preprocessing.image.load_img(img_path)\n    if number == 1:\n        image = tf.keras.preprocessing.image.random_rotation(Image.img_to_array(image),rg=30)\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_rot_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n            \n    if number == 2:\n        image = tf.image.flip_left_right(Image.img_to_array(image))\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_flip_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n                \n    if number ==3:\n        image= tf.keras.preprocessing.image.random_shift(Image.img_to_array(image),0.2,0.1)\n        tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\"_trans_\"+str(i)+\".jpeg\",image,file_format=\"jpeg\")\n    i += 1\n              ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:18:20.676864Z","iopub.execute_input":"2021-07-20T13:18:20.677202Z","iopub.status.idle":"2021-07-20T13:20:36.016401Z","shell.execute_reply.started":"2021-07-20T13:18:20.677165Z","shell.execute_reply":"2021-07-20T13:20:36.015582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_paths = sorted([os.path.abspath(os.path.join('/kaggle/working/real/train/Atypical', p)) for p in os.listdir('/kaggle/working/real/train/Atypical')])\n\n\nfinal_path = \"/kaggle/working/augmented/train/Atypical/\"\nfor img_path in imgs_paths:\n    image = tf.keras.preprocessing.image.load_img(img_path)\n    tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\".jpeg\",image,file_format=\"jpeg\")\n    i += 1\n              ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:20:36.018086Z","iopub.execute_input":"2021-07-20T13:20:36.018396Z","iopub.status.idle":"2021-07-20T13:20:42.141610Z","shell.execute_reply.started":"2021-07-20T13:20:36.018363Z","shell.execute_reply":"2021-07-20T13:20:42.140811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_paths = sorted([os.path.abspath(os.path.join('/kaggle/working/real/train/Typical', p)) for p in os.listdir('/kaggle/working/real/train/Typical')])\n\n\nfinal_path = \"/kaggle/working/augmented/train/Typical/\"\nfor img_path in imgs_paths:\n    image = tf.keras.preprocessing.image.load_img(img_path)\n    tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\".jpeg\",image,file_format=\"jpeg\")\n    i += 1\n              ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:20:42.144415Z","iopub.execute_input":"2021-07-20T13:20:42.144700Z","iopub.status.idle":"2021-07-20T13:21:19.052948Z","shell.execute_reply.started":"2021-07-20T13:20:42.144671Z","shell.execute_reply":"2021-07-20T13:21:19.052092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_paths = sorted([os.path.abspath(os.path.join('/kaggle/working/real/train/Indeterminate', p)) for p in os.listdir('/kaggle/working/real/train/Indeterminate')])\n\n\nfinal_path = \"/kaggle/working/augmented/train/Indeterminate/\"\nfor img_path in imgs_paths:\n    image = tf.keras.preprocessing.image.load_img(img_path)\n    tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\".jpeg\",image,file_format=\"jpeg\")\n    i += 1\n              ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:21:19.054359Z","iopub.execute_input":"2021-07-20T13:21:19.054725Z","iopub.status.idle":"2021-07-20T13:21:32.267982Z","shell.execute_reply.started":"2021-07-20T13:21:19.054690Z","shell.execute_reply":"2021-07-20T13:21:32.267121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgs_paths = sorted([os.path.abspath(os.path.join('/kaggle/working/real/train/Negative', p)) for p in os.listdir('/kaggle/working/real/train/Negative')])\n\n\nfinal_path = \"/kaggle/working/augmented/train/Negative/\"\nfor img_path in imgs_paths:\n    image = tf.keras.preprocessing.image.load_img(img_path)\n    tf.keras.preprocessing.image.save_img(final_path + img_path[:-4].split('/')[-1]+\".jpeg\",image,file_format=\"jpeg\")\n    i += 1\n              ","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:21:32.269205Z","iopub.execute_input":"2021-07-20T13:21:32.269535Z","iopub.status.idle":"2021-07-20T13:21:54.577114Z","shell.execute_reply.started":"2021-07-20T13:21:32.269503Z","shell.execute_reply":"2021-07-20T13:21:54.576102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real = 0\nfor i in os.listdir(\"/kaggle/working/real/train/\"):\n    print(i,len(os.listdir(\"/kaggle/working/real/train/\"+i)))\n    real += len(os.listdir(\"/kaggle/working/real/train/\"+i))\nprint(\"-\"*20)\nprint(\"total = \",real)\nprint()\nprint()\nprint()\n\naugmented = 0\nfor i in os.listdir(\"/kaggle/working/augmented/train/\"):\n    print(i,len(os.listdir(\"/kaggle/working/augmented/train/\"+i)))\n    augmented += len(os.listdir(\"/kaggle/working/augmented/train/\"+i))\nprint(\"-\"*20)\nprint(\"total = \",augmented)\nprint()\nprint()\nprint()","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:21:54.579363Z","iopub.execute_input":"2021-07-20T13:21:54.579777Z","iopub.status.idle":"2021-07-20T13:21:54.613089Z","shell.execute_reply.started":"2021-07-20T13:21:54.579728Z","shell.execute_reply":"2021-07-20T13:21:54.612047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}