{"cells":[{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm \nimport gc\nimport glob, os\nimport pydicom\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport gdcm\nimport pickle\nimport skimage.measure\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\n\ntrain_out = pd.DataFrame()\nimg_array = []\ninstance_array = []\nimg_dict = {}\nfor Patient in tqdm(train.Patient.unique()) :\n    for filename in glob.glob('../input/osic-pulmonary-fibrosis-progression/train/'+Patient+'/*'):\n#         img = skimage.measure.block_reduce(pydicom.dcmread(filename).pixel_array, (3,3), np.max)\n        d = pydicom.dcmread(filename)\n        instance = d.InstanceNumber\n        img = cv2.resize((d.pixel_array * d.RescaleSlope + d.RescaleIntercept)/1000, (512, 512))\n#         img = img.flatten() #To avoid imagepooling\n        img = skimage.measure.block_reduce(img, (2,2), np.max).flatten()\n        img_array.append(img)\n        instance_array.append(instance)\n    df_tmp = pd.DataFrame(img_array).astype('float16')\n    df_tmp['Instance'] = instance_array\n    df_tmp['Patient'] = Patient\n    train_out = pd.concat([train_out, df_tmp])\n#     img_dict[Patient] = img_array\n    img_array = []\n    instance_array = []\ngc.collect()\n\nfor Patient in tqdm(test.Patient.unique()) :\n    for filename in glob.glob('../input/osic-pulmonary-fibrosis-progression/test/'+Patient+'/*'):\n#         img = skimage.measure.block_reduce(pydicom.dcmread(filename).pixel_array, (3,3), np.max)\n        d = pydicom.dcmread(filename)\n        instance = d.InstanceNumber\n        img = cv2.resize((d.pixel_array * d.RescaleSlope + d.RescaleIntercept)/1000, (512, 512))\n#         img = img.flatten() #To avoid imagepooling\n        img = skimage.measure.block_reduce(img, (2,2), np.max).flatten()\n        img_array.append(img)\n        instance_array.append(instance)\n    df_tmp = pd.DataFrame(img_array).astype('float16')\n    df_tmp['Instance'] = instance_array\n    df_tmp['Patient'] = Patient\n    train_out = pd.concat([train_out, df_tmp])\n#     img_dict[Patient] = img_array\n    img_array = []\n    instance_array = []\ngc.collect()\n        \nprint('Train Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_out.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_out.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_out.to_pickle(\"train_out.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_out = pd.DataFrame()\ni=0\nimg_array = []\ninstance_array = []\nimg_dict = {}\nfor Patient in tqdm(test.Patient.unique()) :\n    for filename in glob.glob('../input/osic-pulmonary-fibrosis-progression/test/'+Patient+'/*.dcm'):\n#         img = skimage.measure.block_reduce(pydicom.dcmread(filename).pixel_array, (3,3), np.max)\n        d = pydicom.dcmread(filename)\n        instance = d.InstanceNumber\n#         img = cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (512, 512))\n        img = cv2.resize((d.pixel_array * d.RescaleSlope + d.RescaleIntercept)/1000, (512, 512))\n#         img = img.flatten() #To avoid imagepooling\n        img = skimage.measure.block_reduce(img, (2,2), np.max).flatten()\n        img_array.append(img)\n        instance_array.append(instance)\n    df_tmp = pd.DataFrame(img_array).astype('float16')\n    df_tmp['Instance'] = instance_array\n    df_tmp['Patient'] = Patient\n    test_out = pd.concat([test_out, df_tmp])\n#     img_dict[Patient] = img_array\n    img_array = []\n    instance_array = []\n    gc.collect()\n\nprint('test Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_out.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_out.to_pickle(\"test_out.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# \\pd.concat([train_out, test_out], ignore_index=True).to_pickle(\"train_out.pkl\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}