{"cells":[{"metadata":{},"cell_type":"markdown","source":"Notebooks refered \n* https://www.kaggle.com/vijaybj/merge-all-slices-into-one-image-per-patient (fixed some errors wrt dimensions)\n* https://www.kaggle.com/ulrich07/osic-keras-starter-with-custom-metrics\n\nAttempt to merge patients 2d slices into a jpeg","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\n#DESIRED_SIZE = 256 # Memory issue\nDESIRED_SIZE = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ntr['basescan'] = tr['Patient']\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\nchunk['basescan'] = chunk['Patient']\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['basescan'] = sub['Patient']\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week', 'basescan']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=[\"Patient\",\"basescan\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.tail(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n#=================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['week'] = (data['Weeks'] - data['Weeks'].min() ) / ( data['Weeks'].max() - data['Weeks'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape, tr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"range(tr.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport scipy.ndimage\nimport matplotlib.pyplot as plt\n\nfrom skimage import measure, morphology\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport sys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the scans in given folder path\ndef load_scan(path):\n    #print(os.listdir(path))\n    slices = [pydicom.read_file(path + '/' + s) for s in os.listdir(path)]\n    #print(slices[0].ImagePositionPatient)\n    #print(slices[0].ImagePositionPatient[2])\n    slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    try:\n        slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n    except:\n        slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n        \n    for s in slices:\n        s.SliceThickness = slice_thickness\n        \n    return slices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_pixels_hu(slices):\n    image = np.stack([s.pixel_array for s in slices])\n    # Convert to int16 (from sometimes int16), \n    # should be possible as values should always be low enough (<32k)\n    image = image.astype(np.int16)\n\n    # Set outside-of-scan pixels to 0\n    # The intercept is usually -1024, so air is approximately 0\n    image[image == -2000] = 0\n    \n    # Convert to Hounsfield units (HU)\n    for slice_number in range(len(slices)):\n        \n        intercept = slices[slice_number].RescaleIntercept\n        slope = slices[slice_number].RescaleSlope\n        \n        if slope != 1:\n            image[slice_number] = slope * image[slice_number].astype(np.float64)\n            image[slice_number] = image[slice_number].astype(np.int16)\n            \n        image[slice_number] += np.int16(intercept)\n    \n    return np.array(image, dtype=np.int16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pillow\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pydicom.pixel_data_handlers import gdcm_handler, pillow_handler\n\npydicom.config.image_handlers = ['pillow_handler']\ndef merge_2d_slices_into_one(slices):\n    #print(\"slices len : %d \" % (len(slices)))\n    arr = np.zeros((512, 512), np.int16)\n    count = 3\n    for im in slices:\n        #print(type(im))\n        #print(im)\n        im = Image.fromarray(im)\n        im = im.resize((512,512)) \n        pixels = list(im.getdata())\n        width, height = im.size\n        pixels = [pixels[i * width:(i + 1) * width] for i in range(height)] \n        im = np.array(pixels)\n        #print(type(im))\n        #print(im.shape)\n        smallest = np.amin(im)\n        biggest = np.amax(im)\n        #print(\" biggest : %d , smallest : %d\" % ( biggest,smallest ))\n        #imarr = np.array(im, dtype=np.int16)        \n        arr = arr + (1 - im)*(np.log(count)/(biggest - smallest))\n\n        #print ((N * 14)/ np.log10(count))\n        count = count + 1\n        #arr = np.array(np.round(arr), dtype=np.uint8)\n        arr = np.array(np.round(arr),dtype=np.uint8)\n    \n    return arr","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\ndef zip_and_remove(zipname, path):\n    ziph = zipfile.ZipFile(f'{zipname}.zip', 'w', zipfile.ZIP_DEFLATED)\n    \n    for root, dirs, files in os.walk(path):\n        print(files)\n        for file in tqdm(files):\n            file_path = os.path.join(root, file)\n            #print(file_path)\n            ziph.write(file_path)\n            os.remove(file_path)\n    \n    ziph.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_path = \"/output/kaggle/working/\"\nimport traceback\nimport sys\n\ndef get_images_3d(df, how=\"train\"):\n    xo = []\n    p = []\n    w  = []\n    img_set = []\n    for i in tqdm(range(df.shape[0])):\n        patient = df.iloc[i,0]\n        week = df.iloc[i,1]\n        basescan = df.iloc[i,7]\n        try:\n            if basescan not in img_set:\n                img_path = f\"{ROOT}/{how}/{basescan}/\"\n                slices = load_scan(img_path)\n                slices_pixel_array = get_pixels_hu(slices)\n                pixel_array = merge_2d_slices_into_one(slices_pixel_array)\n                #print(basescan)\n                #print(pixel_array.shape)\n                im = Image.fromarray(pixel_array)\n                im = im.resize((DESIRED_SIZE,DESIRED_SIZE)) \n                out_path = f\"../working/{how}/{basescan}.jpg\"\n                im.save(out_path)\n                im = np.array(im)\n                xo.append(im[np.newaxis,:,:])\n                p.append(patient)\n                w.append(week)\n                img_set.append(basescan)\n        except Exception as e:\n            print(e)\n            traceback.print_exc()\n            pass\n    data = pd.DataFrame({\"Patient\":p,\"Weeks\":w})\n    return xo, data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = \"../working/train/\" \nos.makedirs(train_path , exist_ok=True)\nx, df_tr = get_images_3d(tr, how=\"train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../working/train\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"zip_and_remove('train' ,train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def list_zip_file_contents(zipfilename):\n    # Create a ZipFile Object and load sample.zip in it\n    with zipfile.ZipFile(zipfilename, 'r') as zipObj:\n       # Get list of files names in zip\n       listOfiles = zipObj.namelist()\n       # Iterate over the list of file names in given list & print them\n       for elem in listOfiles:\n           print(elem)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_zip_file_contents(\"train.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path=\"../working/test/\"\nos.makedirs(test_path , exist_ok=True)\nx_test, df_test = get_images_3d(sub, how=\"test\")\nzip_and_remove('test' ,test_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_zip_file_contents(\"test.zip\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Patient'].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['Patient'].head(100).nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}