{"cells":[{"metadata":{"_uuid":"5d3755d2-6514-4312-a7f3-796627ed5416","_cell_guid":"d083fedc-ee8e-4754-b869-59449596062e","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#print(\"test\")\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n    #for filename in filenames:\n        #print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4689c695-0e37-4b54-96f4-0f4c89360bba","_cell_guid":"dd803e1f-0c50-446e-9944-e80ee800d745","trusted":true},"cell_type":"code","source":"import pydicom\nimport plotly\nfrom plotly.graph_objs import *\n#import chart_studio.plotly as py\nimport matplotlib.pyplot as plt\nimport numpy as np\n#import tensorflow_io as tfio\nimport tensorflow as tf\nimport scipy.ndimage\nfrom skimage import measure, morphology\n#from mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport keras_preprocessing\nfrom keras_preprocessing import image\nfrom keras_preprocessing.image import ImageDataGenerator\nimport keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\nimport sys\nimport pandas as pd\nfrom pandas import DataFrame\n#from tensorflow import Session\n\n\ndef get_data(filename):\n    with open(filename) as file_to_read:\n        #first_line = file_to_read.readline()\n        row = file_to_read.readline()\n        count = 0\n        while (row):\n            #print(row)\n            row = file_to_read.readline()\n            count = count+1\n    file_to_read.close()\n    return count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#total number of files under train = 33026, number of patients = 173, on average 9 weeks training data per patient, so each image will be used 9 times\n#so total training paths = 33026 times 9 approx althought image cropping will likely work 90% of times only\n#for steps per epoch calc, assume images = 33026 times 9 times 0.9 = approx 250k \n#to avoid memory allocation error,assume batch size of 32 so may be 8000 steps per epoch!\n\ndef get_images_patient_count(path):\n    patient_count=0\n    images_count=0\n    patients_to_filter_out_while_setting_pipeline = ['ID00010637202177584971671','ID00011637202177653955184','ID00012637202177665765362','ID00014637202177757139317','ID00015637202177877247924',\n                                                     'ID00019637202178323708467','ID00020637202178344345685','ID00023637202179104603099','ID00025637202179541264076','ID00026637202179561894768','ID00027637202179689871102','ID00030637202181211009029',\n                                                     'ID00032637202181710233084','ID00035637202182204917484','ID00038637202182690843176','ID00042637202184406822975','ID00047637202184938901501','ID00048637202185016727717','ID00051637202185848464638',\n                                                     'ID00052637202186188008618','ID00060637202187965290703','ID00061637202188184085559','ID00062637202188654068490','ID00067637202189903532242','ID00068637202190879923934','ID00072637202198161894406',\n                                                     'ID00073637202198167792918','ID00075637202198610425520','ID00076637202199015035026','ID00077637202199102000916','ID00078637202199415319443','ID00082637202201836229724','ID00086637202203494931510',\n                                                     'ID00089637202204675567570','ID00090637202204766623410','ID00093637202205278167493','ID00094637202205333947361','ID00099637202206203080121','ID00102637202206574119190','ID00104637202208063407045',\n                                                     'ID00105637202208831864134','ID00108637202209619669361','ID00109637202210454292264','ID00110637202210673668310','ID00111637202210956877205','ID00115637202211874187958','ID00117637202212360228007',\n                                                     'ID00119637202215426335765','ID00122637202216437668965','ID00123637202217151272140','ID00124637202217596410344','ID00125637202218590429387','ID00126637202218610655908','ID00127637202219096738943',\n                                                     'ID00128637202219474716089','ID00129637202219868188000','ID00130637202220059448013','ID00131637202220424084844','ID00132637202222178761324','ID00133637202223847701934','ID00134637202223873059688',\n                                                     'ID00135637202224630271439','ID00136637202224951350618','ID00138637202231603868088','ID00139637202231703564336','ID00140637202231728595149','ID00149637202232704462834','ID00161637202235731948764',\n                                                     'ID00165637202237320314458','ID00167637202237397919352','ID00168637202237852027833','ID00169637202238024117706','ID00170637202238079193844','ID00172637202238316925179','ID00173637202238329754031',\n                                                     'ID00180637202240177410333','ID00183637202241995351650','ID00184637202242062969203','ID00186637202242472088675','ID00190637202244450116191','ID00192637202245493238298','ID00196637202246668775836',\n                                                     'ID00197637202246865691526','ID00199637202248141386743','ID00202637202249376026949','ID00207637202252526380974','ID00210637202257228694086','ID00213637202257692916109','ID00214637202257820847190',\n                                                     'ID00216637202257988213445','ID00218637202258156844710','ID00219637202258203123958','ID00221637202258717315571','ID00222637202259066229764','ID00224637202259281193413','ID00225637202259339837603',\n                                                     'ID00228637202259965313869','ID00229637202260254240583','ID00232637202260377586117','ID00233637202260580149633','ID00234637202261078001846','ID00235637202261451839085','ID00240637202264138860065',\n                                                     'ID00241637202264294508775','ID00242637202264759739921','ID00248637202266698862378','ID00249637202266730854017','ID00251637202267455595113','ID00255637202267923028520','ID00264637202270643353440',\n                                                     'ID00267637202270790561585','ID00273637202271319294586','ID00275637202271440119890','ID00276637202271694539978','ID00279637202272164826258','ID00283637202278714365037','ID00285637202278913507108',\n                                                     'ID00288637202279148973731','ID00290637202279304677843','ID00291637202279398396106','ID00294637202279614924243','ID00296637202279895784347','ID00298637202280361773446','ID00299637202280383305867',\n                                                     'ID00305637202281772703145','ID00307637202282126172865','ID00309637202282195513787','ID00312637202282607344793','ID00317637202283194142136','ID00319637202283897208687','ID00322637202284842245491',\n                                                     'ID00323637202285211956970','ID00329637202285906759848','ID00331637202286306023714','ID00335637202286784464927','ID00336637202286801879145','ID00337637202286839091062','ID00339637202287377736231',\n                                                     'ID00340637202287399835821','ID00341637202287410878488','ID00342637202287526592911','ID00343637202287577133798','ID00344637202287684217717','ID00351637202289476567312','ID00355637202295106567614',\n                                                     'ID00358637202295388077032','ID00360637202295712204040','ID00364637202296074419422','ID00365637202296085035729','ID00367637202296290303449','ID00368637202296470751086','ID00370637202296737666151',\n                                                     'ID00371637202296828615743','ID00376637202297677828573','ID00378637202298597306391','ID00381637202299644114027','ID00383637202300493233675','ID00388637202301028491611','ID00392637202302319160044',\n                                                     'ID00393637202302431697467','ID00398637202303897337979','ID00400637202305055099402','ID00401637202305320178010','ID00405637202308359492977','ID00407637202308788732304','ID00408637202308839708961',\n                                                     'ID00411637202309374271828',\n                                                     'ID00414637202310318891556','ID00417637202310901214011','ID00419637202311204720264','ID00421637202311550012437','ID00422637202311677017371','ID00423637202312137826377','ID00426637202313170790466']\n\n    for dirname, _, filenames in os.walk(path):\n        patient_id = dirname[len(path)+1:]\n        #if(patient_id in patients_to_filter_out_while_setting_pipeline):\n           #patient_count = patient_count-1\n        if(patient_id.find(\"ID0\") == 0):\n           patient_count = patient_count+1 \n        #dir_name_minus_path = \n        #print(\"directory:\", dirname)\n        #for filename in filenames:\n            #images_count = images_count+1\n            #if(patient_id in patients_to_filter_out_while_setting_pipeline):\n                #images_count = images_count-1\n            \n    return patient_count\n#print(count)\n#patient_count, images_count = get_images_patient_count('/kaggle/input/osic-pulmonary-fibrosis-progression/test')\n#print(\"patients, images\", patient_count,images_count )\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"74fed044-69fa-4778-9321-6ea773327115","_cell_guid":"9ef1bfde-f837-4e1a-a1f1-c8768835134a","trusted":true},"cell_type":"code","source":"def load_dcm_img_bytes(path):\n    image_bytes = [tf.io.read_file(path+\"/\"+ slice) for slice in os.listdir(path)]\n    return image_bytes\n\n        \n    #patient_dicoms = load_dcmFiles(patient_dicom_path)\n     #   patient_images = convert_dcm_slices_to_HU_images(patient_dicoms)\ndef load_dcmFiles(path):\n    slices = [pydicom.dcmread(path + \"/\" + s) for s in               \n              os.listdir(path)]\n    #for oneslice in slices:\n        #print(oneslice.dir())\n        #print(oneslice)\n        #print(\"Pixel Spacing\",oneslice.PixelSpacing)\n    \n    slices = [s for s in slices ]\n    slices.sort(key = lambda x: int(x.InstanceNumber))\n    try: \n        if \"ImagePositionPatient\" in slices[0]:\n            slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n        if \"SliceThickness\" in slices[0]:\n            slice_thickness = slices[0].SliceThickness\n    except:\n        slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n    for s in slices:\n        s.SliceThickness = slice_thickness\n    #for s in slices:\n        #print(s.dir())\n        #print(s.ImagePositionPatient)\n    #print(\"Pixel Spacing\",slices[1].PixelSpacing)\n    return slices\n\ndef load_dcm_Files_file_list(path, file_list):\n    slices = [pydicom.dcmread(path + \"/\" + s) for s in         \n              file_list]\n    #for oneslice in slices:\n        #print(oneslice.dir())\n        #print(oneslice)\n        #print(\"Pixel Spacing\",oneslice.PixelSpacing)\n    \n    slices = [s for s in slices ]\n    slices.sort(key = lambda x: int(x.InstanceNumber))\n    try: \n        if \"ImagePositionPatient\" in slices[0]:\n            slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n        if \"SliceThickness\" in slices[0]:\n            slice_thickness = slices[0].SliceThickness\n    except:\n        slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n    for s in slices:\n        s.SliceThickness = slice_thickness\n    #for s in slices:\n        #print(s.dir())\n        #print(s.ImagePositionPatient)\n    #print(\"Pixel Spacing\",slices[1].PixelSpacing)\n    return slices\n\ndef load_dcm_single_file(single_file):\n    slices = []\n    slices.append(pydicom.dcmread(single_file)) \n    return slices\n\ndef convert_dcm_slices_to_HU_images(scans):\n    image = np.stack([s.pixel_array for s in scans])\n    image = image.astype(np.int16)\n    # Set outside-of-scan pixels to 0\n    # The intercept is usually -1024, so air is approximately 0\n    image[image == -2000] = 0\n    \n    # Convert to Hounsfield units (HU)\n    intercept = scans[0].RescaleIntercept\n    slope = scans[0].RescaleSlope\n    \n    if slope != 1:\n        image = slope * image.astype(np.float64)\n        image = image.astype(np.int16)\n        \n    image += np.int16(intercept)\n    \n    return np.array(image, dtype=np.int16)\n\ndef resample(image, scan, new_spacing=[1,1,1]):\n    # Determine current pixel spacing\n    #spacing = np.array([scan[0].SliceThickness] + scan[0].PixelSpacing, dtype=np.float32)\n    #changed spacing method to be compatible with newer pydicom versions\n    spacing = np.array([scan[0].SliceThickness, scan[0].PixelSpacing[0], scan[0].PixelSpacing[1]], dtype=np.float32)\n    resize_factor = spacing / new_spacing\n    new_real_shape = image.shape * resize_factor\n    new_shape = np.round(new_real_shape)\n    real_resize_factor = new_shape / image.shape\n    new_spacing = spacing / real_resize_factor\n    \n    image = scipy.ndimage.interpolation.zoom(image, real_resize_factor, mode='nearest')\n    \n    return image, new_spacing\n\ndef plot_3d(image, threshold=-300):\n    \n    # Position the scan upright, \n    # so the head of the patient would be at the top facing the camera\n    p = image.transpose(2,1,0)\n    \n    #verts, faces = measure.marching_cubes(p, threshold)\n    #new version for skimage.measure does not have marching_cubes, using marching_cubes_lewiner instead\n    verts, faces, extraxx1, extraxx2 = measure.marching_cubes_lewiner(p, threshold)\n    \n\n    fig = plt.figure(figsize=(10, 10))\n    ax = fig.add_subplot(111, projection='3d')\n\n    # Fancy indexing: `verts[faces]` to generate a collection of triangles\n    mesh = Poly3DCollection(verts[faces], alpha=0.70)\n    face_color = [0.45, 0.45, 0.75]\n    mesh.set_facecolor(face_color)\n    ax.add_collection3d(mesh)\n\n    ax.set_xlim(0, p.shape[0])\n    ax.set_ylim(0, p.shape[1])\n    ax.set_zlim(0, p.shape[2])\n\n    plt.show()\n\ndef largest_label_volume(im, bg=-1):\n    vals, counts = np.unique(im, return_counts=True)\n\n    counts = counts[vals != bg]\n    vals = vals[vals != bg]\n\n    if len(counts) > 0:\n        return vals[np.argmax(counts)]\n    else:\n        return None\n\ndef segment_lung_mask(image, fill_lung_structures=True):\n    \n    # not actually binary, but 1 and 2. \n    # 0 is treated as background, which we do not want\n    binary_image = np.array(image > -320, dtype=np.int8)+1\n    labels = measure.label(binary_image)\n    \n    # Pick the pixel in the very corner to determine which label is air.\n    #   Improvement: Pick multiple background labels from around the patient\n    #   More resistant to \"trays\" on which the patient lays cutting the air \n    #   around the person in half\n    background_label = labels[0,0,0]\n    \n    #Fill the air around the person\n    binary_image[background_label == labels] = 2\n    \n    \n    # Method of filling the lung structures (that is superior to something like \n    # morphological closing)\n    if fill_lung_structures:\n        # For every slice we determine the largest solid structure\n        for i, axial_slice in enumerate(binary_image):\n            axial_slice = axial_slice - 1\n            labeling = measure.label(axial_slice)\n            l_max = largest_label_volume(labeling, bg=0)\n            \n            if l_max is not None: #This slice contains some lung\n                binary_image[i][labeling != l_max] = 1\n\n    \n    binary_image -= 1 #Make the image actual binary\n    binary_image = 1-binary_image # Invert it, lungs are now 1\n    \n    # Remove other air pockets insided body\n    labels = measure.label(binary_image, background=0)\n    l_max = largest_label_volume(labels, bg=0)\n    if l_max is not None: # There are air pockets\n        binary_image[labels != l_max] = 0\n \n    return binary_image\n\nMIN_BOUND = -1000.0\nMAX_BOUND = 400.0\n    \ndef normalize(image):\n    #print(\"normalize start\")\n    image = (image - MIN_BOUND) / (MAX_BOUND - MIN_BOUND)\n    image[image>1] = 1.\n    image[image<0] = 0.\n    #print(\"normalized\")\n    return image\n\n#PIXEL_MEAN = 0.25\nPIXEL_MEAN = -676.4\n#-676.4050579921004 as pixel_mean for this training dataset = does that sound right? Lungs are in -500 space and air in -1000, so maybe it is right\n\ndef zero_center(image):\n    image = image - PIXEL_MEAN\n    #print(\"zero centered\")\n    return image\n\ndef create_zerocenter_norm_images(image):\n    return normalize(zero_center(image))\ndef cropped_masked_lung_images_with_status(masks, images,x_shape=200,y_shape=300 ):\n    return_status = False\n    imglist=[]\n    for i in range(0, len(masks)):\n        segmented_lung_slice_mask = masks[i]\n        shape1, shape2 = segmented_lung_slice_mask.shape\n        x,y=np.nonzero(segmented_lung_slice_mask)\n        masked_image = segmented_lung_slice_mask*images[i]\n        \n        try:\n            x_start = x[0]\n            y_start = y[0]\n            start_offset = shape1-(x_start+x_shape)\n            y_offset = shape2-20-y_shape\n            if(y_offset>0):\n                y_offset=0\n            if(start_offset>0):\n                start_offset=0\n            #print(x_start)\n            crop_image = masked_image[x_start+start_offset:x_start+x_shape,y_offset+20:y_offset+20+y_shape]\n            #print(\"crop image shape\\t\",crop_image.shape)\n            #print(x[0],y[0],x[28672],y[28672])\n            #print(x,y,x.shape,y.shape)\n            totalshape=shape1*shape2\n            mask_ratio = np.sum(segmented_lung_slice_mask)/totalshape\n            imglist.append(crop_image)\n            #if(mask_ratio>0.03 and mask_ratio<0.3):\n                #imglist.append(crop_image)\n                #return_status = True\n                #print(\"good lung image found\")\n            #else:\n                #return_status = False\n                #print(\"cant see much of lungs here, skipping\")\n        except:\n            pass\n    return np.array(imglist, dtype=np.int16), True\n\ndef cropped_masked_lung_images(masks, images,x_shape=200,y_shape=300 ):\n    imglist=[]\n    for i in range(0, len(masks)):\n        segmented_lung_slice_mask = masks[i]\n        shape1, shape2 = segmented_lung_slice_mask.shape\n        x,y=np.nonzero(segmented_lung_slice_mask)\n        masked_image = segmented_lung_slice_mask*images[i]\n        \n        try:\n            x_start = x[0]\n            y_start = y[0]\n            start_offset = shape1-(x_start+x_shape)\n            if(start_offset>0):\n                start_offset=0\n            y_offset = shape2-20-y_shape\n            if(y_offset>0):\n                y_offset=0\n\n            #print(x_start)\n            crop_image = masked_image[x_start+start_offset:x_start+x_shape,y_offset+20:y_offset+20+y_shape]\n            #print(\"crop image shape\\t\",crop_image.shape)\n            #print(x[0],y[0],x[28672],y[28672])\n            #print(x,y,x.shape,y.shape)\n            totalshape=shape1*shape2\n            mask_ratio = np.sum(segmented_lung_slice_mask)/totalshape\n            imglist.append(crop_image)\n            #if(mask_ratio>0.03 and mask_ratio<0.3):\n                #if(crop_image.shape == (200,300)):\n                    #imglist.append(crop_image)\n                #print(\"good lung image found\")\n            #else:\n                #print(\"cant see much of lungs here, skipping\")\n        except:\n            pass\n    return np.array(imglist, dtype=np.int16)\n        \n# paitent id = ID00123637202217151272140 - no good lung seg\n# paitent id = ID00405637202308359492977 - works for plot3d; no good lung seg\n# patient id = ID00012637202177665765362\n#ID00014637202177757139317 - works for plot3d, rest have memory issues\n#ID00398637202303897337979 - works for plot3d and has good lung segmentation\n#ID00011637202177653955184 - GDCM issue\n#ID00329637202285906759848 - no good lung seg\n#'ID00323637202285211956970','','ID00331637202286306023714','ID00335637202286784464927','ID00336637202286801879145','ID00337637202286839091062','ID00339637202287377736231'\n\n#patient_dicoms = load_dcmFiles(\"/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00422637202311677017371\")\n#full_file_list = os.listdir(\"/kaggle/input/osic-pulmonary-fibrosis-progression/test/ID00419637202311204720264\")\n#first_X_files = full_file_list[0:4]\n#patient_dicoms = load_dcm_Files_file_list(patient_dicom_path,full_file_list)\n#patient_images = convert_dcm_slices_to_HU_images(patient_dicoms)\n#pix_resampled, spacing = resample(patient_images, patient_dicoms, [1,1,1])\n#print(\"Shape before resampling\\t\", patient_images.shape)\n#print(\"Shape after resampling\\t\", pix_resampled.shape)\n#print (\"patient_images shape=\" ,patient_images.shape)\n#print (\"patient_dicom len=\" ,len(patient_dicoms))\n#segmented_lungs = segment_lung_mask(pix_resampled, False)\n#segmented_lungs_fill = segment_lung_mask(pix_resampled, True)\n#print(\"Shape after segmentation\\t\", segmented_lungs_fill.shape)\n#cropped_images = cropped_masked_lung_images(segmented_lungs_fill,pix_resampled)\n#zerocenter_norm_ima = create_zerocenter_norm_images(cropped_images)\n#print(\"Shape after cropping\\t\", zerocenter_norm_ima.shape)\n#plt.imshow(cropped_images[1], cmap=plt.cm.bone)\n#plt.imshow(zerocenter_norm_ima[1], cmap=plt.cm.bone)\n\n#print(\"crop image shape\\t\",cropped_images[1].shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2afed3c2-6209-4ca6-a9f4-db3c2e503c15","_cell_guid":"8cddbf50-2878-4d6f-98a3-f1c4f96f476e","trusted":true},"cell_type":"code","source":"#find mean pixel value for the whole training dataset\ndef get_mean_pixel_value():\n    from matplotlib import pyplot as plt\n    import pandas as pd\n    import numpy as np\n    import numpy.ma as ma\n    returnDF = None\n    filename = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv\"\n    pixelvaluestotal = 0\n    pixelcount = 0\n    training_data = pd.read_csv(filename)\n    training_data.drop_duplicates(keep=False, inplace=True, subset=['Patient', 'Weeks'])\n    tr_data_per_patient = training_data.groupby('Patient')\n    #TODO: figure out how to do GDCM binding - it seems it is a C++ library and not that direct outside Conda\n    patients_needing_GDCM_binding = ['ID00011637202177653955184','ID00052637202186188008618']\n    #print(tr_data_per_patient)\n    for patient, patient_df in tr_data_per_patient:\n        #print(patient)\n        if (patient in patients_needing_GDCM_binding):\n            continue\n        #if (patient in patients_to_filter_out_while_setting_pipeline):\n            #continue\n        sorted_patient_df = patient_df.sort_values('Weeks')\n        sorted_patient_df.fillna(0, inplace=True)\n        #let's now load patient images\n        #print(\"loading images\")\n        patient_dicom_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/\"+patient\n        patient_dicoms = load_dcmFiles(patient_dicom_path)\n        patient_images = convert_dcm_slices_to_HU_images(patient_dicoms)\n        #print(len(patient_images))\n        imglist=[]\n        for img in patient_images:\n            pixelvaluestotal = pixelvaluestotal+np.sum(img)\n            pixelcount = pixelcount+ma.count(img)\n    return pixelvaluestotal/pixelcount\n\n#print(get_mean_pixel_value())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"39be1035-0e35-4af7-a43a-a2702e6b677f","_cell_guid":"29a47ac7-7a05-47ab-bb1b-38d5d1de097b","trusted":true},"cell_type":"code","source":"\nTRAINING_DIR = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/\"\nMODEL_CONFIDENCE=0.025\nMAX_SCANS_PER_PATIENT_FOR_TRAINING = 15\n\npolicy = mixed_precision.Policy('mixed_float16')\nmixed_precision.set_policy(policy)\n\n# Note the input shape is the desired size of the image 200x300 with 1 bytes color\ninput_images = tf.keras.layers.Input(shape=[200, 300, 1],name='image')\n#input_images = Input(shape=[512, 512,1],name='image')\n# This is the first convolution\n#conv2point5_model = tf.keras.layers.Conv2D(16, (3,3), activation='relu')(input_images)\nconv2point5_model = layers.Conv2D(16, (3,3), activation='relu')(input_images)\nconv2point5_model = layers.MaxPooling2D(2, 2)(conv2point5_model)\n# The second convolution\nconv2point5_model = layers.Conv2D(32, (3,3), activation='relu')(conv2point5_model)\nconv2point5_model = layers.MaxPooling2D(2, 2)(conv2point5_model)\n# The third convolution\nconv2point5_model = layers.Conv2D(64, (3,3), activation='relu')(conv2point5_model)\nconv2point5_model = layers.MaxPooling2D(2, 2)(conv2point5_model)\n# The fourth convolution\nconv2point5_model = layers.Conv2D(64, (3,3), activation='relu')(conv2point5_model)\nconv2point5_model = layers.MaxPooling2D(2, 2)(conv2point5_model)\n# The fifth convolution\nconv2point5_model = layers.Conv2D(64, (3,3), activation='relu')(conv2point5_model)\nconv2point5_model = layers.GlobalMaxPooling2D()(conv2point5_model)\nconv2point5_model = layers.Dense(3, activation=\"relu\")(conv2point5_model)\n\n\nconv2point5_model = Model(inputs=input_images,outputs=conv2point5_model)\n\n#print(conv2point5_model.summary())\n#Asumme tabular inputs will be 5 features(normalized?)  Age, Sex, SmokingStatus, weeksSinceLastFVC, lastFVC \ninput_tabular = tf.keras.layers.Input(shape=[4,],name='metadata_and_lastFVC')\n#input_tabular = Input(shape=[3,],name='metadata_and_lastFVC')\ntabular_model = layers.Dense(20, activation=\"relu\")(input_tabular)\n#tabular_model = tf.keras.layers.GlobalMaxPooling1D()(tabular_model)\ntabular_model = layers.Dense(3, activation=\"relu\")(tabular_model)\n\n#tabular_model = tf.keras.layers.Conv2D(64, (3,3), activation='relu')(tabular_model)\n#tabular_model = tf.keras.layers.MaxPooling2D(2, 2)(tabular_model)\n\ntabular_model = Model(inputs=input_tabular,outputs=tabular_model)\n\n#print(tabular_model.summary())\n# combine the output of the two branches\nconcat = layers.Concatenate()\nimg_tabular_combined_out = concat([conv2point5_model.output, tabular_model.output])\n#img_tabular_combined_out = layers.concatenate([conv2point5_model.output, tabular_model.output])\n\n# apply a FC # 512 neuron hidden layer and then a regression prediction on the combined outputs\nz = layers.Dense(512, activation=\"relu\")(img_tabular_combined_out)\n#quantiles 0, 1 and 2\n#q0 = layers.Dense(1, activation=\"linear\",name='q0')(z)\nq1 = layers.Dense(1, activation=\"linear\",dtype='float32',name='q1')(z)\n#q2 = layers.Dense(1, activation=\"linear\",name='q2')(z)\n# our model will accept the inputs of the two branches and then output a single value\n#conv_and_tabular_model = tf.keras.Model((inputs=[conv2point5_model.input, tabular_model.input], outputs=z))\n#inputs = {\"image\": shape=[512,512,3],\"metadata_and_lastFVC\": shape=[3]}\n#conv_and_tabular_model = Model(inputs=[conv2point5_model.input, tabular_model.input], outputs=[q0,q1,q2])\nconv_and_tabular_model = Model(inputs=[conv2point5_model.input, tabular_model.input], outputs=q1)\n#conv_and_tabular_model = tf.keras.Model(inputs=inputs, outputs=z))\n#print(\"so far so good\")\n#print(conv_and_tabular_model.summary())\n#print(tabular_model.input.shape)\n\n#import torch\ndef quantile_loss2(target, preds):\n    target_c = tf.keras.backend.cast(target, 'float32') \n    preds_c = tf.keras.backend.cast(preds, 'float32')  \n    quantiles = (0.2, 0.5, 0.8)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target_c - preds_c[:, i]\n        errors1 = (q - 1) * errors\n        errors1_c = tf.keras.backend.cast(errors1, 'int32')\n        errors2 = q  * errors\n        errors2_c = tf.keras.backend.cast(errors2, 'int32')\n        maxerror = torch.max(errors1_c,errors2_c)\n        #print(\"errors2_c shape\",errors2_c.shape)\n        #print(\"errors2 shape\",errors2.shape)\n        #losses.append(tf.math.reduce_max(errors1_c, errors2_c))\n        #max1 = torch.max((q - 1) * errors)\n        #max2 = torch.max((q ) * errors)\n        print(\"max1=\",max1)\n        print(\"max2=\",max2)\n        \n        losses.append(maxerror.unsqueeze(1))\n        #losses.append(tf.math.reduce_max((q - 1) * errors, q * errors))\n        #losses.append(tf.keras.backend.max((q - 1) * errors, q * errors))\n        \n    #loss = tf.math.reduce_mean(tf.math.reduce_sum(tf.concat(losses, 1), dim=1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss\ndef quantile_loss(y,output):\n    y_c = tf.keras.backend.cast(y, 'float32') \n    output_c = tf.keras.backend.cast(output, 'float32') \n    error = tf.subtract(y_c, output_c)\n    q0=0.2\n    q1=0.5\n    q2=0.8\n    losses = []\n    lossq0 = tf.reduce_mean(tf.maximum(q0*error, (q0-1)*error), axis=-1)\n    lossq1 = tf.reduce_mean(tf.maximum(q1*error, (q1-1)*error), axis=-1)\n    lossq2 = tf.reduce_mean(tf.maximum(q2*error, (q2-1)*error), axis=-1)\n    losses.append(lossq0)\n    losses.append(lossq1)\n    losses.append(lossq2)\n    \n    loss = tf.reduce_mean(tf.add_n(losses))\n    return loss\ndef minus_laplace_log_loss(fvc_true,fvc_pred):\n    #cast both fvc_true and fvc_pred to either both int or both float\n    fvc_true_c = tf.keras.backend.cast(fvc_true, 'float32') \n    fvc_pred_c = tf.keras.backend.cast(fvc_pred, 'float32') \n\n    confidence = MODEL_CONFIDENCE * fvc_pred_c\n    confidence_clipped = confidence\n    confidence_clipped = tf.math.maximum(confidence_clipped, tf.constant(70.0))\n    #if(confidence>70):\n        #confidence_clipped = 70.0\n    delta = tf.abs(tf.subtract(fvc_true_c,fvc_pred_c))\n    delta = tf.math.maximum(delta, tf.constant(1000.0))\n    #if(delta>1000):\n        #delta = 1000.0\n    metric_part1 = tf.math.divide_no_nan(tf.math.multiply_no_nan(tf.sqrt(2.0),delta),confidence_clipped)\n    \n    metric_part2 = tf.math.log(tf.math.multiply_no_nan(tf.sqrt(2.0),confidence_clipped))\n    metric =  tf.add(metric_part1,metric_part2)\n    return metric\n        \n    \n#from tensorflow.keras.optimizers import RMSprop\nconv_and_tabular_model.compile(\n    optimizer=keras.optimizers.RMSprop(1e-3),\n    metrics=[tf.keras.metrics.MeanSquaredError()],\n    loss='mse'\n#    loss=minus_laplace_log_loss\n#    loss=quantile_loss\n#    loss={'q0':'mae',\n#           'q1':'mae',\n#           'q2':'mae'},\n#    loss_weights={'q0':0.2,'q1':0.5,'q2':0.8}\n    #loss_weights={'q0':1,'q1':1,'q2':1}\n        )\nkeras.utils.plot_model(conv_and_tabular_model, \"multi_input_and_output_model.png\", show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00695e80-1606-441b-81d0-a8444ef5f533","_cell_guid":"cd5e8493-fed7-441f-bf93-c8aea055bf77","trusted":true},"cell_type":"code","source":"@tf.function\ndef load_training_datapoint(datapoint1,datapoint2):\n    image = datapoint1['image']\n    metadata_and_lastFVC = datapoint1['metadata_and_lastFVC']\n    fvc = datapoint2\n    x = {\"image\": image, \"metadata_and_lastFVC\":metadata_and_lastFVC }\n    y = fvc\n    return x,y\n\n\ndef get_training_data_gen():\n  # The first line contains the column headers\n  # Each successive line contians 7 columns: Patient,Weeks, FVC, PercentAge, Sex, SmokingStatus\n  # We also need to add two extra columns : lastFVC and weeksSinceLastFVC  \n  # We should groupby for a patient, sort by weeks and then for the minimum week, copy FVC value into lastFVC \n  # also and weeksSinceLastFVC = 0 for the minimum week. For next week value copy lastFVC from previous week and \n  # calculate weeksSinceLastFVC as difference between this weeks's value and previous one. \n  # Then FVC value is the label\n  # The function will return labels, dicom_slices and metadata_and_lastFVC\n  # \n    from matplotlib import pyplot as plt\n    import pandas as pd\n    import numpy as np\n    labels_list = []\n    dicom_slices_list = []\n    metadata_and_lastFVC = []\n    returnDF = None\n    filename = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv\"\n    training_data = pd.read_csv(filename)\n\n    training_data.drop_duplicates(keep=False, inplace=True, subset=['Patient', 'Weeks'])\n    #Initialize lastFVC as FVC and weeksSinceLastFVC as Weeks\n    training_data['lastFVC'] = training_data['FVC']\n    training_data['weeksSinceLastFVC'] = training_data['Weeks']\n    training_data['MaleFemale'] = training_data['Weeks']\n    training_data['codedSmokingStatus'] = training_data['Weeks']\n    tr_data_per_patient = training_data.groupby('Patient')\n\n    #training_complete_copy = training_data.copy()\n    #training_data = training_complete_copy.sample(frac=(1-validation_split), random_state=0)\n    #testing_data = training_complete_copy.drop(training_data.index)\n    \n    #if(validation_data):\n        #tr_data_per_patient = training_data.groupby('Patient')\n    #else:\n        #tr_data_per_patient = testing_data.groupby('Patient')\n        \n    #TODO: figure out how to do GDCM binding - it seems it is a C++ library and not that direct outside Conda\n    patients_needing_GDCM_binding = ['ID00011637202177653955184','ID00052637202186188008618']\n    #only to restrict data while setting and testing pipeline\n    patients_to_filter_out_while_setting_pipeline = ['ID00010637202177584971671','ID00011637202177653955184','ID00012637202177665765362','ID00014637202177757139317','ID00015637202177877247924',\n                                                     'ID00019637202178323708467','ID00020637202178344345685','ID00023637202179104603099','ID00025637202179541264076','ID00026637202179561894768','ID00027637202179689871102','ID00030637202181211009029',\n                                                     'ID00032637202181710233084','ID00035637202182204917484','ID00038637202182690843176','ID00042637202184406822975','ID00047637202184938901501','ID00048637202185016727717','ID00051637202185848464638',\n                                                     'ID00052637202186188008618','ID00060637202187965290703','ID00061637202188184085559','ID00062637202188654068490','ID00067637202189903532242','ID00068637202190879923934','ID00072637202198161894406',\n                                                     'ID00073637202198167792918','ID00075637202198610425520','ID00076637202199015035026','ID00077637202199102000916','ID00078637202199415319443','ID00082637202201836229724','ID00086637202203494931510',\n                                                     'ID00089637202204675567570','ID00090637202204766623410','ID00093637202205278167493','ID00094637202205333947361','ID00099637202206203080121','ID00102637202206574119190','ID00104637202208063407045',\n                                                     'ID00105637202208831864134','ID00108637202209619669361','ID00109637202210454292264','ID00110637202210673668310','ID00111637202210956877205','ID00115637202211874187958','ID00117637202212360228007',\n                                                     'ID00119637202215426335765','ID00122637202216437668965','ID00123637202217151272140','ID00124637202217596410344','ID00125637202218590429387','ID00126637202218610655908','ID00127637202219096738943',\n                                                     'ID00128637202219474716089','ID00129637202219868188000','ID00130637202220059448013','ID00131637202220424084844','ID00132637202222178761324','ID00133637202223847701934','ID00134637202223873059688',\n                                                     'ID00135637202224630271439','ID00136637202224951350618','ID00138637202231603868088','ID00139637202231703564336','ID00140637202231728595149','ID00149637202232704462834','ID00161637202235731948764',\n                                                     'ID00165637202237320314458','ID00167637202237397919352','ID00168637202237852027833','ID00169637202238024117706','ID00170637202238079193844','ID00172637202238316925179','ID00173637202238329754031',\n                                                     'ID00180637202240177410333','ID00183637202241995351650','ID00184637202242062969203','ID00186637202242472088675','ID00190637202244450116191','ID00192637202245493238298','ID00196637202246668775836',\n                                                     'ID00197637202246865691526','ID00199637202248141386743','ID00202637202249376026949','ID00207637202252526380974','ID00210637202257228694086','ID00213637202257692916109','ID00214637202257820847190',\n                                                     'ID00216637202257988213445','ID00218637202258156844710','ID00219637202258203123958','ID00221637202258717315571','ID00222637202259066229764','ID00224637202259281193413','ID00225637202259339837603',\n                                                     'ID00228637202259965313869','ID00229637202260254240583','ID00232637202260377586117','ID00233637202260580149633','ID00234637202261078001846','ID00235637202261451839085','ID00240637202264138860065',\n                                                     'ID00241637202264294508775','ID00242637202264759739921','ID00248637202266698862378','ID00249637202266730854017','ID00251637202267455595113','ID00255637202267923028520','ID00264637202270643353440',\n                                                     'ID00267637202270790561585','ID00273637202271319294586','ID00275637202271440119890','ID00276637202271694539978','ID00279637202272164826258','ID00283637202278714365037','ID00285637202278913507108',\n                                                     'ID00288637202279148973731','ID00290637202279304677843','ID00291637202279398396106','ID00294637202279614924243','ID00296637202279895784347','ID00298637202280361773446','ID00299637202280383305867',\n                                                     'ID00305637202281772703145','ID00307637202282126172865','ID00309637202282195513787','ID00312637202282607344793','ID00317637202283194142136','ID00319637202283897208687','ID00322637202284842245491',\n                                                     'ID00323637202285211956970','ID00329637202285906759848','ID00331637202286306023714','ID00335637202286784464927','ID00336637202286801879145','ID00337637202286839091062','ID00339637202287377736231',\n                                                     'ID00340637202287399835821','ID00341637202287410878488','ID00342637202287526592911','ID00343637202287577133798','ID00344637202287684217717','ID00351637202289476567312','ID00355637202295106567614',\n                                                     'ID00358637202295388077032','ID00360637202295712204040','ID00364637202296074419422','ID00365637202296085035729','ID00367637202296290303449','ID00368637202296470751086','ID00370637202296737666151',\n                                                     'ID00371637202296828615743','ID00376637202297677828573','ID00378637202298597306391','ID00381637202299644114027','ID00383637202300493233675','ID00388637202301028491611','ID00392637202302319160044',\n                                                     'ID00393637202302431697467','ID00398637202303897337979','ID00400637202305055099402','ID00401637202305320178010','ID00405637202308359492977','ID00407637202308788732304','ID00408637202308839708961',\n                                                     'ID00411637202309374271828',\n                                                     'ID00414637202310318891556','ID00417637202310901214011','ID00419637202311204720264','ID00421637202311550012437','ID00422637202311677017371','ID00423637202312137826377','ID00426637202313170790466']\n\n    #print(tr_data_per_patient)\n    df_row_count = 0\n    rows=[]\n    rows_image=[]\n    rows_fvc=[]\n    dtype = np.float32\n    for patient, patient_df in tr_data_per_patient:\n        #print(patient)\n        if (patient in patients_needing_GDCM_binding):\n            continue\n        #TODO comment once pipeline has been tested\n        #if (patient in patients_to_filter_out_while_setting_pipeline):\n            #continue\n        sorted_patient_df = patient_df.sort_values('Weeks')\n        sorted_patient_df.lastFVC =  sorted_patient_df.FVC.shift(1)\n        sorted_patient_df.weeksSinceLastFVC =  sorted_patient_df.Weeks -sorted_patient_df.Weeks.shift(1)\n        try:\n            sorted_patient_df.lastFVC[0] = sorted_patient_df.FVC[0]\n        except:\n            pass\n        sorted_patient_df.fillna(0, inplace=True)\n        #add column MaleFemale with 1 where Male and 2 where Female\n        sorted_patient_df.MaleFemale = np.where(sorted_patient_df['Sex']=='Male', 1, 2)\n        #Currently smokes=1 Ex-smoker=2 else 3\n        sorted_patient_df.codedSmokingStatus = np.where(sorted_patient_df['SmokingStatus']=='Currently smokes', 1, np.where(sorted_patient_df['SmokingStatus']=='Ex-smoker',2,3))\n        #normalize age simply by divide-by-100 and storing that in column called AgeNorm\n        sorted_patient_df['AgeNorm'] = (sorted_patient_df['Age']) / 100\n         \n        #let's now load patient images\n        #print(\"loading images\")\n        patient_dicom_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/train/\"+patient\n        full_file_list = os.listdir(patient_dicom_path)\n        if(len(full_file_list)>MAX_SCANS_PER_PATIENT_FOR_TRAINING):\n            first_X_files = full_file_list[0:MAX_SCANS_PER_PATIENT_FOR_TRAINING]\n            #print(\"will get zero center images for\", patient)\n            #patient_zerocenter_norm_im = get_fixed_number_of_cropped_images(patient_dicom_path,NUM_CROPS)\n            #patient_dicoms = load_dcmFiles(patient_dicom_path)\n            patient_dicoms = load_dcm_Files_file_list(patient_dicom_path,first_X_files)\n        else:\n            patient_dicoms = load_dcmFiles(patient_dicom_path)\n        #patient_dicoms = load_dcmFiles(patient_dicom_path)\n        patient_images = convert_dcm_slices_to_HU_images(patient_dicoms)\n        pix_resampled, spacing = resample(patient_images, patient_dicoms, [1,1,1])\n        segmented_lungs_fill = segment_lung_mask(pix_resampled, True)\n        #print(\"Shape after segmentation\\t\", segmented_lungs_fill.shape)\n        cropped_images = cropped_masked_lung_images(segmented_lungs_fill,pix_resampled)\n        zerocenter_norm_im = create_zerocenter_norm_images(cropped_images)\n\n        \n        if(len(zerocenter_norm_im)>MAX_SCANS_PER_PATIENT_FOR_TRAINING):\n            zerocenter_norm_im_trim = zerocenter_norm_im[0:MAX_SCANS_PER_PATIENT_FOR_TRAINING]\n        \n        for row in sorted_patient_df.itertuples():\n            for img in zerocenter_norm_im_trim:\n                image = np.expand_dims(img,2)#first add the single channel at the end\n                tabular_input = [row.Weeks,row.AgeNorm,row.MaleFemale,row.codedSmokingStatus]\n                label = row.FVC\n                yield (np.asarray(image,dtype=dtype),np.asarray(tabular_input,dtype=dtype)),np.asarray(label,dtype=dtype)\n    \n #--------END of train_gen       \n \n\n    \nclass myCallback (tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs={}):\n        print(\"epoch callback\")\n        #if(logs.get('accuracy')>0.8):\n            #print(\"\\nReached 80% accuracy, so stopping further epochs\")\n            #self.model.stop_training = True\n\ncallbacks = myCallback()\n#conv_and_tabular_model.summary()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32\nBUFFER_SIZE = 1000\nVALIDATION_SPLIT = 0.2\n\ntraining_dataset_full = tf.data.Dataset.from_generator(get_training_data_gen, output_types=((tf.float32,tf.float32),tf.float32))\ntrain_patient_count = get_images_patient_count('/kaggle/input/osic-pulmonary-fibrosis-progression/train')\nvalidation_size = int((VALIDATION_SPLIT)*MAX_SCANS_PER_PATIENT_FOR_TRAINING*train_patient_count*8) #assuming 8 weeks per patient\n\ntraining_size = int((1-VALIDATION_SPLIT)*MAX_SCANS_PER_PATIENT_FOR_TRAINING*train_patient_count*8 )\n\ntrain_dataset = training_dataset_full.take(training_size)\nvalidation_dataset = training_dataset_full.skip(training_size)\nvalidation_dataset = validation_dataset.take(validation_size)\n#training_dataset = training_dataset.map(load_training_datapoint)\ntrain_dataset_cached = train_dataset.cache().batch(BATCH_SIZE).repeat()\ntrain_dataset = train_dataset_cached.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\nvalidation_dataset = validation_dataset.batch(BATCH_SIZE)\n#Steps per epoch = number of training images/batch-size in training generator. Likewise validation steps\nif(training_size==0):\n    training_size = 36000 #dummy number to avoid issues\n\nif(validation_size==0):\n    validation_size = 200 #dummy number to avoid issues\n\nnumber_of_epochs = 30\n\nsteps_per_epoch = training_size//BATCH_SIZE\nvalidation_steps = validation_size//BATCH_SIZE\n\nmodel_history = conv_and_tabular_model.fit(train_dataset, epochs=number_of_epochs,\n                         steps_per_epoch=steps_per_epoch,\n                         validation_steps=validation_steps,\n                         validation_data=validation_dataset,\n                         #verbose=2,\n                         callbacks=[callbacks])\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e008aa76-d83d-47b4-b432-6966ccaa2fbd","_cell_guid":"b17ddde5-4ca1-4a09-a4c3-0e84a95ff9b7","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nloss = model_history.history['loss']\n#val_loss = model_history.history['val_loss']\n\nepochs = range(len(loss))\n\n\nplt.figure()\n\nplt.plot(epochs, loss, 'r', label='Training Loss')\n#plt.plot(epochs, val_loss, 'b', label='Validation Loss')\nplt.title('Training and validation loss')\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"conv_and_tabular_model.save(\"/kaggle/working/osic_model\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/working'):\n   for filename in filenames:\n        path = os.path.join(dirname, filename)\n        size = os.stat(path).st_size\n        print(filename, size)\n#multi_input_and_output_model.png 142881\n#__notebook_source__.ipynb 263\n#saved_model.pb 276973\n#variables.index 3689\n#variables.data-00000-of-00001 825341\n#Looks like it is taking about 1MB to save model\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f37f1c3-c3c0-4c4c-8dad-87fae34fab92","_cell_guid":"55d48ea0-adf8-4d13-9932-369b702c5a9f","trusted":true},"cell_type":"code","source":"def update_submission(patient,week,fvc, confidence):\n    patient_week_to_updt = patient + \"_\" + str(week)\n    #print(patient_week_to_updt)\n    #subs_df.FVC = np.where(subs_df['Patient_Week']=='patient_week_to_updt', fvc, 2000)\n    #subs_df.Confidence = np.where(subs_df['Patient_Week']=='patient_week_to_updt', confidence, 100)\n    subs_df.loc[subs_df['Patient_Week'] == patient_week_to_updt, 'FVC'] = fvc\n    subs_df.loc[subs_df['Patient_Week'] == patient_week_toupdt, 'Confidence'] = confidence\n    #subs_df['Patient_Week'==patient_week_to_updt].FVC = fvc\n    #subs_df['Patient_Week'==patient_week_to_updt].Confidence = confidence\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"783d85b1-4b54-4380-8cab-fd668e186a8a","_cell_guid":"2361c802-042f-456c-a62e-53a882509678","trusted":true},"cell_type":"code","source":"def nextFVC_and_confidence(patient_df, zerocenter_norm_im, lastFVC,weekincrement):\n\n    AgeNorm = patient_df.AgeNorm\n    MaleFemale = patient_df.MaleFemale\n    codedSmokingStatus = patient_df.codedSmokingStatus\n    Percent = patient_df.Percent\n    inputs2 = np.asarray([[weekincrement,lastFVC,AgeNorm,MaleFemale,codedSmokingStatus]])\n    q0FVC=0\n    q1FVC=0\n    q2FVC=0\n    q1min=0\n    q1max=0\n    #print(inputs2)\n    for img in zerocenter_norm_im:\n        #print(inputs2)\n        image = np.asarray([img])\n        weekFVC = conv_and_tabular_model.predict({\"image\": image, \"metadata_and_lastFVC\": inputs2})\n        #q0FVC = q0FVC+weekFVC[0]\n        #q1FVC = q1FVC+weekFVC[1]\n        q1FVC = q1FVC+weekFVC\n        #q2FVC = q2FVC+weekFVC[2]\n        if(weekFVC>q1max):\n            q1max=weekFVC\n        if(weekFVC<q1min):\n            q1min=weekFVC\n    datapoints = len(zerocenter_norm_im)\n    if(datapoints>0):\n        predictedFVC = q1FVC/datapoints\n        #confidence = (q2FVC-q0FVC)/datapoints\n        confidence = MODEL_CONFIDENCE*predictedFVC\n        lower_confidence = 0.015*predictedFVC\n        upper_confidence = 0.04*predictedFVC\n        if(confidence<lower_confidence):\n            confidence = lower_confidence\n        if(confidence>upper_confidence):\n            confidence = upper_confidence\n            \n    else:\n        #dummy data if we could not extract ANY lung images\n        predictedFVC = 2000\n        confidence = 100\n        return predictedFVC, confidence\n        \n    #print(type(predictedFVC))\n    return predictedFVC.item(), confidence.item()\n\ndef get_fvc_and_confidence(patient_df,zerocenter_norm_im, week):\n\n    AgeNorm = patient_df.AgeNorm\n    MaleFemale = patient_df.MaleFemale\n    codedSmokingStatus = patient_df.codedSmokingStatus\n    Percent = patient_df.Percent\n    inputs2 = np.asarray([[week,AgeNorm,MaleFemale,codedSmokingStatus]])\n    q0FVC=0\n    q1FVC=0\n    q2FVC=0\n    q1min=0\n    q1max=0\n    #print(inputs2)\n    for img in zerocenter_norm_im:\n        #print(inputs2)\n        #print(\"image found:\",img)\n        image = np.asarray([img])\n        weekFVC = conv_and_tabular_model.predict({\"image\": image, \"metadata_and_lastFVC\": inputs2})\n        #q0FVC = q0FVC+weekFVC[0]\n        #q1FVC = q1FVC+weekFVC[1]\n        q1FVC = q1FVC+weekFVC\n        #q2FVC = q2FVC+weekFVC[2]\n        if(weekFVC>q1max):\n            q1max=weekFVC\n        if(weekFVC<q1min):\n            q1min=weekFVC\n    datapoints = len(zerocenter_norm_im)\n    if(datapoints>0):\n        predictedFVC = q1FVC/datapoints\n        #confidence = (q2FVC-q0FVC)/datapoints\n        confidence = MODEL_CONFIDENCE*predictedFVC\n        lower_confidence = 0.015*predictedFVC\n        upper_confidence = 0.04*predictedFVC\n        if(confidence<lower_confidence):\n            confidence = lower_confidence\n        if(confidence>upper_confidence):\n            confidence = upper_confidence\n            \n    else:\n        #dummy data if we could not extract ANY lung images\n        predictedFVC = 2000\n        confidence = 100\n        return predictedFVC, confidence\n        \n    #print(type(predictedFVC))\n    return predictedFVC.item(), confidence.item()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#prepare submissions file - move this block after nextFVC_and_conf method\n#The number of patients should be read from test.csv\ndef get_fixed_number_of_cropped_images(patient_dicom_path,NUM_CROPS):\n    \n    imglist = []\n    crop_count = 0\n    for single_file in os.listdir(patient_dicom_path):\n        #print(\"single_file\", single_file)\n        single_file_path = patient_dicom_path + \"/\" + single_file\n        patient_dicoms = load_dcm_single_file(single_file_path)\n        #print(\"dicom for single file loaded\",single_file_path)\n\n        patient_images = convert_dcm_slices_to_HU_images(patient_dicoms)\n        #print(\"hu for single file done\")\n        pix_resampled, spacing = resample(patient_images, patient_dicoms, [1,1,1])\n        #print(\"Shape before resampling\\t\", patient_images.shape)\n        #print(\"Shape after resampling\\t\", pix_resampled.shape)\n        #print (\"patient_images shape=\" ,patient_images.shape)\n        segmented_lungs_fill = segment_lung_mask(pix_resampled, True)\n        #print(\"Shape after segmentation\\t\", segmented_lungs_fill.shape)\n        cropped_image, success_in_cropping = cropped_masked_lung_images_with_status(segmented_lungs_fill,pix_resampled)\n        #print(\"Shape after cropping\\t\", cropped_image.shape)\n        zerocenter_image = create_zerocenter_norm_images(cropped_image)\n        #print(\"Shape after zero center\\t\", zerocenter_image.shape)\n        imglist.append(zerocenter_image)\n        #print(\"crop success:\",success_in_cropping)\n        if(success_in_cropping):\n            crop_count = crop_count +1\n            #plt.imshow(cropped_image, cmap=plt.cm.bone)\n        if(crop_count>NUM_CROPS):\n            print(\"breaking crop count\",crop_count)\n            break\n    #patient_zerocenter_norm_im = np.array(imglist, dtype=np.int16)\n    return imglist\n    \n\nfilename = \"/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv\"\ntest_data = pd.read_csv(filename)\ntest_data['MaleFemale'] = test_data['Weeks']\ntest_data['codedSmokingStatus'] = test_data['Weeks']\n#add column MaleFemale with 1 where Male and 2 where Female\ntest_data.MaleFemale = np.where(test_data['Sex']=='Male', 1, 2)\n#Currently smokes=1 Ex-smoker=2 else 3\ntest_data.codedSmokingStatus = np.where(test_data['SmokingStatus']=='Currently smokes', 1, np.where(test_data['SmokingStatus']=='Ex-smoker',2,3))\n#normalize age simply by divide-by-100 and storing that in column called AgeNorm\ntest_data['AgeNorm'] = (test_data['Age']) / 100\n#total_patients_to_predict_on = len(test_data.index)\ntotal_patients_to_predict_on = 2\nsubmission_row_list_for_one_row = []\nsubmission_data_list = []\nsub_patient_week_list = []\nsub_fvc_list = []\nsub_conf_list = []\nsub_patient_list=[]\nsub_week_list=[]\nNUM_CROPS = 3\nNUM_DCM = 3\nfor patient_number in range(0,total_patients_to_predict_on,1):\n    patientdf = test_data.iloc[patient_number]\n    patient = patientdf.Patient\n    try:\n        patient_dicom_path = \"/kaggle/input/osic-pulmonary-fibrosis-progression/test/\"+patient\n        #attempt to speed up prediction - instead of predicting on all slices, restrict to first NUM_CROPS OR NUM_DCM\n        full_file_list = os.listdir(patient_dicom_path)\n        if(len(full_file_list)>NUM_DCM):\n            first_X_files = full_file_list[0:NUM_DCM]\n            #print(\"will get zero center images for\", patient)\n            #patient_zerocenter_norm_im = get_fixed_number_of_cropped_images(patient_dicom_path,NUM_CROPS)\n            #patient_dicoms = load_dcmFiles(patient_dicom_path)\n            patient_dicoms = load_dcm_Files_file_list(patient_dicom_path,first_X_files)\n            \n            patient_images = convert_dcm_slices_to_HU_images(patient_dicoms)\n            pix_resampled, spacing = resample(patient_images, patient_dicoms, [1,1,1])\n            segmented_lungs_fill = segment_lung_mask(pix_resampled, True)\n            cropped_images = cropped_masked_lung_images(segmented_lungs_fill,pix_resampled)\n            #print(\"Shape after cropping\\t\", cropped_images.shape)\n            #print(\"length=\",len(cropped_images))\n            \n            patient_zerocenter_norm_im = create_zerocenter_norm_images(cropped_images)\n            start_index = NUM_DCM\n            \n            while(len(patient_zerocenter_norm_im)==0):\n                #in this case keep trying with 4 extra DCMs till the length is not zero or files exhausted\n                end_index = start_index + NUM_DCM\n                if end_index>(len(full_file_list)-1):\n                    break\n                first_X_files = full_file_list[start_index:end_index]\n                #print(\"will get zero center images for\", patient)\n                #patient_zerocenter_norm_im = get_fixed_number_of_cropped_images(patient_dicom_path,NUM_CROPS)\n                #patient_dicoms = load_dcmFiles(patient_dicom_path)\n                patient_dicoms = load_dcm_Files_file_list(patient_dicom_path,first_X_files)\n                patient_images = convert_dcm_slices_to_HU_images(patient_dicoms)\n                pix_resampled, spacing = resample(patient_images, patient_dicoms, [1,1,1])\n                segmented_lungs_fill = segment_lung_mask(pix_resampled, True)\n                cropped_images = cropped_masked_lung_images(segmented_lungs_fill,pix_resampled)\n                #print(\"Shape after cropping\\t\", cropped_images.shape)\n                #print(\"length=\",len(cropped_images))\n                patient_zerocenter_norm_im = create_zerocenter_norm_images(cropped_images)\n                start_index = end_index\n                    \n                \n                \n        print(\"patient images for a patient prepared for prediction----------\")\n\n        #print(\"patient\"+ patient_number+\" images loaded---------------------------\")\n    except:\n        imglist = []\n        #print(\"exception happened\")\n        patient_zerocenter_norm_im = np.array(imglist, dtype=np.int16)\n    \n    #if(len(patient_zerocenter_norm_im)>0):\n        #print(\"A sample image being used while predicting:\")\n        #plt.imshow(patient_zerocenter_norm_im[1], cmap=plt.cm.bone)\n    \n    starting_week_fwd = patientdf.Weeks+1\n    starting_week_bwd = patientdf.Weeks-1\n    #MAX_WEEK=9\n    #MIN_WEEK=5\n    MIN_WEEK=-13\n    MAX_WEEK=134\n\n    predictedFVC =patientdf.FVC\n    patient_week_given = patient + \"_\" + str(patientdf.Weeks)\n    confidence_given = 1#dont know if there is a divide by zero issue in eval script and hence\n    sub_patient_week_list.append(patient_week_given)\n    sub_fvc_list.append(predictedFVC)\n    sub_conf_list.append(confidence_given)\n    sub_patient_list.append(patient)\n    sub_week_list.append(patientdf.Weeks)\n    for week in range(starting_week_fwd,MAX_WEEK,1):\n        patient_week = patient + \"_\" + str(week)\n        #print(patient_week)\n        #predictedFVC, confidence = nextFVC_and_confidence(patientdf,patient_zerocenter_norm_im,predictedFVC,1)\n        predictedFVC, confidence = get_fvc_and_confidence(patientdf,patient_zerocenter_norm_im, week)\n        print(patient_week,predictedFVC,confidence)\n        sub_patient_week_list.append(patient_week)\n        sub_fvc_list.append(int(predictedFVC))\n        sub_conf_list.append(int(confidence))\n        sub_patient_list.append(patient)\n        sub_week_list.append(week)\n\n        #submission_row_list_for_one_row.append([patient_week,predictedFVC,confidence])\n        #submission_data_list.append(submission_row_list_for_one_row)\n        #update_submission(patient0df.Patient,week,predictedFVC, confidence)\n    #now walk back from week 5 to week -12\n    predictedFVC=patientdf.FVC\n    for week in range(starting_week_bwd,MIN_WEEK,-1):\n        patient_week = patient + \"_\" + str(week)\n        #print(patient_week)\n        #predictedFVC, confidence = nextFVC_and_confidence(patientdf,patient_zerocenter_norm_im,predictedFVC,-1)\n        predictedFVC, confidence = get_fvc_and_confidence(patientdf,patient_zerocenter_norm_im, week)\n        print(patient_week,predictedFVC,confidence)\n        sub_patient_week_list.append(patient_week)\n        sub_fvc_list.append(int(predictedFVC))\n        sub_conf_list.append(int(confidence))\n        sub_patient_list.append(patient)\n        sub_week_list.append(week)\n\n        #submission_row_list_for_one_row.append([patient_week,predictedFVC,confidence])\n        #submission_data_list.append(submission_row_list_for_one_row)\n        #update_submission(patient0df.Patient,week,predictedFVC, confidence)\nsubs_data = {'Patient_Week':sub_patient_week_list, 'FVC':sub_fvc_list,'Confidence':sub_conf_list,'Patient':sub_patient_list,'Week':sub_week_list} \nsubmission_df = pd.DataFrame(subs_data)\nsubmission_df = submission_df.sort_values(by=['Patient', 'Week'], ascending=True)\nsubmission_df = submission_df.drop(columns=['Patient', 'Week'])\n\nsubmission_df\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}