{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"###=================================== Importing the required Modules ======================================================###\nimport numpy as np \nimport pandas as pd \nimport pydicom # this one is to read the dicom files \nimport os \nimport scipy.ndimage\nimport matplotlib.pyplot as plt \nimport sklearn\nfrom sklearn.preprocessing import normalize\nfrom tqdm.auto import tqdm \n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom skimage import measure, morphology \n#from mpl_toolkits.mplot3d.art3d import Poly3DCollection\nfrom sklearn.preprocessing import normalize\n\n#from mpl_toolkits.mplot3d.art3d import Poly3DCollection \nfrom torch.utils.data import DataLoader \nfrom torch.utils.data import TensorDataset ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"### Listing the number of ct scan in the train folder of dataset \nTRAIN_FOLDER = '../data/train'\n\n\n\n\n# Load the scans in given folder path \n\ndef load_scan(path): # Here path == (../input/osic-pulmonary-fibrosis-progression/train/patientId)\n\ttry:\n\t\tslices = [pydicom.dcmread(path+os.sep+s) for s in os.listdir(path)]\n\t\tslices.sort(key = lambda x:float(x.ImagePositionPatient[2]))\n\texcept:\n\t\tfiles = os.listdir(path)\n\t\tfiles.sort()\n\t\tslices = [pydicom.dcmread(path+os.sep+s) for s in files]\n\n\t# \ttry:\n\t# \t\tslice_thickness = np.abs(slices[0].ImagePositionPatient[2]-\\\n\t# \t\t\t\t\t\t\t\tslices[1].ImagePositionPatient[2])\n\t# \texcept:\n\t# \t\tslice_thickness = np.abs(slices[0].SliceLocation - \\\n\t# \t\t\t\t\t\t\t\tslices[1].SliceLocation)\n\n\t# \tfor s in slices:\n\t# \t\ts.SliceThickness = slice_thickness         \n\treturn slices \n\n\n### Load the slices of all patient and make a dict\n'''\npatientIDs = ['ID00007637202177411956430', 'ID00009637202177434476278']\n'''\ndef make_dict_with_slices(patientIDs): # this will be very huge memory consuming . don't use it \n\tpatient_slices_dict = {}\n\tfailed_slices_patiendIDs = []\n\n\tfor patientID in patientIDs:\n\t\tpath = TRAIN_FOLDER + os.sep + patientID\n\n\t\ttry:\n\t\t\tpatient_slices_dict[patientID] = load_scan(path)\n\t\texcept:\n\t\t\tfailed_slices_patiendIDs.append(patientID)\n\n\treturn patient_slices_dict, failed_slices_patiendIDs \n\n\n\n\n\n####================= Taking the data into Hounsfield Unit (HU) =============\n'''\nhere slices is a pydicom.dcmread() file combinations \n'''\ndef get_pixels_hu(slices):\n\timage = np.stack([s.pixel_array for s in slices])\n\t# Convert to int16\n\timage = image.astype(np.int16)\n\ttry:\n\t\t# Set outside-of-scan pixel to 0 \n\t\timage[image <= -2000] = 0\n\t\t# Convert to Hounsfield units (HU)\n\t\tfor slice_number in range(len(slices)):\n\t\t\tintercept = slices[slice_number].RescaleIntercept\n\t\t\tslope = slices[slice_number].RescaleSlope\n\n\t\t\tif slope !=1:\n\t\t\t\timage[slice_number] =slope *image[slice_number].astype(np.float64)\n\t\t\t\timage[slice_number] = image[slice_number].astype(np.int16)\n\t\t\timage[slice_number] += np.int16(intercept)\n\texcept:\n\t\tprint('HU conversion Failed!!')\n\n\t# here return a 3d numpy array \n\treturn np.array(image, dtype = np.int16)\n\n\n#### ============================ visualization of data =========================\n'''\nHere input is either pydicom.dcmread class or numpy3d array \n'''\ndef plot_show_slice(slices):\n\tif not isinstance(slices, type(np.array([]))):\n\t\tfirst_patient_pixels = get_pixels_hu(slices) ## conversion to np array and HU unit \n\telse:\n\t\tfirst_patient_pixels = slices \n\n\tprint('Number of Total Slices in this Scan:', len(slices))\n\ttry:\n\t\tprint('Shape of the the Image is:', slices.shape[1], slices.shape[2])\n\texcept:\n\t\tprint('Shape of the the Image is:BLANK')\n\tfig = plt.figure(figsize=(10,10))  \n\tfor i,slice in enumerate(first_patient_pixels[:16]):\n\t\ty=fig.add_subplot(4,4, i+1)\n\t\ty.imshow(slice, cmap='gray')\n\tplt.show()\n\n#### =================== Make the array of desired shape ======================\n'''\nHere input slices are numpy array not a pydicom.dcmread class \nreturned a numpy 3darray \n'''\ndef resize_along_zaxis(slices, target_dimension=30):\n\tpresent_dimension = len(slices)\n\tif target_dimension == present_dimension:\n\t\treturn slices\n\n\tzoom_factor = float(target_dimension)/float(present_dimension)\n\tresize_image=scipy.ndimage.zoom(slices, [zoom_factor, 1., 1.])\n\t\n\treturn resize_image\n\n####### ================= Making array of desird shape along all axis ============= \n'''\ninput: 3d numpy array\noutput: 3d numpy array \n\n'''\n\ndef resize_along_allaxis(slices, target_dimensionZ=30, target_dimensionY= 100, target_dimensionX= 100):\n\tpresent_dimensionZ, present_dimensionY, present_dimensionX = slices.shape[0], slices.shape[1] ,slices.shape[2]\n\tif target_dimensionZ == present_dimensionZ and\\\n            target_dimensionY == present_dimensionY and target_dimensionX == present_dimensionX:        \n\t\treturn slices\n\tzoom_factorZ = float(target_dimensionZ)/float(present_dimensionZ)\n\tzoom_factorY = float(target_dimensionY)/float(present_dimensionY)\n\tzoom_factorX = float(target_dimensionX)/float(present_dimensionX)\n                         \n\tresize_image=scipy.ndimage.zoom(slices, [zoom_factorZ,zoom_factorY , zoom_factorX], mode='nearest')\n\t\n\treturn resize_image\n\n\nMIN_BOUND = -1000.0\nMAX_BOUND = 400.0\n    \ndef image_normalize(image):\n    image = (image - MIN_BOUND) / (MAX_BOUND - MIN_BOUND)\n    image[image>1] = 1.\n    image[image<0] = 0.\n    return image\n##### ======================== Saving the data in npy format =========================== ### \n'''\npatient_folder = './osic-pulmonary-fibrosis-progression/train'\n\t\t\t### where the patientId folder and their slices in it \noutput_folder = 'Relative path with respect to PWD where data will be saved'\n'''\ndef save_array(patientID_folder, output_folder,Z=100,Y=200,X=200): #patientID_folder, output_folder\n\t#TRAIN_FOLDER = './osic-pulmonary-fibrosis-progression/train'\n\tpatientIDs = os.listdir(patientID_folder)\n\tpatientIDs.sort()\n\tSave_dir = output_folder # './trainset'\n\tif not os.path.exists(Save_dir):\n\t\tprint('The output directory doesnt exists')\n\t\traise Exception \n\t\t\n\tfor i,patientID in enumerate(patientIDs):\n\t\tpath = TRAIN_FOLDER + os.sep + patientIDs[i]\n\t\tslices = load_scan(path)\n\t\ttry:\n\t\t\timage_array = get_pixels_hu(slices) # HU unit conversion + nparray\n\t\t\tctimage_resizedAll = resize_along_allaxis(image_array, target_dimensionX=X,\n\t\t\t\t\t\t\t\t\t\t\t\t\ttarget_dimensionY=Y,target_dimensionZ=Z)\n\t\t\timage=(image_normalize(ctimage_resizedAll)*255.0).astype('uint8')\n# \t\t\t#saving the array \n# \t\t\tif Normalize:\n# \t\t\t\tnpSlices= []\n# \t\t\t\tfor npSlice in ctimage_resizedAll:\n# \t\t\t\t\tnpSlices.append(normalize(npSlice))\n# \t\t\t\tctimage_resizedAll = np.array(npSlices)\n            \n\t\t\tnp.save(Save_dir+os.sep+patientID+'.npy', image)\n\t\texcept:\n\t\t\tprint('PatientId:%s couldnt be save and converted'%(patientID))\n\n### ========================== Loading the data from npy format ====================== #### \ndef load_array(path): ### path='./traindataset/ID00052637202186188008618.npy'\n\ttry:\n\t\tif(path.endswith('.npy')):\n\t\t\timage_array = np.load(path)\n\t\telse:\n\t\t\tpath= path+'.npy'\n\t\t\timage_array = np.load(path)\n\texcept:\n\t\t\n\t\tprint('The file in the Path:%s doesnetexists!!'%(path.split('\\\\')[-1].split('/')[-1].split('.')[0]))\n\t\treturn []\n\treturn image_array\n\n### ========================== Loading the data from npy format ====================== ####\ndef read_image(dir_name,patientid,Z=100,Y=200,X=200): #patientID_folder, output_folder\n\tpath = dir_name + os.sep + patientid\n\tslices = load_scan(path)\n\ttry:\n\t\timage_array = get_pixels_hu(slices) # HU unit conversion + nparray\n\t\tctimage_resizedAll = resize_along_allaxis(image_array, target_dimensionX=X,\n\t\t\t\t\t\t\t\t\t\t\t\ttarget_dimensionY=Y,target_dimensionZ=Z+10)\n\t\t#saving the array \n\t\tctimage = ctimage_resizedAll[5:105][:][:]\n\t\timage=(image_normalize(ctimage)*255.0).astype('uint8')\n\t\treturn image\n\texcept:\n\t\tprint('PatientId:%s couldnt be converted'%(patientid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def csv_preprocess (data):\n    \n    #Healthy FVC\n    data['Healthy-FVC']=round((data['FVC']*100)/data['Percent'])\n    FE=[]\n    FE.append('Healthy-FVC')\n    \n    #Create Male, Female, Ex-smoker, Current-smoker and Never smoked\n    COLS = ['Sex','SmokingStatus']\n    for col in COLS:\n        for mod in data[col].unique():\n            FE.append(mod)\n            data[mod] = (data[col] == mod).astype(int)\n    \n    data =  data[['Patient','Weeks','FVC','Age']+FE]\n    data = data.sort_values(['Patient','Weeks'], ascending=True).reset_index(drop=True)\n    \n    FE1=['Male','Female','Ex-smoker','Never smoked','Currently smokes']\n    #Rename base_Weeks and base_FVC\n    rename_col={'Weeks':'base_Weeks','FVC':'base_FVC'}\n    data=data.rename(columns=rename_col)\n    \n    #Weeks biasing Week=-12 to Week =0 and Week = 133 to Week = 145\n    #data.base_Weeks+=t_min_week\n    #Add new fields Week and actual_FCV\n    npData=pd.DataFrame(columns=['Patient','base_Weeks','base_FVC','Age']+FE1+['Week','Healthy-FVC','actual_FVC'])\n    \n    for pid in data['Patient'].unique():\n        weeks=data.loc[data['Patient']==pid].base_Weeks\n        fvc = data.loc[data['Patient']==pid].base_FVC\n        index = data.loc[data['Patient']==pid].index\n        weeks.reset_index(inplace = True, drop = True)\n        fvc.reset_index(inplace = True, drop = True)\n        for k in range(len(weeks)):\n            npData=pd.concat([npData,data.loc[data.index==index[0]]], sort=False)\n            npData.iloc[-1, npData.columns.get_loc('Week')]=weeks[k]\n            npData.iloc[-1, npData.columns.get_loc('actual_FVC')]=fvc[k]\n    npData.reset_index(inplace = True, drop = True)\n    npData=npData.fillna(0)\n    \n#     npData['Week']=npData['Week']-npData['base_Weeks']\n#     npData['base_Weeks']=0.0\n    #Random Shuffle\n    #npData=sklearn.utils.shuffle(npData)\n    #npData.reset_index(inplace = True, drop = True)\n    \n    return npData","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#################################################### Network Model########################################\nclass Flatten(nn.Module):\n\tdef forward(self, input):\n\t\treturn input.view(input.size(0), -1)\n\nclass ds_3d_conv(nn.Module):\n\tdef __init__(self, nin, nout, kernel_size, padding, kernels_per_layer):\n\t\tsuper(ds_3d_conv, self).__init__()\n\t\tself.depthwise = nn.Conv3d(nin, nin * kernels_per_layer, kernel_size=kernel_size, padding=padding, groups=nin)\n\t\tself.pointwise = nn.Conv3d(nin * kernels_per_layer, nout, kernel_size=1)\n\n\tdef forward(self, x):\n\t\tout = self.depthwise(x)\n\t\tout = self.pointwise(out)\n\t\treturn out\n\n# class SIGMA(nn.Module):\n# \tdef __init__(self):\n# \t\tsuper(SIGMA, self).__init__()\n# \t\tself.data_net1=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(42,64),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(64,118),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t#nn.Dropout(0.5)\n# \t\t\t\t\t\t)\n# \t\tself.data_net2=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(128,256),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(256,502),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t#nn.Dropout(0.5)\n# \t\t\t\t\t\t)\n# \t\tself.data_net3=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(512,256),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(256,118),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t#nn.Dropout(0.5)\n# \t\t\t\t\t\t)\n# \t\tself.data_net4=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(780,256),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\tnn.Linear(256,64),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(64,2),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t)\n\n# \tdef forward(self, data_i, image_o):\n# \t\tx = torch.cat((data_i, image_o), dim=-1)\n# \t\tout1 = self.data_net1(x)\n# \t\tout2 = torch.cat((data_i,out1), dim=-1)\n# \t\tout2 = self.data_net2(out2)\n# \t\tout3 = torch.cat((data_i,out2), dim=-1)\n# \t\tout3 = self.data_net3(out3)\n# \t\tout4 = torch.cat((x,out1,out2,out3), dim=-1)\n# \t\tout = self.data_net4(out4)\n# \t\treturn out\n\n# class DATA(nn.Module):\n# \tdef __init__(self):\n# \t\tsuper(DATA, self).__init__()\n# \t\tself.data_net1=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(41,100),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(100,100),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t#nn.Dropout(0.5)\n# \t\t\t\t\t\t)\n# \t\tself.data_net2=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(100,3),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t)\n        \n# \t\tself.data_net3=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(100,3)\n# \t\t\t\t\t\t)\n\n# \tdef forward(self, data_i):\n# \t\tout1 = self.data_net1(data_i)\n# \t\tout2 = self.data_net2(out1)\n# \t\tout3 = self.data_net3(out1)\n# \t\tout = out2 + out3\n# \t\treturn out\n\n\nclass IMAGE(nn.Module):\n\tdef __init__(self, channel_number=[32, 64, 128, 256, 256, 64], output_dim=16, dropout=True):\n\t\tsuper(IMAGE, self).__init__()\n\t\tn_layer = len(channel_number)\n\t\tself.feature_extractor = nn.Sequential()\n\t\tfor i in range(n_layer):\n\t\t\tif i == 0:\n\t\t\t\tin_channel = 1\n\t\t\telse:\n\t\t\t\tin_channel = channel_number[i-1]\n\t\t\tout_channel = channel_number[i]\n\t\t\tif i < n_layer-1:\n\t\t\t\tself.feature_extractor.add_module('conv_%d' % i,\n\t\t\t\t\t\t\t\t\t\t\t\t  self.conv_layer(in_channel,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  out_channel,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  maxpool=True,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  kernel_size=3,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  padding=1,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  kernels_per_layer=1))\n\t\t\telse:\n\t\t\t\tself.feature_extractor.add_module('conv_%d' % i,\n\t\t\t\t\t\t\t\t\t\t\t\t  self.conv_layer(in_channel,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  out_channel,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  maxpool=False,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  kernel_size=1,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  padding=0,\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t  kernels_per_layer=1))\n\t\tself.classifier = nn.Sequential()\n\t\t#avg_shape = [3, 3, 3]\n\t\t#self.classifier.add_module('average_pool', nn.AvgPool3d(avg_shape))\n\t\tif dropout is True:\n\t\t\tself.classifier.add_module('dropout', nn.Dropout(0.5))\n\t\ti = n_layer\n\t\tin_channel = channel_number[-1]\n\t\tout_channel = output_dim\n\t\tself.classifier.add_module('conv_%d' % i,\n\t\t\t\t\t\t\t\t   nn.Conv3d(in_channel, out_channel, padding=0, kernel_size=1)\n\t\t\t\t\t\t\t\t   #ds_3d_conv(in_channel, out_channel, kernel_size=1, padding=0, kernels_per_layer=1)\n\t\t\t\t\t\t\t\t  )\n\t\tself.flat=nn.Sequential(\n\t\t\tFlatten(),\n\t\t\tnn.Linear(1728,511),\n\t\t\tnn.ReLU()\n\t\t)\n\t\tself.f=nn.Sequential(\n\t\t\tnn.Linear(512,128),\n\t\t\tnn.ReLU(),\n\t\t\tnn.Linear(128,32),\n\t\t\tnn.ReLU(),\n\t\t\tnn.Linear(32,2),\n\t\t\tnn.ReLU()\n\t\t)\n\t@staticmethod\n\tdef conv_layer(in_channel, out_channel, maxpool=True, kernel_size=3, padding=1, kernels_per_layer=1, maxpool_stride=2):\n\t\tif maxpool is True:\n\t\t\tlayer = nn.Sequential(\n\t\t\t\t#nn.Conv3d(in_channel, out_channel, padding=padding, kernel_size=kernel_size),\n\t\t\t\tds_3d_conv(in_channel, out_channel, kernel_size, padding, kernels_per_layer),\n\t\t\t\tnn.BatchNorm3d(out_channel),\n\t\t\t\tnn.MaxPool3d(2, stride=maxpool_stride),\n\t\t\t\tnn.ReLU(),\n\t\t\t)\n\t\telse:\n\t\t\tlayer = nn.Sequential(\n\t\t\t\t#nn.Conv3d(in_channel, out_channel, padding=padding, kernel_size=kernel_size),\n\t\t\t\tds_3d_conv(in_channel, out_channel, kernel_size, padding, kernels_per_layer),\n\t\t\t\tnn.BatchNorm3d(out_channel),\n\t\t\t\tnn.ReLU()\n\t\t\t)\n\t\treturn layer\n\n\tdef forward(self, image_i, data_i):\n\t\timage_o = self.feature_extractor(image_i)\n\t\timage_o = self.classifier(image_o)\n\t\timage_o = self.flat(image_o)\n\t\timage_o = torch.cat((image_o,data_i), dim=-1)\n\t\timage_o = self.f(image_o)\n\t\treturn image_o\n\t\n# class Combined_NET(nn.Module):\n# \tdef __init__(self):\n# \t\tsuper(Combined_NET, self).__init__()\n# \t\tself.image = IMAGE()\n# \t\tself.data = SIGMA()\n\t\t\n# \tdef forward(self, image_i, data_i):\n# \t\timage_o = self.image(image_i)\n# \t\tdata_o = self.data(data_i,image_o)\n# \t\t#print(x.shape)\n# \t\treturn data_o","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = torch.tensor(70, dtype=torch.float32), torch.tensor(1000, dtype= torch.float32)\n\ndef score(y_true, y_pred):\n\t# tf.dtypes.cast(y_true, tf.float32)\n\t# tf.dtypes.cast(y_pred, tf.float32)\n\tsigma = y_pred[:, 0]\n\tfvc_pred = y_pred[:, 1]\n\t\n\t#sigma_clip = sigma + C1\n\t#sigma_clip = tf.maximum(sigma, C1)\n\tc1_same_shape = torch.ones(sigma.size(),device=y_pred.device)*C1 \n\tsigma_clip = torch.max(sigma, c1_same_shape)\n\tdelta = (y_true[:, 0] - fvc_pred).abs()\n\n\tc2_same_shape = torch.ones(delta.size(),device=y_pred.device)*C2 \n\tdelta = torch.min(delta, c2_same_shape)\n\n\tsq2 = torch.tensor(2.).sqrt()\n\tmetric = (delta / sigma_clip)*sq2 + (sigma_clip* sq2).log()\n\treturn (metric).mean()\n\n\n\ndef quartile_loss(y_true, y_pred): # 0.65 \n\t#def loss(y_true, y_pred): # (batch_size, 1), (batch_size , 3)\n\tloss = score(y_true, y_pred) # 0.35\n\treturn loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def train_combined_net(epochs, batch_size, npTrain, npValid, processed_img_dir, model, train_device ='cpu'):\n\n# \tx_train_values_df = npTrain[['base_FVC','Age','Male','Female','Ex-smoker','Never smoked','Currently smokes','Week','Healthy-FVC']]\n\n# \tx_train_values = x_train_values_df.values\n# \ty_train_values = npTrain['actual_FVC'].values\n\n# \tx_valid_values_df = npValid[['base_FVC','Age','Male','Female','Ex-smoker','Never smoked','Currently smokes','Week','Healthy-FVC']]\n\n# \tx_valid_values = x_valid_values_df.values\n# \ty_valid_values = npValid['actual_FVC'].values\n\t\n# \tx_train_patientId_df = npTrain['Patient'] # df of only patientId \n# \tx_valid_patientId_df = npValid['Patient'] # df of only patientId \n\n\n# # \tabnormaltraindata=pd.DataFrame(columns=npTrain.columns)\n# # \tabnormalvaliddata=pd.DataFrame(columns=npValid.columns)\n\n# \ttrain_patientIDs_name = x_train_patientId_df.values \n# \tindex_of_patientIDS_train = np.arange(len(train_patientIDs_name)) \n\n\n\n# \tvalid_patientIDs_name = x_valid_patientId_df.values \n# \tindex_of_patientIDS_valid = np.arange(len(valid_patientIDs_name))\n\n\n\n# \tx_train_values = torch.tensor(x_train_values, dtype = torch.float)\n# \ty_train_values = torch.tensor(y_train_values, dtype = torch.float).unsqueeze(1)\n# \tindex_of_patientIDS_train = torch.tensor(index_of_patientIDS_train, dtype= torch.int16)\n\n\n# \tx_valid_values = torch.tensor(x_valid_values, dtype = torch.float)\n# \ty_valid_values = torch.tensor(y_valid_values, dtype = torch.float).unsqueeze(1)\n# \tindex_of_patientIDS_valid = torch.tensor(index_of_patientIDS_valid, dtype= torch.int16)\n    \n# \t#print('train input',x_train_values.size(0))\n# \t#print('train target: ',y_train_values.size(0))\n# \t#print('train patient id: ',index_of_patientIDS_train.size(0))\n# \t#print('valid input',x_valid_values.size(0))\n# \t#print('valid target: ',y_valid_values.size(0))\n# \t#print('valid patient id: ',index_of_patientIDS_valid.size(0))\n\n\n# \ttrain_ds = TensorDataset(x_train_values, index_of_patientIDS_train , y_train_values)\n# \tvalid_ds = TensorDataset(x_valid_values, index_of_patientIDS_valid , y_valid_values)\n\n# \ttrain_dl = DataLoader(train_ds, batch_size = batch_size, shuffle = True)\n# \tvalid_dl = DataLoader(valid_ds , batch_size = batch_size)\n\n\n\n\n# \tif train_device =='cuda':\n# \t\tdevice = torch.device(\"cuda\")\n# \t\tmodel.to(device)\n\n\n# \ttorch.backends.cudnn.benchmark = True\n\n# \tfor epoch in range(epochs):\n\n# \t\ttrain_acc_total = 0. \n# \t\tscore_train_total = 0. \n# \t\tpbar = tqdm(train_dl)\n# \t\tmodel.train()\n# \t\tfor i, (xb_meta, xb_patients_idxs, Y_target) in enumerate(pbar):     \n\n# \t\t\txb_patients_names = train_patientIDs_name[xb_patients_idxs.to('cpu').numpy()]\n# \t\t\txb_ct = []\n\n# \t\t\tfor patientId in xb_patients_names:\n# \t\t\t\tpath = processed_img_dir + os.sep + patientId\n# \t\t\t\timage_3darray = load_array(path)\n# \t\t\t\txb_ct.append(image_3darray)\n\n# \t\t\txb_image =torch.tensor(xb_ct).float() \n\n# \t\t\tif train_device == 'cuda':\n# \t\t\t\txb_image = xb_image.cuda()\n# \t\t\t\txb_meta = xb_meta.cuda()\n# \t\t\t\tY_target = Y_target.cuda()\n# \t\t\txb_image = xb_image.unsqueeze(1)\n# \t\t\tprediction = model(xb_image, xb_meta)\n\n\n# \t\t\tloss = quartile_loss(Y_target, prediction)\n\n# \t\t\tloss.backward()\n# \t\t\toptimizer.step()\n\n\n# \t\t\twith torch.no_grad():\n# \t\t\t\taccuracy =(1- ((prediction[:,1]- Y_target)/Y_target).abs()).mean()\n# \t\t\t\tscore_train = score(Y_target, prediction).item()\n\n# # \t\t\tif (accuracy.data.item() < 0.75 or score_train > 8.5 ):\n# # \t\t\t\tabnormaltraindata=abnormaltraindata.append(npTrain.iloc[xb_patients_idxs.to('cpu').numpy()])\n                \n# \t\t\ts = ('Epochs: %5d/%d , Steps: %8d/%d , train_loss: %5.3f  ,trian_accuracy: %5.3f, score: %4.4f'%\\\n# \t\t\t\t(epoch, epochs, i, len(train_dl), loss.data.item(), accuracy.data.item(), score_train))\n# \t\t\tpbar.set_description(s)  \n# \t\t\toptimizer.zero_grad()\n# \t\t\ttrain_acc_total += accuracy.data.item()*len(xb_meta)\n# \t\t\tscore_train_total += score_train * len(xb_meta)\n# \t\t\tdel prediction\n\t\t\t\n# \t\tavg_train_acc = (train_acc_total)/len(train_ds) \n# \t\tavg_score_train = score_train_total/ len(train_ds)\n# \t\tprint('Average trianing accuracy:', avg_train_acc)\n# \t\tprint(\"Average training Score:\", avg_score_train)\n\n# \t\t#abnormaltraindata.to_csv('abnormaltraindata.csv',index=False)\n\n\t \n\t\t\t \n\t\t\t\n\t\t\t \n\n\n# \t\t## Validation\n# \t\tscore_valid_total = 0.\n# \t\tval_acc_total = 0.\n# \t\tpbar2 = tqdm(valid_dl)\n# \t\tmodel.eval()\n# \t\tfor i, (xb_meta, xb_patients_idxs, Y_target) in enumerate(pbar2):  \n\n# \t\t\txb_patients_names = valid_patientIDs_name[xb_patients_idxs.to('cpu').numpy()]\n# \t\t\txb_ct = []\n# \t\t\t#print(xb_patients_names)           \n# \t\t\tfor patientId in xb_patients_names:\n# \t\t\t\t#print(patientId)\n# \t\t\t\tpath = processed_img_dir + os.sep + patientId\n# \t\t\t\timage_3darray = load_array(path)\n# \t\t\t\txb_ct.append(image_3darray)\n\n# \t\t\txb_image =torch.tensor(xb_ct).float() \n\n# \t\t\tif train_device == 'cuda':\n# \t\t\t\txb_image = xb_image.cuda()\n# \t\t\t\txb_meta = xb_meta.cuda()\n# \t\t\t\tY_target = Y_target.cuda()\n# \t\t\txb_image = xb_image.unsqueeze(1)\n# \t\t\tprediction = model(xb_image, xb_meta)\n\n\n# \t\t\tloss = quartile_loss(Y_target, prediction)\n\t\t\t\n# \t\t\twith torch.no_grad():\n# \t\t\t\taccuracy =(1- ((prediction[:,1]- Y_target)/Y_target).abs()).mean()\n# \t\t\t\tscore_valid = score(Y_target, prediction).item()\n                \n# # \t\t\tif (accuracy.data.item() < 0.75 or score_valid > 8.5 ):\n# # \t\t\t\tabnormalvaliddata=abnormalvaliddata.append(npValid.iloc[xb_patients_idxs.to('cpu').numpy()])\n               \n                \n# \t\t\ts = ('Epochs: %5d/%d , Steps: %8d/%d ,Valid_loss: %5.3f  ,valid_accuracy: %5.3f, Score: %4.4f'%\\\n# \t\t\t\t  (epoch, epochs, i, len(valid_dl), loss.data.item(), accuracy.data.item(), score_valid))\n# \t\t\tpbar2.set_description(s)  \n\t\t\t\n# \t\t\tval_acc_total += accuracy.data.item()*len(xb_meta)\n# \t\t\tscore_valid_total += score_valid * len(xb_meta)\n# \t\t\tdel prediction\n\t\t\t\n# \t\tavg_valid_acc = (val_acc_total)/len(valid_ds) \n# \t\tavg_score_valid = score_valid_total/ len(valid_ds)\n# \t\tprint('Average validation accuracy:', avg_valid_acc)\n# \t\tprint('Average validation score :', avg_score_valid)\n\n# \t\t#abnormalvaliddata.to_csv('abnormalvaliddata.csv',index=False)\n\n\n# \t\tPATH = 'Epoch'+'%s_'%epoch+'Score'+'%s_'%avg_score_valid+'Acc'+'%s'%avg_valid_acc + '.pth'\n# \t\ttorch.save(model.state_dict(), './model_weight'+os.sep+PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#     return npEval\n######### ----------------- Making the Evaluation Data -------------------- #######\ndef make_eval_data(npEval, model, device = 'cuda'):\n    x_features = npEval[['Week']]\n    x_features = torch.tensor(x_features.values).float()\n    x_patientids_name = npEval[['Patient']].values\n    \n    unique_patients = npEval.Patient.unique()\n    loaded_images = {}\n    dir_name_of_patientid= '../input/osic-pulmonary-fibrosis-progression/train'\n    \n    for unique_patient in unique_patients:\n        loaded_images[unique_patient] = read_image(dir_name_of_patientid, unique_patient, Z=100,Y=200,X=200)\n    \n    #y_labels = npEval['actual_FVC'].values \n    #patientsID = npEval['Patient'].values\n    \n    \n    \n    if torch.cuda.is_available() and device == 'cuda':\n        model.to('cuda')\n    model.eval()\n    predictions = []\n    #raw_submission_df = pd.DataFrame([]) \n    for i, patientid in enumerate(x_patientids_name):\n        #x_image = read_image(dir_name_of_patientid, patientid,Z=100,Y=200,X=200, Normalize = True)\n        x_image = loaded_images[patientid[0]]\n#         x_image = np.zeros((100,200,200))\n        x_image = torch.tensor(x_image, dtype = torch.float32).unsqueeze(0).unsqueeze(0) # one for channel another for batch \n        x_feature = x_features[i] # taking the feature for the particular patientid and week of npEval dataframe \n        x_feature = x_feature.unsqueeze(0)\n        \n        \n        if torch.cuda.is_available() and device == 'cuda':\n            x_image = x_image.cuda()\n            x_feature = x_feature.cuda()\n            \n        prediction = model(x_image, x_feature)\n        predictions.append(prediction.to('cpu').detach().numpy()[0]) # since it returns batched output , and we use batch_size =1, taking the [0] output \n    \n    predictions = np.array(predictions) # shape = [none, 3]\n    npEval['FVC'] = predictions[:,1]\n    npEval['Confidence'] = predictions[:,0]\n    \n    return npEval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ndata_test = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\nsubmission  = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['Patient']=submission['Patient_Week'].apply(lambda x:x.split('_')[0])\nsubmission['Weeks']=submission['Patient_Week'].apply(lambda x:x.split('_')[1]).astype(int)\nsubmission=submission.sort_values(by=['Patient','Weeks'], ascending=True ).reset_index(drop=True)\n\n\nmerge=pd.merge(data_test,submission,on=['Patient'],how='left').sort_values(['Patient','Weeks_y']).reset_index(drop=True)\nmerge=merge.drop(['FVC_y'],axis=1)\nmerge=merge.rename(columns={'FVC_x':'base_FVC','Weeks_y':'Week','Weeks_x':'base_Weeks'})\n\ndel data_test\ndel submission\n\ndata_test=merge.loc[:,['Patient','base_Weeks','base_FVC','Percent','Age','Sex','SmokingStatus','Week']]\nsubmission=merge.loc[:,['Patient_Week','base_FVC','Confidence']]\nsubmission=submission.rename(columns={'base_FVC':'FVC'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data_test.copy()\ndata['Healthy-FVC']=round((data['base_FVC']*100)/data['Percent'])\nFE=[]\nFE.append('Healthy-FVC')\n\n#Create Male, Female, Ex-smoker, Current-smoker and Never smoked\nCOLS = ['Sex','SmokingStatus']\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\nFE1=['Male','Female','Ex-smoker','Never smoked','Currently smokes']\nnpData=pd.DataFrame(columns=['Patient','base_Weeks','base_FVC','Age','Healthy-FVC']+FE1+['Week'])\nnpData=npData.append(data, sort=True)\nnpData=npData.fillna(0)\n\n# npData['Week']=npData['Week']-npData['base_Weeks']\n# npData['base_Weeks']=0.0\n\ndel data_test, data\ndata_test = npData[['Patient','base_Weeks','base_FVC','Age','Healthy-FVC','Male','Female','Ex-smoker','Never smoked','Currently smokes','Week']]\ndel npData   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #train, valid, test = csv_split(data_train, v=15, t=0)\n\ndata_train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ndata_train=csv_preprocess(data_train)\n\n# #npTrain=csv_preprocess(train)\n# #npValid=csv_preprocess(valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data=data_train[:1]\n# print(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# min_week = min(data_train['Week'].min(), data_test['Week'].min())\n# max_week = max(data_train['Week'].max(), data_test['Week'].max())\n\n# min_age = min(data_train['Age'].min(), data_test['Age'].min()) \n# max_age = max(data_train['Age'].max(), data_test['Age'].max())\n\n# min_Healthy_FVC = min(data_train['Healthy-FVC'].min(), data_test['Healthy-FVC'].min()) \n# max_Healthy_FVC = max(data_train['Healthy-FVC'].max(), data_test['Healthy-FVC'].max())\n\n# min_FVC = min(data_train['base_FVC'].min(), data_test['base_FVC'].min()) \n# max_FVC = max(data_train['base_FVC'].max(), data_test['base_FVC'].max())\n\n# def Normalize (data,min_age,max_age,min_week,max_week,min_FVC,max_FVC,min_Healthy_FVC,max_Healthy_FVC):\n    \n#     data['Age'] = (data['Age'] - min_age ) / ( max_age - min_age )\n#     data['Week'] = (data['Week'] - min_week ) / ( max_week - min_week )\n#     data['base_FVC'] = (data['base_FVC'] - min_FVC ) / ( max_FVC - min_FVC )\n#     data['Healthy-FVC'] = (data['Healthy-FVC'] - min_Healthy_FVC ) / ( max_Healthy_FVC - min_Healthy_FVC )\n    \n#     return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = IMAGE()\nmodel.load_state_dict(torch.load('../input/image-w/Epoch61_Score6.620608233458159_Acc0.9449220416166925.pth'))\n#model.eval()\n# for the validation inputset of sigma \n# ndf = pd.concat([npValid,npTest])\n# outdf = make_eval_data(df_test, model)\n# for the training inputset of sigma \n#train_inp_sigma = make_eval_data(df_train, model)\n# for the final test inputset of sigma\ntest = make_eval_data(data_test.copy(), model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# start = torch.cuda.Event(enable_timing=True)\n# end = torch.cuda.Event(enable_timing=True)\n\n# start.record()\n# train = make_eval_data(data.copy(), model)\n# end.record()\n# torch.cuda.synchronize()\n# print(start.elapsed_time(end))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install torch-summary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from torchvision import models\n# from torchsummary import summary\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# summary(model, ((1, 100, 200, 200), (10,)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train=train.rename(columns={'Confidence':'pred_sigma','FVC':'pred_FVC'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train=train[['Patient','Week','actual_FVC','pred_FVC','pred_sigma']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train.head(8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# FVC and confidence correction at base week \nfor nid in test.Patient.unique():\n    index = test[(test.Patient==nid) & (test.Week==test.base_Weeks)].index.values\n    test.iloc[index[0],test.columns.get_loc('FVC')]=test.iloc[index[0],test.columns.get_loc('base_FVC')]\n    test.iloc[index[0],test.columns.get_loc('Confidence')] = 70","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Week bias correction and confidence clipping at 70\n# test.base_Weeks += -12\n# test.Week += -12\ntest.loc[test.Confidence < 70, 'Confidence' ] = 70","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.loc[:, 'FVC']=test.FVC\nsubmission.loc[:, 'Confidence']= test.Confidence\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}