{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport subprocess\ndef install(package):\n    subprocess.check_call([sys.executable, \"-m\", \"pip\",\"install\",package])\ninstall(\"../input/fastremap/fastremap-1.10.2-cp37-cp37m-manylinux1_x86_64.whl\")\ninstall(\"../input/fillvoids/fill_voids-2.0.0-cp37-cp37m-manylinux1_x86_64.whl\")\ninstall(\"../input/finalmask\")\ninstall(\"pydicom\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#importing libraries for the analysis\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport glob\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport keras.backend as kb\nimport pydicom\nfrom pydicom.data import get_testdata_files\n\nimport os\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import OneHotEncoder \nfrom sklearn import preprocessing \nfrom tensorflow import keras\nimport keras.backend as kb\nfrom sklearn.model_selection import train_test_split\n\nsns.set(style=\"whitegrid\")\nsns.set_context(\"paper\")\nfrom lungmask import mask\nimport SimpleITK as sitk\nimport math\nimport time\nfrom skimage.transform import resize\nfrom skimage import data\nfrom skimage.util import pad","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport random\ndef seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\ntrain=pd.read_csv(f\"{ROOT}/train.csv\")\n\ntrain['Patient_Week']=train['Patient']+'_'+train['Weeks'].astype(str)\nlists=train['Patient_Week'][train.duplicated(['Patient_Week'], keep=False)].unique().tolist()\nfor patient_week in lists:\n  new_row=train.loc[train['Patient_Week']==patient_week].groupby(['Patient','Weeks','Age','Sex','SmokingStatus','Patient_Week']).mean().copy()\n  train=train[train['Patient_Week']!=patient_week]\n  train=train.append(new_row.reset_index())\nadd=train.copy()\nadd.rename(columns={'Weeks':'base_weeks','FVC':'base_fvc'},inplace=True)\nfinal=train.merge(add,on='Patient')\nfinal.drop(['Patient_Week_x','Age_y','Sex_y','SmokingStatus_y','Percent_y'],axis=1,inplace=True)\nfinal.rename(columns={'Weeks':'base_week','FVC':'base_fvc','base_fvc':'FVC','Percent_x':'base_percent','Patient_Week_y':'Patient_Week','Age_x':'Age','Sex_x':'sex','SmokingStatus_x':'smokingstatus','base_weeks':'predict_week'},inplace=True)\nfinal['weeks_passed']=final['predict_week']-final['base_week']\ncols=['Patient','Patient_Week', 'base_week', 'base_fvc', 'base_percent', 'Age', 'sex','smokingstatus','predict_week','weeks_passed', 'FVC']\nfinal=final[cols]\nfinal=final.loc[final['weeks_passed']!=0]\nfinal.reset_index(drop=True,inplace=True)\nfinal.head()\nsample_submission=pd.read_csv(f\"{ROOT}/sample_submission.csv\")\n\n\ntest=pd.read_csv(f\"{ROOT}/test.csv\")\n\n\ntest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Percent': 'base_Percent', 'Age': 'base_Age'},inplace=True)\nWeek=sample_submission['Patient_Week'].apply(lambda x : x.split('_')[1]).unique()\nWeek=np.tile(Week, len(test['Patient']))\ntest=test.loc[test.index.repeat(146)].reset_index(drop=True)\ntest['predict_week']=Week\n\ntest['Patient_Week']=test['Patient']+'_'+test['predict_week']\n\n\ntest['weeks_passed']=test['predict_week'].astype(int)-test['base_Week'].astype(int)\n\ntest.rename(columns={'base_Week':'base_week','base_FVC':'base_fvc','base_Percent':'base_percent','base_Age':'Age','Sex':'sex','SmokingStatus':'smokingstatus'},inplace=True)\n\ncols=['Patient','Patient_Week','base_week','base_fvc','base_percent','Age','sex','smokingstatus','predict_week','weeks_passed']\n\ntest=test[cols]\n\ndef bounding_box(img3d):\n        mid_img = img3d\n        same_first_row = (mid_img[0, :] == mid_img[0, 0]).all()\n        same_first_col = (mid_img[:, 0] == mid_img[0, 0]).all()\n        if same_first_col and same_first_row and (mid_img!=0).any():\n            return True\n        else:\n            return False\n\n\ndef crop_bounding(input):\n  x=0\n  dataset=pydicom.dcmread(input)\n  image1=dataset.pixel_array\n  while x==0:\n    bounding=bounding_box(image1)\n    if (image1[0, :] == image1[0, 0]).all()==True:\n      image1=image1[1:-1,:]\n    elif (image1[:, 0] == image1[0, 0]).all()==True:\n      image1=image1[:,1:-1]\n    else:\n      x=x+1\n  return image1\n\ndef masking(input):\n  input_image = sitk.ReadImage(input)\n  dataset = pydicom.dcmread(input)\n  if bounding_box(dataset.pixel_array)==False:\n    segmentation = mask.apply(input_image)\n    dataset = pydicom.dcmread(input)\n    segment_image=dataset.pixel_array\n    segment_image=np.where(segmentation[0]==0,dataset.pixel_array.min(),segment_image)\n  else:\n    array_ct = sitk.GetArrayFromImage(input_image)\n    column_dif=int((array_ct.shape[1]-crop_bounding(input).shape[0])/2)\n    row_dif=int((array_ct.shape[2]-crop_bounding(input).shape[1])/2)\n    input_image=input_image[row_dif:-row_dif,column_dif:-column_dif]\n    segmentation = mask.apply(input_image)\n    dataset = pydicom.dcmread(input)\n    segment_image=sitk.GetArrayFromImage(input_image)[0]\n    segment_image=np.where(segmentation[0]==0,dataset.pixel_array.min(),segment_image)\n  return(segment_image)\n\ndef pixscale(input):\n  dataset = pydicom.dcmread(input)\n  scaled_image=dataset.pixel_array*dataset.RescaleSlope + dataset.RescaleIntercept\n  return(scaled_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patients=test['Patient'].unique().tolist()\n\n### find length of lung:\npatient=\"ID00419637202311204720264\"\ndef lunglength(patient):\n  directory=f\"{ROOT}/test/\"+patient+\"/\"\n  images=glob.glob(directory+'*')\n  images.sort(key = lambda x: int(x.split('/')[5].split('.dcm')[0]))\n  first_image=pydicom.dcmread(images[0])\n  last_image=pydicom.dcmread(images[-1])\n  lung_length=-(last_image.ImagePositionPatient[2]-first_image.ImagePositionPatient[2])\n  return(lung_length)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#patient='ID00007637202177411956430'\nno_of_images=[]\nslice_thickness=[]\nrescaletype=[]\nRescale_slope=[]\nRescale_intercept=[]\nPixel_spacing=[]\nrows=[]\ncolumns=[]\npadding=[]\nimage_position=[]\nfor patient in patients:\n  directory=f\"{ROOT}/test/\"+patient+\"/\"\n  no_of_images.append(len(glob.glob(directory+'*')))\n  dataset=pydicom.dcmread(glob.glob(directory+'*')[0])\n  slice_thickness.append(dataset.SliceThickness)\n  #rescale_type.append(dataset.RescaleType)\n  Rescale_slope.append(dataset.RescaleIntercept)\n  Rescale_intercept.append(dataset.RescaleSlope)\n  Pixel_spacing.append(dataset.PixelSpacing)\n  rows.append(dataset.Rows)\n  columns.append(dataset.Columns)\n  #image_position.append(dataset.ImagePositionPatient)\n  #padding_value.append(dataset.PixelPaddingValue)\nfor patient in patients:\n  directory=f\"{ROOT}/test/\"+patient+\"/\"\n  dataset=pydicom.dcmread(glob.glob(directory+'*')[0])\n  try:\n    padding.append(dataset.PixelPaddingValue)\n  except:\n    padding.append(np.nan)\n  rescaletype=[]\nfor patient in patients:\n  directory=f\"{ROOT}/test/\"+patient+\"/\"\n  dataset=pydicom.dcmread(glob.glob(directory+'*')[0])\n  try:\n    rescaletype.append(dataset.RescaleType)\n  except:\n    rescaletype.append(np.nan)\nfor patient in patients:\n  directory=f\"{ROOT}/test/\"+patient+\"/\"\n  dataset=pydicom.dcmread(glob.glob(directory+'*')[0])\n  try:\n    image_position.append(dataset.ImagePositionPatient)\n  except:\n    image_position.append(np.nan)\nlung_lengths=[]\nfor patient in patients:\n  try:\n   lung_lengths.append(abs(lunglength(patient)))\n  except:\n    lung_lengths.append(np.nan)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata=pd.DataFrame({'patient_ID':test['Patient'].unique(),'no_of_images':no_of_images,\n'slice_thickness':slice_thickness,\n'rescale_type':rescaletype,\n'Rescale_slope':Rescale_slope,\n'Rescale_intercept':Rescale_intercept,\n'Pixel_spacing':Pixel_spacing,\n'rows':rows,\n\"columns\":columns,\n\"padding_value\":padding,\n\"image_position\":image_position,\n\"lung_lengths\":lung_lengths})\nmetadata['lung_lengths'].fillna(metadata['lung_lengths'].mean(),inplace=True)\nextra=[]\nfor i in range(len(test['Patient'].unique())):\n    extra.append((((no_of_images[i]-1)*max(metadata['lung_lengths'].to_list())/metadata['lung_lengths'].to_list()[i])-no_of_images[i])+1)\nmetadata['extra']=extra\n\nto_be_extracted=20*metadata['no_of_images']/(metadata['extra']+metadata['no_of_images'])\nto_be_extracted=[math.ceil(value) for value in to_be_extracted]\nmetadata['to_be_extracted']=to_be_extracted\n\nmetadata.to_csv('metadata.csv',index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_nn_convert(patient):\n  directory=f\"{ROOT}/test/\"+patient+\"/\"\n  images=glob.glob(directory+'*')\n  images.sort(key = lambda x: int(x.split('/')[5].split('.dcm')[0]))\n  image_number=[]\n  for value in np.linspace(1,len(images),metadata['to_be_extracted'].loc[metadata['patient_ID']==patient].to_list()[0],endpoint=True):\n    image_number.append(int(round(value)))\n  revised_images=[]\n  for i in image_number:\n    revised_images.append(images[i-1])\n  normalized=[]\n  for input in tqdm(revised_images):\n    dataset = pydicom.dcmread(input)\n    slope=dataset.RescaleSlope\n    intercept=dataset.RescaleIntercept\n    masked_image=masking(input)\n    masked_image=(masked_image*slope) + intercept\n    masked_image=np.where((masked_image>=1000)|(masked_image<=-1000),-1000,masked_image)\n    new_shape_row=round(float(dataset.PixelSpacing[0])*masked_image.shape[0])\n    new_shape_column=round(float(dataset.PixelSpacing[1])*masked_image.shape[1])\n    resized=resize(masked_image, (new_shape_row, new_shape_column),preserve_range=True)\n    resized=pad(resized,((math.ceil((512-resized.shape[0])/2),math.floor((512-resized.shape[0])/2)),((math.ceil((512-resized.shape[1])/2),math.floor((512-resized.shape[1])/2)))),mode='edge')\n    resized=resize(resized,(128,128),preserve_range=True)\n    normalized.append(resized)\n    empty_images=np.ones([128,128])*-1000\n  while np.array(normalized).shape[0]<20:\n    normalized=np.insert(normalized,0,empty_images,0)\n    if np.array(normalized).shape[0]<20:\n      normalized=np.insert(np.array(normalized),np.array(normalized).shape[0],empty_images,0)\n    else:\n      pass\n  return(normalized)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_images=[]\n#patient=\"ID00009637202177434476278\"\nfor patient in tqdm(patients):\n  start = time.time()\n  compy=image_nn_convert(patient)\n  all_images.append(compy)\n  end = time.time()\n  print(end - start)\nall_images=np.array(all_images)\n\nfig=plt.figure(figsize=(50, 24))\nfor i in range(len(compy)):\n    img = compy[i]\n    fig.add_subplot(4, 5,i+1)\n    plt.imshow(img, cmap=plt.cm.bone)\n    plt.title(i, fontsize = 9)\n    plt.axis('off');\n\nnp.save('data_new.npy', all_images) # save\nnew_num_arr = np.load('data_new.npy') #","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_num_arr = np.load('../input/image-to-np/data.npy')\ntrain_num_arr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_num_arr=np.concatenate((train_num_arr,new_num_arr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" def score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = abs(y_pred[:,2] - y_pred[:,0])\n    fvc_pred = y_pred[:,1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, 70)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, 1000)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return metric\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.8,0.5,0.2]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return kb.mean(v)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder_input = keras.Input(shape=(20,128,128,1), name=\"input\")\nx = layers.Conv3D(16, kernel_size=(3, 3, 3),padding='same')(encoder_input)\nx=layers.MaxPooling3D(pool_size=(2, 2, 2))(x)\nx = layers.Conv3D(32, 3,padding='same')(x)\nx=layers.MaxPooling3D(pool_size=(2, 2, 2))(x)\n\nx=layers.Conv3D(16, kernel_size=(3, 3, 3),padding='same')(x)\n\nx=layers.Flatten()(x)\n#x=layers.Dense(500, activation='relu')(x)\nencoder_output=layers.Dense(10)(x)\n#encoder_output=layers.LeakyReLU()(x)\n\nencoder = keras.Model(encoder_input, encoder_output, name=\"encoder\")\n#encoder.summary()\nx=layers.Dense(81920)(encoder_output)\nx=layers.LeakyReLU()(x)\nx = layers.Reshape(( 5, 32, 32,16))(x)\nx = layers.UpSampling3D((4,4,4))(x)\nx = layers.Conv3DTranspose(16, 3,padding='same')(x)\nx=layers.Conv3DTranspose(32, kernel_size=(3, 3, 3),padding='same')(x)\nx=layers.Conv3DTranspose(32, kernel_size=(3, 3, 3),padding='same')(x)\ndecoder_output = layers.Conv3DTranspose(1, kernel_size=(3, 3, 3),padding='same')(x)\n\nautoencoder = keras.Model(encoder_input,decoder_output, name=\"autoencoder\")\nautoencoder.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"autoencoder.compile(loss='mse',optimizer='adam')\nautoencoder.fit(final_num_arr,final_num_arr,epochs=13,batch_size=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoded_train=encoder.predict(train_num_arr)\nencoded_test=encoder.predict(new_num_arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"encoder.predict(new_num_arr)[4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image1=autoencoder.predict(new_num_arr)[4]\nplt.figure(figsize=(10,10))\nplt.imshow(np.reshape(image1[10],(128,128)),cmap=plt.cm.bone)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X1=final[['base_fvc','base_percent','Age','sex','smokingstatus','weeks_passed','base_week']].copy()\ny=final.FVC.copy()\nenc = OneHotEncoder(handle_unknown='ignore')\nenc.fit(X1[['sex','smokingstatus']])\nencoded=pd.DataFrame(enc.transform(X1[['sex','smokingstatus']]).toarray())\ncounts=final.groupby(('Patient')).count()['Patient_Week'].to_numpy()\nX1=X1.join(encoded)\nX1.drop(['smokingstatus','sex'],axis=1,inplace=True)\nX1=np.concatenate((X1,np.repeat(encoded_train,counts,0)),axis=1)\nscaler=preprocessing.MinMaxScaler().fit(X1)\nX1=pd.DataFrame(scaler.transform(X1))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test=test[['base_fvc','base_percent','Age','sex','smokingstatus','weeks_passed','base_week']].copy()\nencoded=pd.DataFrame(enc.transform(X_test[['sex','smokingstatus']]).toarray())\nX_test=X_test.join(encoded)\nX_test.drop(['smokingstatus','sex'],axis=1,inplace=True)\ncounts=test.groupby(('Patient')).count()['Patient_Week'].to_numpy()\nX_test=np.concatenate((X_test,np.repeat(encoded_test,counts,0)),axis=1)\nX_test=pd.DataFrame(scaler.transform(X_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X1=X1.astype(np.float32)\ny1=y.astype(np.float32)\n\ninputs= keras.Input(shape=[20])\ndense = layers.Dense(100, activation=\"relu\")\nx = dense(inputs)\nx = layers.Dense(100, activation=\"relu\")(x)\nx = layers.Dense(100, activation=\"relu\")(x)\noutput1 = layers.Dense(3,activation='linear')(x)\nmodel = keras.Model(inputs=inputs, outputs=output1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=mloss(0.8),optimizer='adam',metrics=score)\nmodel.fit(X1,y1,batch_size=128,epochs=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_high=model.predict(X_test)[:,0]\npreds_low=model.predict(X_test)[:,2]\npreds=model.predict(X_test)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_set=pd.DataFrame({'preds_high':preds_high})\npreds_set['preds']=preds\npreds_set['preds_low']=preds_low\npreds_set['sigma_pred']=abs(preds_set['preds_high']-preds_set['preds_low'])\npreds_set.reset_index(inplace=True,drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.DataFrame({'Patient_Week':test['Patient_Week'],'FVC': preds_set['preds'],'Confidence':preds_set['sigma_pred']})\nsubmission['FVC']=submission['FVC'].apply(lambda x: round(x, 4))\nsubmission['Confidence']=submission['Confidence'].apply(lambda x: round(x, 4))/1.2\n                         ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}