{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport os\nimport numpy as np\nfrom scipy.ndimage import zoom\nimport keras\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Flatten, Conv2D, MaxPooling2D,LSTM,TimeDistributed, Input,BatchNormalization, Activation, Conv3D,MaxPooling3D, Dropout, concatenate\nfrom tensorflow.keras.callbacks import ModelCheckpoint,CSVLogger\nfrom tensorflow.math import abs, sqrt, minimum, maximum, log, multiply, divide_no_nan, add\nfrom tensorflow.keras.backend import greater, zeros_like, variable, constant, int_shape, mean, shape, ones_like\nfrom tensorflow.keras.regularizers import l2\nfrom keras.utils import to_categorical\nimport tensorflow as tf\nimport h5py\nimport cv2\nimport pandas as pd\nimport skimage\nfrom sklearn import preprocessing\nfrom pydicom import dcmread\n#from lungmask import mask\n#import SimpleITK as sitk\nimport sys\nfrom tensorflow_addons.losses import PinballLoss\nfrom tensorflow.keras.losses import MSE\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nimport keras\nfrom keras.utils.vis_utils import model_to_dot\nfrom sklearn.linear_model import LinearRegression\nfrom tensorflow import boolean_mask, where\nfrom tensorflow_addons.losses import pinball_loss\nfrom sklearn.model_selection import train_test_split\nsetup=0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loc='/kaggle/input/osic-pulmonary-fibrosis-progression/train/'\nloctest='/kaggle/input/osic-pulmonary-fibrosis-progression/test/'\n\ndirs=os.listdir('/kaggle/input/osic-pulmonary-fibrosis-progression/train/')\ndirstest=os.listdir('/kaggle/input/osic-pulmonary-fibrosis-progression/test/')\nsetup=0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def cropImg(img):\n    \n    if not (sum(img[0])==sum(img[round(len(img)/2)])).all():\n        r_min=0\n        r_max=len(img)\n        c_min=0\n        c_max=len(img[0])\n        cropImg=[r_min,r_max,c_min,c_max]\n        return cropImg\n    r_min, r_max = None, None\n    c_min, c_max = None, None\n    \n    for row in range(len(img)):\n        if not (img[row,:]==img[0,0]).all() and r_min is None:\n            r_min=row\n        if (img[row,:]==img[0,0]).all() and r_min is not None and r_max is None:\n            r_max=row\n                \n    flipImg=np.rot90(img)\n    for col in range(len(flipImg)):\n        if not (flipImg[col,:]==flipImg[0,0]).all() and c_min is None:\n            c_min=col\n        if (flipImg[col,:]==flipImg[0,0]).all() and c_min is not None and c_max is None:\n            c_max=col\n    cropImg=[r_min,r_max,c_min,c_max]\n    return cropImg\n\ndef resizeImg(img,x3d,y3d,z3d):\n    reX = x3d/len(img)\n    reY = y3d/len(img[0])\n    reZ = z3d/len(img[0][0])\n    reImg=zoom(img,(reX,reY,reZ))\n    return reImg\n\n\ndef LLL_metric(y_true, y_pred,return_values = False):\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    sd_clipped = maximum(sigma,70)\n    delta= minimum(abs(y_pred[:,1] - y_true[:,0]), 1000)\n    metric=- sqrt(2.0) * delta / sd_clipped - log(sqrt(2.0) * sd_clipped)\n        #print('\\n')\n        #print(metric)\n    return metric\n        \ndef get_patient_data(ID):\n    return table[table['Patient'] == ID]\n\ndef plot_patients(patient_list, age, sex, smoke):\n    patient_length = len(patient_list)\n    cols = 8\n\n    fig, ax = plt.subplots(patient_length//cols+1, cols, sharey = True, sharex = 'none', figsize=(30,15))\n    for num, patient in enumerate(patient_list):\n        ax[num//cols,num%cols].plot(get_patient_data(patient)['Weeks'], get_patient_data(patient)['FVC'])\n        if smoke[num] == 'Ex-smoker':\n            smoke_str = 'EX'\n        elif smoke[num] == 'Never smoked':\n            smoke_str = 'NA'\n        elif smoke[num] == 'Currently smokes':\n            smoke_str = 'SM'\n        ax[num//cols,num%cols].set_title(str(age[num])+'_'+sex[num]+'_'+smoke_str)\n\n    plt.show()\n\ndef get_patients(lower, upper):\n    patient_list = table['Patient'].unique()[lower:upper]\n\n    age = []\n    sex = []\n    smoke = []\n\n    for patient in patient_list:\n        subset = table[table['Patient'] == patient].reset_index()\n        age.append(subset['Age'][1])\n        sex.append(subset['Sex'][1])\n        smoke.append(subset['SmokingStatus'][1])\n\n    return patient_list, age,sex,smoke\n\n# Loss function\ndef LLL_metric2(y_true, y_pred):\n  zeros = zeros_like(y_true)\n  bool_mask = greater(y_true, [0])\n  y_pred = where(bool_mask, y_pred, y_true)\n\n  diff = abs(y_pred - y_true)\n  sigma = constant(value = 200, shape = [146])\n\n  delta = minimum(diff, constant(value = 1000, shape = [146]))\n  delta = diff\n  sqrt2 = constant(value = 1.414, shape = [146])\n  \n  loss = - divide_no_nan(sqrt2 * delta, sigma) - log(where(bool_mask, sqrt2 * sigma, ones_like(y_true)))\n  avg_loss = mean(boolean_mask(loss, bool_mask), axis = 0)\n  \n  return avg_loss\n\ndef LLL_metric_neg(y_true, y_pred):\n  zeros = zeros_like(y_true)\n  bool_mask = greater(y_true, [0])\n  y_pred = where(bool_mask, y_pred, y_true)\n\n  diff = abs(y_pred - y_true)\n  sigma = constant(value = 70, shape = [146])\n\n  delta = minimum(diff, constant(value = 1000, shape = [146]))\n  delta = diff\n  sqrt2 = constant(value = 1.414, shape = [146])\n  \n  loss = - divide_no_nan(sqrt2 * delta, sigma) - log(where(bool_mask, sqrt2 * sigma, ones_like(y_true)))\n  avg_loss = mean(boolean_mask(loss, bool_mask), axis = 0)\n  \n  return -(avg_loss)\n\ndef pinball2(y_true, y_pred):\n  zeros = zeros_like(y_true)\n  bool_mask = greater(y_true, [0])\n  y_pred = where(bool_mask, y_pred, y_true)\n  #tf.print(y_pred, summarize = 20)\n\n  return pinball_loss(y_true, y_pred)\n\ndef MSE2(y_true, y_pred):\n  zeros = zeros_like(y_true)\n  bool_mask = greater(y_true, [0])\n  y_pred = where(bool_mask, y_pred, y_true)\n  #tf.print(y_pred, summarize = 20)\n\n  return MSE(y_true, y_pred)\n\ndef createLine(outPred):\n    click=0\n    first=0\n    last=0\n    for i in range(len(outPred)):\n        if outPred[i]<=1001 and click==0:\n            first=i+1\n        elif outPred[i]<=1001 and click==1 and i>20:\n            last=i+1\n            break\n        if not (outPred[i]<=1001):\n            click=1\n\n    y=np.array(outPred[first:last])\n    x=np.array(list(range(first-12,last-12)))\n    xmean=np.mean(x)\n    ymean=np.mean(y)\n\n    xycov = (x - xmean) * (y - ymean)\n    xvar = (x - xmean)**2\n    beta = xycov.sum() / xvar.sum()\n    alpha = ymean - (beta * xmean)\n    line=[i*beta+alpha for i in np.array(list(range(-12,134)))]\n    return line","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_data_test = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv', delimiter=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"setup=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d1 = os.listdir(loctest+dirstest[2])\nd = sorted(os.listdir(loctest+dirstest[2]), key=lambda v:int(v.split('.')[0]))\nd = {i+1:dcm for i,dcm in enumerate(d)}\ncenter = len(d)//2\nloctest+dirstest[0]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"table = my_data_test\ntable.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ntable = table.reset_index(drop = True)\npatient_list, age, sex, smoke = get_patients(50, 70)\n#plot_patients(patient_list,age,sex,smoke)\ndf = pd.DataFrame(columns = ['Patient','1st Read','1st Wk','Coef', 'Age','Sex','Smoker'])\nfor num, patient in enumerate(patient_list):\n    patient_data = get_patient_data(patient).reset_index()\n    x = patient_data['Weeks'].values.reshape(-1,1)\n    y = patient_data['FVC'].values\n    lin_reg = LinearRegression().fit(x,y)\n\n    df = df.append({'Patient':patient,\n                    '1st Read':patient_data['FVC'][0],\n                    '1st Wk':patient_data['Weeks'][0],\n                    'Coef':lin_reg.coef_,\n                    'Age':age[num],\n                    'Sex':sex[num],\n                    'Smoker':smoke[num]}, ignore_index = True)\n    \nprint(table.shape)\ntable.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# One-hot encode Sex and SmokingStatus\nonehot = OneHotEncoder(sparse = False)\ntrf = onehot.fit_transform(table[['Sex','SmokingStatus']])\n\ncat_list = []\n\nfor item in onehot.categories_:\n  temp = item.tolist()\n  cat_list.extend(temp)\n\ntrf_df = pd.DataFrame(trf, columns = cat_list)\n\nmerge = pd.concat([table, trf_df], axis = 1).drop(['Sex','SmokingStatus'], axis = 1)\n\n# Standardize FVC, Age, and Percent\n#norm_FVC = StandardScaler().fit(merge.FVC.values.reshape(-1,1))\n#norm_age = StandardScaler().fit(merge.Age.values.reshape(-1,1))\n#norm_percent = StandardScaler().fit(merge.Percent.values.reshape(-1,1))\n\n#merge.FVC = pd.DataFrame(norm_FVC.transform(merge.FVC.values.reshape(-1,1)))\n#merge.Age = pd.DataFrame(norm_age.transform(merge.Age.values.reshape(-1,1)))\n#merge.Percent = pd.DataFrame(norm_percent.transform(merge.Percent.values.reshape(-1,1)))\n\nfinal_df = pd.merge(merge.groupby('Patient')['Weeks'].apply(list).to_frame(),\n                    merge.groupby('Patient')['FVC'].apply(list).to_frame(),\n                    on = 'Patient')\n\nfinal_df = pd.merge(final_df, merge.groupby('Patient')['Percent'].apply(list).to_frame(), on = 'Patient')\nfinal_df = pd.merge(final_df, merge.groupby('Patient')['Age'].first().to_frame(), on = 'Patient')\ntry:\n    final_df = pd.merge(final_df, merge.groupby('Patient')['Male'].first().to_frame(), on = 'Patient')\nexcept:\n    final_df['Male']=0\n\ntry:\n    final_df = pd.merge(final_df, merge.groupby('Patient')['Female'].first().to_frame(), on = 'Patient')\nexcept:\n    final_df['Female']=0\ntry:\n    final_df = pd.merge(final_df, merge.groupby('Patient')['Currently smokes'].first().to_frame(), on = 'Patient')\nexcept:\n    final_df['Currently smokes']=0\ntry:\n    final_df = pd.merge(final_df, merge.groupby('Patient')['Ex-smoker'].first().to_frame(), on = 'Patient')\nexcept:\n    final_df['Ex-smoker']=0\ntry:\n    final_df = pd.merge(final_df, merge.groupby('Patient')['Never smoked'].first().to_frame(), on = 'Patient')\nexcept:\n    final_df['Never smoked']=0\nfinal_df = final_df.reset_index()\n\nfinal_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if setup==1:\n    xImg=[]\n    time1=1\n    for folder in final_df.Patient:\n        print(time1)\n        time1=time1+1\n        folderdir=loctest+folder\n        images=sorted(os.listdir(loctest+folder), key=lambda v:int(v.split('.')[0]))\n        try:\n            imagecrop=folderdir+\"/\"+images[round(len(images)/2)]\n            ds1 = dcmread(imagecrop)\n            imgCrop= cropImg(ds1.pixel_array)\n            imgC = ds1.pixel_array[imgCrop[0]:imgCrop[1],imgCrop[2]:imgCrop[3]]\n\n        except:\n            imgCrop=[0,48,0,48]\n        \n        imglist=[]\n        for img in images:\n            try:\n                ds = dcmread(folderdir+\"/\"+img)\n                imgC = ds.pixel_array[imgCrop[0]:imgCrop[1],imgCrop[2]:imgCrop[3]]\n            except:\n                imgC=np.zeros([np.abs(imgCrop[1]-imgCrop[0]),np.abs(imgCrop[3]-imgCrop[4])])\n            imglist.append(imgC)\n        \n        imgCR=resizeImg(imglist,48,48,48)\n        xImg.append(imgCR)\n            \n    xImg=np.array(xImg)\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Setup dataset\n\nfeature_num = 9\n\nx = np.zeros([final_df.shape[0], feature_num])\nfor index, row in final_df.iterrows():\n  x[index, 0] = row['Weeks'][0] + 12\n  x[index, 1] = row['FVC'][0]\n  x[index, 2] = row['Percent'][0]\n  x[index, 3] = row['Age']\n  x[index, 4] = row['Male']\n  x[index, 5] = row['Female']\n  x[index, 6] = row['Currently smokes']\n  x[index, 7] = row['Ex-smoker']\n  x[index, 8] = row['Never smoked']\n\ny = np.zeros((final_df.shape[0], 146))\n\nfor index, row in final_df.iterrows():\n  y[index, :][np.array(row['Weeks']) + 12] = row['FVC']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pickle import load\n\nscaler = load(open('/kaggle/input/convmodel/scaler.pkl', 'rb'))\nx_scaled=x.copy()\nx_scaled[:,:4]=scaler.transform(x_scaled[:,:4])\nx_scaled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xt=[]\nfor k in range(len(xImg)):\n    a1=[]\n    for j in range(len(xImg[k])):\n        rImg=xImg[k][j]/32768*255\n        aa=np.dstack((rImg,rImg,rImg))\n        a1.append(aa)\n    xt.append(a1)\nxImg=np.array(xt)\nxImg.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xImg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model3=tf.keras.models.load_model('/kaggle/input/convmodel/ConvRegResBLK.hdf5',compile=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.DataFrame()\nfor i in range(len(y)):\n    outPred=np.array(model3.predict([np.array([xImg[i]]),np.array([x_scaled[i]])])[0])\n    patientList=[]\n    confidenceList=np.ones(len(outPred))*200\n    line=createLine(outPred)\n    outPred2=[outPred[i] if np.abs(outPred[i]-line[i])<300 else line[i] for i in range(len(outPred))]\n    \n    for j in range(len(outPred2)):\n        patientList.append(final_df.Patient[i]+'_'+str(j-12))\n    data=[]\n    for j in range(len(outPred2)):\n        data.append([patientList[j],outPred2[j],200])\n        \n    newDF=pd.DataFrame(data,columns=['Patient_Week','FVC','Confidence'])\n    if i==0:\n        submission=newDF\n        print(0)\n    else:\n        submission=submission.append(newDF)\n        print(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outPred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.mean(outPred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nplt.plot([i*beta+alpha for i in np.array(list(range(-12,134)))])\nplt.plot(outPred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"line=[i*beta+alpha for i in np.array(list(range(-12,134)))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xImg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}