{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install ../input/convmodel/Keras_Applications-1.0.8-py3-none-any.whl\n! pip install ../input/convmodel/efficientnet-1.1.1-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport pydicom\nfrom pydicom import dcmread\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport random\nfrom numpy import dstack\nfrom scipy.ndimage import zoom\nimport skimage\nimport tensorflow as tf\nfrom tensorflow_addons.losses import pinball_loss\nfrom tensorflow_addons.optimizers import RectifiedAdam\nfrom tensorflow.keras.optimizers import Nadam\nfrom PIL import Image\nfrom tensorflow.keras.backend import greater, zeros_like, variable, constant, int_shape, mean, shape, ones_like\nfrom tensorflow import boolean_mask, where\nfrom sklearn.datasets.samples_generator import make_blobs\nfrom sklearn.metrics import accuracy_score, mean_absolute_error\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.linear_model import LogisticRegression\nfrom keras.models import load_model\nfrom keras.utils import to_categorical, Sequence\nfrom pickle import load\nfrom keras.layers import Dense, Flatten, Conv2D, MaxPooling2D,LSTM,TimeDistributed, Input,BatchNormalization, Activation, Conv3D,MaxPooling3D, Dropout, concatenate,GlobalAveragePooling2D,Conv2D,AveragePooling2D,LeakyReLU,Concatenate\nfrom keras.models import Sequential, Model\nfrom tensorflow.math import abs, sqrt, minimum, maximum, log, multiply, divide_no_nan, add\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nimport seaborn as sns\nfrom tqdm.notebook import tqdm \nfrom sklearn.model_selection import train_test_split\n\nfrom shutil import copyfile,copy\ncopyfile(src = \"../input/convmodel/applinearmodel/src/linear_model.py\", dst = \"../working/linear_model.py\")\ncopyfile(src = \"../input/convmodel/convmdlnosave.py\", dst = \"../working/convmdlnosave.py\")\ncopyfile(src = \"../input/convmodel/EfficientNet_Pred_v1.1.py\", dst = \"../working/EfficientNet_Pred.py\")\ncopyfile(src = \"../input/convmodel/fold-2_best.h5\", dst = \"../working/fold-2_best.h5\")\n\nfrom linear_model import NoTransformer, load_huber_models, huber_predict\nfrom convmdlnosave import conv3dModel\nfrom EfficientNet_Pred import start_predict\nfrom tensorflow.keras.layers import (\n    Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n    LeakyReLU, Concatenate \n)\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def pinball2(y_true, y_pred):\n  zeros = zeros_like(y_true)\n  bool_mask = greater(y_true, [0])\n  y_pred = where(bool_mask, y_pred, y_true)\n  #tf.print(y_pred, summarize = 20)\n\n  return pinball_loss(y_true, y_pred)\n\ndef LLL_metric2(y_true, y_pred):\n  zeros = zeros_like(y_true)\n  bool_mask = greater(y_true, [0])\n  y_pred = where(bool_mask, y_pred, y_true)\n\n  diff = abs(y_pred - y_true)\n  sigma = constant(value = 200, shape = [146])\n\n  delta = minimum(diff, constant(value = 1000, shape = [146]))\n  delta = diff\n  sqrt2 = constant(value = 1.414, shape = [146])\n  \n  loss = - divide_no_nan(sqrt2 * delta, sigma) - log(where(bool_mask, sqrt2 * sigma, ones_like(y_true)))\n  avg_loss = mean(boolean_mask(loss, bool_mask), axis = 0)\n  \n  return avg_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Chun How Model"},{"metadata":{},"cell_type":"markdown","source":"# Data Prep"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\na=np.array([6,6,6])\nb=np.array([4,3,2])\na-b","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ensemble"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create stacked model input dataset as outputs from the ensemble\n# Returns the output of all the different models to be used as the inputs for the ensemble\ndef stacked_dataset(inputX):\n    stacked = None\n    stackX = []\n    # make prediction\n    failed=[]\n    stackConf=[]\n    for i in range(len(inputX)):\n        print(i)\n        age=inputX[i][0]\n        sex=inputX[i][1]\n        week=inputX[i][2]\n        fvc=inputX[i][3]\n        percent=inputX[i][4]\n        smokingstatus=inputX[i][5]\n        imgpath=inputX[i][6]+'/'\n        modelpath=inputX[i][7]\n        scalerpath=inputX[i][8]\n        patient=inputX[i][9]\n        try:\n            convMdl=conv3dModel(age,sex,week,fvc,percent,smokingstatus,imgpath,modelpath,scalerpath)\n            effNetMdlOut=start_predict(fvc,week,percent,age,sex,smokingstatus,imgpath)\n            # initialize models\n            effNetMdl=[i[0] for i in effNetMdlOut[1::3]]\n            effNetConf=[i[0] for i in effNetMdlOut[2::3]]\n            lower_huber, mid_huber, upper_huber, datawrangler = load_huber_models('../input/convmodel/applinearmodel/model_weights/linear_model/lower_huber.pkl',\n                                                                                '../input/convmodel/applinearmodel/model_weights/linear_model/mid_huber.pkl',\n                                                                                '../input/convmodel/applinearmodel/model_weights/linear_model/upper_huber.pkl',                                                                                '../input/convmodel/applinearmodel/model_weights/linear_model/datawrangler.pkl')\n\n            # prediction for linear model\n            df = huber_predict(lower_huber, mid_huber, upper_huber, datawrangler, patient, week, fvc, percent, age, sex, smokingstatus)\n            df = df[['FVC','Lower','Upper']]\n\n            huberMdl=list(df.FVC)\n            huberConf=np.abs(df.Lower-df.Upper)\n            confTotal=(np.array(effNetConf)+huberConf)/2\n            stacked = dstack((convMdl,huberMdl,effNetMdl))\n            stackX.append(stacked)\n            stackConf.append(confTotal)\n        except:\n            failed.append(i)\n            continue\n    stackX=np.array([i[0] for i in stackX])\n    return stackX,stackConf,failed\n\n \n# fit a model based on the outputs from the ensemble members\ndef fit_stacked_model(inputX, inputy,testX,testy):\n    model=create_stacked_model()\n    # create dataset using ensemble\n    stackedX,failed = stacked_dataset(inputX)\n    stackedXtest,failedtest = stacked_dataset(inputX)\n    inputy=[inputy[i] for i in range(len(inputy)) if i not in failed]\n    testy=[texty[i] for i in range(len(texty)) if i not in failedtest]\n    # fit standalone model\n    model.fit(stackedX, inputy,validation_data=(stackedXtest,testy), epochs=10)\n    return model\n \n# make a prediction with the stacked model\ndef stacked_prediction(inputX):\n    model=tf.keras.models.load_model('../input/convmodel/mkEnsemble.hdf5',compile=False)\n    # create dataset using ensemble\n    stackedX,conf,failed = stacked_dataset(inputX)\n    # make a prediction\n    yhat = model.predict(np.array(stackedX))\n    return yhat,conf\n\ndef create_stacked_model():\n    inputs=Input(shape=(146,3),name=\"ensemble_input\")\n    flatten=Flatten()(inputs)\n    dense1=Dense(256, activation='relu')(flatten)\n    dense2=Dense(256, activation='relu')(dense1)\n    outputs=Dense(146, activation='linear',name=\"FVC\")(dense2)\n    model=Model(inputs=inputs,outputs=outputs,name=\"stacked_model\")\n    model.compile(loss=pinball2, optimizer='adam', metrics=LLL_metric2)\n    return model\n\ndef createLine(outPred):\n    click=0\n    first=0\n    last=0\n    for i in range(len(outPred)):\n        if outPred[i]<=1001 and click==0:\n            first=i+1\n        elif outPred[i]<=1001 and click==1 and i>20:\n            last=i+1\n            break\n        if not (outPred[i]<=1001):\n            click=1\n\n    y=np.array(outPred[first:last])\n    x=np.array(list(range(first-12,last-12)))\n    xmean=np.mean(x)\n    ymean=np.mean(y)\n\n    xycov = (x - xmean) * (y - ymean)\n    xvar = (x - xmean)**2\n    beta = xycov.sum() / xvar.sum()\n    alpha = ymean - (beta * xmean)\n    line=[i*beta+alpha for i in np.array(list(range(-12,134)))]\n    return line","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Prep to test"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nloc=\"../input/osic-pulmonary-fibrosis-progression/\"\nmy_data = pd.read_csv(loc+'test.csv', delimiter=',')\nmy_data['imgloc']=\"/kaggle/input/osic-pulmonary-fibrosis-progression/test/\"+my_data.Patient\nage=list(my_data.Age)\nsex=list(my_data.Sex)\nweek=list(my_data.Weeks)\n\ncat_list = []\nmerge =my_data\nfinal_df = pd.merge(merge.groupby('Patient')['Weeks'].apply(list).to_frame(),\n                    merge.groupby('Patient')['FVC'].apply(list).to_frame(),\n                    on = 'Patient')\nfinal_df = pd.merge(final_df, merge.groupby('Patient')['Percent'].apply(list).to_frame(), on = 'Patient')\nfinal_df = pd.merge(final_df, merge.groupby('Patient')['Age'].first().to_frame(), on = 'Patient')\nfinal_df = pd.merge(final_df, merge.groupby('Patient')['Sex'].first().to_frame(), on = 'Patient')\nfinal_df = pd.merge(final_df, merge.groupby('Patient')['SmokingStatus'].first().to_frame(), on = 'Patient')\nfinal_df = pd.merge(final_df, merge.groupby('Patient')['imgloc'].first().to_frame(), on = 'Patient')\nfinal_df['modelpath']=\"../input/convmodel/\"\nfinal_df['scalerpath']=\"../input/convmodel/\"\nfinal_df = final_df.reset_index()\n\ny = np.zeros((final_df.shape[0], 146))\n\nfor index, row in final_df.iterrows():\n    y[index][np.array(row['Weeks']) + 12] = row['FVC']\n    \nage=list(final_df.Age)\nsex=list(final_df.Sex)\nweek=list(final_df.Weeks[0])\nfvc=list(final_df.FVC[0])\npercent=list(final_df.Percent)\nsmokingstatus=list(final_df.SmokingStatus)\nimgpath=list(final_df.imgloc)\nmodelpath=list(final_df.modelpath)\nscalerpath=list(final_df.scalerpath)\npatient=list(final_df.Patient)\n\n\ninputs = []\nfor index, row in final_df.iterrows():\n    x1=[]\n    x1.append(row['Age'])\n    x1.append(row['Sex'])\n    x1.append(row['Weeks'][0] + 12)\n    x1.append(row['FVC'][0])\n    x1.append(row['Percent'][0])\n    x1.append(row['SmokingStatus'])\n    x1.append(row['imgloc'])\n    x1.append(\"../input/convmodel/\")\n    x1.append(\"../input/convmodel/\")\n    x1.append(row['Patient'])\n    inputs.append(x1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model Callback  # Step 1\nmodelname='mkEnsemble'\nfolderpath      = '../working/'\nfilepath        = folderpath + \"mkEnsemble.hdf5\"\ncheckpoint      = tf.keras.callbacks.ModelCheckpoint(filepath, \n                                  monitor='val_LLL_metric2', \n                                  verbose=0, \n                                  save_best_only=True, \n                                  mode='max')\n\ncsv_logger      = tf.keras.callbacks.CSVLogger(folderpath+modelname +'.csv')                       # Step 2\ncallbacks_list  = [checkpoint,csv_logger]                                       # Step 3\n\nprint(\"Callbacks created:\")\nprint(callbacks_list[0])\nprint(callbacks_list[1])\nprint('')\nprint(\"Path to model:\", filepath)\nprint(\"Path to log:  \", folderpath+modelname+'.csv')\n#keras.utils.plot_model(model3, \"multi_input_and_output_model.png\", show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X=inputs\ny=y\n#out=stacked_prediction([X[1]])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.DataFrame()\nfor i in range(len(y)): #len(y)\n    outPred,confidenceList=stacked_prediction([X[i]])\n    outPred=np.array(outPred[0])\n    patientList=[]\n    line=createLine(outPred)\n    outPred2=[outPred[i] if np.abs(outPred[i]-line[i])<300 else line[i] for i in range(len(outPred))]\n    for j in range(len(outPred2)):\n        patientList.append(final_df.Patient[i]+'_'+str(j-12))\n    data=[]\n    for j in range(len(outPred2)):\n        data.append([patientList[j],outPred2[j],np.round(confidenceList[0][j])])\n        \n    newDF=pd.DataFrame(data,columns=['Patient_Week','FVC','Confidence'])\n    if i==0:\n        submission=newDF\n        print(0)\n    else:\n        submission=submission.append(newDF)\n        print(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}