{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pydicom\nimport os\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom sklearn.ensemble import RandomForestRegressor\nfrom skimage import morphology\nfrom skimage import measure\nfrom skimage.transform import resize\nimport tensorflow as tf\nfrom sklearn.cluster import KMeans\nimport matplotlib.patches as patches\nimport tensorflow.keras.backend as k","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# making new features"},{"metadata":{},"cell_type":"markdown","source":"FEV1 on FVC and age\nfemales:\nFEV1 = 0.77FVC + 0.28 – 0.0052age\nmales:\nFEV1 = 0.77FVC + 0.32 – 0.0069age"},{"metadata":{},"cell_type":"markdown","source":"Note: FROM THE WORD \"BASE\" I MEAN THE FIRST OBSERVATION GIVEN IN THE CSV FILES . "},{"metadata":{},"cell_type":"markdown","source":"(FVC = 0.077 H – 9.03)*1000                   #height in cm fvc in ml"},{"metadata":{"trusted":true},"cell_type":"code","source":"###########################  BASE WEEK RECORDED #################################\nbase_week=train_csv.groupby('Patient')['Weeks'].min()\nbase_week_list=[]\nfor i in range(len(train_csv)):\n    base_week_list.append(base_week[train_csv.iloc[i,0]])\ntrain_csv['base_week']=base_week_list\n\n\n###########################  COUNT FROM BASE WEEK RECORDED  #################################\nbase_week=train_csv.groupby('Patient')['Weeks'].min()\ncount_from_base_week=[]\nfor i in range(len(train_csv)):\n    count_from_base_week.append(train_csv.iloc[i,1]-base_week[train_csv.iloc[i,0]])\ntrain_csv['count_from_base_week']=count_from_base_week\n\n###########################  CONFIDENCE(FOR MODEL)  #################################\nconfidence=np.zeros(train_csv.shape[0])\ntrain_csv['confidence']=confidence\n\n###########################  BASE FVC  #################################\nbase_fvc_dict={}\nfor id in train_csv['Patient'].unique():\n    base_fvc_dict[id]=np.array(train_csv[(train_csv['Patient']==id) & (train_csv['Weeks']==base_week[id])]['FVC'])[0]\nbase_fvc=[]\nfor i in range(len(train_csv)):\n    base_fvc.append(base_fvc_dict[train_csv.iloc[i,0]])\ntrain_csv['base_fvc']=base_fvc\n\n###########################  BASE FEV1  #################################\nbase_fev1_dict={}\nfor id in train_csv['Patient'].unique():\n    A=train_csv[train_csv['Patient']==id][\"base_fvc\"].unique()[0]\n    B=train_csv[train_csv['Patient']==id][\"Age\"].unique()[0]\n    if train_csv[train_csv['Patient']==id][\"Sex\"].unique()[0]=='Male':\n        base_fev1_dict[id]=0.77*A+0.32+0.0069*B\n    else:\n        base_fev1_dict[id]=0.77*A+0.28+0.0052*B\n        \nbase_fev1=[]\nfor i in range(len(train_csv)):\n    base_fev1.append(base_fev1_dict[train_csv.iloc[i,0]])\ntrain_csv['base_fev1']=base_fev1\n\n########################## BASE WEEK PERCENT RECORDED ############################### ONLY USE IF NOT USING PERCENT COLUMN ######\nbase_week_percent_dict={}\nfor id in train_csv['Patient'].unique():\n    base_week_percent_dict[id]=np.array(train_csv[(train_csv['Patient']==id) & (train_csv['Weeks']==base_week[id])]['Percent'])[0]\n    \nbase_week_percent=[]\nfor i in range(len(train_csv)):\n    base_week_percent.append(base_week_percent_dict[train_csv.iloc[i,0]])\ntrain_csv['base_week_percent']=base_week_percent\n\n######################## BASE FEV1/FVC ####################\ntrain_csv['base fev1/base fvc']=train_csv['base_fev1']/train_csv['base_fvc']\n\n####################### BASE HEIGHT ########################\ntrain_csv['base_height']=(train_csv['base_fvc']+9030)/77.0\n\n###################### BASE WEIGHT #######################\n\nbase_weight_dict={}\nfor id in train_csv['Patient'].unique():\n    FVC=train_csv[train_csv['Patient']==id][\"base_fvc\"].unique()[0]\n    A=train_csv[train_csv['Patient']==id][\"Age\"].unique()[0]\n    H=train_csv[train_csv['Patient']==id][\"base_height\"].unique()[0]\n    if train_csv[train_csv['Patient']==id][\"Sex\"].unique()[0]=='Male':\n        base_weight_dict[id]=(FVC+5458-49*H+8*A)/12.0\n    else:\n        base_weight_dict[id]=(FVC+3863-37*H+6*A)/14.0\nbase_weight=[]\nfor i in range(len(train_csv)):\n    base_weight.append(base_weight_dict[train_csv.iloc[i,0]])\ntrain_csv['base_weight']=base_weight\n\n###################### BASE BMI ########################\ntrain_csv['base_bmi']=train_csv['base_weight']/((train_csv['base_height']/100)**2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nlb=LabelEncoder()#sex\ntrain_csv.iloc[:,5]=lb.fit_transform(train_csv.iloc[:,5])\nlb2=LabelEncoder()#ss\ntrain_csv.iloc[:,6]=lb2.fit_transform(train_csv.iloc[:,6])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\n#smoking status\noh1=OneHotEncoder(handle_unknown='ignore')\nsmoke_cat=pd.DataFrame(oh1.fit_transform(train_csv[['SmokingStatus']]).toarray(),columns=['smoking cat 0','smoking cat 1','smoking cat 2'])\ntrain_csv=pd.concat([train_csv,smoke_cat],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nsc=StandardScaler()\ntrain_scaled=pd.DataFrame(sc.fit_transform(train_csv[['Weeks','Age','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi']]),columns=['Weeks','Age','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi'])\ntrain_scaled['Sex']=train_csv['Sex']\ntrain_scaled['smoking cat 0']=train_csv['smoking cat 0']\ntrain_scaled['smoking cat 1']=train_csv['smoking cat 1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_scaled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# making submission format"},{"metadata":{},"cell_type":"markdown","source":"* FOR TEST SET:\n* WEEKS : -12,133\n* AGE : COPIED FROM SINGLE ENTRY IN TEST.CSV\n* SEX : COPIED FROM SINGLE ENTRY IN TEST.CSV\n* SMOKING STATUS : COPIED FROM SINGLE ENTRY IN TEST.CSV\n* BASE WEEK : COPIED FROM SINGLE ENTRY IN TEST.CSV\n* COUNT FROM BASE WEEK : WEEKS - BASE WEEK\n* BASE_FVC : THE ONLY FVC GIVEN IN TEST.CSV\n* BASE_FEV1 : FROM BASE FEV1\n* BASE_PERCENT : THE ONLY PERCENT GIVEN IN TEST.CSV"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\ntest_csv=pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_week=[]\npatient_id=[]\nfor i in range(len(sub)):\n    test_week.append(int(sub.iloc[i,0].split('_')[-1]))\n    patient_id.append(sub.iloc[i,0].split('_')[0])\nsub['Patient']=patient_id\nsub['Weeks']=test_week\nsub.drop(['FVC','Confidence'],axis=1,inplace=True)\n\n########## BASE FVC ##########\nbase_fvc=test_csv.groupby('Patient')['FVC'].min()\nfvc=[]\nfor i in range(len(sub)):\n    fvc.append(base_fvc[sub.iloc[i,1]])\nsub['base_fvc']=fvc\n\n######### BASE FEV1 ##########\nbase_fev1_dict_test={}\nfor id in sub['Patient'].unique():\n    A=sub[sub['Patient']==id][\"base_fvc\"].unique()[0]\n    B=test_csv[test_csv['Patient']==id][\"Age\"].unique()[0]\n    if test_csv[test_csv['Patient']==id][\"Sex\"].unique()[0]=='Male':\n        base_fev1_dict_test[id]=0.77*A+0.32+0.0069*B\n    else:\n        base_fev1_dict_test[id]=0.77*A+0.28+0.0052*B\n        \nbase_fev1_test=[]\nfor i in range(len(sub)):\n    base_fev1_test.append(base_fev1_dict_test[sub.iloc[i,1]])\nsub['base_fev1']=base_fev1_test\n\n###############################\n\nsub['base_height']=(sub['base_fvc']+9030)/77.0\n\n##############################\n\n\nbase_weight_dict_test={}\nfor id in sub['Patient'].unique():\n    FVC=sub[sub['Patient']==id][\"base_fvc\"].unique()[0]\n    A=test_csv[test_csv['Patient']==id][\"Age\"].unique()[0]\n    H=sub[sub['Patient']==id][\"base_height\"].unique()[0]\n    if test_csv[test_csv['Patient']==id][\"Sex\"].unique()[0]=='Male':\n        base_weight_dict_test[id]=(FVC+5458-49*H+8*A)/12.0\n    else:\n        base_weight_dict_test[id]=(FVC+3863-37*H+6*A)/14.0\nbase_weight_test=[]\nfor i in range(len(sub)):\n    base_weight_test.append(base_weight_dict_test[sub.iloc[i,1]])\nsub['base_weight']=base_weight_test\n\n##############################\n\ntest_csv.iloc[:,5]=lb.transform(test_csv.iloc[:,5])\ntest_csv.iloc[:,6]=lb2.transform(test_csv.iloc[:,6])\n\n##############################\npercent_dict={}\nfor id in test_csv['Patient'].unique():\n    percent_dict[id]=float(test_csv[test_csv['Patient']==id]['Percent'])\n    \nsex_dict={}\nfor id in test_csv['Patient'].unique():\n    sex_dict[id]=int(test_csv[test_csv['Patient']==id]['Sex'])\n\nage_dict={}\nfor id in test_csv['Patient'].unique():\n    age_dict[id]=int(test_csv[test_csv['Patient']==id]['Age'])\n    \nss_dict={}\nfor id in test_csv['Patient'].unique():\n    ss_dict[id]=int(test_csv[test_csv['Patient']==id]['SmokingStatus'])\n\npercent=[]\nsex=[]\nage=[]\nss=[]\nfor i in range(len(sub)):\n    percent.append(percent_dict[sub.iloc[i,1]])\n    sex.append(sex_dict[sub.iloc[i,1]])\n    age.append(age_dict[sub.iloc[i,1]])\n    ss.append(ss_dict[sub.iloc[i,1]])    \nsub['base_week_percent']=percent\nsub['Age']=age\nsub['Sex']=sex\nsub['SmokingStatus']=ss\n\n###############################\n\nbase_week_test=test_csv.groupby('Patient')['Weeks'].min()\ncount_from_base_week_test=[]\nbase_week=[]\nfor i in range(len(sub)):\n    count_from_base_week_test.append(sub.iloc[i,2]-base_week_test[sub.iloc[i,1]])\n    base_week.append(base_week_test[sub.iloc[i,1]])\nsub['count_from_base_week']=count_from_base_week_test\nsub['base_week']=base_week\n\n###############################\n\nsub['base fev1/base fvc']=sub['base_fev1']/sub['base_fvc']\n\n###############################\nsub['base_bmi']=sub['base_weight']/((sub['base_height']/100.0)**2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"smoke_cat_test=pd.DataFrame(oh1.transform(sub[['SmokingStatus']]).toarray(),columns=['smoking cat 0','smoking cat 1','smoking cat 2'])\nsub=pd.concat([sub,smoke_cat_test],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_scaled=pd.DataFrame(sc.transform(sub[['Weeks','Age','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi']]),columns=['Weeks','Age','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi'])\nsub_scaled['Sex']=sub['Sex']\nsub_scaled['smoking cat 0']=sub['smoking cat 0']\nsub_scaled['smoking cat 1']=sub['smoking cat 1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_scaled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x=np.array(train_csv[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi','smoking cat 0','smoking cat 1']])\n#x=np.array(train_csv[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_week_percent','base_height','smoking cat 0','smoking cat 1']])\n#x=np.array(train_scaled[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_week_percent','base_height','smoking cat 0','smoking cat 1']])\nx=np.array(train_scaled[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi','smoking cat 0','smoking cat 1']])\ny=np.array(train_csv[['FVC','confidence']])\n\nfrom sklearn.model_selection import train_test_split\nxtrain,xvalid,ytrain,yvalid=train_test_split(x,y,test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric(actual_fvc, predicted_fvc, confidence, return_values = False):\n    \"\"\"\n    Calculates the modified Laplace Log Likelihood score for this competition.\n    \"\"\"\n    sd_clipped = np.maximum(confidence, 70)\n    delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)\n    metric = - np.sqrt(2) * delta / sd_clipped - np.log(np.sqrt(2) * sd_clipped)\n\n    if return_values:\n        return metric\n    else:\n        return np.mean(metric)\n    \n    \ndef model_loss(ytrue,ypred):   # this loss penalises both prediction and confidence value\n    eps=1.0\n    \n    fvc_pred=ypred[:,0]\n    sigmas=ypred[:,1]+eps     # so as to avoid log(0) . these are predicted connfidences\n    \n    ans=tf.math.log(sigmas)\n    ans=ans+((ytrue[:,0]-fvc_pred)**2)/(2*sigmas**2)\n    \n    return tf.reduce_mean(ans)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n\ndef score(y_true, y_pred):\n    \n    sigma = y_pred[:,1]\n    fvc_pred = y_pred[:,0]\n    \n\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:,0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt(tf.dtypes.cast(2, dtype=tf.float32))\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return tf.reduce_mean(metric)\n\ndef huber_loss(y_true, y_pred):\n    \n    error = y_true[:,0] - y_pred[:,0]\n    is_small_error = tf.abs(error) <= 1000.0\n    quad_loss = tf.square(error) / 2\n    linear_loss = 1000*tf.abs(error) - tf.square(C2)*0.5\n    return tf.reduce_mean(tf.where(is_small_error, quad_loss, linear_loss))\n\n\ndef custom_loss(y_true, y_pred):\n    return 0.01*huber_loss(y_true, y_pred) + score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_scheduler=tf.keras.callbacks.ReduceLROnPlateau(factor=0.2,monitor='val_loss',mode='min',patience=150,verbose=0)\nclass best_weights(tf.keras.callbacks.Callback):\n    def __init__(self):\n        self.metric_op=-30.0\n        self.weights_op=None\n        self.epoch_op=-1\n    def on_epoch_end(self,epoch,logs={}):\n        if logs['val_metric']>=self.metric_op:\n            self.metric_op=logs['val_metric']\n            self.epoch_op=epoch\n            self.weights_op=self.model.get_weights()\n    def on_train_end(self,logs={}):\n        self.model.set_weights(self.weights_op)\n        print('BEST_EPOCH = {}   BEST_SCORE_ON_VALID_SET = {}'.format(self.epoch_op+1,self.metric_op))\n        \n        \n\nclass metrics_call(tf.keras.callbacks.Callback):\n    def __init__(self,mertic,xtrain,ytrain,xvalid,yvalid):\n        self.metric=metric\n        self.xtrain=xtrain\n        self.ytrain=ytrain\n        self.xvalid=xvalid\n        self.yvalid=yvalid\n        \n    def on_epoch_end(self,epoch,logs={}):\n        train_preds=self.model.predict(self.xtrain)\n        val_preds=self.model.predict(self.xvalid)\n        #print('\\r  metric on train set: ',self.metric(self.ytrain[:,0],train_preds[:,0],train_preds[:,1]),end='')\n        logs['val_metric']=self.metric(self.yvalid[:,0],val_preds[:,0],val_preds[:,1])\n        #print('  metric on valid set: ',self.metric(self.yvalid[:,0],val_preds[:,0],val_preds[:,1]))\n        \n        \n\ndef run_model(xtrain,ytrain,xvalid,yvalid,epoch=50):\n    input=tf.keras.layers.Input(shape=xtrain.shape[1:])\n    noisy=tf.keras.layers.GaussianNoise(0.6)(input)\n    \n    d1=tf.keras.layers.Dense(128,activation='relu')(noisy)\n    d2=tf.keras.layers.Dense(128,activation='relu')(d1)\n    d3=tf.keras.layers.Dense(128,activation='relu')(d2)\n    mean_out1=tf.keras.layers.Dense(1)(d3)\n    std_den1=tf.keras.layers.Dense(1)(d3)\n    \n    d4=tf.keras.layers.Dense(128,activation='relu')(noisy)\n    d5=tf.keras.layers.Dense(128,activation='relu')(d4)\n    d6=tf.keras.layers.Dense(128,activation='relu')(d5)\n    mean_out2=tf.keras.layers.Dense(1)(d6)\n    std_den2=tf.keras.layers.Dense(1)(d6)\n    \n    d7=tf.keras.layers.Dense(128,activation='relu')(noisy)\n    d8=tf.keras.layers.Dense(128,activation='relu')(d7)\n    d9=tf.keras.layers.Dense(128,activation='relu')(d8)\n    mean_out3=tf.keras.layers.Dense(1)(d9)\n    std_den3=tf.keras.layers.Dense(1)(d9)\n    \n    mean_combine=tf.keras.layers.Concatenate()([mean_out1,mean_out2,mean_out3])\n    std_combine=tf.keras.layers.Concatenate()([std_den1,std_den2,std_den3])\n    mean_final=tf.keras.layers.Dense(1)(mean_combine)\n    std_final_den=tf.keras.layers.Dense(1)(std_combine)\n    std_final=tf.keras.layers.Lambda(lambda x: tf.abs(x))(std_final_den)\n    output=tf.keras.layers.Concatenate()([mean_final,std_final])\n    model=tf.keras.models.Model(inputs=input,outputs=output)\n    \n    model.compile(loss=lambda ytrue,ypred: custom_loss(ytrue,ypred),optimizer=tf.keras.optimizers.Adam(lr=0.0001))\n    print(model.summary())\n    history=model.fit(xtrain,ytrain,epochs=epoch,batch_size=256,validation_data=(xvalid,yvalid),verbose=0,callbacks=[metrics_call(metric,xtrain,ytrain,xvalid,yvalid),best_weights()])\n    pd.DataFrame(history.history).plot(figsize=(8, 5))\n    plt.ylim(-10,10)\n    plt.grid(True)\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=run_model(xtrain,ytrain,xvalid,yvalid,epoch=1000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#xtest=np.array(sub[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi','smoking cat 0','smoking cat 1']])\n#xtest=np.array(sub[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_week_percent','base_height','smoking cat 0','smoking cat 1']])\nxtest=np.array(sub_scaled[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi','smoking cat 0','smoking cat 1']])\n#xtest=np.array(sub_scaled[['Weeks','Age','Sex','base_week','count_from_base_week','base_fvc','base_week_percent','base_height','smoking cat 0','smoking cat 1']])\n\nyans=model.predict(xtest)\n\nsub['FVC']=yans[:,0]\nsub['Confidence']=yans[:,1]\nsub.drop(['Patient','Weeks','Age','Sex','SmokingStatus','base_week','count_from_base_week','base_fvc','base_fev1','base_week_percent','base fev1/base fvc','base_height','base_weight','base_bmi','smoking cat 0','smoking cat 1','smoking cat 2'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}