{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport pydicom # this one is to read the dicom files \nimport os \nimport scipy.ndimage\nimport matplotlib.pyplot as plt \nimport sklearn\nfrom sklearn.preprocessing import normalize\nfrom tqdm.auto import tqdm \n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom skimage import measure, morphology \n#from mpl_toolkits.mplot3d.art3d import Poly3DCollection\nfrom sklearn.preprocessing import normalize\n\n#from mpl_toolkits.mplot3d.art3d import Poly3DCollection \nfrom torch.utils.data import DataLoader \nfrom torch.utils.data import TensorDataset \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def csv_preprocess (data):\n    \n    #Healthy FVC\n    data['Healthy-FVC']=round((data['FVC']*100)/data['Percent'])\n    FE=[]\n    FE.append('Healthy-FVC')\n    \n    #Create Male, Female, Ex-smoker, Current-smoker and Never smoked\n    COLS = ['Sex','SmokingStatus']\n    for col in COLS:\n        for mod in data[col].unique():\n            FE.append(mod)\n            data[mod] = (data[col] == mod).astype(int)\n    \n    data =  data[['Patient','Weeks','FVC','Age']+FE]\n    data = data.sort_values(['Patient','Weeks'], ascending=True).reset_index(drop=True)\n    \n    FE1=['Male','Female','Ex-smoker','Never smoked','Currently smokes']\n    #Rename base_Weeks and base_FVC\n    rename_col={'Weeks':'base_Weeks','FVC':'base_FVC'}\n    data=data.rename(columns=rename_col)\n    \n    #Weeks biasing Week=-12 to Week =0 and Week = 133 to Week = 145\n    #data.base_Weeks+=t_min_week\n    #Add new fields Week and actual_FCV\n    npData=pd.DataFrame(columns=['Patient','base_Weeks','base_FVC','Age']+FE1+['Week','Healthy-FVC','actual_FVC'])\n    \n    for pid in data['Patient'].unique():\n        weeks=data.loc[data['Patient']==pid].base_Weeks\n        fvc = data.loc[data['Patient']==pid].base_FVC\n        index = data.loc[data['Patient']==pid].index\n        weeks.reset_index(inplace = True, drop = True)\n        fvc.reset_index(inplace = True, drop = True)\n        for k in range(len(weeks)):\n            npData=pd.concat([npData,data.loc[data.index==index[0]]], sort=False)\n            npData.iloc[-1, npData.columns.get_loc('Week')]=weeks[k]\n            npData.iloc[-1, npData.columns.get_loc('actual_FVC')]=fvc[k]\n    npData.reset_index(inplace = True, drop = True)\n    npData=npData.fillna(0)\n    \n#     npData['Week']=npData['Week']-npData['base_Weeks']\n#     npData['base_Weeks']=0.0\n    #Random Shuffle\n    #npData=sklearn.utils.shuffle(npData)\n    #npData.reset_index(inplace = True, drop = True)\n    \n    return npData","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #################################################### Network Model########################################\n# class DATA(nn.Module):\n# \tdef __init__(self):\n# \t\tsuper(DATA, self).__init__()\n# \t\tself.data_net1=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(10,42),\n# \t\t\t\t\t\tnn.ReLU(),            \n# \t\t\t\t\t\tnn.Linear(42,64),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(64,118),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t#nn.Dropout(0.5)\n# \t\t\t\t\t\t)\n# \t\tself.data_net2=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(128,256),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(256,502),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t#nn.Dropout(0.5)\n# \t\t\t\t\t\t)\n# \t\tself.data_net3=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(512,256),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(256,118),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t#nn.Dropout(0.5)\n# \t\t\t\t\t\t)\n# \t\tself.data_net4=nn.Sequential(\n# \t\t\t\t\t\tnn.Linear(748,256),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\tnn.Linear(256,64),\n# \t\t\t\t\t\tnn.ReLU(),\n# \t\t\t\t\t\t#nn.Dropout(0.5),\n# \t\t\t\t\t\tnn.Linear(64,2),\n# \t\t\t\t\t\tnn.ReLU()\n# \t\t\t\t\t\t)\n\n# \tdef forward(self, data_i):\n# \t\tout1 = self.data_net1(data_i)\n# \t\tout2 = torch.cat((data_i,out1), dim=-1)\n# \t\tout2 = self.data_net2(out2)\n# \t\tout3 = torch.cat((data_i,out2), dim=-1)\n# \t\tout3 = self.data_net3(out3)\n# \t\tout4 = torch.cat((data_i,out1,out2,out3), dim=-1)\n# \t\tout = self.data_net4(out4)\n# \t\treturn out\n\n#################################################### Network Model########################################\nclass DATA(nn.Module):\n\tdef __init__(self):\n\t\tsuper(DATA, self).__init__()\n\t\tself.data_net1=nn.Sequential(\n\t\t\t\t\t\tnn.Linear(10,32),\n\t\t\t\t\t\tnn.ReLU(),            \n\t\t\t\t\t\tnn.Linear(32,64),\n\t\t\t\t\t\tnn.ReLU(),\n\t\t\t\t\t\t#nn.Dropout(0.5),\n\t\t\t\t\t\tnn.Linear(64,128),\n\t\t\t\t\t\tnn.ReLU(),\n\t\t\t\t\t\tnn.Linear(128,64),\n\t\t\t\t\t\tnn.ReLU(),\n\t\t\t\t\t\tnn.Linear(64,2),\n\t\t\t\t\t\tnn.ReLU()\n\t\t\t\t\t\t#nn.Dropout(0.5)\n\t\t\t\t\t\t)\n\n\tdef forward(self, data_i):\n\t\tout1 = self.data_net1(data_i)\n\t\treturn out1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = torch.tensor(70, dtype=torch.float32), torch.tensor(1000, dtype= torch.float32)\n\ndef score(y_true, y_pred):\n\t# tf.dtypes.cast(y_true, tf.float32)\n\t# tf.dtypes.cast(y_pred, tf.float32)\n\tsigma = y_pred[:, 0]\n\tfvc_pred = y_pred[:, 1]\n\t\n\t#sigma_clip = sigma + C1\n\t#sigma_clip = tf.maximum(sigma, C1)\n\t#c1_same_shape = torch.ones(sigma.size(),device=y_pred.device)*C1 \n\t#sigma_clip = torch.max(sigma, c1_same_shape)\n\tdelta = (y_true[:, 0] - fvc_pred).abs()\n\n\t#c2_same_shape = torch.ones(delta.size(),device=y_pred.device)*C2 \n\t#delta = torch.min(delta, c2_same_shape)\n\n\tsq2 = torch.tensor(2.).sqrt()\n\tmetric = (delta / sigma)*sq2 + (sigma* sq2).log()\n\treturn (metric).mean()\n\n\n\ndef quartile_loss(y_true, y_pred): # 0.65 \n\t#def loss(y_true, y_pred): # (batch_size, 1), (batch_size , 3)\n\tloss = score(y_true, y_pred) # 0.35\n\treturn loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#     return npEval\n######### ----------------- Making the Evaluation Data -------------------- #######\ndef make_eval_data(npEval, model, device = 'cuda'):\n    x_features = npEval[['base_Weeks','base_FVC','Age','Male','Female','Ex-smoker','Never smoked','Currently smokes','Week','Healthy-FVC']]\n    x_features = torch.tensor(x_features.values).float()\n    x_patientids_name = npEval[['Patient']].values\n    \n\n    #y_labels = npEval['actual_FVC'].values \n    #patientsID = npEval['Patient'].values\n    \n    \n    \n    if torch.cuda.is_available() and device == 'cuda':\n        model.to('cuda')\n    model.eval()\n    predictions = []\n    #raw_submission_df = pd.DataFrame([]) \n    for i, patientid in enumerate(x_patientids_name):\n\n        x_feature = x_features[i] # taking the feature for the particular patientid and week of npEval dataframe \n        x_feature = x_feature.unsqueeze(0)\n        \n        \n        if torch.cuda.is_available() and device == 'cuda':\n            x_feature = x_feature.cuda()\n            \n        prediction = model(x_feature)\n        predictions.append(prediction.to('cpu').detach().numpy()[0]) # since it returns batched output , and we use batch_size =1, taking the [0] output \n    \n    predictions = np.array(predictions) # shape = [none, 3]\n    npEval['FVC'] = predictions[:,1]\n    npEval['Confidence'] = predictions[:,0]\n    \n    return npEval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ndata_test = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\nsubmission  = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['Patient']=submission['Patient_Week'].apply(lambda x:x.split('_')[0])\nsubmission['Weeks']=submission['Patient_Week'].apply(lambda x:x.split('_')[1]).astype(int)\nsubmission=submission.sort_values(by=['Patient','Weeks'], ascending=True ).reset_index(drop=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merge=pd.merge(data_test,submission,on=['Patient'],how='left').sort_values(['Patient','Weeks_y']).reset_index(drop=True)\nmerge=merge.drop(['FVC_y'],axis=1)\nmerge=merge.rename(columns={'FVC_x':'base_FVC','Weeks_y':'Week','Weeks_x':'base_Weeks'})\n\ndel data_test\ndel submission\n\ndata_test=merge.loc[:,['Patient','base_Weeks','base_FVC','Percent','Age','Sex','SmokingStatus','Week']]\nsubmission=merge.loc[:,['Patient_Week','base_FVC','Confidence']]\nsubmission=submission.rename(columns={'base_FVC':'FVC'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data_test.copy()\ndata['Healthy-FVC']=round((data['base_FVC']*100)/data['Percent'])\nFE=[]\nFE.append('Healthy-FVC')\n\n#Create Male, Female, Ex-smoker, Current-smoker and Never smoked\nCOLS = ['Sex','SmokingStatus']\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\nFE1=['Male','Female','Ex-smoker','Never smoked','Currently smokes']\nnpData=pd.DataFrame(columns=['Patient','base_Weeks','base_FVC','Age','Healthy-FVC']+FE1+['Week'])\nnpData=npData.append(data, sort=True)\nnpData=npData.fillna(0)\n\n# npData['Week']=npData['Week']-npData['base_Weeks']\n# npData['base_Weeks']=0.0\n\ndel data_test, data\ndata_test = npData[['Patient','base_Weeks','base_FVC','Age','Healthy-FVC','Male','Female','Ex-smoker','Never smoked','Currently smokes','Week']]\ndel npData   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = DATA()\nmodel.load_state_dict(torch.load('../input/meta-42/Epoch42_Score6.829206974134532_Acc0.9300930269189462.pth'))\n#model.eval()\n# for the validation inputset of sigma \n# ndf = pd.concat([npValid,npTest])\n# outdf = make_eval_data(df_test, model)\n# for the training inputset of sigma \n#train_inp_sigma = make_eval_data(df_train, model)\n# for the final test inputset of sigma\ntest = make_eval_data(data_test.copy(), model)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# FVC and confidence correction at base week \nfor nid in test.Patient.unique():\n    index = test[(test.Patient==nid) & (test.Week==test.base_Weeks)].index.values\n    test.iloc[index[0],test.columns.get_loc('FVC')]=test.iloc[index[0],test.columns.get_loc('base_FVC')]\n    test.iloc[index[0],test.columns.get_loc('Confidence')] = 70","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Week bias correction and confidence clipping at 70\n# test.base_Weeks += -12\n# test.Week += -12\ntest.loc[test.Confidence < 70, 'Confidence' ] = 70","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.loc[:, 'FVC']=test.FVC\nsubmission.loc[:, 'Confidence']= test.Confidence\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}