{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nimport pydicom # this one is to read the dicom files \nimport scipy.ndimage\nimport matplotlib.pyplot as plt \nimport sklearn\nfrom sklearn.preprocessing import normalize\nfrom tqdm.auto import tqdm \n\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom skimage import measure, morphology \n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### CSV Functions\ndef csv_split (data, v, t):\n    \n    #remove duplicate\n    data.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\n    \n    #Drop rows with patientids if needed\n    drop_patientID = ['']\n    for i in drop_patientID:\n        ind=data.Patient[data.Patient == i ].index.tolist()\n        for j in ind:\n            #print(j)\n            data=data.drop([j], axis=0)\n    data.reset_index(inplace = True, drop = True)\n    \n    #split\n    unique_patient=data.Patient.unique()\n    unique_patient_val=unique_patient[-v:]\n    unique_patient_test=unique_patient[-(v+t):-v]\n    unique_patient_train=unique_patient[:-(v+t)]\n    \n    valid=pd.DataFrame()\n    for id in unique_patient_val:\n        valid_x=data.loc[data['Patient']==id]\n        valid=pd.concat([valid,valid_x])\n    test=pd.DataFrame()\n    for id in unique_patient_test:\n        test_x=data.loc[data['Patient']==id]\n        test=pd.concat([test,test_x])\n    train=pd.DataFrame()\n    for id in unique_patient_train:\n        train_x=data.loc[data['Patient']==id]\n        train=pd.concat([train,train_x])\n    \n    valid.reset_index(inplace = True, drop = True)\n    test.reset_index(inplace = True, drop = True)\n    train.reset_index(inplace = True, drop = True)\n    \n    return train, valid, test\n\ndef csv_preprocess (data):\n    \n    #Healthy FVC\n    data['Healthy-FVC']=round((data['FVC']*100)/data['Percent'])\n    FE=[]\n    FE.append('Healthy-FVC')\n    \n    #Create Male, Female, Ex-smoker, Current-smoker and Never smoked\n    COLS = ['Sex','SmokingStatus']\n    for col in COLS:\n        for mod in data[col].unique():\n            FE.append(mod)\n            data[mod] = (data[col] == mod).astype(int)\n    \n    data =  data[['Patient','Weeks','FVC','Age']+FE]\n    \n    FE1=['Male','Female','Ex-smoker','Never smoked','Currently smokes']\n    #Rename base_Weeks and base_FVC\n    rename_col={'Weeks':'base_Weeks','FVC':'base_FVC'}\n    data=data.rename(columns=rename_col)\n    \n    #Weeks biasing Week=-12 to Week =0 and Week = 133 to Week = 145\n    data.base_Weeks+=12\n    #Add new fields Week and actual_FCV\n    npData=pd.DataFrame(columns=['Patient','base_Weeks','base_FVC','Age','Healthy-FVC']+FE1+['Week','actual_FVC'])\n    \n    for pid in data['Patient'].unique():\n        weeks=data.loc[data['Patient']==pid].base_Weeks\n        fvc = data.loc[data['Patient']==pid].base_FVC\n        index = data.loc[data['Patient']==pid].index\n        weeks.reset_index(inplace = True, drop = True)\n        fvc.reset_index(inplace = True, drop = True)\n        for j in index:\n            for k in range(len(weeks)):\n                if (weeks[k] == data.at[j,'base_Weeks']):\n                    continue\n                else:\n                    npData=pd.concat([npData,data.loc[data.index==j]], sort=False)\n                    npData.iloc[-1, npData.columns.get_loc('Week')]=weeks[k]\n                    npData.iloc[-1, npData.columns.get_loc('actual_FVC')]=fvc[k]\n    npData.reset_index(inplace = True, drop = True)\n    npData=npData.fillna(0)\n    \n    #Random Shuffle\n    npData=sklearn.utils.shuffle(npData)\n    npData.reset_index(inplace = True, drop = True)\n    \n    return npData","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"###Evaluation Metric Function\ndef laplace_log_likelihood(actual_fvc, predicted_fvc, confidence, return_values = False):\n    \"\"\"\n    Calculates the modified Laplace Log Likelihood score for this competition.\n    \"\"\"\n    sd_clipped = np.maximum(confidence, 70)\n    delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)\n    metric = - np.sqrt(2) * delta / sd_clipped - np.log(np.sqrt(2) * sd_clipped)\n\n    if return_values:\n        return metric\n    else:\n        return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigma_generator (data):\n    confidence=np.arange(70,1000,1)\n    data['actual_sigma']=np.nan\n    FVC=data['actual_FVC'].values\n    Pred=data['Prediction'].values\n    #data=pd.DataFrame(columns=['prediction_FVC', 'sigma'])\n    for j in range(len(FVC)):\n        score=laplace_log_likelihood(FVC[j], Pred[j], confidence, return_values = True)\n        ind=np.where(score == score.max())\n        i = int(ind[0])\n        actual_sigma=confidence[i]\n        data.at[j, 'actual_sigma']= actual_sigma\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DATA(nn.Module):\n\tdef __init__(self):\n\t\tsuper(DATA, self).__init__()\n\t\t\n\t\tself.layer1 = nn.Linear(10,64)\n\t\tself.layer2 = nn.ReLU()\n\t\t#nn.Dropout(0.5),\n\t\tself.layer3 = nn.Linear(64,128)\n\t\tself.layer4 = nn.ReLU()\n\t\t#nn.Dropout(0.5),\n\t\tself.layer5 = nn.Linear(128,256)\n\t\tself.layer6 = nn.ReLU()\n\t\t#nn.Dropout(0.5),\n\t\tself.layer7 = nn.Linear(256,512)\n\t\tself.layer8 = nn.ReLU()\n\t\t#nn.Dropout(0.5),\n\t\tself.layer9 = nn.Linear(512,512)\n\t\tself.layer10 = nn.ReLU()\n\t\t### \n\t\tself.layer11 = nn.Linear(512,512)\n\t\tself.layer12 = nn.ReLU()\n\t\tself.layer13 = nn.Linear(512, 512)\n\t\tself.layer14 = nn.ReLU()\n\t\t### \n\t\t#nn.Dropout(0.5),\n\t\tself.layer15 = nn.Linear(512,128)\n\t\tself.layer16 = nn.ReLU()\n\t\t#nn.Dropout(0.5),\n\t\tself.layer17 = nn.Linear(128,64)\n\t\tself.layer18 = nn.ReLU()\n\t\t#nn.Dropout(0.5),\n\t\tself.layer19 = nn.Linear(64,1)\n\t\tself.layer20 = nn.ELU()\n\n\tdef forward(self, x):\n\t\tx = self.layer1(x)\n\t\tx = self.layer2(x)\n\t\tx = self.layer3(x)\n\t\tx = self.layer4(x)\n\t\tx = self.layer5(x)\n\t\tx = self.layer6(x)\n\t\tx =x1= self.layer7(x)\n\t\t#print(x.shape)\n\t\tx = self.layer8(x)\n\t\tx = self.layer9(x)\n\t\tx = self.layer10(x)\n\t\tx = self.layer11(x)\n\n\t\tx = self.layer12(x)\n\t\tx = self.layer13(x)\n\t\tx = self.layer14(x)\n\t\tx = x+x1  \n\t\t#print(x1.shape)\n\t\tx = self.layer15(x)\n\t\tx = self.layer16(x)\n\t\tx = self.layer17(x)\n\t\tx = self.layer18(x)\n\t\tx = self.layer19(x)\n\t\tx = self.layer20(x)\n\t\t#x = self.layer21(x)\n\t\treturn x\n\nclass SIGMA(nn.Module):\n    def __init__(self):\n        super(SIGMA, self).__init__()\n        self.data_net1=nn.Sequential(\n                        nn.Linear(10,64),\n                        nn.ReLU(),\n                        #nn.Dropout(0.5),\n                        nn.Linear(64,118),\n                        nn.ReLU()\n                        #nn.Dropout(0.5)\n                        )\n        self.data_net2=nn.Sequential(\n                        nn.Linear(128,256),\n                        nn.ReLU(),\n                        #nn.Dropout(0.5),\n                        nn.Linear(256,502),\n                        nn.ReLU()\n                        #nn.Dropout(0.5)\n                        )\n        self.data_net3=nn.Sequential(\n                        nn.Linear(512,256),\n                        nn.ReLU(),\n                        #nn.Dropout(0.5),\n                        nn.Linear(256,118),\n                        nn.ReLU()\n                        #nn.Dropout(0.5)\n                        )\n        self.data_net4=nn.Sequential(\n                        nn.Linear(748,64),\n                        nn.ReLU(),\n                        #nn.Dropout(0.5),\n                        nn.Linear(64,1),\n                        nn.ReLU()\n                        )\n\n    def forward(self, data_i):\n        out1 = self.data_net1(data_i)\n        out2 = torch.cat((data_i,out1), dim=-1)\n        out2 = self.data_net2(out2)\n        out3 = torch.cat((data_i,out2), dim=-1)\n        out3 = self.data_net3(out3)\n        out4 = torch.cat((data_i,out1,out2,out3), dim=-1)\n        out = self.data_net4(out4)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"######## Meta Data Model Trainer ##############################\n\ndef train_data_net(epochs, batch_size, npTrain,npValid, model, train_device ='cpu'):\n\n\n    x_train_values_df = npTrain[['base_Weeks', 'base_FVC', 'Age', 'Male', 'Female', 'Ex-smoker','Never smoked', 'Currently smokes', 'Week', 'Healthy-FVC']]\n    #x_train_values_df = npTrain.drop(['Patient','actual_FVC'], axis = 1) # dataframw without patientId \n    x_train_values = x_train_values_df.values # ndarray of train metadata \n    y_train_values = npTrain['actual_FVC'].values # ndarray of metadata label \n\n    x_valid_values_df = npValid[['base_Weeks', 'base_FVC', 'Age', 'Male', 'Female', 'Ex-smoker','Never smoked', 'Currently smokes', 'Week', 'Healthy-FVC']] # dataframw without patientId \n    x_valid_values = x_valid_values_df.values # ndarray of train metadata \n    y_valid_values = npValid['actual_FVC'].values # ndarray of metadata label\n    \n    if train_device =='cuda':\n        device = torch.device(\"cuda\")\n        model.to(device)\n\n    \n    for epoch in range(epochs):\n        #cudnn.benchmark = True\n        torch.backends.cudnn.benchmark = True\n\n        ## Training \n        n = len(x_train_values)\n        model.train()\n        #pbar = tqdm(range((n-1)// batch_size +1), total = (n-1)// batch_size +1)\n        Steps = (n-1)// batch_size +1\n        pbar = tqdm(range(Steps), total= Steps)\n        for i in pbar:     \n            #range((n-1)// batch_size +1):\n            # path = processed_img_dir + os.sep + patientId\n            # image_3darray = load_array(path)\n            start_i = i * batch_size\n            end_i = start_i + batch_size  \n            xb_meta =  torch.tensor(x_train_values[start_i:end_i]).float()\n            Y_target = torch.tensor(y_train_values[start_i:end_i]).float().unsqueeze(1)\n            #print(Y_target)\n            #print(Y_target.shape) \n\n            if train_device == 'cuda':\n                xb_meta = xb_meta.cuda()\n                Y_target = Y_target.cuda()\n            prediction = model(xb_meta)\n            loss = compute_loss(prediction, Y_target)\n\n            loss.backward()\n            optimizer.step()\n            with torch.no_grad():\n                accuracy =(1- ((prediction- Y_target)/Y_target).abs())\n\n            s = ('Epochs: %5d/%d , Steps: %8d/%d , train_loss: %5.3f  ,trian_accuracy: %5.3f'%\\\n                  (epoch, epochs, i, Steps, loss.data, accuracy.data.item()))\n            pbar.set_description(s)\n            optimizer.zero_grad()\n            del prediction\n             \n            \n             \n\n\n        ## Validation\n        val_acc_total = 0.\n        n = len(x_valid_values)\n        Steps = (n-1)// batch_size +1\n        pbar = tqdm(range(Steps), total= Steps)\n        model.eval()\n        for i in pbar:  #range((n-1)//batch_size +1)\n            start_i = i * batch_size\n            end_i = start_i + batch_size \n            xb_meta =  torch.tensor(x_valid_values[start_i:end_i]).float()\n            Y_target = torch.tensor(y_valid_values[start_i:end_i]).float().unsqueeze(1)\n            \n            if train_device == 'cuda':\n                xb_meta = xb_meta.cuda()\n                Y_target = Y_target.cuda()\n            prediction = model(xb_meta)\n            loss = compute_loss(prediction, Y_target)\n            with torch.no_grad():\n                accuracy =(1- ((prediction- Y_target)/Y_target).abs())\n            \n\n            s = ('Epochs: %5d/%d , Steps: %8d/%d , val_loss: %5.3f  ,val_accuracy: %5.3f'%\\\n                  (epoch, epochs, i, Steps, loss.data, accuracy.data.item()))\n            pbar.set_description(s)\n            \n            val_acc_total += accuracy.data.item()\n            del prediction\n        avg_val_acc = (val_acc_total)/n \n        print('Average Validation accuracy:', avg_val_acc)\n             \n        #PATH = 'Epoch'+'%s_'%epoch+'%s'%avg_val_acc + '.pth'\n        #torch.save(model.state_dict(), './checkpoint_data_models'+os.sep+PATH)\n\n######## Sigma Model Trainer ##############################\n\ndef train_sigma_net(epochs, batch_size, npTrain, npValid, model, train_device ='cpu'):\n\n\n    \n    x_train_values_df = npTrain[['base_Weeks','base_FVC','Age','Male','Female','Ex-smoker','Never smoked','Currently smokes','Healthy-FVC','Prediction']]\n    x_train_values = x_train_values_df.values\n    y_train_values = npTrain['actual_sigma'].values\n\n    x_valid_values_df = npValid[['base_Weeks','base_FVC','Age','Male','Female','Ex-smoker','Never smoked','Currently smokes','Healthy-FVC','Prediction']]\n    x_valid_values = x_valid_values_df.values\n    y_valid_values = npValid['actual_sigma'].values\n    \n    if train_device =='cuda':\n        device = torch.device(\"cuda\")\n        model.to(device)\n\n    \n    for epoch in range(epochs):\n        #cudnn.benchmark = True\n        torch.backends.cudnn.benchmark = True\n\n        ## Training \n        n = len(x_train_values)\n        model.train()\n        #pbar = tqdm(range((n-1)// batch_size +1), total = (n-1)// batch_size +1)\n        Steps = (n-1)// batch_size +1\n        pbar = tqdm(range(Steps), total= Steps)\n        for i in pbar:     #range((n-1)// batch_size +1):\n            # path = processed_img_dir + os.sep + patientId\n            # image_3darray = load_array(path)\n            start_i = i * batch_size\n            end_i = start_i + batch_size  \n            xb_meta =  torch.tensor(x_train_values[start_i:end_i]).float()\n            Y_target = torch.tensor(y_train_values[start_i:end_i]).float().unsqueeze(1)\n            #print(Y_target)\n            #print(Y_target.shape) \n\n            if train_device == 'cuda':\n                xb_meta = xb_meta.cuda()\n                Y_target = Y_target.cuda()\n            prediction = model(xb_meta)\n            loss = ((prediction- Y_target)/Y_target).abs()\n\n            loss.backward()\n            optimizer.step()\n            with torch.no_grad():\n                accuracy =(1- ((prediction- Y_target)/Y_target).abs())\n\n            s = ('Epochs: %5d/%d , train_loss: %5.3f  ,trian_accuracy: %5.3f'%\\\n                  (epoch, epochs,  loss.data, accuracy.data.item()))\n            pbar.set_description(s)\n            optimizer.zero_grad()\n            del prediction\n             \n            \n             \n\n\n        ## Validation\n        val_loss=0\n        val_acc_total = 0.\n        n = len(x_valid_values)\n        Steps = (n-1)// batch_size +1\n        pbar = tqdm(range(Steps), total= Steps)\n        model.eval()\n        for i in pbar:  #range((n-1)//batch_size +1)\n            start_i = i * batch_size\n            end_i = start_i + batch_size \n            xb_meta =  torch.tensor(x_valid_values[start_i:end_i]).float()\n            Y_target = torch.tensor(y_valid_values[start_i:end_i]).float().unsqueeze(1)\n            \n            if train_device == 'cuda':\n                xb_meta = xb_meta.cuda()\n                Y_target = Y_target.cuda()\n            prediction = model(xb_meta)\n            loss = ((prediction- Y_target)/Y_target).abs()\n            with torch.no_grad():\n                accuracy =(1- ((prediction- Y_target)/Y_target).abs())\n            \n\n            s = ('Epochs: %5d/%d , val_loss: %5.3f  ,val_accuracy: %5.3f'%\\\n                  (epoch, epochs,  loss.data, accuracy.data.item()))\n            pbar.set_description(s)\n            \n            val_loss += loss.data.item()\n            val_acc_total += accuracy.data.item()\n            del prediction\n        avg_loss = val_loss/n\n        avg_val_acc = (val_acc_total)/n \n        print('Average Validation accuracy:', avg_val_acc)\n        print('Average Validation loss:', avg_loss)\n             \n        #PATH = 'Epoch'+'%s_'%epoch+'%s'%avg_val_acc + '.pth'\n        #torch.save(model.state_dict(), './checkpoint_sigma_models'+os.sep+PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"######### ----------------- Making the Evaluation Data -------------------- #######\ndef make_eval_data(npEval, model):\n    x_features = npEval[['base_Weeks', 'base_FVC', 'Age', 'Male', 'Female', 'Ex-smoker','Never smoked', 'Currently smokes', 'Week', 'Healthy-FVC']]\n    x_features = torch.tensor(x_features.values).float()\n    #y_labels = npEval['actual_FVC'].values \n    patientsID = npEval['Patient'].values \n    \n    predictions = []\n    for x_feature in x_features:\n        x_feature = x_feature.unsqueeze(0)\n        #print(x_feature.shape)\n        prediction = model(x_feature)\n        predictions.append(prediction.data.item())\n    #     eval_data_df = pd.DataFrame([])\n    #     eval_data_df['PatientID'] = patientsID\n    #     eval_data_df['Actual_Value'] = y_labels \n    npEval['Prediction'] = predictions\n    #eval_data_df.set_index('PatientID')\n    npEval.reset_index(inplace = True)\n    #npEval.to_csv('myoutputtrain.csv', sep='\\t')  \n    return npEval\n\n\n\n######### ----------------- Making the Evaluation Data -------------------- #######\ndef make_eval_sigma(npEval, model):\n    x_features = npEval[['base_Weeks','base_FVC','Age','Male','Female','Ex-smoker','Never smoked','Currently smokes','Healthy-FVC','Prediction']]\n\n    x_features = torch.tensor(x_features.values).float()\n    #y_labels = npEval['actual_FVC'].values \n    patientsID = npEval['Patient'].values \n    \n    confidences = []\n    #print(x_features.shape)\n\n    for i in range(x_features.shape[0]):\n        x_feature  = x_features[i]\n        x_feature = x_feature.unsqueeze(0)\n        #print(x_feature.shape)\n        confidence = model(x_feature)\n        #         print(confidence.item())\n        #         print(x_feature)\n        confidences.append(confidence.data.item())\n        \n    #     eval_data_df = pd.DataFrame([])\n    #     eval_data_df['PatientID'] = patientsID\n    #     eval_data_df['Actual_Value'] = y_labels \n    npEval['confidence'] = confidences\n    #eval_data_df.set_index('PatientID')\n    #npEval.reset_index(inplace = True)\n    #npEval.to_csv('myoutputtrain.csv', sep='\\t')  \n    return npEval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubmission['Patient']=submission['Patient_Week'].apply(lambda x:x.split('_')[0])\nsubmission['Weeks']=submission['Patient_Week'].apply(lambda x:x.split('_')[1]).astype(int)\n\n# Week bias add\nsubmission.Weeks += 12\n\n\ntestdf = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nmerge=pd.merge(testdf,submission,on=['Patient'],how='left').sort_values(['Weeks_y','Patient']).reset_index(drop=True)\nmerge=merge.drop(['FVC_y'],axis=1)\nmerge=merge.rename(columns={'FVC_x':'base_FVC','Weeks_y':'Week','Weeks_x':'base_Weeks'})\n\ndel testdf\ndel submission\n\ntestdf=merge.loc[:,['Patient','base_Weeks','base_FVC','Percent','Age','Sex','SmokingStatus','Week']]\nsubmission=merge.loc[:,['Patient_Week','base_FVC','Confidence']]\nsubmission=submission.rename(columns={'base_FVC':'FVC'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = testdf.copy()\ndata['Healthy-FVC']=round((data['base_FVC']*100)/data['Percent'])\nFE=[]\nFE.append('Healthy-FVC')\n\n#Create Male, Female, Ex-smoker, Current-smoker and Never smoked\nCOLS = ['Sex','SmokingStatus']\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\nFE1=['Male','Female','Ex-smoker','Never smoked','Currently smokes']\nnpData=pd.DataFrame(columns=['Patient','base_Weeks','base_FVC','Age','Healthy-FVC']+FE1+['Week'])\nnpData=npData.append(data)\nnpData=npData.fillna(0)\n\ndel testdf\ntestdf = npData[['Patient','base_Weeks','base_FVC','Age','Healthy-FVC','Male','Female','Ex-smoker','Never smoked','Currently smokes','Week']]\n   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(testdf.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_FVC = DATA()\nmodel_FVC.load_state_dict(torch.load('../input/metadatapreweights/metadata_checkpoint.pth'))\nmodel_FVC.eval()\n# for the validation inputset of sigma \n# ndf = pd.concat([npValid,npTest])\n# outdf = make_eval_data(df_test, model)\n# for the training inputset of sigma \n#train_inp_sigma = make_eval_data(df_train, model)\n# for the final test inputset of sigma\n\ntest_inp_sigma = make_eval_data(testdf.copy(), model_FVC)\n\n#test_inp_sigma.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_sigma = SIGMA()\n# model_sigma.load_state_dict(torch.load('../input/sigmapreweight/sigma_preweight.pth'))\n# model_sigma.eval()\n\n\n# test_confidence_df = make_eval_sigma(test_inp_sigma.copy(), model_sigma)\n\n# #test_confidence_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = test_inp_sigma.copy()\n# df = pd.DataFrame(columns=['Patient','base_Weeks','confidence'])\n# tid=train.Patient.unique()\n# i=0\n# for pid in tid:\n#     weeks=train.loc[train.Patient==pid].base_Weeks.unique()\n#     for w in weeks:\n#         temp=train[train.Patient==pid]\n#         value=temp[temp.base_Weeks==w].Prediction.values\n#         diff = (value - np.mean(value))\n#         var = np.square(diff).sum()/len(value)\n#         sd = np.sqrt(var)\n#         c = sd*1.96/np.sqrt(len(value))\n#         #print(6*c)\n#         df.at[i,'Patient']=pid\n#         df.at[i,'base_Weeks']=w\n#         df.at[i,'confidence']=8*c\n#         i=i+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge = pd.merge(df, train, on=['Patient','base_Weeks'], how='left')\n# result=merge[['Patient', 'base_Weeks', 'base_FVC', 'Age', 'Healthy-FVC',\n#        'Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes',\n#        'Week', 'Prediction', 'confidence']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# submission.loc[:, 'FVC']=result.Prediction\n# submission.loc[:, 'Confidence']=result.confidence\n\n\n#print(submission.tail())\n#temp=temp.sort_values(by=['Week'], ascending = True)\n#temp.reset_index(inplace=True , drop=True)\n\n# temp_df = pd.DataFrame([])\n# temp_df['Patient_Week'] = temp['Patient'].apply(lambda x: str(x)) +'_'+ temp['Week'].apply(lambda x: str(x))\n# temp['Patient_Week'] = temp_df['Patient_Week'].values\n# sample_submission_df = pd.DataFrame([])\n# sample_submission_df['Patient_Week'] = temp['Patient_Week'].values\n# sample_submission_df['FVC'] = temp['Prediction'].values\n# sample_submission_df['Confidence'] = temp['confidence'].values\n#print(sample_submission_df.tail(20))\n# sample_submission_df = test_confidence_df[['Patient_Week','Prediction', 'confidence']]\n# sample_submission_df.rename(columns ={'Prediction':'FVC','confidence':'Confidence'}, inplace=True) \n# sample_submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # ### CSV Load, Read and Process\n# path = \"../input/osic-pulmonary-fibrosis-progression\"\n\n# data = pd.read_csv(f\"{path}/train.csv\")                        \n# final_test  = pd.read_csv(f\"{path}/test.csv\")  \n\n# valid, train, test = csv_split(data, v=0, t=0)\n# print(train.shape)\n# #print(valid.shape)\n# #print(test.shape)\n# npTrain=csv_preprocess(train)\n# #npValid=csv_preprocess(valid)\n# #npTest=csv_preprocess(test)\n# print(npTrain.shape)\n# #print(npValid.shape)\n# #print(npTest.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_FVC = DATA()\n# model_FVC.load_state_dict(torch.load('../input/metadatapreweights/metadata_checkpoint.pth'))\n# model_FVC.eval()\n# # for the validation inputset of sigma \n# # ndf = pd.concat([npValid,npTest])\n# # outdf = make_eval_data(df_test, model)\n# # for the training inputset of sigma \n# #train_inp_sigma = make_eval_data(df_train, model)\n# # for the final test inputset of sigma\n\n# train_inp_sigma = make_eval_data(npTrain.copy(), model_FVC)\n\n# train_inp_sigma.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_sigma = SIGMA()\n# model_sigma.load_state_dict(torch.load('../input/sigmapreweight/sigma_preweight.pth'))\n# model_sigma.eval()\n\n\n# train_confidence_df = make_eval_sigma(train_inp_sigma.copy(), model_sigma)\n\n# train_confidence_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# actual_FVC = train_inp_sigma.actual_FVC.values\n# Prediction = train_inp_sigma.Prediction.values\n# sigma = train_confidence_df.confidence.values\n# #print(len(actual_FVC))\n# #print(len(Prediction))\n# #print(len(sigma))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# temp = sigma_generator(train_inp_sigma)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# actual_sigma=temp.actual_sigma.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# score = laplace_log_likelihood(actual_FVC, Prediction, actual_sigma, return_values = True)\n\n# print(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# score = laplace_log_likelihood(actual_FVC, Prediction, 235, return_values = True)\n\n# print(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# score = laplace_log_likelihood(actual_FVC, Prediction, sigma, return_values = True)\n\n# print(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#===========================================================\ndef run_single_lightgbm(param, train_df, valid_df, test_df, categorical=False):\n    \n    #trn_idx = folds[folds.fold != fold_num].index\n    #val_idx = folds[folds.fold == fold_num].index\n    #logger.info(f'len(trn_idx) : {len(trn_idx)}')\n    #logger.info(f'len(val_idx) : {len(val_idx)}')\n    # ['base_FVC', 'base_Percent', 'base_Age', 'Week_passed', 'Sex', 'SmokingStatus']\n    #[['Patient', 'base_Weeks', 'base_FVC', 'Age', 'Healthy-FVC', 'Male','Female', 'Ex-smoker', 'Never smoked', 'Currently smokes', 'Week','actual_FVC', 'Prediction']]\n    train_target=train_df[['actual_sigma']]\n    valid_target=valid_df[['actual_sigma']]\n    train_df = train_df[['base_FVC','Age','Healthy-FVC','Prediction','Male','Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']]\n    valid_df = valid_df[['base_FVC','Age','Healthy-FVC','Prediction','Male','Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']]\n    test_df = test_df[['base_FVC','Age','Healthy-FVC','Prediction','Male','Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']]\n    \n    if categorical == False:\n        trn_data = lgb.Dataset(train_df,\n                               label=train_target)\n        val_data = lgb.Dataset(valid_df,\n                               label=valid_target)\n    else:\n        trn_data = lgb.Dataset(train_df,\n                               label=train_target,\n                               categorical_feature=['Male','Female', 'Ex-smoker', 'Never smoked', 'Currently smokes'])\n        val_data = lgb.Dataset(valid_df,\n                               label=valid_target,\n                               categorical_feature=['Male','Female', 'Ex-smoker', 'Never smoked', 'Currently smokes'])\n\n    train_confidence = np.zeros(len(train_df))\n    \n    valid_confidence = np.zeros(len(valid_df))\n    \n    test_confidence = np.zeros(len(test_df))\n\n    num_round = 10000\n\n    clf = lgb.train(param,\n                    trn_data,\n                    num_round,\n                    valid_sets=[trn_data, val_data],\n                    verbose_eval=100,\n                    early_stopping_rounds=100)\n\n    train_confidence = clf.predict(train_df, num_iteration=clf.best_iteration)\n\n    valid_confidence = clf.predict(valid_df, num_iteration=clf.best_iteration)\n    \n    test_confidence = clf.predict(test_df, num_iteration=clf.best_iteration)\n    \n    # RMSE\n    print(\"Valid RMSE score: {:<8.5f}\".format( np.sqrt(mean_squared_error(valid_target, valid_confidence))))\n    \n    return train_confidence, valid_confidence, test_confidence\n\n\ndef run_kfold_lightgbm(param, train, valid, test, epochs=5, categorical=False):\n    \n    #logger.info(f\"================================= {n_fold}fold lightgbm =================================\")\n    \n    train_confidence = np.zeros(len(train))\n    valid_confidence = np.zeros(len(valid))\n    test_confidence = np.zeros(len(test))\n    \n    valid_target = valid[['actual_sigma']]\n\n    for fold_ in range(epochs):\n        print(\"Epoch {}\".format(fold_))\n        _train_confidence, _valid_confidence, _test_confidence  = run_single_lightgbm(param,\n                                                                     train,\n                                                                     valid,test,\n                                                                     categorical=categorical)\n        train_confidence += _train_confidence/epochs\n        valid_confidence += _valid_confidence/epochs\n        test_confidence += _test_confidence/epochs\n\n    # RMSE\n    print(\"Final Valid RMSE score: {:<8.5f}\".format(np.sqrt(mean_squared_error(valid_target, valid_confidence))))\n\n    print(f\"=========================================================================================\")\n    \n    return train_confidence, valid_confidence, test_confidence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train=pd.read_csv(\"../input/trainallpatient/outputtrain.csv\")\n#data=data[['PatientID','Actual_Value','Prediction']]\n#data=data.sort_values(by=['Patient','base_Weeks'], ascending=True )\n#data.reset_index(inplace =True, drop=True)\ntrain = train[['Patient', 'base_Weeks', 'base_FVC', 'Age', 'Healthy-FVC', 'Male',\n       'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes', 'Week',\n       'actual_FVC', 'Prediction']]\nvalid=pd.read_csv(\"../input/validallpatient/outputvalid.csv\")\n#data=data[['PatientID','Actual_Value','Prediction']]\n#data=data.sort_values(by=['Patient','base_Weeks'], ascending=True )\n#data.reset_index(inplace =True, drop=True)\nvalid = valid[['Patient', 'base_Weeks', 'base_FVC', 'Age', 'Healthy-FVC', 'Male',\n       'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes', 'Week',\n       'actual_FVC', 'Prediction']]\ntest = test_inp_sigma.copy()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy as sp\nfrom functools import partial\nimport math\n\n#row['FVC']=train['actual_FVC'].values\n#row['Pred']=train['Prediction'].values\n\ndef loss_func(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = abs(row['actual_FVC'] - row['Prediction'])\n    delta = min(diff, 1000)\n    score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n    return -score\n\nresults = []\ntk0 = tqdm(train.iterrows(), total=len(train))\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [100]\n    #bounds = [(70, 100)]\n    #result = sp.optimize.minimize(loss_partial, weight, method='SLSQP', bounds=bounds)\n    result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n    x = result['x']\n    results.append(x[0])\ntrain['actual_sigma']=results\nresults = []\ntk0 = tqdm(valid.iterrows(), total=len(valid))\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [100]\n    #bounds = [(70, 100)]\n    #result = sp.optimize.minimize(loss_partial, weight, method='SLSQP', bounds=bounds)\n    result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n    x = result['x']\n    results.append(x[0])\nvalid['actual_sigma']=results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_param = {'objective': 'regression',\n             'metric': 'rmse',\n             'boosting_type': 'gbdt',\n             'learning_rate': 0.01,\n             'seed': 0,\n             'max_depth': -1,\n             'verbosity': -1,\n            }\n\ntrain_confidence, valid_confidence, test_confidence = run_kfold_lightgbm(lgb_param, train, valid, test, epochs=5, categorical=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['confidence']=train_confidence\nvalid['confidence']=valid_confidence\ntest['confidence']=test_confidence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"score=laplace_log_likelihood(train.actual_FVC.values, train.Prediction.values, train.confidence.values.astype(np.float64), return_values = True)\n#print(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.loc[:, 'FVC']=test.Prediction\nsubmission.loc[:, 'Confidence']=test.confidence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}