{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is in continuation of my previous notebook where I Convered the EDA part. Link [here](https://www.kaggle.com/sawans/basic-eda-for-osic)\n\nCredit to [this](https://www.kaggle.com/maunish/osic-super-cool-eda-and-pytorch-baseline) great notebook by Maunish Dave.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport glob\nimport tqdm\nimport cv2\nimport pydicom \nimport numpy as np\nimport pandas as pd\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading Data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\n\ntrain_dir = '../input/osic-pulmonary-fibrosis-progression/train'\ntest_dir = '../input/osic-pulmonary-fibrosis-progression/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Checking dimensions of all datasets\n\ntrain.shape, test.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Exploring Train data\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Importing some more libraries**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler,LabelEncoder,OneHotEncoder,PowerTransformer\nfrom sklearn.model_selection import train_test_split, cross_val_score,cross_validate, KFold,GroupKFold\nfrom sklearn.metrics import make_scorer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# getting base week for patient\n# here min_week is the minimum number of week present for a patient\n# baseline_week is the number of weeks counting from the min week for a patient\ndef get_baseline_week(data):\n    df = data.copy()\n    df['Weeks'] = df['Weeks'].astype(int)\n    df['min_week'] = df.groupby('Patient')['Weeks'].transform('min')\n    df['baseline_week'] = df['Weeks'] - df['min_week']\n    return df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Lets understand the above method with below example. Here we are showing the min_week and baseline_week\ncolumns for patient ID ID00026637202179561894768.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df_1 = get_baseline_week(train)\ndf_1[df_1['Patient']=='ID00026637202179561894768']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#getting FVC for base week and setting it as base_FVC for patient\ndef get_base_FVC(data):\n    df = data.copy()\n    base = df.loc[df['Weeks']==df['min_week']][['Patient','FVC']].copy()\n    base.columns =['Patient','base_FVC']\n    \n    base['nb'] = 1\n    base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\n    \n    base = base[base['nb']==1]\n    base.drop('nb',axis=1,inplace=True)\n    df = df.merge(base,on='Patient',how='left')\n    df.drop(['min_week'],axis=1)\n    return df\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop_duplicates(keep=False,inplace=True,subset=['Patient','Weeks'])\ntrain_data = get_baseline_week(train)\ntrain_data = get_base_FVC(train_data)\ntrain_data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As you can see, we have created an additional column 'base_FVC' which will have FVC value measured at base week for\nevery patient.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Exploring the content of the submission file\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Exploring the content of test file\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Processing submission file\n# Dropping FVC column and merging with Test data\nsub.drop('FVC',axis=1,inplace=True)\nsub[['Patient','Weeks']] = sub['Patient_Week'].str.split(\"_\",expand=True)\nsub = sub.merge(test.drop('Weeks',axis=1), on ='Patient', how='left')\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['min_weeks'] = np.nan\nsub = get_baseline_week(sub)\nsub = get_base_FVC(sub)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split the train columns and train label\ntrain_columns = ['baseline_week','base_FVC','Percent','Age','Sex','SmokingStatus']\ntrain_label = ['FVC']\n\n# Columns in submission file\nsub_columns = ['Patient_Week','FVC','Confidence']\n\ntrain = train_data[train_columns]\ntest = sub[train_columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Scaling and Handling of Categorical Features**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"We will be using sklearn's ColumnTransformer to transform the columns. ColumTransformer lets us define all the transform operations (scaling, one-hot encoding etc.) and the list of columns where we want to apply those operations. You can read more about ColumnTransformer in [this](https://towardsdatascience.com/columntransformer-in-scikit-for-labelencoding-and-onehotencoding-in-machine-learning-c6255952731b) blog post.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# applying standard scaling on numeric columns 0,1,2,3\n# applying One-hot encoding on categorical columns 4,5 \n# each transformation will be defined in a tuple\ntransformer = ColumnTransformer([('s',StandardScaler(),[0,1,2,3]),('o',OneHotEncoder(),[4,5])])\n\ntarget = train_data[train_label].values\ntrain = transformer.fit_transform(train)\ntest = transformer.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transformed train data\ntrain[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that all the train features are on the same scale now.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Creating Pytorch Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing Pytorch libraries\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import TensorDataset, DataLoader\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define class for our torch model\n# This class will be inherited from nn.Module\nclass Model(nn.Module):\n    \n    def __init__(self,n):\n        super(Model,self).__init__()\n        \n        #Define the metwork layers\n        self.layer1 = nn.Linear(n,200)  # n inputs nodes, 200 output nodes\n        self.layer2 = nn.Linear(200,100) # 200 input nodes, 100 output nodes\n        \n        self.out1 = nn.Linear(100,3) # 100 input nodes, 3 output nodes\n        self.relu3 = nn.ReLU()\n        self.out2 = nn.Linear(100,3) # 100 input nodes, 3 output nodes\n        \n    \n    def forward(self,x):\n        x1 = F.leaky_relu(self.layer1(x))\n        x1 = F.leaky_relu(self.layer2(x1))\n        \n        o1 = self.out1(x1)\n        o2 = F.relu(self.out2(x1))\n        \n        return o1+ torch.cumsum(o2,dim=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training Model**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def run():\n    \n    # function to calculate metircs score\n    def score(outputs,target):\n        confidence = outputs[:,2] - outputs[:,0]\n        clip = torch.clamp(confidence,min=70) # Condidence is clipped at min value of 70\n        target = torch.reshape(target, outputs[:,1].shape)\n        delta = torch.abs(outputs[:,1] - target)\n        delta = torch.clamp(delta,max=1000) # delta is clipped at max value of 1000\n        \n        # calculating the metrics as provided in the challenge\n        sqrt_2 = torch.sqrt(torch.tensor([2.])).to(device)\n        metrics = (delta*sqrt_2/clip) + torch.log(clip*sqrt_2)\n        return torch.mean(metrics)\n        \n    \n    def qloss(outputs,target):\n        qs = [0.25,0.5,0.75]\n        qs = torch.tensor(qs,dtype=torch.float).to(device)\n        e =  target - outputs\n        e.to(device)\n        v = torch.max(qs*e,(qs-1)*e)\n        v = torch.sum(v,dim=1)\n        return torch.mean(v)\n    \n    def loss_fn(outputs,target,l):\n        return l*qloss(outputs,target) + (1-l) * score(outputs,target)\n    \n    def train_loop(train_loader,model,loss_fn, device,optimizer,lr_scheduler=None):\n        model.train()\n        losses = []\n        metrics = []\n        \n        for i, (inputs,labels) in enumerate(train_loader):\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            \n            optimizer.zero_grad()\n            with torch.set_grad_enabled(True):\n                outputs = model(inputs)\n                metric = score(outputs,labels)\n                \n                loss = loss_fn(outputs,labels,0.8)\n                metrics.append(metric.cpu().detach().numpy())\n                losses.append(loss.cpu().detach().numpy())\n                \n                loss.backward()\n                optimizer.step()\n                \n                if lr_scheduler !=None:\n                    lr_scheduler.step()\n                    \n        return losses,metrics\n    \n    def valid_loop(valid_loader,model,loss_fn, device):\n        model.eval()\n        losses = []\n        metrics = []\n        \n        for i, (inputs, labels) in enumerate(valid_loader):\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            \n            outputs = model(inputs)                 \n            metric = score(outputs,labels)\n            \n            loss = loss_fn(outputs,labels,0.8)\n            metrics.append(metric.cpu().detach().numpy())\n            losses.append(loss.cpu().detach().numpy())\n            \n        return losses,metrics\n    \n    \n    NFOLDS = 5\n    kfold = KFold(NFOLDS,shuffle=True,random_state=42)\n    \n    #generate kfolds\n    for k,(train_idx,valid_idx) in enumerate(kfold.split(train)):\n        batch_size =64\n        epochs = 50\n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        print(f'{device} is used')\n        \n        #Split into train and validation\n        X_train,X_valid,y_train,y_valid = train[train_idx,:], train[valid_idx,:], target[train_idx], target[valid_idx]\n        n = X_train.shape[1] #number of inputs (records)\n        model = Model(n)\n        model.to(device)\n        lr = 0.1\n        optimizer = optim.Adam(model.parameters(),lr=lr)\n        lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)\n        \n        # create tensors for training\n        train_tensor = torch.tensor(X_train, dtype = torch.float)\n        y_train_tensor = torch.tensor(y_train,dtype=torch.float)\n        \n        train_ds = TensorDataset(train_tensor, y_train_tensor)\n        train_dl = DataLoader(train_ds,batch_size=batch_size, num_workers=4, shuffle=True)\n        \n        # create tensors for validation\n        valid_tensor = torch.tensor(X_valid,dtype=torch.float)\n        y_valid_tensor = torch.tensor(y_valid,dtype=torch.float)\n        \n        valid_ds = TensorDataset(valid_tensor,y_valid_tensor)\n        valid_dl = DataLoader(valid_ds,\n                             batch_size = batch_size,\n                             num_workers=4,\n                             shuffle=False\n                             )\n        \n        print(f\"Fold {k}\")\n        \n        for i in range(epochs):\n            losses, metrics = train_loop(train_dl,model,loss_fn,device,optimizer,lr_scheduler)\n            valid_losses,valid_metrics = valid_loop(valid_dl,model,loss_fn,device)\n            \n            if (i+1)%5==0:\n                print(f\"epoch:{i} Training | loss:{np.mean(losses)} score: {np.mean(metrics)}| \\n Validation | loss:{np.mean(valid_losses)} score:{np.mean(valid_metrics)}|\")\n        \n        # save the model for current fold\n        torch.save(model.state_dict(),f'model{k}.bin')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference():\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    nfold = 5\n    all_prediction = np.zeros((test.shape[0],3))\n    \n    for i in range(nfold):\n        n = train.shape[1]\n        \n        model = Model(n)\n        model.load_state_dict(torch.load(f\"model{i}.bin\"))\n        predictions = list()\n        model.to(device)\n        test_tensor = torch.tensor(test,dtype=torch.float)\n        test_dl = DataLoader(test_tensor,\n                        batch_size=64,\n                        num_workers=2,\n                        shuffle=False)\n        \n        with torch.no_grad():\n            for i, inputs in enumerate(test_dl):\n                inputs = inputs.to(device, dtype=torch.float)\n                outputs= model(inputs) \n                predictions.extend(outputs.cpu().detach().numpy())\n        \n        all_prediction += np.array(predictions)/nfold\n        \n    return all_prediction\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = inference()\nsub['Confidence'] = np.abs(prediction[:,2]-prediction[:,0])\nsub['FVC'] = prediction[:,1]\nsubmission = sub[sub_columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}