{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport torch.nn as nn\nfrom torch import tensor\nimport torch.optim as torch_optim\nimport torch.nn.functional as F\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in data\ntrain = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ndep_var = 'FVC'\n\n# Drop the dependent variable\n# CURRENTLY DROPPING PERCENT. NEED TO LOOK INTO IF WE NEED TO KEEP OR NOT\nX_df = train.drop(labels=[dep_var, 'Percent'], axis=1)\ny_df = train[dep_var]\n\ndef encode_objects(df):\n    obj_cols = []\n    le = LabelEncoder()\n    for col, dtype in zip(df, df.dtypes):\n        if dtype == 'object':\n            df[col] = le.fit_transform(df[col])\n            df[col] = df[col].astype('category')\n            obj_cols.append(col)\n    \n    print(\"Converted {0} columns from objects to categories\".format(obj_cols))\n\n# Encode 'object' dtype columns\nencode_objects(X_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X_df, y_df, test_size=0.10, shuffle=True)\nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_embeddings(df):\n    embedded_cols = {}\n    for n,col in X_df.items():\n        if str(X_df[n].dtype) == 'category':\n            embedded_cols[n] = len(col.cat.categories)\n            \n    # Create dict of emb names\n    embedded_col_names = embedded_cols.keys()\n\n    embedding_sizes = [(n_categories, min(50, (n_categories+1)//2)) for _,n_categories in embedded_cols.items()]\n    \n    return embedded_col_names, embedding_sizes\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedded_col_names, embedding_sizes = create_embeddings(X_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Create Dataset\n\nclass OSICTabularDataset(Dataset):\n    \"\"\"\n    Args:\n        csv_file : str \n            Path to the csv file with annotations.\n        y_name : str\n            Name of dependent variable\n    \"\"\"\n    def __init__(self, X, Y=None, embedded_col_names=None):\n        # Break it into the numberical and categoricals for the model later\n        self.X1 = X.loc[:,embedded_col_names].copy().values.astype(np.int64) #categorical columns\n        self.X2 = X.drop(columns=embedded_col_names).copy().values.astype(np.float32) #numerical columns\n        if Y is not None:\n            self.y = Y.values.astype(np.float32)\n        else:\n            self.y = None\n                        \n    def __len__(self):\n        if self.y is None:\n            return len(self.X1)\n        else:\n            return len(self.y)\n    \n    def __getitem__(self, idx):\n        if self.y is None:\n            return self.X1[idx], self.X2[idx]\n        else:\n            return self.X1[idx], self.X2[idx], self.y[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = OSICTabularDataset(X_train, y_train, embedded_col_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = OSICTabularDataset(X_val, y_val, embedded_col_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity check loop through train_dataset\nfor i in range(10):\n    sample = train_ds[i] \n    print(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity check loop through train_dataset\nfor i in range(10):\n    sample = valid_ds[i] \n    print(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"z_cat = torch.randn(50, 53)\nz_cont = torch.randn(50, 3)\nprint(z_cont.size(), z_cat.size())\n\ncomb = torch.cat((z_cat, z_cont), 1)\nprint(comb.size())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_default_device():\n    \"\"\"Pick GPU if available, else CPU\"\"\"\n    if torch.cuda.is_available():\n        return torch.device('cuda')\n    else:\n        return torch.device('cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_device(data, device):\n    \"\"\"Move tensor(s) to chosen device\"\"\"\n    if isinstance(data, (list,tuple)):\n        return [to_device(x, device) for x in data]\n    return data.to(device, non_blocking=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DeviceDataLoader():\n    \"\"\"Wrap a dataloader to move data to a device\"\"\"\n    def __init__(self, dl, device):\n        self.dl = dl\n        self.device = device\n        \n    def __iter__(self):\n        \"\"\"Yield a batch of data after moving it to device\"\"\"\n        for b in self.dl: \n            yield to_device(b, self.device)\n\n    def __len__(self):\n        \"\"\"Number of batches\"\"\"\n        return len(self.dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = get_default_device()\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Network Architecture\n\nclass OSICModel(nn.Module):\n    def __init__(self, embedding_sizes, n_cont):\n        super().__init__()\n        self.embeddings = nn.ModuleList([nn.Embedding(categories, size) for categories,size in embedding_sizes])\n        n_emb = sum(e.embedding_dim for e in self.embeddings) #length of all embeddings combined\n        self.n_emb, self.n_cont = n_emb, n_cont\n        self.lin1 = nn.Linear(55, 100)\n        self.lin2 = nn.Linear(100, 100)\n        self.lin3 = nn.Linear(100, 1)\n#         self.bn1 = nn.BatchNorm1d(self.n_cont)\n#         self.bn2 = nn.BatchNorm1d(200)\n#         self.bn3 = nn.BatchNorm1d(70)\n#         self.emb_drop = nn.Dropout(0.6)\n#         self.drops = nn.Dropout(0.3)\n\n        self.sigma = nn.Linear(100, 1)\n        \n    def forward(self, x_cat, x_cont):\n        out = [e(x_cat[:,i]) for i,e in enumerate(self.embeddings)]\n        # Concat into one tensor\n        out = torch.cat(out, 1)\n        out = torch.cat((x_cont, out), 1)\n        \n        out = F.relu(self.lin1(out))\n        out = F.relu(self.lin2(out))\n        fvc = F.relu(self.lin3(out))\n        sigma = F.relu(self.sigma(out))\n        return fvc,sigma\n        \n    def metric_loss(self,pred_fvc,true_fvc,pred_sigma):\n        true_fvc=torch.reshape(true_fvc,pred_fvc.shape)\n        sigma_clipped=torch.clamp(pred_sigma,min=1e-3)\n        delta=torch.clamp(torch.abs(pred_fvc-true_fvc),max=1000)\n        metric=torch.div(-torch.sqrt(torch.tensor([2.0]).to(device))*delta,sigma_clipped)-torch.log(torch.sqrt(torch.tensor([2.0]).to(device))*sigma_clipped)\n        return -metric\n    \n    def fvc_loss(self,pred_fvc,true_fvc):\n        true_fvc=torch.reshape(true_fvc,pred_fvc.shape)\n        fvc_err=torch.abs(pred_fvc-true_fvc)\n        return fvc_err","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = OSICModel(embedding_sizes, 3)\n\nto_device(model, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I have a dataloader waiting in the wings\ndataloader_valid = DataLoader(train_ds, batch_size=6, shuffle=True, num_workers=4)\n\nfor i_batch, (cats, nums, dep_var) in enumerate(dataloader_valid):\n    print(\"Batch:\", i_batch)\n    print(\"Categoricals:\\n {0}\".format(cats))\n    print(\"Numericals:\\n {0}\".format(nums))\n    print(\"Ground Truth:\\n {0}\".format(dep_var))\n    print(\"-------------------\\n\")\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Optimizer (ADAM)\ndef get_optimizer(model, lr = 0.01, wd = 0.0):\n    parameters = filter(lambda p: p.requires_grad, model.parameters())\n    optim = torch_optim.Adam(parameters, lr=lr, weight_decay=wd)\n    return optim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, optim, data):\n    model.train()\n    total = 0\n    train_loss=0\n    train_metric=0\n    for x1, x2, y in data:\n        batch = y.shape[0]\n        fvc, sigma = model(x1, x2)\n        \n        fvc_loss = model.fvc_loss(fvc, y.to(device)).mean()\n        metric_loss = model.metric_loss(fvc, y.to(device),sigma).mean()\n        loss = metric_loss\n        optim.zero_grad()\n        loss.backward()\n        \n        train_loss += fvc_loss.item()\n        train_metric += metric_loss.item()\n        optim.step()\n        total += batch\n        \n    return train_loss, train_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def val_result(model, valid_dl):\n    model.eval()\n    val_loss=0\n    val_metric=0\n    for x1, x2, y in valid_dl:\n        # FVC and Sigma are the \"predictions\"\n        fvc, sigma = model(x1, x2)\n        fvc_loss = model.fvc_loss(fvc, y.to(device)).mean()\n        metric_loss = model.metric_loss(fvc, y.to(device),sigma).mean()\n        \n        loss = metric_loss\n        val_loss += fvc_loss.item()\n        val_metric += metric_loss.item()\n    return val_loss, val_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epoch_train_metric=[]\nepoch_val_metric=[]\nepoch_train_loss=[]\nepoch_val_loss=[]\n\ndef train_loop(model, train_data, valid_data=None, epochs=50, lr=0.01, wd=0.0):\n    optim = get_optimizer(model, lr = lr, wd = wd)\n    for epoch in range(epochs): \n        train_loss, train_metric = train_model(model, optim, train_data)\n        print('\\n====> Epoch: {}'.format(epoch))\n        print('-------------------------------')\n        \n        print('Average TRAIN fvc loss: {:.4f}'.format(\n              train_loss / len(train_dl)))\n        print('Average TRAIN metric: {:.4f}'.format(\n              train_metric / len(train_dl)))\n        \n        val_loss, val_metric = val_result(model, valid_dl)\n        print('Average VALIDATION fvc loss: {:.4f}'.format(\n              val_loss / len(valid_dl)))\n        print('Average VALIDATION metric: {:.4f}'.format(\n              val_metric / len(valid_dl)))\n        \n        epoch_train_loss.append(train_loss/ len(train_dl))\n        epoch_val_loss.append(val_loss / len(valid_dl))\n        epoch_train_metric.append(train_metric/ len(train_dl))\n        epoch_val_metric.append(val_metric / len(valid_dl))\n        \n    print(\"Min TRAIN metric:\", min(epoch_train_metric))\n    print(\"Min VALID metric:\", min(epoch_val_metric))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 50\ntrain_dl = DataLoader(train_ds, batch_size=batch_size,shuffle=True)\nvalid_dl = DataLoader(valid_ds, batch_size=batch_size,shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = DeviceDataLoader(train_dl, device)\nvalid_dl = DeviceDataLoader(valid_dl, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loop(model, train_dl, valid_dl, epochs=50, lr=0.05, wd=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot results\ndef plot_training_loss(train, val,title='loss'):\n    plt.figure()\n    plt.plot(train, label='Train')\n    plt.plot(val, label='Val')\n    if title=='loss':\n        plt.title('Model Training Loss')\n    else:\n        plt.title('Model Metric Loss')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.yscale('log')\n    plt.legend()\n    plt.savefig('training_loss')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_training_loss(epoch_train_loss, epoch_val_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_training_loss(epoch_train_metric, epoch_val_metric, title='metric')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataframe Cleanup\n\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\ntest = test.rename(columns={'Weeks': 'base_Weeks', 'FVC': 'base_FVC','Percent': 'base_Percent'})\n\n# Adding Sample Submission\nsubmission = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n\n# In submisison file, format: ID_'week', using lambda to split the ID\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])\n\n# In submisison file, format: ID_'week', using lambda to split the Week\nsubmission['Weeks'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1]).astype(int)\n\ntest = submission.drop(columns = [\"FVC\", \"Confidence\"]).merge(test, on = 'Patient')\n\ntest['Week_passed'] = test['Weeks'] - test['base_Weeks']\n\ntest=test[train.columns.drop(['FVC','Percent'])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cleanup dataframe\nencode_objects(test)\nembedded_test_col_names, embedding_test_sizes = create_embeddings(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create dataset\ntest_ds = OSICTabularDataset(test, embedded_col_names=embedded_test_col_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity check loop through test_dataset\nfor i in range(5):\n    sample = test_ds[i] \n    print(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dl = DataLoader(test_ds, batch_size=batch_size,shuffle=True)\ntest_dl = DeviceDataLoader(test_dl, device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def test_eval(model, test_data):\n    model.eval()\n    fvc_pred = []\n    sigma_pred = []\n    with torch.no_grad():\n        for x1, x2 in test_dl:\n            # FVC and Sigma are the \"predictions\"\n            fvc, sigma = model(x1, x2)\n            fvc_pred.append(fvc)\n            sigma_pred.append(sigma)\n    fvc_pred=torch.cat(fvc_pred, dim=0)\n    sigma_pred=torch.cat(sigma_pred, dim=0)\n    \n    return fvc_pred, sigma_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fvc_pred, sigma_pred = test_eval(model, test_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    test['FVC']=fvc_pred.cpu().numpy()\n    test['Confidence']=sigma_pred.cpu().numpy()\n\n    test['Patient_Week']=test[\"Patient\"]\n\n    final_submission = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n    test['FVC']\n\n    kaggle_submission = pd.concat([final_submission['Patient_Week'], test['FVC'], test['Confidence']], axis=1)\n    kaggle_submission.to_csv('./submission.csv', index=False, float_format='%.1f')\n    \nexcept: \n    raise RuntimeError","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle_submission = pd.concat([final_submission['Patient_Week'], test['FVC'], test['Confidence']], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kaggle_submission.to_csv('./submission.csv', index=False, float_format='%.1f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nfor dirname, _, filenames in os.walk('/kaggle/working/'):\n   for filename in filenames:\n       print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}