{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys, os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom sklearn import preprocessing\nfrom sklearn import model_selection","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/osic-pulmonary-fibrosis-progression/\"\nK_FOLDS = 5\nNUM_EPOCHS = 1000\nTRAIN_BATCH_SIZE = 16\nTEST_BATCH_SIZE = 4\nQUANTILES = [0.1, 0.5, 0.9]\nLEARNING_RATE = 4e-5\nPRINT_EVERY = 50\n\nSCALE_COLUMNS = ['Weeks', 'FVC', 'Percent', 'Age']\nSEX_COLUMNS = ['Male', 'Female']\nSMOKING_STATUS_COLUMNS = ['Currently smokes', 'Ex-smoker', 'Never smoked']\n# the feature vector (passed into the model)\nFV = SEX_COLUMNS + SMOKING_STATUS_COLUMNS + SCALE_COLUMNS\n\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define splitter and min_max_scaler\nkf = model_selection.KFold(K_FOLDS)\nMIN_MAX_SCALER = preprocessing.MinMaxScaler()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# read train.csv and test.csv\ntrain_df = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\ntrain_df.drop_duplicates(keep=False, inplace=True, subset=['Patient', 'Weeks'])\ntest_df = pd.read_csv(os.path.join(DATA_DIR, \"test.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fit the scaler and transform the data using the fit_transform function\ntrain_df[SCALE_COLUMNS] = MIN_MAX_SCALER.fit_transform(train_df[SCALE_COLUMNS])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# specify the categorical columns and the categories incase any class is missing (in test)\n# convert into one-hot-encoding\ntrain_df['Sex'] = pd.Categorical(train_df['Sex'], categories=SEX_COLUMNS)\ntrain_df['SmokingStatus'] = pd.Categorical(train_df['SmokingStatus'], categories=SMOKING_STATUS_COLUMNS)\ntrain_df = train_df.join(pd.get_dummies(train_df['Sex']))\ntrain_df = train_df.join(pd.get_dummies(train_df['SmokingStatus']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(os.path.join(DATA_DIR, \"sample_submission.csv\"))\n# get the patient_id and the week from the Patient_Week column\nsub_df['Patient'] = sub_df['Patient_Week'].apply(lambda x: x.split('_')[0])\nsub_df['Weeks'] = sub_df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = sub_df.drop(\"FVC\", axis=1).merge(test_df.drop('Weeks', axis=1), on='Patient')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# have to make it categorical coz sub's sex column has males only\nsub_df['Sex'] = pd.Categorical(sub_df['Sex'], categories=SEX_COLUMNS)\nsub_df['SmokingStatus'] = pd.Categorical(sub_df['SmokingStatus'], categories=SMOKING_STATUS_COLUMNS)\nsub_df = sub_df.join(pd.get_dummies(sub_df['Sex']))\nsub_df = sub_df.join(pd.get_dummies(sub_df['SmokingStatus']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use the global min_max_scaler with values from train.csv to scale the columns for the submission\nsub_df[SCALE_COLUMNS] = MIN_MAX_SCALER.transform(sub_df[SCALE_COLUMNS])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PulmonaryDataset(torch.utils.data.Dataset):\n    def __init__(self, df, FV, test=False):\n        self.df = df\n        self.test = test\n        self.FV = FV\n\n    def __getitem__(self, idx):\n        return {\n            'features': torch.tensor(self.df[self.FV].iloc[idx].values),\n            'target': torch.tensor(self.df['FVC'].iloc[idx])\n        }\n\n    def __len__(self):\n        return len(self.df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PulmonaryModel(nn.Module):\n    def __init__(self, in_features=9, out_quantiles=3):\n        super(PulmonaryModel, self).__init__()\n        self.fc1 = nn.Linear(in_features, 100)\n        self.fc2 = nn.Linear(100, 100)\n        self.fc3 = nn.Linear(100, out_quantiles)\n    \n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quantile_loss(preds, target, quantiles):\n    assert not target.requires_grad\n    assert preds.size(0) == target.size(0)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target - preds[:, i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_one_epoch(model, train_data_loader, optimizer, train_loss):\n    model.train()\n\n    for i, data in enumerate(train_data_loader):\n        features = data['features']\n        targets = data['target']\n\n        features = features.to(DEVICE).float()\n        targets = targets.to(DEVICE).float()\n\n        model.zero_grad()\n        out = model(features)\n        loss = quantile_loss(out, targets, QUANTILES)\n        train_loss.update(loss, features.size(0))\n        loss.backward()\n        optimizer.step()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def eval_one_epoch(model, valid_data_loader, valid_loss, lr_scheduler):\n    model.eval()\n\n    with torch.no_grad():\n        for i, data in enumerate(valid_data_loader):\n            features = data['features']\n            targets = data['target']\n\n            features = features.to(DEVICE).float()\n            targets = targets.to(DEVICE).float()\n            \n            out = model(features)\n            loss = quantile_loss(out, targets, QUANTILES)\n            valid_loss.update(loss, features.size(0))\n    \n    if lr_scheduler is not None:\n        lr_scheduler.step(valid_loss.avg)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# REMOVE THE ONES FROM THE TRAIN_DF THAT ARE PRESENT IN TEST_DF AS WELL\nTEST_PATIENTS = test_df['Patient'].unique().tolist()\nvalid_df = train_df[train_df['Patient'].isin(TEST_PATIENTS)]\ntrain_df = train_df[~train_df['Patient'].isin(TEST_PATIENTS)]\nTRAIN_PATIENTS = train_df['Patient'].unique().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold, (train_index, test_index) in enumerate(kf.split(TRAIN_PATIENTS)):\n    model = PulmonaryModel(len(FV))\n    model = model.to(DEVICE)\n\n    df_train = train_df.iloc[train_index].reset_index(drop=True)\n    df_valid = train_df.iloc[test_index].reset_index(drop=True)\n\n    train_dataset = PulmonaryDataset(df_train, FV)\n    valid_dataset = PulmonaryDataset(df_valid, FV)\n\n    train_data_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TRAIN_BATCH_SIZE,\n        shuffle=True,\n        num_workers=4\n    )\n\n    valid_data_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=TEST_BATCH_SIZE,\n        shuffle=False,\n        num_workers=4\n    )\n\n    optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n    lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=10, factor=0.05, verbose=True)\n\n    best_valid_loss = float('inf')\n    \n    train_loss = AverageMeter()\n    valid_loss = AverageMeter()\n    \n    for epoch in range(NUM_EPOCHS):\n        train_one_epoch(model, train_data_loader, optimizer, train_loss)\n        eval_one_epoch(model, valid_data_loader, valid_loss, lr_scheduler)\n        \n        if epoch % PRINT_EVERY == 0:\n            print(f\"Fold {fold} Epoch {epoch}/{NUM_EPOCHS}, train_loss: {train_loss.avg}, val_loss: {valid_loss.avg}\")\n\n        if valid_loss.avg < best_valid_loss:\n            best_valid_loss = valid_loss.avg\n            torch.save({\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n            }, f\"model_fold_{fold}.pt\")\n    print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\nfor fold in range(K_FOLDS):\n    model = PulmonaryModel(len(FV))\n    model = model.to(DEVICE)\n    checkpoint = torch.load(f\"model_fold_{fold}.pt\")\n    model.load_state_dict(checkpoint['model_state_dict'])\n    models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = PulmonaryDataset(sub_df, FV)\ntest_data_loader = torch.utils.data.DataLoader(\n    test_dataset,\n    batch_size=TEST_BATCH_SIZE,\n    shuffle=False,\n    num_workers=4\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_preds = np.zeros((len(test_dataset), len(QUANTILES)))\nwith torch.no_grad():\n    for model in models:\n        preds = []\n        for j, test_data in enumerate(test_data_loader):\n            features = test_data['features']\n            targets = test_data['target']\n\n            features = features.to(DEVICE).float()\n            targets = targets.to(DEVICE).float()\n\n            out = model(features)\n            preds.append(out)\n        preds = torch.cat(preds, dim=0).cpu().numpy()\n        avg_preds += preds\navg_preds /= len(models)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# inverse the scaling operation for FVC\navg_preds -= MIN_MAX_SCALER.min_[SCALE_COLUMNS.index('FVC')]\navg_preds /= MIN_MAX_SCALER.scale_[SCALE_COLUMNS.index('FVC')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"avg_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df['FVC'] = avg_preds[:, 1]\nsub_df['Confidence'] = np.abs(avg_preds[:, 2] - avg_preds[:, 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df.head(25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df[['Patient_Week', 'FVC', 'Confidence']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}