{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import libraries\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, StepLR, CosineAnnealingLR\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold,  GroupKFold\nfrom tqdm import tqdm\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OrdinalEncoder\n\nimport warnings\n\nwarnings.simplefilter('ignore')\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nquantiles = [0.2, 0.5, 0.8]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# dataframe for loader\ndef make_X(dt, dense_cols, cat_feats):\n    X = {\"dense\": dt[dense_cols].to_numpy()}\n    for i, v in enumerate(cat_feats):\n        X[v] = dt[[v]].to_numpy()\n    return X\n\n# loader for embedding layers\nclass Loader:\n\n    def __init__(self, X, y, shuffle=True, batch_size=64, cat_cols=[]):\n\n        self.X_cont = X[\"dense\"]\n        self.X_cat = np.concatenate([X[k] for k in cat_cols], axis=1)\n        self.y = y\n\n        self.shuffle = shuffle\n        self.batch_size = batch_size\n        self.n_conts = self.X_cont.shape[1]\n        self.len = self.X_cont.shape[0]\n        n_batches, remainder = divmod(self.len, self.batch_size)\n\n        if remainder > 0:\n            n_batches += 1\n        self.n_batches = n_batches\n        self.remainder = remainder  # for debugging\n\n        self.idxes = np.array([i for i in range(self.len)])\n\n    def __iter__(self):\n        self.i = 0\n        if self.shuffle:\n            ridxes = self.idxes\n            np.random.shuffle(ridxes)\n            self.X_cat = self.X_cat[[ridxes]]\n            self.X_cont = self.X_cont[[ridxes]]\n            if self.y is not None:\n                self.y = self.y[[ridxes]]\n\n        return self\n\n    def __next__(self):\n        if self.i >= self.len:\n            raise StopIteration\n\n        if self.y is not None:\n            y = torch.FloatTensor(self.y[self.i:self.i + self.batch_size].astype(np.float32))\n\n        else:\n            y = None\n\n        xcont = torch.FloatTensor(self.X_cont[self.i:self.i + self.batch_size])\n        xcat = torch.LongTensor(self.X_cat[self.i:self.i + self.batch_size])\n\n        batch = (xcont, xcat, y)\n        self.i += self.batch_size\n        return batch\n\n    def __len__(self):\n        return self.n_batches","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass model_nn(nn.Module):\n\n    def __init__(self, hidden_dim, output_dim, emb_dims, n_cont):\n        super().__init__()\n\n        self.emb_layers = nn.ModuleList([nn.Embedding(x, y) for x, y in emb_dims])\n        n_embs = sum([y for x, y in emb_dims])\n\n        self.n_embs = n_embs  # + t_embs\n        self.n_cont = n_cont\n\n        inp_dim = n_embs + n_cont\n        self.inp_dim = inp_dim\n        \n        self.fs0 = nn.Linear(inp_dim, hidden_dim)\n        self.relufs0 = nn.ELU()\n        self.fs1 = nn.Linear(hidden_dim, inp_dim)\n        self.fs2 = nn.Sigmoid()\n        \n        self.fc0 = nn.Linear(inp_dim, hidden_dim)\n        self.relu0 = nn.ELU()\n        self.fc1 = nn.Linear(hidden_dim, hidden_dim)\n        self.relu1 = nn.ELU()\n\n        self.fc2 = nn.Linear(hidden_dim, output_dim)\n        self.fc3 = nn.Linear(hidden_dim, output_dim)\n        self.fc4 = nn.Linear(hidden_dim, output_dim)\n\n    def encode_and_combine_data(self, cat_data):\n        xcat = [el(cat_data[:, k]) for k, el in enumerate(self.emb_layers)]\n        xcat = torch.cat(xcat, 1)\n        return xcat\n\n    def forward(self, cont_data, cat_data):\n        cont_data = cont_data.to(device)\n        cat_data = cat_data.to(device)\n\n        cat_data = self.encode_and_combine_data(cat_data)\n\n        x = torch.cat([cont_data, cat_data], dim=1)\n        \n        w = self.fs0(x)\n        w = self.relufs0(w)\n        w = self.fs1(w)\n        w = self.fs2(w)\n        \n        wx = w*x + x\n        \n        hz = self.fc0(wx)\n        hz = self.relu0(hz)\n        hz = self.fc1(hz)\n        hz = self.relu1(hz)\n\n        out1 = self.fc2(hz)\n        out2 = self.fc3(hz)\n        out3 = self.fc4(hz)\n        \n        return torch.cat([out1, out2, out3], dim=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#loss function\ndef quantile_loss(preds, target, quantiles):\n    assert not target.requires_grad\n    assert preds.size(0) == target.size(0)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target - preds[:, i]\n        losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss\n\n#competation metric for numpy array\ndef metric(outputs, target):\n\n    confidence = np.abs(outputs[:, 2] - outputs[:, 0])\n    clip = np.where(confidence > 70, confidence, 70)\n    delta = np.abs(outputs[:, 1] - target)\n    delta = np.where(delta > 1000, 1000, delta)\n\n    metrics = (delta*np.sqrt(2)/clip) + np.log(clip*np.sqrt(2))\n\n    return np.mean(metrics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False, delta=0):\n\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n\n    def __call__(self, val_loss, model, path):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model, path)\n        elif score < self.best_score - self.delta:\n            self.counter += 1\n            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model, path)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model, path):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model, path)\n        self.val_loss_min = val_loss\n\ndef model_training(model, train_loader, val_loader, epochs,\n                   batch_size=64, lr=0.001, patience=10,\n                   model_path='model.pth'):\n    \n    if os.path.isfile(model_path):\n\n        # load the last checkpoint with the best model\n        model = torch.load(model_path)\n\n        return model\n\n    else:\n\n        # Loss and optimizer\n        optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n        scheduler = ReduceLROnPlateau(optimizer, mode='min', patience=2,\n                                      factor=0.4, verbose=True)\n\n        train_losses = []\n        val_losses = []\n        early_stopping = EarlyStopping(patience=patience, verbose=True)\n\n        for epoch in tqdm(range(epochs)):\n            train_loss, val_loss = 0, 0\n            # Training phase\n            model.train()\n            bar = tqdm(train_loader)\n\n            for i, (X_cont, X_cat, y) in enumerate(bar):\n                preds = model(X_cont, X_cat)\n                loss = quantile_loss(preds, y.to(device), quantiles)\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                with torch.no_grad():\n                    train_loss += loss.item() / len(train_loader)\n                    # bar.set_description(f\"{loss.item():.3f}\")\n\n            # Validation phase\n            val_preds = []\n            true_y = []\n            model.eval()\n            with torch.no_grad():\n                for phase in [\"valid\"]:\n                    if phase == \"train\":\n                        loader = train_loader\n                    else:\n                        loader = val_loader\n\n                    for i, (X_cont, X_cat, y) in enumerate(loader):\n                        preds = model(X_cont, X_cat)\n\n                        val_preds.append(preds)\n                        true_y.append(y)\n\n                        loss = quantile_loss(preds, y.to(device), quantiles)\n                        val_loss += loss.item() / len(loader)\n\n                val_preds = torch.cat(val_preds, dim=0).detach().cpu().numpy()\n                true_y = torch.cat(true_y, dim=0).detach().cpu().numpy()\n                score = metric(val_preds, true_y)\n\n            print(f\"[{phase}] Epoch: {epoch} | Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f} | Val score: {score:.4f}\")\n\n            early_stopping(score, model, path=model_path)\n\n            if early_stopping.early_stop:\n                print(\"Early stopping\")\n                break\n\n            train_losses.append(train_loss)\n            val_losses.append(val_loss)\n            scheduler.step(val_loss)\n\n        model = torch.load(model_path)\n\n        return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\nModel_Root = 'models'  #this root for the prediction phase ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")\n\ntr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])\n\ndata['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n\nbase = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient', 'FVC', 'Percent']].copy()\nbase.columns = ['Patient','min_FVC', 'min_percent']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\n\ndata = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base\n\nCOLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n\ndata['age_week'] = data['Age'].values + data['base_week'].values/53\n        \ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['min_percent_norm'] = (data['min_percent'] - data['min_percent'].min() ) / ( data['min_percent'].max() - data['min_percent'].min() )\ndata['age_week_norm'] = (data['age_week'] - data['age_week'].min() ) / ( data['age_week'].max() - data['age_week'].min() )\n\ncat_feat = ['Sex', 'SmokingStatus']\n\nuniques = []\nfor i, v in enumerate(cat_feat):\n    data[v] = OrdinalEncoder(dtype=\"int\").fit_transform(data[[v]])\n    uniques.append(len(data[v].unique()))\n\ntr = data.loc[data.WHERE == 'train']\nchunk = data.loc[data.WHERE == 'val']\nsub = data.loc[data.WHERE == 'test']\n\nFE += ['age', 'week', 'BASE', 'min_percent_norm', 'age_week_norm']\n\nprint(list(tr))\nprint(list(sub))\nprint(FE)\nprint(cat_feat)\nprint(uniques)\n\ndims = [1, 2]\nemb_dims = [(x, y) for x, y in zip(uniques, dims)]\nn_cont = len(FE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model training and submission print\n\nhidden_dim = 256\nout_dim = 1\n\nkfold = 6\ngroups = np.asarray(tr['Patient'].values)\nskf = GroupKFold(n_splits=kfold)\n\navg_preds = np.zeros((len(sub), len(quantiles)))\nmodels = []\n\nfor i, (train_index, test_index) in enumerate(skf.split(tr, tr['FVC'].values, groups=groups)):\n    print('[Fold %d/%d]' % (i + 1, kfold))\n\n    model_path = f\"{Model_Root}/nn_model_%s.pth\" % i\n\n    if os.path.isfile(model_path):\n\n        final_model = torch.load(model_path)\n\n        X_test = make_X(sub, FE, cat_feat)\n        test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)\n\n        preds = []\n        # model = model_nn(hidden_dim, out_dim, emb_dims, n_cont).to(device)\n        # model.load_state_dict(torch.load(model_path))\n\n\n        for i, (X_cont, X_cat, y) in enumerate(tqdm(test_loader)):\n            print(X_cont.shape, X_cat.shape)\n            out = final_model(X_cont, X_cat)\n            preds.append(out)\n\n        preds = torch.cat(preds, dim=0).detach().cpu().numpy()\n        avg_preds += preds\n\n    else:\n\n        X_train, X_valid = tr.iloc[train_index], tr.iloc[test_index]\n        y_train, y_valid = tr.iloc[train_index]['FVC'].values, tr.iloc[test_index]['FVC'].values\n\n        X_train = make_X(X_train.reset_index(), FE, cat_feat)\n        X_valid = make_X(X_valid.reset_index(), FE, cat_feat)\n\n        train_loader = Loader(X_train, y_train, cat_cols=cat_feat, batch_size=16, shuffle=True)\n        val_loader = Loader(X_valid, y_valid, cat_cols=cat_feat, batch_size=64, shuffle=True)\n\n        model = model_nn(hidden_dim, out_dim, emb_dims, n_cont).to(device)\n\n        final_model = model_training(model, train_loader, val_loader, epochs=1000,\n                                     batch_size=64, lr=0.01, patience=20,\n                                     model_path='nn_model_%s.pth' % i)\n\n        models.append(final_model)\n\nfor model in models:\n\n    X_test = make_X(sub, FE, cat_feat)\n    test_loader = Loader(X_test, None, cat_cols=cat_feat, batch_size=256, shuffle=False)\n\n    preds = []\n    # model = model_nn(hidden_dim, out_dim, emb_dims, n_cont).to(device)\n    # model.load_state_dict(torch.load('nn_model_%s.pth' % i))\n    with torch.no_grad():\n        for i, (X_cont, X_cat, y) in enumerate(tqdm(test_loader)):\n            out = model(X_cont, X_cat)\n            preds.append(out)\n\n    preds = torch.cat(preds, dim=0).detach().cpu().numpy()\n    avg_preds += preds\n\nprint(avg_preds)\navg_preds = avg_preds/kfold\nprint(avg_preds)\n\nsub['Patient_Week'] = sub['Patient_Week'].values\nsub['FVC'] = avg_preds[:, 1]\nsub['Confidence'] = avg_preds[:, 2]-avg_preds[:, 0]\n\nsub[['Patient_Week', 'FVC', 'Confidence']].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}