{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport random\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch.optim as optim\nfrom tqdm.notebook import tqdm\nimport sklearn.model_selection\nimport torch.optim.lr_scheduler as lr\nimport sys\nsys.path.append(\"../input/osic-vae\")\nfrom preprocess import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#img = get_img_3d(\"ID00419637202311204720264\", file_path=\"../input/osic-pulmonary-fibrosis-progression/test/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#features = get_img_features(img, latent_features=100, model_path=\"../input/osic-vae/full_chest_model.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ntrain[\"FVC\"] = train.groupby([\"Patient\", \"Weeks\"])[\"FVC\"].transform(\"mean\")\ntrain[\"Percent\"] = train.groupby([\"Patient\", \"Weeks\"])[\"Percent\"].transform(\"mean\")\ntrain.drop_duplicates(inplace=True, ignore_index=True)\n\nchunk = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\n\ntest = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\ntest[\"Patient\"] = test[\"Patient_Week\"].apply(lambda x: x.split(\"_\")[0])\ntest[\"Weeks\"] = test[\"Patient_Week\"].apply(lambda x: int(x.split(\"_\")[1]))\ntest = test[[\"Patient\", \"Weeks\", \"Confidence\", \"Patient_Week\"]]\ntest = test.merge(chunk.drop(\"Weeks\", axis=1), on=\"Patient\")\n\ntrain[\"WHERE\"] = \"train\"\nchunk[\"WHERE\"] = \"val\"\ntest[\"WHERE\"] = \"test\"\ndf = train.append([chunk, test])\n\ndf[\"Male\"] = df[\"Sex\"].apply(lambda x: int(x == \"Male\"))\ndf[\"Female\"] = df[\"Sex\"].apply(lambda x: int(x == \"Female\"))\n\ndf[\"ExSmoker\"] = df[\"SmokingStatus\"].apply(lambda x: int(x == \"Ex-smoker\"))\ndf[\"NeverSmoked\"] = df[\"SmokingStatus\"].apply(lambda x: int(x == \"Never smoked\"))\ndf[\"CurrentlySmokes\"] = df[\"SmokingStatus\"].apply(lambda x: int(x == \"Currently smokes\"))\n\n\ndf[\"MinWeeks\"] = df[\"Weeks\"]\ndf.loc[df.WHERE == \"test\", \"MinWeeks\"] = np.nan\ndf[\"MinWeeks\"] = df.groupby(\"Patient\")[\"MinWeeks\"].transform(\"min\")\n\n\ndf = df.merge(\n    df.loc[df[\"Weeks\"] == df[\"MinWeeks\"], [\"Patient\", \"FVC\", \"Percent\", \"WHERE\"]].rename({\"FVC\": \"StartFVC\", \"Percent\": \"StartPercent\"}, axis=1),\n    on=[\"Patient\", \"WHERE\"]\n)\n\ndf[\"Weeks\"] = df[\"Weeks\"] - df[\"MinWeeks\"]\n\ndf[\"Weeks\"] = (df[\"Weeks\"] - df[\"Weeks\"].mean()) / df[\"Weeks\"].std()\ndf[\"Age\"] = (df[\"Age\"] - df[\"Age\"].mean()) / df[\"Age\"].std()\ndf[\"StartFVC\"] = (df[\"StartFVC\"] - df[\"StartFVC\"].mean()) / df[\"StartFVC\"].std()\ndf[\"StartPercent\"] = (df[\"StartPercent\"] - df[\"StartPercent\"].mean()) / df[\"StartPercent\"].std()\n\ntrain = df.loc[df.WHERE == \"train\"]\nchunk = df.loc[df.WHERE == \"val\"]\ntest = df.loc[df.WHERE == \"test\"]\n\ndel df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [x for x in test.columns if x not in [\"Patient_Week\", \"Patient\", \"FVC\", \"Percent\", \"Confidence\", \"Sex\", \"SmokingStatus\", \"WHERE\", \"MinWeeks\"]]\ntarget = \"FVC\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[features]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyDataset(torch.utils.data.Dataset):\n    def __init__(self, df, mode=\"train\"):\n        self.df = df\n        self.mode = mode\n        self.cache = {}\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        patient_id = self.df.loc[idx, \"Patient\"]\n\n        if self.mode == \"train\":\n            tab_features = self.df.loc[idx, features].values\n            img_features = np.load(\"../input/osic-train-image-features/{}.npy\".format(patient_id))\n\n            return np.concatenate((tab_features, img_features)).astype(np.float32), self.df.loc[idx, target].astype(np.float32).reshape(1)\n        elif self.mode == \"test\":\n            tab_features = self.df.loc[idx, features].values\n\n            if patient_id in self.cache:\n                img_features = self.cache[patient_id]\n            else:\n                img = get_img_3d(patient_id, file_path=\"../input/osic-pulmonary-fibrosis-progression/test/\")\n                img_features = get_img_features(img, latent_features=100, model_path=\"../input/osic-vae/full_chest_model.pth\")\n                \n                self.cache[patient_id] = img_features\n\n            return np.concatenate((tab_features, img_features)).astype(np.float32)\n        else:\n            return None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyModel(nn.Module):\n    def __init__(self, in_features, out_features, args):\n        super().__init__()\n        \n        def get_activation(args):\n            if args[\"activation\"] == \"ReLU\":\n                return nn.ReLU(inplace=True)\n            elif args[\"activation\"] == \"LeakyReLU\":\n                return nn.LeakyReLU(inplace=True)\n            elif args[\"activation\"] == \"Tanh\":\n                return nn.Tanh()\n            elif args[\"activation\"] == \"Swish\":\n                return Swish()\n            else:\n                return None\n\n        self.my_model = nn.Sequential(\n            nn.Linear(in_features, args[\"mid_features\"]),\n            get_activation(args),\n            nn.Dropout(p=args[\"dropout\"]),\n            \n            nn.Linear(args[\"mid_features\"], args[\"mid_features\"]),\n            get_activation(args),\n            nn.Dropout(p=args[\"dropout\"]),\n            \n            nn.Linear(args[\"mid_features\"], out_features),\n            get_activation(args)\n        )\n\n    def forward(self, x):\n        return self.my_model(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quantile_loss(preds, targets, quantiles):\n    errors = targets - preds\n    losses = [torch.max((q - 1) * errors[:, i], q * errors[:, i]).unsqueeze(1) for i, q in enumerate(quantiles)]\n\n    return torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quantile_metric(preds, targets):\n    sigma = preds[:, 2:3] - preds[:, 0:1]\n    sigma[sigma < 70] = 70\n\n    delta = (targets - preds[:, 1:2]).abs()\n    delta[delta > 1000] = 1000\n\n    return (-np.sqrt(2) * delta / sigma - torch.log(np.sqrt(2) * sigma)).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_lr(optimizer):\n    for p in optimizer.param_groups:\n        return p[\"lr\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main(args, fold, train_df, dev_df):\n    seed_everything(args[\"seed\"])\n\n    data_loaders = {\n        \"train\": torch.utils.data.DataLoader(MyDataset(train_df), batch_size=args[\"batch_size\"], shuffle=True, num_workers=3, pin_memory=True),\n        \"dev\": torch.utils.data.DataLoader(MyDataset(dev_df), batch_size=args[\"batch_size\"], shuffle=True, num_workers=3, pin_memory=True)\n    }\n\n    model = MyModel(len(features) + 200, 3, args).to(device=args[\"device\"], dtype=args[\"dtype\"])\n    optimizer = optim.AdamW(model.parameters(), lr=args[\"lr\"], weight_decay=args[\"weight_decay\"])\n    scheduler = lr.StepLR(optimizer, step_size=args[\"step_size\"], gamma=args[\"gamma\"])\n\n    best_metric = -10**10\n\n    pbar = tqdm(range(args[\"epochs\"]), disable=(not args[\"progress_bar\"]))\n\n    for epoch in pbar:\n        stats = {\n            \"train\": {\n                \"loss\": 0,\n                \"metric\": 0\n            },\n            \"dev\": {\n                \"loss\": 0,\n                \"metric\": 0\n            }\n        }\n\n        model = model.train()\n\n        for X, y in data_loaders[\"train\"]:\n            X = X.to(device=args[\"device\"], dtype=args[\"dtype\"])\n            y = y.to(device=args[\"device\"], dtype=args[\"dtype\"])\n\n            y_pred = model(X)\n            loss = quantile_loss(y_pred, y, args[\"quantiles\"])\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n        model = model.eval()\n\n        with torch.no_grad():\n            for mode in [\"train\", \"dev\"]:\n                for X, y in data_loaders[mode]:\n                    X = X.to(device=args[\"device\"], dtype=args[\"dtype\"])\n                    y = y.to(device=args[\"device\"], dtype=args[\"dtype\"])\n\n                    y_pred = model(X)\n                    loss = quantile_loss(y_pred, y, args[\"quantiles\"])\n                    metric = quantile_metric(y_pred, y)\n\n                    stats[mode][\"loss\"] += loss.item() / len(data_loaders[mode])\n                    stats[mode][\"metric\"] += metric.item() / len(data_loaders[mode])\n\n        if stats[\"dev\"][\"metric\"] > best_metric:\n            best_metric = stats[\"dev\"][\"metric\"]\n\n            if args[\"save\"]:\n                torch.save(model.state_dict(), \"model_fold_{}.pth\".format(fold))\n\n        scheduler.step()\n\n        pbar.set_description(\"best: {:.4f} current: {:.4f} lr: {}\".format(best_metric, stats[\"dev\"][\"metric\"], get_lr(optimizer)))\n\n        if get_lr(optimizer) < args[\"min_lr\"]:\n            break\n\n    return -best_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def main_folds(args):\n    kfold = sklearn.model_selection.GroupKFold(n_splits=args[\"n_splits\"])\n\n    avg_metric = 0\n\n    for fold, (train_idx, dev_idx) in enumerate(kfold.split(train, groups=train[\"Patient\"])):\n        train_df = train.loc[train_idx].reset_index(drop=True)\n        dev_df = train.loc[dev_idx].reset_index(drop=True)\n\n        avg_metric += main(args, fold, train_df, dev_df) / args[\"n_splits\"]\n        #break\n\n    print(avg_metric, args)\n\n    return avg_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from hyperopt import hp, fmin, tpe\n\nspace = {\n    \"seed\": 42,\n    \"n_splits\": 5,\n    \"batch_size\": hp.choice(\"batch_size\", [32, 64, 128, 256, 512, 1024, 2048]),\n    \"epochs\": 25,\n    \"lr\": hp.loguniform(\"lr\", -10, 0),\n    \"weight_decay\": hp.loguniform(\"weight_decay\", -5, 0),\n    \"quantiles\": [0.2, 0.5, 0.8],\n    \"mid_features\": hp.choice(\"mid_features\", [128, 256, 512, 1024, 2048]),\n    \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n    \"dtype\": torch.float,\n    \"step_size\": hp.randint(\"step_size\", 2, 20),\n    \"gamma\": hp.loguniform(\"gamma\", -5, 0),\n    \"min_lr\": 1e-8,\n    \"dropout\": 0.5,\n    \"activation\": hp.choice(\"activation\", [\"ReLU\", \"LeakyReLU\", \"Tanh\", \"Swish\"]),\n    \"progress_bar\": False,\n    \"save\": False\n}\n\n#best = fmin(main_folds, space, algo=tpe.suggest, max_evals=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ARGS = {\n    \"seed\": 42,\n    \"n_splits\": 5,\n    \"batch_size\": 32,\n    \"epochs\": 100,\n    \"lr\": 0.0027214256541696524,\n    \"weight_decay\": 0.5435493827830923,\n    \"quantiles\": [0.2, 0.5, 0.8],\n    \"mid_features\": 1024,\n    \"device\": torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\"),\n    \"dtype\": torch.float,\n    \"step_size\": 9,\n    \"gamma\": 0.6642775979489709,\n    \"min_lr\": 1e-8,\n    \"dropout\": 0.5,\n    \"activation\": \"LeakyReLU\",\n    \"progress_bar\": True,\n    \"save\": True\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#main_folds(ARGS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference(args):\n    data_loaders = {\n        \"test\": torch.utils.data.DataLoader(MyDataset(test.reset_index(drop=True), mode=\"test\"), batch_size=args[\"batch_size\"], shuffle=False),\n    }\n\n    preds = {\n        \"FVC\": np.zeros((len(test), 1)),\n        \"Confidence\": np.zeros((len(test), 1))\n    }\n\n    for fold in range(args[\"n_splits\"]):\n        fold_preds = {\n            \"FVC\": [],\n            \"Confidence\": []\n        }\n\n        model = MyModel(len(features) + 200, 3, args).to(device=args[\"device\"], dtype=args[\"dtype\"])\n        #model.load_state_dict(torch.load(\"model_fold_{}.pth\".format(fold)))\n        model.load_state_dict(torch.load(\"../input/osic-trained/model_fold_{}.pth\".format(fold)))\n        model = model.eval()\n\n        with torch.no_grad():\n            for i, X in enumerate(tqdm(data_loaders[\"test\"], disable=(not args[\"progress_bar\"]))):\n                X = X.to(device=args[\"device\"], dtype=args[\"dtype\"])\n\n                y_pred = model(X).cpu().detach().numpy()\n\n                fold_preds[\"FVC\"].extend(y_pred[:, 1:2])\n                fold_preds[\"Confidence\"].extend(y_pred[:, 2:3] - y_pred[:, 0:1])\n\n        preds[\"FVC\"] += np.array(fold_preds[\"FVC\"]) / args[\"n_splits\"]\n        preds[\"Confidence\"] += np.array(fold_preds[\"Confidence\"]) / args[\"n_splits\"]\n\n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = inference(ARGS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"FVC\"] = preds[\"FVC\"]\ntest[\"Confidence\"] = preds[\"Confidence\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[[\"Patient_Week\", \"FVC\", \"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}