{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport typing as tp\n\nimport numpy as np\nimport pandas as pd\n\nimport cv2\nimport pydicom\nfrom PIL import Image\n\nimport sklearn\nfrom sklearn import model_selection\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import mean_squared_error\n\nimport lightgbm as lgb\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 42\n\nif os.path.exists('/kaggle/input'):\n    DATA_DIR = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\nelse:\n    DATA_DIR = '../data/raw/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing(data: pd.DataFrame, is_test: bool = True) -> pd.DataFrame:\n    # Create Common Features.\n    features = pd.DataFrame()\n    for patient, u_data in data.groupby('Patient'):\n        feature = pd.DataFrame({\n            'current_FVC': u_data['FVC'],\n            'current_Percent': u_data['Percent'],\n            'current_Age': u_data['Age'],\n            'current_Week': u_data['Weeks'],\n            'Patient': u_data['Patient'],\n            'Sex': u_data['Sex'].map({'Female': 0, 'Male': 1}),\n            'SmokingStatus': u_data['SmokingStatus'].map({'Currently smokes': 0, 'Never smoked': 1, 'Ex-smoker': 2}),\n        })\n        features = pd.concat([features, feature])\n    # Create Label Data.\n    if is_test:\n        label = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), usecols=['Patient_Week'])\n        label['Patient'] = label['Patient_Week'].apply(lambda x: x.split('_')[0])\n        label['pred_Weeks'] = label['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\n        label['FVC'] = np.nan\n\n        dst_data = pd.merge(label, features, how='left', on='Patient')\n    else:\n        label = pd.DataFrame({\n            'Patient_Week': data['Patient'].astype(str) + '_' + data['Weeks'].astype(str),\n            'Patient': data['Patient'],\n            'pred_Weeks': data['Weeks'],\n            'FVC': data['FVC']\n        })\n\n        dst_data = pd.merge(label, features, how='outer', on='Patient')\n        dst_data = dst_data.query('pred_Weeks!=current_Week')\n        \n    dst_data['passed_Weeks'] = dst_data['current_Week'] - dst_data['pred_Weeks']\n    return dst_data\n\ntrain = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))\ntrain = preprocessing(train, is_test=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metric","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class OSICLossForLGBM:\n    \"\"\"\n    Custom Loss for LightGBM.\n    \n    * Objective: return grad & hess of NLL of gaussian\n    * Evaluation: return competition metric\n    \"\"\"\n    \n    def __init__(self, epsilon: float=1) -> None:\n        \"\"\"Initialize.\"\"\"\n        self.name = \"osic_loss\"\n        self.n_class = 2  # FVC & Confidence\n        self.epsilon = epsilon\n    \n    def __call__(self, preds: np.ndarray, labels: np.ndarray, weight: tp.Optional[np.ndarray]=None) -> float:\n        \"\"\"Calc loss.\"\"\"\n        sigma_clip = np.maximum(preds[:, 1], 70)\n        Delta = np.minimum(np.abs(preds[:, 0] - labels), 1000)\n        loss_by_sample = - np.sqrt(2) * Delta / sigma_clip - np.log(np.sqrt(2) * sigma_clip)\n        loss = np.average(loss_by_sample, weight)\n        \n        return loss\n    \n    def _calc_grad_and_hess(\n        self, preds: np.ndarray, labels: np.ndarray, weight: tp.Optional[np.ndarray]=None\n    ) -> tp.Tuple[np.ndarray]:\n        \"\"\"Calc Grad and Hess\"\"\"\n        mu = preds[:, 0]\n        sigma = preds[:, 1]\n        \n        sigma_t = np.log(1 + np.exp(sigma))\n        grad_sigma_t = 1 / (1 + np.exp(- sigma))\n        hess_sigma_t = grad_sigma_t * (1 - grad_sigma_t)\n        \n        grad = np.zeros_like(preds)\n        hess = np.zeros_like(preds)\n        grad[:, 0] = - (labels - mu) / sigma_t ** 2\n        hess[:, 0] = 1 / sigma_t ** 2\n        \n        tmp = ((labels - mu) / sigma_t) ** 2\n        grad[:, 1] = 1 / sigma_t * (1 - tmp) * grad_sigma_t\n        hess[:, 1] = (\n            - 1 / sigma_t ** 2 * (1 - 3 * tmp) * grad_sigma_t ** 2\n            + 1 / sigma_t * (1 - tmp) * hess_sigma_t\n        )\n        if weight is not None:\n            grad = grad * weight[:, None]\n            hess = hess * weight[:, None]\n        return grad, hess\n    \n    def loss(self, preds: np.ndarray, data: lgb.Dataset) -> tp.Tuple[str, float, bool]:\n        \"\"\"Return Loss for lightgbm\"\"\"\n        labels = data.get_label()\n        weight = data.get_weight()\n        n_example = len(labels)\n        \n        # # reshape preds: (n_class * n_example,) => (n_class, n_example) =>  (n_example, n_class)\n        preds = preds.reshape(self.n_class, n_example).T\n        # # calc loss\n        loss = self(preds, labels, weight)\n        \n        return self.name, loss, True\n    \n    def grad_and_hess(self, preds: np.ndarray, data: lgb.Dataset) -> tp.Tuple[np.ndarray]:\n        \"\"\"Return Grad and Hess for lightgbm\"\"\"\n        labels = data.get_label()\n        weight = data.get_weight()\n        n_example = len(labels)\n        \n        # # reshape preds: (n_class * n_example,) => (n_class, n_example) =>  (n_example, n_class)\n        preds = preds.reshape(self.n_class, n_example).T\n        # # calc grad and hess.\n        grad, hess =  self._calc_grad_and_hess(preds, labels, weight)\n\n        # # reshape grad, hess: (n_example, n_class) => (n_class, n_example) => (n_class * n_example,) \n        grad = grad.T.reshape(n_example * self.n_class)\n        hess = hess.T.reshape(n_example * self.n_class)\n        \n        return grad, hess","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class LGBM_Wrapper():\n\n    def __init__(self):\n        self.model = None\n        self.importance = None\n\n        self.train_bin_path = 'tmp_train_set.bin'\n        self.valid_bin_path = 'tmp_valid_set.bin'\n\n    def _remove_bin_file(self, filename):\n        if os.path.exists(filename):\n            os.remove(filename)\n\n    def dataset_to_binary(self, train_dataset, valid_dataset):\n        # Remove Binary Cache.\n        self._remove_bin_file(self.train_bin_path)\n        self._remove_bin_file(self.valid_bin_path)\n        # Save Binary Cache.\n        train_dataset.save_binary(self.train_bin_path)\n        valid_dataset.save_binary(self.valid_bin_path)\n        # Reload Binary Cache.\n        train_dataset = lgb.Dataset(self.train_bin_path)\n        valid_dataset = lgb.Dataset(self.valid_bin_path)\n        return train_dataset, valid_dataset\n\n    def fit(self, params, train_param,\n            X_train, y_train, X_valid, y_valid, categorical=None,\n            train_weight=None, valid_weight=None):\n        train_dataset = lgb.Dataset(\n            X_train, y_train, feature_name=X_train.columns.tolist(),\n            categorical_feature=categorical, weight=train_weight\n        )\n        valid_dataset = lgb.Dataset(\n            X_valid, y_valid, weight=valid_weight, \n            categorical_feature=categorical, reference=train_dataset\n        )\n\n        train_dataset, valid_dataset = self.dataset_to_binary(train_dataset, valid_dataset)\n\n        self.model = lgb.train(\n            params,\n            train_dataset,\n            valid_sets=[train_dataset, valid_dataset],\n            **train_param\n        )\n        # Remove Binary Cache.\n        self._remove_bin_file(self.train_bin_path)\n        self._remove_bin_file(self.valid_bin_path)\n\n    def predict(self, data):\n        return self.model.predict(data, num_iteration=self.model.best_iteration)\n\n    def model_importance(self):\n        imp_df = pd.DataFrame(\n            [self.model.feature_importance()],\n            columns=self.model.feature_name(),\n            index=['Importance']\n        ).T\n        imp_df.sort_values(by='Importance', inplace=True)\n        return imp_df\n\n    def plot_importance(self, filepath, max_num_features=50, figsize=(18, 25)):\n        imp_df = self.model_importance()\n        # Plot Importance DataFrame.\n        plt.figure(figsize=figsize)\n        imp_df[-max_num_features:].plot(\n            kind='barh', title='Feature importance', figsize=figsize,\n            y='Importance', align=\"center\"\n        )\n        plt.show()\n        # plt.savefig(filepath)\n        # plt.close('all')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"custom_loss = OSICLossForLGBM()\n\nparams = {\n    'model_params':{\n        'num_class': 2,\n        # 'objective': 'regression',\n        'metric': 'None',\n        'boosting_type': 'gbdt',\n        'learning_rate': 5e-02,\n        'seed': SEED,\n        \"subsample\": 0.4,\n        \"subsample_freq\": 1,\n        'max_depth': 1,\n        'verbosity': -1\n    },\n    'train_params': {\n        \"num_boost_round\": 10000,\n        \"verbose_eval\":100,\n        \"early_stopping_rounds\": 100,\n        'fobj': custom_loss.grad_and_hess,\n        'feval': custom_loss.loss\n    }\n}\n\n\nu_idx = train['Patient_Week']\n\ncategorical_cols = ['Sex', 'SmokingStatus']\ndrop_cols = ['Patient', 'Patient_Week', 'FVC']\nfeatures = [c for c in train.columns.tolist() if c not in drop_cols]\n\nX = train[features]\ny = train['FVC']\ngroups = train['Patient']\n\nn_fold = 5\ng_kfold = model_selection.GroupKFold(n_splits=n_fold)\n\nmodels = []\noof = np.zeros((train.shape[0], 2))\nfor i, (train_idx, valid_idx) in enumerate(g_kfold.split(X, y, groups)):\n    print('\\n' + '#'*20)\n    print('#'*5, f' {i+1}-Fold')\n    print('#'*20 + '\\n')\n    \n    print(f'Train Size: {len(train_idx)}')\n    print(f'Valid Size: {len(valid_idx)}', '\\n')\n    \n    X_train, y_train = X.iloc[train_idx, :], y.iloc[train_idx]\n    X_valid, y_valid = X.iloc[valid_idx, :], y.iloc[valid_idx]\n    \n    lgb_model = LGBM_Wrapper()\n    lgb_model.fit(\n        params['model_params'],\n        params['train_params'],\n        X_train,\n        y_train,\n        X_valid,\n        y_valid,\n        categorical_cols\n    )\n    # add to oof\n    oof[valid_idx] = lgb_model.predict(X_valid)\n    # Add Model\n    models.append(lgb_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f_imp = np.array([m.model_importance().sort_index().values for m in models])\nf_name = models[0].model_importance().sort_index().index\n\nimp_df = pd.DataFrame(f_imp.reshape(-1, len(f_name)).T, index=f_name)\nimp_df['AVG_importance'] = imp_df.iloc[:, :len(models)].mean(axis=1)\nimp_df['STD_importance'] = imp_df.iloc[:, :len(models)].std(axis=1)\nimp_df.sort_values(by='AVG_importance', inplace=True)\n\nimp_df.plot(\n    kind='barh', \n    y='AVG_importance', \n    xerr='STD_importance', \n    capsize=4, \n    figsize=(5, 6)\n)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def score(fvc_true, fvc_pred, sigma):\n    sigma_clip = np.maximum(sigma, 70)\n    delta = np.minimum(np.abs(fvc_true - fvc_pred), 1000)\n    metric = - (np.sqrt(2) * delta / sigma_clip) - np.log(np.sqrt(2) * sigma_clip)\n    return np.mean(metric)\n\nscore(train['FVC'], oof[:, 0], oof[:, 1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(os.path.join(DATA_DIR, 'test.csv'))\ntest = preprocessing(test, is_test=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_idx = test['Patient_Week'].to_numpy().reshape(-1, 1)\npred = np.mean([m.predict(test[features]) for m in models], axis=0)\n\npred_df = pd.DataFrame(\n    np.concatenate((test_idx, pred), axis=1), \n    columns=['Patient_Week', 'FVC', 'Confidence']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'))\n\nsub_df = submission.drop(columns=['FVC', 'Confidence'])\nsub_df = sub_df.merge(pred_df[['Patient_Week', 'FVC', 'Confidence']], on='Patient_Week')\nsub_df.columns = submission.columns\n\n\nif os.path.exists('/kaggle/input'):\n    sub_df.to_csv('submission.csv', index=False)\n\nprint(sub_df.shape)\nsub_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}