{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!cp ../input/ngboost/autograd-1.3.xyz /tmp/pip/cache/autograd-1.3.tar.gz\n!pip install --no-index --find-links /tmp/pip/cache/ autograd\n\n!pip install \"../input/ngboost/autograd_gamma-0.4.2-py3-none-any.whl\"\n!pip install \"../input/ngboost/lifelines-0.25.2-py3-none-any.whl\"\n!pip install \"../input/ngboost/ngboost-0.2.3-py3-none-any.whl\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os, random\nimport typing as tp\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\nfrom itertools import combinations\nfrom itertools import combinations\nfrom sklearn import model_selection\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import mean_squared_error\n\n# NGBoost\n# -------------------------------------------\n#! pip install ngboost\nfrom ngboost import NGBRegressor\nfrom ngboost.learners import default_tree_learner\nfrom ngboost.distns import Normal\nfrom ngboost.scores import MLE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f\"../input/osic-pulmonary-fibrosis-progression/train.csv\")\ntest = pd.read_csv(f\"../input/osic-pulmonary-fibrosis-progression/test.csv\")\nsample_submission = pd.read_csv(f\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 5\ngroups = train[\"Patient\"].values\nkfold = model_selection.GroupKFold(n_splits=FOLDS)\nfor n, (train_index, valid_index) in enumerate(kfold.split(train, train[\"FVC\"], groups)):\n    train.loc[valid_index, 'kfold'] = int(n)\ntrain[\"kfold\"] = train[\"kfold\"].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid = train.copy()\ntrain = train.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pgb = train.groupby(\"Patient\")\ntrain = pd.DataFrame()\ntk0 = tqdm(pgb, total=len(pgb))\nfor _, user_df in tk0:\n    user_df = user_df.reset_index(drop=True)\n    for index in list(combinations(user_df.index, 2)):\n        df1 = user_df.iloc[[index[0]]].copy()\n        df2 = user_df.iloc[[index[1]]].copy()\n        # df1\n        df1[\"base_Weeks\"] = df2[\"Weeks\"].iloc[0]\n        df1[\"base_FVC\"] = df2[\"FVC\"].iloc[0]\n        df1[\"base_Percent\"] = df2[\"Percent\"].iloc[0]\n        train = pd.concat([train, df1])\n\n        #df2\n        df2[\"base_Weeks\"] = df1[\"Weeks\"].iloc[0]\n        df2[\"base_FVC\"] = df1[\"FVC\"].iloc[0]\n        df2[\"base_Percent\"] = df1[\"Percent\"].iloc[0]\n        train = pd.concat([train, df2])\ntrain = train.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pgb = valid.groupby(\"Patient\")\nvalid = pd.DataFrame()\ntk0 = tqdm(pgb, total=len(pgb))\nfor _, user_df in tk0:\n    user_df = user_df.reset_index(drop=True)\n    index = random.choice(user_df.index.values)\n    df1 = user_df.iloc[index].copy()\n    user_df[\"base_Weeks\"] = df1[\"Weeks\"]\n    user_df[\"base_FVC\"] = df1[\"FVC\"]\n    user_df[\"base_Percent\"] = df1[\"Percent\"]\n    valid = pd.concat([valid, user_df])\nvalid = valid.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.rename(columns={'Weeks': 'base_Weeks', 'FVC': 'base_FVC', 'Percent' : 'base_Percent'})\ntrain[\"Patient_Week\"] = train[\"Patient\"] + '_' + train[\"Weeks\"].astype(str)\nvalid[\"Patient_Week\"] = valid[\"Patient\"] + '_' + valid[\"Weeks\"].astype(str)\n\nsample_submission['Patient'] = sample_submission['Patient_Week'].apply(lambda x: x.split('_')[0])\nsample_submission['Weeks'] = sample_submission['Patient_Week'].apply(lambda x: int(x.split('_')[1]))\n\ntest = pd.merge(sample_submission, test, how='left', left_on='Patient', right_on='Patient')\n\nsample_submission.drop(columns=['Weeks', 'Patient'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'FVC'\ncol_drop = ['Patient', 'Patient_Week', 'kfold', 'Percent']\ncategorical_columns = []\ncategorical_dims = {}\nfor col in train.columns[train.dtypes == object]:\n    if col not in col_drop + [target]:\n        print(col, train[col].nunique())\n        l_enc = LabelEncoder()\n        train[col] = train[col].fillna(\"VV_likely\")\n        train[col] = l_enc.fit_transform(train[col].values)\n        \n        valid[col] = valid[col].fillna(\"VV_likely\")\n        valid[col] = l_enc.transform(valid[col].values)\n        \n        test[col] = test[col].fillna(\"VV_likely\")\n        test[col] = l_enc.transform(test[col].values)\n        \n        categorical_columns.append(col)\n        categorical_dims[col] = len(l_enc.classes_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def score(actual_fvc, predicted_fvc, confidence):\n    \n    sd_clipped = np.maximum(confidence, 70)\n    delta = np.minimum(np.abs(actual_fvc - predicted_fvc), 1000)\n    metric = - np.sqrt(2) * delta / sd_clipped - np.log(np.sqrt(2) * sd_clipped)\n    \n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"quantiles = (0.2, 0.5, 0.8)\ndef quantile_loss(target, preds):\n    #assert not target.requires_grad\n    assert preds.size(0) == target.size(0)\n    losses = []\n    for i, q in enumerate(quantiles):\n        errors = target[:, i] - preds[:, i]\n        losses.append(torch.max((q-1) * errors, q * errors).unsqueeze(1))\n    loss = torch.mean(torch.sum(torch.cat(losses, dim=1), dim=1))\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_one_fold(clf,train_df,valid_df,test_df,features,targets,categorical=[],fold=0):\n    \n    trn_idx = train_df[train_df.kfold != fold].index\n    val_idx = train_df[train_df.kfold == fold].index\n    print(f'len(trn_idx) : {len(trn_idx)}')\n    print(f'len(val_idx) : {len(val_idx)}')\n\n    X_train = train_df.iloc[trn_idx][features]\n    X_valid = train_df.iloc[val_idx][features]\n    X_test  = test_df[features].values\n    \n\n    y_train = train_df.iloc[trn_idx][targets].values#.reshape(-1, 1)\n    y_valid = train_df.iloc[val_idx][targets].values#.reshape(-1, 1)\n\n    \n    fold_oof_df = pd.DataFrame()\n    predictions = np.zeros((len(test_df), 2))\n    \n\n    clf.fit(X_train, y_train,\n        X_val = X_valid , Y_val = y_valid ,\n        sample_weight = None ,\n        val_sample_weight = None ,\n        train_loss_monitor = None ,\n        val_loss_monitor = None ,\n        early_stopping_rounds = 500\n    )\n    \n    \n    fold_oof_df[\"pred_FVC\"] = clf.predict(X_valid.values)\n    \n    y_dists = clf.pred_dist(X_valid.values)\n    #print(np.array(list(y_dists.dist.interval(alpha=0.8)[0])).shape) #- np.array(list(y_dists.dist.interval(0.2))))\n    fold_oof_df[\"Confidence\"] = np.abs(np.array(list(y_dists.dist.interval(alpha=0.8)[0])) - np.array(list(y_dists.dist.interval(0.2)[0])))\n    #fold_oof_df[\"Confidence\"] = clf.pred_dist(X_valid.values).params['scale']\n    fold_oof_df[\"fold\"] = fold\n    fold_oof_df[\"Patient_Week\"] = train_df.iloc[val_idx][\"Patient_Week\"].values\n    fold_oof_df[\"FVC\"] = y_valid\n    \n    y_test_dists = clf.pred_dist(X_test)\n    predictions[:, 0] = clf.predict(X_test)\n    predictions[:, 1] =  np.abs(np.array(list(y_test_dists.dist.interval(alpha=0.8)[0])) - np.array(list(y_test_dists.dist.interval(0.2)[0])))\n\n    # RMSE\n    print(\"fold{} RMSE score: {:<8.5f}\".format(\n        fold, np.sqrt(mean_squared_error(y_valid, fold_oof_df[\"pred_FVC\"].values))))\n    # Competition Metric\n    print(\"fold{} Metric: {:<8.5f}\".format(\n        fold, score(fold_oof_df['FVC'].values, fold_oof_df[\"pred_FVC\"].values, fold_oof_df['Confidence'])\n    ))\n    \n    return fold_oof_df, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_kfold(\n    clf, \n    train,\n    valid,\n    test,\n    features, \n    target, \n    n_fold=FOLDS, \n    categorical=[], \n    my_loss=None\n):\n          \n          \n    \n    print(f\"================================= FOLDS : {n_fold} =================================\")\n    \n    oof_df = pd.DataFrame()\n    predictions = np.zeros((len(test), 2))\n\n    for fold_ in range(n_fold):\n        \n        print(\"Fold {}\".format(fold_))\n        fold_oof_df, fold_predictions = run_one_fold(\n                clf, \n                train,\n                valid, \n                test,\n                features, \n                target, \n                fold=fold_\n        )\n          \n        oof_df = pd.concat([oof_df, fold_oof_df], axis=0)\n        predictions += fold_predictions        \n\n    # RMSE\n    print(\"CV RMSE score: {:<8.5f}\".format(np.sqrt(mean_squared_error(oof_df[\"FVC\"], oof_df[\"pred_FVC\"]))))\n    # Metric\n    print(\"CV Metric: {:<8.5f}\".format(\n        score(oof_df['FVC'].values, oof_df[\"pred_FVC\"].values, oof_df['Confidence'].values)\n    ))\n    \n    predictions = predictions / n_fold\n\n    print(f\"=========================================================================================\")\n    \n    return oof_df, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from lightgbm import LGBMRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets = 'FVC'\nfeatures = [col for col in train.columns if col not in col_drop + [target]]\n\n\npram = {\n    #'Base' : LGBMRegressor(\n      #**{\n      #      'learning_rate': 0.03585185547472276,\n      #      'max_depth': 2,\n      #      'n_estimators': 4558,\n      #      'num_leaves': 459\n      #   }\n    #),\n    #'col_sample': 1.0, \n    #'minibatch_frac': 0.55, \n    'n_estimators': 1000\n}\n\nclf = NGBRegressor(**pram)\n\noof_df, predictions = run_kfold(\n    clf,\n    train,\n    valid,\n    test,\n    features,\n    targets,\n    n_fold=5,\n    categorical=None\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission[\"FVC\"] = predictions[:, 0]\nsample_submission['Confidence'] = predictions[:, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}