{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom sklearn.preprocessing import OneHotEncoder, MinMaxScaler\nfrom sklearn.compose import ColumnTransformer \nfrom sklearn.model_selection import GroupKFold\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        break\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Config","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"BASE_DIR = '/kaggle/input/osic-pulmonary-fibrosis-progression/'\nBASE_PATIENT_DIR = '/kaggle/input/osic-pulmonary-fibrosis-progression/train/'\nWORKING_DIR = '/kaggle/working/'\nTEMP_DIR = '/kaggle/temp/'\n\nNFOLDS = 5\nTUNING = False   # when submission make it False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_score(fvc_true, fvc_pred, confidence, return_vec=False):\n    '''modified Laplace Log Likelihood'''\n    sigma_clipped = np.maximum(confidence, 70.)\n    delta = np.minimum(np.abs(fvc_true-fvc_pred), 1000.)\n    metric = -np.sqrt(2)*delta/sigma_clipped - np.log(np.sqrt(2)*sigma_clipped)\n    if return_vec:\n        return metric\n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data preparation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Sex_mapper = {'Male':1, 'Female':0}\n\n\n# prepare training data\ndef load_train():\n    train_df = pd.read_csv(os.path.join(BASE_DIR, 'train.csv'))\n    train_df['Percent'] /= 100.\n    \n    # delete duplicate rows by averaging FVC and Percent\n    train_df[['FVC', 'Percent']] = train_df.groupby(['Patient', 'Weeks'])[['FVC', 'Percent']].transform('mean').values\n    train_df.drop_duplicates(subset=['Patient', 'Weeks'], inplace=True)\n\n    # set baseline weeks\n    train_df['base_Weeks'] = train_df.groupby('Patient')['Weeks'].transform('min')\n    train_df['Weeks_passed'] = train_df['Weeks'] - train_df['base_Weeks']\n    \n    # set baseline FVC and Percent\n    base_df = train_df.loc[train_df.Weeks_passed==0, ['Patient', 'FVC', 'Percent']]\n    base_df.columns = ['Patient', 'base_FVC', 'base_Percent']\n    base_df.reset_index(drop=True, inplace=True)\n    train_df = train_df.merge(base_df, on='Patient')\n\n    train_df['ref_FVC'] = train_df['base_FVC'] / train_df['base_Percent'] \n    train_df['Sex'] = train_df['Sex'].map(Sex_mapper)\n    train_df['target_ratio'] = train_df['FVC'] / train_df['base_FVC']\n    \n    # mark last 3 visits\n    def f(x):\n        result = np.zeros_like(x, dtype='int')\n        result[-3:] = 1\n        return result.astype(bool)\n\n    train_df['last_3_visits'] = train_df.groupby('Patient')['FVC'].transform(f).values\n    \n    # add sample weights :\n    train_df['weights'] =  1./train_df.groupby('Patient')['FVC'].transform('std')\n    #train_df.loc[train_df.last_3_visits, 'weights'] = ... \n    \n    train_df = train_df.reset_index(drop=True)\n    \n    return train_df \n\n\n# prepare test data\ndef load_test():\n    test_df = pd.read_csv(os.path.join(BASE_DIR, 'test.csv'))\n    submit_df = pd.read_csv(os.path.join(BASE_DIR, 'sample_submission.csv'))\n\n    test_df = test_df.rename(columns={'Weeks':'base_Weeks', 'FVC':'base_FVC', 'Percent':'base_Percent'})\n\n    submit_df['Patient'] = submit_df.Patient_Week.str.split('_').str[0]\n    submit_df['Weeks'] = submit_df.Patient_Week.str.split('_').str[1].astype(int)\n\n    test_df = test_df.merge(submit_df, on='Patient')\n    test_df['Weeks_passed'] = test_df['Weeks'] - test_df['base_Weeks']\n    test_df['base_Percent'] /= 100.\n    test_df['ref_FVC'] = test_df['base_FVC'] / test_df['base_Percent']\n    test_df['Sex'] = test_df['Sex'].map(Sex_mapper)\n    test_df = test_df.set_index('Patient_Week')\n    return test_df, submit_df[['Patient_Week', 'FVC', 'Confidence']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = load_train()\ntest_df, submit_df = load_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Keras Model & custom metrics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import callbacks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def laplace_log_likelihood(y_true, y_pred):\n    # y_pred=[fvc_ratio, log_sigma], y_true=[fvc_ratio, FVC_baseline]\n    fvc_true = y_true[:, 0] * y_true[:,1]\n    fvc_pred = y_pred[:, 0] * y_true[:,1]\n\n    log_sigma = y_pred[:, 1]\n    sigma = K.exp(log_sigma)\n\n    sigma_clipped = K.maximum(sigma, K.constant(70., dtype='float32'))\n    delta = K.minimum(K.abs(fvc_true - fvc_pred), K.constant(1000., dtype='float32'))\n\n    sqrt2 = K.sqrt(K.constant(2, dtype='float32'))\n    metric = -sqrt2*(delta/sigma_clipped) - K.log(sqrt2*sigma_clipped)\n\n    return K.mean(metric)\n    \ndef nll_gaussian_loss(y_true, y_pred):\n    # negative loglilelihood loss (NLL) of gaussian\n    # reference: https://www.kaggle.com/ttahara/osic-baseline-lgbm-with-custom-metric?scriptVersionId=38578211\n    fvc_true = y_true[:, 0] * y_true[:,1]\n    fvc_pred = y_pred[:, 0] * y_true[:, 1]\n    log_sigma = y_pred[:, 1]\n\n    term1 = -K.constant(0.5, dtype='float32') * K.square((fvc_true-fvc_pred)/K.exp(log_sigma))\n    term2 = -K.log(K.sqrt(K.constant(2.0*np.pi, dtype='float32'))) - log_sigma\n    loss = -(term1 + term2)\n    \n    # try sample weights\n    if y_true.shape[1] == 3:\n        weights = y_true[:, 2]\n        return K.sum(loss * weights) / K.sum(weights)\n    else:\n        return K.mean(loss)\n\n\ndef create_model_v3(input_dim):\n    \n    # default model - without parameters tuning\n    \n    K.clear_session()\n    x_in = layers.Input(shape=input_dim)\n    x = layers.Dense(128, activation='relu')(x_in)\n    x = layers.Dropout(.25)(x)\n    x = layers.Dense(128, activation='relu')(x)\n    x = layers.Dropout(.25)(x)\n    x_out = layers.Dense(2, activation=None, name='pred')(x)   \n    \n    m = models.Model(inputs=x_in, outputs=x_out, name='NeuralNet')\n    m.compile(optimizer=Adam(lr=.0005), loss=nll_gaussian_loss, metrics=[laplace_log_likelihood])\n    \n    return m\n\ndef create_model_v4(input_dim, params):\n    \n    # for parameters tuning\n    # params: num_layers, num_units, learning_rate, dropout_rate, activation\n\n    n_units = params['num_units']\n    n_layers = params['num_layers']\n    dropout_rate = params['dropout_rate']\n    activation = params['activation']\n    lr = params['learning_rate']\n    \n    K.clear_session()\n    x_in = layers.Input(shape=input_dim)\n    x = x_in\n    for _ in range(n_layers):\n        x = layers.Dense(n_units, activation=activation)(x)\n        x = layers.Dropout(dropout_rate)(x)\n    x_out = layers.Dense(2, activation=None, name='pred')(x)   \n    m = models.Model(inputs=x_in, outputs=x_out, name='NeuralNet')\n    m.compile(optimizer=Adam(lr=lr), loss=nll_gaussian_loss, metrics=[laplace_log_likelihood])\n    return m","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Tuning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model_cv(train_df, train_params, test_df=None, nfolds=NFOLDS):\n    '''\n    return: train_preds, oof_preds, [test_preds], trained_models, training_history\n    '''\n    cat_cols = ['SmokingStatus']\n    num_cols = ['base_Weeks', 'Weeks_passed', 'Age']# 'ref_FVC']\n    pass_cols = ['base_Percent', 'Sex']\n    all_cols = cat_cols + num_cols + pass_cols\n\n    target_cols = ['target_ratio', 'base_FVC', 'weights']\n\n\n    X = train_df[all_cols].copy()\n    y = train_df[target_cols]\n    group_train = train_df.Patient.values\n    \n    if test_df is not None:\n        X_test = test_df[all_cols].copy()\n        test_fvc_baseline = test_df['base_FVC'].values\n        test_preds = np.zeros(shape=(len(X_test), 2))\n    \n    transformer = ColumnTransformer([\n            ('cat',OneHotEncoder(),cat_cols),\n            ('num',MinMaxScaler(), num_cols)\n        ], remainder='passthrough')\n    \n    oof_preds = pd.DataFrame(np.zeros(shape=(len(X), 2)), index=X.index, columns=['FVC', 'Confidence'])\n    tr_preds = pd.DataFrame(np.zeros(shape=(len(X), 2)), index=X.index, columns=['FVC', 'Confidence'])\n    trained_models = dict()\n    histories = dict()\n\n    cv = GroupKFold(n_splits=nfolds)\n    pbar = tqdm(desc='Group K-folds', total=nfolds)\n    for i, (tr_idx, val_idx) in enumerate(cv.split(X, y, groups=group_train), start=1):\n        X_tr = X.iloc[tr_idx]\n        y_tr = y.iloc[tr_idx]\n        X_val = X.iloc[val_idx]\n        y_val = y.iloc[val_idx]\n\n        X_tr_trans = transformer.fit_transform(X_tr)\n        X_val_trans = transformer.transform(X_val)\n\n        neuralnet = create_model_v4(input_dim=X_tr_trans.shape[1], params=train_params)\n\n        hx = neuralnet.fit(X_tr_trans, y_tr, \n                       batch_size=128, \n                       epochs=5000, \n                       validation_data=(X_val_trans, y_val), \n                       verbose=0,\n                       callbacks=[callbacks.EarlyStopping(monitor='val_loss', patience=1000, mode='min', restore_best_weights=True)]\n                    )\n\n        trained_models[f'cv{i}'] = neuralnet\n        histories[f'cv{i}'] = hx\n        \n        \n        \n        tr_pred = neuralnet.predict(X_tr_trans)\n        tr_pred[:, 0] *= y_tr.iloc[:, 1].values\n        tr_pred[:, 1] = np.exp(tr_pred[:, 1])\n\n        oof_pred = neuralnet.predict(X_val_trans)\n        oof_pred[:, 0] *= y_val.iloc[:, 1].values\n        oof_pred[:, 1] = np.exp(oof_pred[:, 1])\n\n        oof_preds.iloc[val_idx, :] = oof_pred\n        tr_preds.iloc[tr_idx, :] += tr_pred\n        \n        if test_df is not None:\n            X_test_trans = transformer.transform(X_test)\n            # pred=>[target_ratio, log_sigma]\n            test_pred = neuralnet.predict(X_test_trans)\n            # convert to predict FVC and sigma\n            test_pred[:, 0] *= test_fvc_baseline\n            test_pred[:, 1] = np.exp(test_pred[:, 1])\n            test_preds += test_pred\n        \n        pbar.update(1)\n    pbar.close()\n    \n    if test_df is not None:\n        test_preds /= nfolds\n        test_preds = pd.DataFrame(test_preds, index=test_df.index)\n    else:\n        test_preds = None\n        \n    tr_preds /= (nfolds - 1)\n    tr_preds['FVC_true'] = (y.target_ratio * y.base_FVC).values #'target_ratio', 'base_FVC'\n    oof_preds['FVC_true'] = (y.target_ratio * y.base_FVC).values\n    \n    return tr_preds, oof_preds, test_preds, trained_models, histories\n\n\n\nimport optuna\n\nclass TuningObjective:\n    \n    def __init__(self, train_df, nfolds):\n        self.train_df = train_df\n        self.nfolds = nfolds\n        \n    def __call__(self, trial: optuna.Trial):\n        \n        tr_params = {\n            'num_units': trial.suggest_int('num_units', 10, 256),\n            'num_layers': trial.suggest_int('num_layers', 1, 3),\n            'dropout_rate': trial.suggest_uniform('dropout_rate', 0., .9),\n            'activation': trial.suggest_categorical('activation', choices=['elu','relu','selu','tanh']),\n            'learning_rate': trial.suggest_loguniform('learning_rate', 1e-5, 1e-2) \n        }\n        \n        try:\n            tr_preds, oof_preds, test_preds, trained_models, histories = train_model_cv(self.train_df, tr_params, test_df=None, nfolds=self.nfolds)\n\n            score = compute_score(oof_preds.FVC_true, oof_preds.FVC, oof_preds.Confidence, return_vec=True) # the higher the better\n            score = np.mean(score[self.train_df.last_3_visits.values])   # only last 3 visits\n\n            return -score  # minimize -> the lower the better\n        except:\n            return np.nan","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if TUNING:\n    study = optuna.create_study()\n    study.optimize(TuningObjective(train_df, NFOLDS), n_trials=20)\n    study_df = study.trials_dataframe()\n    print(f\"best params:\\n{study.best_params}\\nbest score:{study.best_value:.4f}\")\n    study_df.to_csv(os.path.join(WORKING_DIR, 'study_df.csv'), index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_params = {\n    'num_units': 137,\n    'num_layers': 2,\n    'dropout_rate': 0.28,\n    'activation': 'tanh',\n    'learning_rate': 0.0055\n}\n\nbest_params = {\n    'num_units': 147,\n     'num_layers': 1,\n     'dropout_rate': 0.12,\n     'activation': 'elu',\n     'learning_rate': 0.00014\n}\n'''\nbest params:\n{'num_units': 256, 'num_layers': 2, 'dropout_rate': 0.01815267204890613, 'activation': 'elu', 'learning_rate': 4.7424362225054834e-05}\nbest score:7.0147\n'''\nbest_params = {'num_units': 256, 'num_layers': 2, 'dropout_rate': 0.018, 'activation': 'elu', 'learning_rate': 4.7e-05}\n#study.best_params","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_preds, oof_preds, test_preds, cv_models, histories = train_model_cv(train_df, best_params, test_df, nfolds=NFOLDS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Error Analysis","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(hx):\n    fig, ax = plt.subplots(ncols=2, figsize=(15, 6))\n\n    xs = range(1, len(hx['loss'])+1)\n\n    pd.Series(hx['loss']).rolling(window=10).mean().plot(ax=ax[0], label='tr', alpha=.7)\n    pd.Series(hx['val_loss']).rolling(window=10).mean().plot(ax=ax[0], label='val', alpha=.7)\n    ax[0].set_xlabel('epoch')\n    ax[0].set_ylabel('loss')\n    ax[0].set_ylim(5, 12)\n    ax[0].legend()\n\n    pd.Series(hx['laplace_log_likelihood']).rolling(window=10).mean().plot(ax=ax[1], label='tr', alpha=.7)\n    pd.Series(hx['val_laplace_log_likelihood']).rolling(window=10).mean().plot(ax=ax[1], label='val', alpha=.7)\n    ax[1].set_xlabel('epoch')\n    ax[1].set_ylabel('score')\n    ax[1].legend()\n    ax[1].set_ylim(-11, -6)\n    plt.show()\n    \ndef print_result(oof_preds, train_df):\n    tmp = oof_preds.copy()\n    X = train_df\n    tmp['predicted_Weeks'] = (X.base_Weeks + X.Weeks_passed).values\n    tmp['Sex'] = X.Sex.values\n    tmp['SmokingStatus'] = X.SmokingStatus.values\n    tmp['Age'] = X.Age.values\n    tmp['last_3_visits'] = X.last_3_visits.values\n    tmp['Patient'] = X.Patient.values\n    \n    # oof-score\n    # ---------\n    fvc_true = tmp[tmp.last_3_visits].FVC_true\n    fvc_pred = tmp[tmp.last_3_visits].FVC\n    conf = tmp[tmp.last_3_visits].Confidence\n    print(f'oof-score: only last 3 visits = {compute_score(fvc_true, fvc_pred, conf):.5f}')\n\n\n    fvc_true = tmp.FVC_true\n    fvc_pred = tmp.FVC\n    sigma = tmp.Confidence\n\n    metric = compute_score(fvc_true, fvc_pred, sigma, return_vec=True)\n    tmp['oof_score'] = metric\n    metric_mean = np.mean(metric)\n    print(\"oof-score: all data {:.5f}\".format(metric_mean))\n    \n    \n    # train-score:\n    # -----------\n    print()\n    fvc_true = tr_preds[tmp.last_3_visits].FVC_true\n    fvc_pred = tr_preds[tmp.last_3_visits].FVC\n    conf = tr_preds[tmp.last_3_visits].Confidence\n    print(f'tr-score: only last 3 visits = {compute_score(fvc_true, fvc_pred, conf):.5f}')\n    print(\"tr-score: all data {:.5f}\".format(compute_score(tr_preds.FVC_true, tr_preds.FVC, tr_preds.Confidence)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hx = histories['cv1'].history \nplot_history(hx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hx = histories['cv2'].history \nplot_history(hx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hx = histories['cv3'].history \nplot_history(hx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hx = histories['cv4'].history \nplot_history(hx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hx = histories['cv5'].history \nplot_history(hx)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_preds.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp = oof_preds.copy()\nX = train_df\ntmp['predicted_Weeks'] = (X.base_Weeks + X.Weeks_passed).values\ntmp['Sex'] = X.Sex.values\ntmp['SmokingStatus'] = X.SmokingStatus.values\ntmp['Age'] = X.Age.values\ntmp['last_3_visits'] = X.last_3_visits.values\ntmp['Patient'] = X.Patient.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp[tmp.Patient=='ID00061637202188184085559']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_result(oof_preds, train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(5,5))\nax = sns.scatterplot(x='FVC', y='FVC_true', data=tmp, ax=plt.gca())\nax.plot([1000, 6000], [1000, 6000], linestyle='--', color='r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"err = tmp.FVC_true - tmp.FVC\n\nplt.figure(figsize=(10, 6))\nax = err.reset_index().plot.scatter(x='index', y=0, ax=plt.gca())\nax.axhline(xmax=len(err), linestyle='--', color='k', linewidth=2)\nax.set_ylabel('Error')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"err[err < -1000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[1146]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.loc[200]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.distplot(tmp.FVC, label='pred')\nax = sns.distplot(tmp.FVC_true, label='true', ax=ax)\nax.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 1\nfig = plt.figure(figsize=(20, 10))\nfor s in tmp.Sex.unique():\n    for smoke in tmp.SmokingStatus.unique():\n        df = tmp[(tmp.Sex==s)&(tmp.SmokingStatus==smoke)]\n        plt.subplot(2, 3, i)\n        ax = sns.distplot(df.FVC, label='pred', ax=plt.gca())\n        ax = sns.distplot(df.FVC_true, label='true', ax=ax)\n        ax.legend()\n        ax.set_title(f\"Sex: {s}, SmokingStatus: {smoke}\")\n        i += 1\n        \nfig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.lmplot(x='predicted_Weeks', y='Confidence', data=tmp, col='SmokingStatus', hue='Sex')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tmp.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# score statistics by patient backgrounds\ntmp['oof_score'] = compute_score(tmp.FVC_true, tmp.FVC, tmp.Confidence, return_vec=True)\n\n(tmp.groupby(['Sex', 'SmokingStatus'])['oof_score']\n     .agg(['mean', 'std', 'median'])\n     .reset_index()\n     .sort_values('mean', ascending=False)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# shap values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit_df = submit_df.merge(test_preds, left_on='Patient_Week', right_index=True)\nsubmit_df = submit_df.drop(columns=['FVC', 'Confidence'])\nsubmit_df.columns = ['Patient_Week', 'FVC', 'Confidence']\nsubmit_df.to_csv('submission.csv', index=False)\nsubmit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}