{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\nimport os\nfrom sklearn.model_selection import KFold\nfrom tqdm.notebook import tqdm\nimport seaborn as sbn\nfrom  datetime import datetime, timedelta\nfrom sklearn import datasets, linear_model\nimport matplotlib.pyplot as plt\nimport statsmodels.api as sm\nimport random\n\nfrom scipy.stats import norm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.mode.chained_assignment = None","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Path"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('/kaggle/input/osic-pulmonary-fibrosis-progression')\nassert path.exists()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"model_path = Path('/kaggle/working/model')\nif os.path.isdir(model_path) == False:\n    os.makedirs(model_path)\nassert model_path.exists()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Read Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_TYPES={\"Patient\": \"category\", \n         \"Weeks\": \"int16\", \"FVC\": \"int32\", 'Percent': 'float32', \"Age\": \"uint8\",\n        \"Sex\": \"category\", \"SmokingStatus\": \"category\" }\nSUBMISSION_TYPES={\"Patient_Week\": \"category\", \"FVC\": \"int32\", \"Confidence\": \"int16\"}\n\ndef read_data(path):\n    train_df = pd.read_csv(path/'train.csv', dtype = TRAIN_TYPES)\n    test_df = pd.read_csv(path/'test.csv', dtype = TRAIN_TYPES)\n    submission_df = pd.read_csv(path/'sample_submission.csv', dtype = SUBMISSION_TYPES)\n    train_df.drop_duplicates(keep='first', inplace=True, subset=['Patient','Weeks'])\n    return train_df, test_df, submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, test_df, submission_df = read_data(path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Feature generation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_submission(df, test_df):\n    df['Patient'] = df['Patient_Week'].apply(lambda x:x.split('_')[0])\n    df['Weeks'] = df['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\n    df = df[['Patient','Weeks','Confidence','Patient_Week']]\n    df = df.merge(test_df.drop('Weeks', axis=1).copy(), on=['Patient'])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = prepare_submission(submission_df, test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df[((submission_df['Patient'] == 'ID00419637202311204720264') & (submission_df['Weeks'] == 6))].head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def adapt_percent_in_submission():\n    previous_match = None\n    for i, r in submission_df.iterrows():\n        in_training = train_df[(train_df['Patient'] == r['Patient']) & (train_df['Weeks'] == r['Weeks'])]\n        if(len(in_training) > 0):\n            previous_match = in_training['Percent'].item()\n            submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match\n        elif previous_match is not None:\n            submission_df.iloc[i, submission_df.columns.get_loc('Percent')] = previous_match","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"adapt_percent_in_submission()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['WHERE'] = 'train'\ntest_df['WHERE'] = 'val'\nsubmission_df['WHERE'] = 'test'\ndata = train_df.append([test_df, submission_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sbn.countplot(base['Sex'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = base[['Patient','FVC', 'Percent']].copy()\nbase.columns = ['Patient','min_FVC', 'min_Percent']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['base_week'] = data['Weeks'] - data['min_week']\ndata['base_week'] = data['base_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[data['Patient'] == 'ID00421637202311550012437']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.rename(columns={\"Age\": \"age\", \"min_FVC\": \"BASE\", \"base_week\": \"week\", \"Percent\": \"percent\"})\nFE += ['age','week','BASE', 'percent']\nFE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = data.loc[data.WHERE=='train']\ntest_df = data.loc[data.WHERE=='val']\nsubmission_df = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.sort_values(['Patient', 'Weeks'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train_df[FE]\nX.head(15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_df['FVC']\ny","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Seed"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### LightGBM and OLS Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"C1_val = 70\nC2_val = 1000\nC1, C2 = C1_val, C2_val\nq = np.array([0.2, 0.50, 0.8])\n\ndef score(y_true, y_pred):\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    #sigma_clip = sigma + C1\n    sigma_clip = np.max(sigma)\n    delta = np.abs(y_true[:, 0] - fvc_pred)\n    delta = np.min(delta)\n    sq2 = np.sqrt(2.)\n    metric = (delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)\n    return np.mean(metric)\n\ndef qloss(y_true, y_pred):\n    print('y_true.shape', y_true.shape)\n    print('y_pred.shape', y_pred.shape)\n    e = y_true - y_pred\n    v = np.max(q*e, (q-1)*e)\n    return np.mean(v)\n\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        y_true = y_true.reshape(-1, 1)\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nn_estimators = 30000\n    \nif not 'sub_row' in locals():\n    sub_row = 0.75\n    \nif not 'bagging_freq' in locals():\n    bagging_freq = 1\n    \nif not 'learning_rate' in locals():\n    learning_rate = 0.4\n\nleave_size = 4\n\nlgb_params = {\n    \"objective\": 'quantile',\n    'n_jobs': 1,\n    'max_depth': leave_size + 1,\n    'num_leaves': 2**leave_size-1,\n    \"min_data_in_leaf\": 2**(leave_size + 1)-1,\n#     'subsample': 0.9,\n    \"n_estimators\": n_estimators,\n    'learning_rate': 8e-3,\n    'colsample_bytree': 0.9,\n    'boosting_type': 'gbdt',\n    \"early_stopping_rounds\": 100,\n    'verbosity': 1000,\n    \"metric\": [\"rmse\", \"mse\"]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_feats = ['Male', 'Female', 'Ex-smoker', 'Never smoked', 'Currently smokes']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD = 5\nkf = KFold(n_splits=NFOLD, shuffle=False)\npred = {a: np.zeros((train_df.shape[0])) for a in q.tolist()}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ensemble_weights = [6./7, 1./7]\nassert np.sum(ensemble_weights) == 1.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def ols_quantile(m, X, q):\n    # m: OLS model.\n    # X: X matrix.\n    # q: Quantile.\n    #\n    # Set alpha based on q. Vectorized for different values of q.\n    mean_pred = m.predict(X)\n    se = np.sqrt(m.scale)\n    return mean_pred + norm.ppf(q) * se","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\nlinear_models = []\nfor cnt, (tr_idx, val_idx) in tqdm(enumerate(kf.split(X)), total=NFOLD):\n    X_train, y_train = X.loc[tr_idx], y.loc[tr_idx]\n    X_valid, y_valid = X.loc[val_idx], y.loc[val_idx]\n    print(f\"FOLD {cnt}\", X_train.shape, y_train.shape, X_valid.shape, y_valid.shape)\n    for qi, quantile_alpha in enumerate(q.tolist()):\n        lgb_params['alpha'] = quantile_alpha\n        m_lgb_regressor = lgb.LGBMRegressor(**lgb_params)\n        m_lgb_regressor.fit(X=X_train, y=y_train, \n                  eval_set=[(X_train, y_train), (X_valid, y_valid)],\n                  eval_names=['train mloss', 'valid mloss'], \n                  eval_metric=lgb_params['metric'],\n                  verbose=lgb_params['verbosity'],\n                  early_stopping_rounds=lgb_params[\"early_stopping_rounds\"],\n                  categorical_feature=cat_feats)\n        lin_model = sm.OLS(y_train, X_train).fit()\n        \n        lin_predict = ols_quantile(lin_model, X_valid, quantile_alpha)\n        lgb_predict = m_lgb_regressor.predict(X_valid)\n        pred[quantile_alpha][val_idx] = np.average([lin_predict, lgb_predict], axis = 0, weights=ensemble_weights)\n        models.append(m_lgb_regressor)\n        linear_models.append(lin_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"full_preds = np.vstack([pred[a] for a in q]).T\nscore(np.array(y).reshape(-1, 1), full_preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = []\nfor i in range(NFOLD):\n    cur_pred = []\n    for j, quantile in enumerate(q):\n        model_idx = i * 3 + j\n        model = models[model_idx]\n        lin_predict = ols_quantile(linear_models[model_idx], submission_df[FE], quantile)\n        dbmc_predict = model.predict(submission_df[FE])\n        cur_pred.append(np.average([lin_predict, dbmc_predict], axis = 0, weights=ensemble_weights))\n    pred.append(np.array(cur_pred).T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_array = np.array(pred)\npreds_array.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_preds = np.mean(preds_array, axis=0)\nfinal_preds.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['FVC1'] = final_preds[:,1]\nsubmission_df['Confidence1'] = final_preds[:, 2] - final_preds[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.loc[~submission_df.FVC1.isnull(),'FVC'] = submission_df.loc[~submission_df.FVC1.isnull(),'FVC1']\nsubmission_df.loc[~submission_df.FVC1.isnull(),'Confidence'] = submission_df.loc[~submission_df.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df['Confidence'] = np.clip(submission_df['Confidence'], a_min=200, a_max=1000)\nsubmission_df['Confidence'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checks"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final_df = pd.read_csv(\"submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final_df.describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for p in test_df['Patient'].unique():\n    submission_final_df[submission_final_df['Patient_Week'].str.find(p) == 0]['FVC'].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for p in test_df['Patient'].unique():\n    fig, ax = plt.subplots()\n    submission_final_df[submission_final_df['Patient_Week'].str.find(p) == 0]['FVC'].plot(ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}