{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ensemble technique in machine learning \n\n### please upvote if you like the approach simple but effective approach ! this will motivate me to make more such notebooks, and also i didnt apply grid search you can do it to tune the hyper parametrs !  This is just an approach if you like mahcine leanring ! \n"},{"metadata":{},"cell_type":"markdown","source":"# Libararies "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom sklearn import linear_model, ensemble\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\n\nimport tensorflow as tf\n\nfrom tqdm.notebook import tqdm\n\nimport os\nfrom PIL import Image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# loading data sets "},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsubmission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['Patient'] = (\n    submission['Patient_Week']\n    .apply(\n        lambda x:x.split('_')[0]\n    )\n)\n\nsubmission['Weeks'] = (\n    submission['Patient_Week']\n    .apply(\n        lambda x: int(x.split('_')[-1])\n    )\n)\n\nsubmission =  submission[['Patient','Weeks', 'Confidence','Patient_Week']]\n\nsubmission = submission.merge(test.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Dataset'] = 'train'\ntest['Dataset'] = 'test'\nsubmission['Dataset'] = 'submission'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data = train.append([test, submission])\n\nall_data = all_data.reset_index()\nall_data = all_data.drop(columns=['index'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.head()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# A quick data analysis "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_patients = train.Patient.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(5, 1, figsize=(10, 20))\n\nfor i in range(5):\n    patient_log = train[train['Patient'] == train_patients[i]]\n\n    ax[i].set_title(train_patients[i])\n    ax[i].plot(patient_log['Weeks'], patient_log['FVC'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"feature engineering "},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data['FirstWeek'] = all_data['Weeks']\nall_data.loc[all_data.Dataset=='submission','FirstWeek'] = np.nan\nall_data['FirstWeek'] = all_data.groupby('Patient')['FirstWeek'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"first_fvc = (\n    all_data\n    .loc[all_data.Weeks == all_data.FirstWeek][['Patient','FVC']]\n    .rename({'FVC': 'FirstFVC'}, axis=1)\n    .groupby('Patient')\n    .first()\n    .reset_index()\n)\n\nall_data = all_data.merge(first_fvc, on='Patient', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data['WeeksPassed'] = all_data['Weeks'] - all_data['FirstWeek']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Patient height and FEV "},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"def calculate_height(row):\n    if row['Sex'] == 'Male':\n        return row['FirstFVC'] / (27.63 - 0.112 * row['Age'])\n    else:\n        return row['FirstFVC'] / (21.78 - 0.101 * row['Age'])\n\nall_data['Height'] = all_data.apply(calculate_height, axis=1)\n\"\"\"\n\ndef calculate_height(row):\n    height = 0\n    if row['Sex'] == 'Male' or 'Female':\n        height = (((row['FirstFVC']/933.33) + 0.026*row['Age'] + 2.89)/0.0443)\n        return int(height) \n\nall_data['Height'] = all_data.apply(calculate_height, axis=1)\n\n\ndef FEV1(row):\n    FEV = 0\n    if row['Sex'] == 'Male':\n        FEV = (0.84 * row['FirstFVC'] - 0.23)\n    else:\n        FEV = (0.84 * row['FirstFVC'] - 0.36)\n    return FEV\nall_data['FEV'] = all_data.apply(FEV1, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Categorical column"},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data = pd.concat([\n    all_data,\n    pd.get_dummies(all_data.Sex),\n    pd.get_dummies(all_data.SmokingStatus)\n], axis=1)\n\nall_data = all_data.drop(columns=['Sex', 'SmokingStatus'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature scaling "},{"metadata":{"trusted":true},"cell_type":"code","source":"def scale_feature(series):\n    return (series - series.min()) / (series.max() - series.min())\n\nall_data['Weeks'] = scale_feature(all_data['Weeks'])\nall_data['Percent'] = scale_feature(all_data['Percent'])\nall_data['Age'] = scale_feature(all_data['Age'])\nall_data['FirstWeek'] = scale_feature(all_data['FirstWeek'])\nall_data['FirstFVC'] = scale_feature(all_data['FirstFVC'])\nall_data['WeeksPassed'] = scale_feature(all_data['WeeksPassed'])\nall_data['Height'] = scale_feature(all_data['Height'])\nall_data['FEV'] = scale_feature(all_data['FEV'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_columns = [\n    'Percent',\n    'Age',\n    'FirstWeek',\n    'FirstFVC',\n    'WeeksPassed',\n    'Female',\n    'Male', \n    'Currently smokes',\n    'Ex-smoker',\n    'Never smoked',\n    'FEV'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = all_data.loc[all_data.Dataset == 'train']\ntest = all_data.loc[all_data.Dataset == 'test']\nsubmission = all_data.loc[all_data.Dataset == 'submission']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[feature_columns].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold \nfrom sklearn.model_selection import cross_val_score\n\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.50, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\nn_folds = 10\n\ndef rmse_cv(model):\n    kf = KFold(n_folds, shuffle=True, random_state=1234).get_n_splits(train[feature_columns])\n    rmse= np.sqrt(-cross_val_score(model, train[feature_columns], train['FVC'], scoring=\"neg_mean_squared_error\", cv = kf))\n    return(rmse)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lasso"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import Lasso\nfrom sklearn.pipeline import make_pipeline \nfrom sklearn.linear_model import Lasso\nlasso = Lasso()\n\nlasso.fit(train[feature_columns], train['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lasso_preds = lasso.predict(train[feature_columns])\nsub = pd.DataFrame()\nsub['lasso_FVC'] = lasso_preds\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import Ridge\nridge = Ridge()\nridge.fit(train[feature_columns], train['FVC'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# NG boost"},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ngboost","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ngboost import NGBRegressor\nngb = NGBRegressor()\nngb.fit(train[feature_columns], train['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ngb_preds = ngb.predict(train[feature_columns])\nsub = pd.DataFrame()\nsub['ngb_FVC'] = ngb_preds\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ridge "},{"metadata":{"trusted":true},"cell_type":"code","source":"ridge_preds = lasso.predict(train[feature_columns])\nsub = pd.DataFrame()\nsub['ridge_FVC'] = ridge_preds\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bayesian Ridge"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import BayesianRidge\nbayesian_ridge = BayesianRidge()\nbayesian_ridge.fit(train[feature_columns], train['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bayesian_ridge_preds = bayesian_ridge.predict(train[feature_columns])\nsub = pd.DataFrame()\nsub['bay_ridge_FVC'] = bayesian_ridge_preds\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Huber"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import HuberRegressor\nHuber = HuberRegressor()\nHuber.fit(train[feature_columns], train['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Huber_preds = Huber.predict(train[feature_columns])\nsub = pd.DataFrame()\nsub['huber_FVC'] = Huber_preds\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cat boost"},{"metadata":{"trusted":true},"cell_type":"code","source":"from catboost import CatBoostRegressor\ncat = CatBoostRegressor()\ncat.fit(train[feature_columns], train['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_preds = cat.predict(train[feature_columns])\nsub = pd.DataFrame()\nsub['cat_FVC'] = cat_preds\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ensemble "},{"metadata":{},"cell_type":"markdown","source":"# weights "},{"metadata":{"trusted":true},"cell_type":"code","source":"ridge_weight = 0.2\nlasso_weight = 0.2\n#cat_weight = 0.3\nhuber_weight = 0.40 \nbayesian_ridge_weight = 0.20 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction1 = 0\nsub = pd.DataFrame()\nsub['ensembled_FVC'] = (ridge_preds*ridge_weight) + (lasso_preds*lasso_weight)  + (Huber_preds*huber_weight ) + (bayesian_ridge_preds*bayesian_ridge_weight)\npredictions = sub['ensembled_FVC'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MAE and MSE"},{"metadata":{"trusted":true},"cell_type":"code","source":"mse = mean_squared_error(\n    train['FVC'],\n    predictions,\n    squared=False\n)\n\nmae = mean_absolute_error(\n    train['FVC'],\n    predictions\n)\n\nprint('MSE Loss: {0:.2f}'.format(mse))\nprint('MAE Loss: {0:.2f}'.format(mae))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def competition_metric(trueFVC, predFVC, predSTD):\n    clipSTD = np.clip(predSTD, 70 , 9e9)  \n    deltaFVC = np.clip(np.abs(trueFVC - predFVC), 0 , 1000)  \n    return np.mean(-1 * (np.sqrt(2) * deltaFVC / clipSTD) - np.log(np.sqrt(2) * clipSTD))\n    \n\nprint(\n    'Competition metric: ', \n    competition_metric(train['FVC'].values, predictions, 285) \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['prediction'] = predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(predictions, train['FVC'])\n\nplt.xlabel('predictions')\nplt.ylabel('FVC (labels)')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"delta = predictions - train['FVC']\nplt.hist(delta, bins=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(5, 1, figsize=(10, 20))\n\nfor i in range(5):\n    patient_log = train[train['Patient'] == train_patients[i]]\n\n    ax[i].set_title(train_patients[i])\n    ax[i].plot(patient_log['WeeksPassed'], patient_log['FVC'], label='truth')\n    ax[i].plot(patient_log['WeeksPassed'], patient_log['prediction'], label='prediction')\n    ax[i].legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission[feature_columns].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lasso predictions "},{"metadata":{"trusted":true},"cell_type":"code","source":"lasso_preds1 = lasso.predict(submission[feature_columns])\nsubmission1 = pd.DataFrame()\nsubmission1['lasso_FVC'] = lasso_preds1\nsubmission1.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ridge Predictions "},{"metadata":{"trusted":true},"cell_type":"code","source":"ridge_preds1 = lasso.predict(submission[feature_columns])\nsubmission2 = pd.DataFrame()\nsubmission2['ridge_FVC'] = ridge_preds1\nsubmission2.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bayesian predictions "},{"metadata":{"trusted":true},"cell_type":"code","source":"bayesian_ridge_preds1 = bayesian_ridge.predict(submission[feature_columns])\nsubmission3 = pd.DataFrame()\nsubmission3['bay_ridge_FVC'] = bayesian_ridge_preds1\nsubmission3.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Huber predictions "},{"metadata":{"trusted":true},"cell_type":"code","source":"Huber_preds1 = Huber.predict(submission[feature_columns])\nsubmission4 = pd.DataFrame()\nsubmission4['huber_FVC'] = Huber_preds1\nsubmission4.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Cat predictions "},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_preds1 = cat.predict(submission[feature_columns])\nsubmission5 = pd.DataFrame()\nsubmission5['cat_FVC'] = cat_preds1\nsubmission5.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_ensemble = pd.DataFrame()\nsubmission_ensemble['pred_FVC'] = (submission2['ridge_FVC'].values*ridge_weight) + (submission1['lasso_FVC']*lasso_weight)  + (submission4['huber_FVC'].values*huber_weight ) + (submission3['bay_ridge_FVC'].values*bayesian_ridge_weight)\n\nfinal_prediction = submission_ensemble['pred_FVC'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_ensemble.shape \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['FVC'] = final_prediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# plot for the first five test sets "},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"test_patients = list(submission.Patient.unique())\nfig, ax = plt.subplots(5, 1, figsize=(10, 20))\n\nfor i in range(5):\n    patient_log = submission[submission['Patient'] == test_patients[i]]\n\n    ax[i].set_title(test_patients[i])\n    ax[i].plot(patient_log['WeeksPassed'], patient_log['FVC'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = submission[['Patient_Week', 'FVC']]\n\nsubmission['Confidence'] = 275","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(submission['FVC'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def competition_metric(trueFVC, predFVC, predSTD):\n    clipSTD = np.clip(predSTD, 70 , 9e9)  \n    deltaFVC = np.clip(np.abs(trueFVC - predFVC), 0 , 1000)  \n    return np.mean(-1 * (np.sqrt(2) * deltaFVC / clipSTD) - np.log(np.sqrt(2) * clipSTD))\n    \n\nprint(\n    'Competition metric: ', \n    competition_metric(submission['FVC'].values, final_prediction, 275) \n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}