{"cells":[{"metadata":{},"cell_type":"markdown","source":"## OSIC Pulmonary Fibrosis Progression\n\n**Data Provided**\n- train.csv : Baseline CT Scan and entire history of FVC\n- test.csv  : Baseline CT and Initial FVC Measurement\n- train/    : Baseline CT scan in DICOM format\n- test/     : Baseline CT Scan in DICOM format\n\n\n**Commit 5**\n- Folds 10"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\n\nfrom tqdm.notebook import tqdm\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nimport category_encoders as ce\n\nfrom sklearn.linear_model import Ridge, ElasticNet\nfrom functools import partial\nimport scipy as sp\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom plotly.subplots import make_subplots\nimport plotly.express as px","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DICT = './'\n\nID = 'Patient_Week'\nTARGET = 'FVC'\nSEED = 2020\nseed_everything(seed=SEED)\n\nN_FOLD = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest_a = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_a.head(8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training Data:',train.info(), end = \"\\n\\n\\n\")\n\nprint('Testing Data:',test_a.info())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualising Train DataSet \nfig = px.histogram(train, x=\"Sex\")\nfig.update_layout(title_text= \"Patient Count in Training Dataset\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.histogram(train, x=\"SmokingStatus\")\nfig.update_layout(title_text= \"Ex-Smoker , Never Smoked, Present Smoker\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Age Distribution\n\nfig = px.histogram(train, x=\"Age\")\nfig.update_layout(title_text= \"Patient Count in Training Dataset\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.histogram(train, y=\"Sex\" , color = \"Age\")\nfig.update_layout(title_text= \"Affected Patient wr Age\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.histogram(train, x=\"Age\" , color = \"SmokingStatus\")\nfig.update_layout(title_text= \"Age wr Smoking Status\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = px.data.gapminder()\nfig = px.area(train, x=\"Weeks\", y=\"Percent\", color = \"SmokingStatus\")\nfig.update_layout(title_text= \"Percent Affected wr Weeks and Smoking Status\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.scatter(x = train[\"Weeks\"] , y = train[\"Percent\"])\n\nfig.update_layout(title_text= \"Weeks vs Percent\")\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.histogram(train, x=\"FVC\", color = \"Sex\")\nfig.update_layout(title_text= \"FVC wr Gender\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.histogram(train, x=\"FVC\", color = \"SmokingStatus\")\nfig.update_layout(title_text= \"FVC wr Smoking Status\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parallel_diagram = train[['Weeks', 'Patient', 'FVC', 'Percent', 'Age', 'Sex', 'SmokingStatus']]\n\nfig = px.parallel_categories(parallel_diagram, color_continuous_scale=px.colors.sequential.Inferno)\nfig.update_layout(title='Parallel category diagram on trainset')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Constructing Training Input\nSince we have common columns in both training and test dataset, concat to add more info also the test dataset has just 5 entries\n\ngroupby 'Patient'\nrename columns as follows\ndrop few cols\nWeeks passed = predict_week - base_week\nMake changes to train dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([train, test_a])\n\noutput = pd.DataFrame()\n\ngb = train.groupby('Patient') # Combines all col data by object name and return mean values respectively\n\n# tqdm => i love you so much in spanish, progress bar for running loops\n\ntk0 = tqdm(gb, total = len(gb))\n\nfor _, usr_df in tk0:\n    usr_output = pd.DataFrame()\n    for week, tmp in usr_df.groupby(\"Weeks\"):\n        rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}\n        \n        tmp = tmp.rename(columns = rename_cols)\n        \n        drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent'] \n        \n        _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient')\n        \n        _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']\n        \n        # Concat the empty DF with edited DF\n        usr_output = pd.concat([usr_output, _usr_output])\n    output = pd.concat([output, usr_output])\n        \ntrain = output[output['Week_passed']!=0].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Constructing Testing Input\n- Rename columns in test dataset 'test_a' to 'test'\n- From sample submission, getting values of week from ID\n- From Patient Week, get values of predict week\n- Drop columns in submission and merge with Test on 'Patient'\n\nWeek_passed in test, week passed = predict week - base week"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test_a.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})\n\n# Adding Sample Submission\nsubmission = pd.read_csv(\"../input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n\n# In submisison file, format: ID_'week', using lambda to split the ID\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])\n\n# In submisison file, format: ID_'week', using lambda to split the Week\nsubmission['predict_Week'] = submission['Patient_Week'].apply(lambda x:x.split('_')[1]).astype(int)\n\ntest = submission.drop(columns = [\"FVC\", \"Confidence\"]).merge(test, on = 'Patient')\n\ntest['Week_passed'] = test['predict_Week'] - test['base_Week']\n\ntest.set_index('Patient_Week', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Folds Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train[['Patient', TARGET]].copy()\nfolds = train[['Patient', TARGET]].copy()\nFold = GroupKFold(n_splits=N_FOLD)\ngroups = folds['Patient'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[TARGET], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Building Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):\n    trn_idx = folds[folds.fold!=fold_num].index\n    val_idx = folds[folds.fold==fold_num].index\n    \n    y_tr = target.iloc[trn_idx].values\n    X_tr = train_df.iloc[trn_idx][features].values\n    y_val = target.iloc[val_idx].values\n    X_val = train_df.iloc[val_idx][features].values\n    \n    oof = np.zeros(len(train_df))\n    predictions = np.zeros(len(test_df))\n    clf.fit(X_tr, y_tr)\n    \n    oof[val_idx] = clf.predict(X_val)\n    predictions += clf.predict(test_df[features])\n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_kfold_model(clf, train, test, folds, features, target, n_fold=9):\n    \n    # n_fold from 5 to 7\n    \n    oof = np.zeros(len(train))\n    predictions = np.zeros(len(test))\n    feature_importance_df = pd.DataFrame()\n\n    for fold_ in range(n_fold):\n\n        _oof, _predictions = run_single_model(clf,train, test, folds, features, target, fold_num = fold_)\n\n        oof += _oof\n        predictions += _predictions/n_fold\n    \n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### FVC Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = train[TARGET]\ntest[TARGET] = np.nan # Displays all Null values\n\n# features\ncat_features = ['Sex', 'SmokingStatus'] # Categorical Features\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)] # Numerical Features\n\nfeatures = num_features + cat_features\ndrop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']\nfeatures = [c for c in features if c not in drop_features]\n\nif cat_features:\n    ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')\n    ce_oe.fit(train)\n    train = ce_oe.transform(train)\n    test = ce_oe.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for alpha1 in [0.3]:\n    for l1s in [0.8]:\n        \n        print(\" For alpha:\",alpha1,\"& l1_ratio:\",l1s)\n        clf = ElasticNet(alpha=alpha1, l1_ratio = l1s)\n        oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)\n\n        train['FVC_pred'] = oof\n        test['FVC_pred'] = predictions\n\n        # baseline score\n        train['Confidence'] = 100\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)\n\n        def loss_func(weight, row):\n            confidence = weight\n            sigma_clipped = max(confidence, 70)\n            diff = abs(row['FVC'] - row['FVC_pred'])\n            delta = min(diff, 1000)\n            score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n            return -score\n\n        results = []\n        tk0 = tqdm(train.iterrows(), total=len(train))\n        for _, row in tk0:\n            loss_partial = partial(loss_func, row=row)\n            weight = [100]\n            result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n            x = result['x']\n            results.append(x[0])\n\n        # optimized score\n        train['Confidence'] = results\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predicting Confidence"},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = 'Confidence'\n\ntarget = train[TARGET]\ntest[TARGET] = np.nan\n\n# features\ncat_features = ['Sex', 'SmokingStatus']\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)]\nfeatures = num_features + cat_features\ndrop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']\nfeatures = [c for c in features if c not in drop_features]\n\noof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Confidence'] = oof\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['Confidence'] = predictions\ntest = test.reset_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = submission[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')\nsub = sub.rename(columns={'FVC_pred': 'FVC'})\n\nfor i in range(len(test_a)):\n    sub.loc[sub['Patient_Week']==test_a.Patient[i]+'_'+str(test_a.Weeks[i]), 'FVC'] = test_a.FVC[i]\n    sub.loc[sub['Patient_Week']==test_a.Patient[i]+'_'+str(test_a.Weeks[i]), 'Confidence'] = 0.1\n    \nsub[sub.Confidence<1]\n\nsub.to_csv('submission.csv', index=False, float_format='%.1f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}