{"cells":[{"metadata":{},"cell_type":"markdown","source":"### V01 is derived from public kernel: https://www.kaggle.com/paulorzp/baseline-ridge/ -runs ElasticNet","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Libraries","execution_count":null},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\n\nfrom tqdm.notebook import tqdm\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nimport category_encoders as ce\n\nfrom sklearn.linear_model import Ridge, ElasticNet\nfrom functools import partial\nimport scipy as sp\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utils","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=777):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DICT = './'\n\nID = 'Patient_Week'\nTARGET = 'FVC'\nSEED = 777\nseed_everything(seed=SEED)\n\nN_FOLD = 7","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Loading","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\notest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct train input\ntrain = pd.concat([train,otest])\noutput = pd.DataFrame()\ngb = train.groupby('Patient')\ntk0 = tqdm(gb, total=len(gb))\nfor _, usr_df in tk0:\n    usr_output = pd.DataFrame()\n    for week, tmp in usr_df.groupby('Weeks'):\n        rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}\n        tmp = tmp.rename(columns=rename_cols)\n        drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']\n        _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient')\n        _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']\n        usr_output = pd.concat([usr_output, _usr_output])\n    output = pd.concat([output, usr_output])\n    \ntrain = output[output['Week_passed']!=0].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct test input\ntest = otest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})\nsubmission = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x: x.split('_')[0])\nsubmission['predict_Week'] = submission['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\ntest = submission.drop(columns=['FVC', 'Confidence']).merge(test, on='Patient')\ntest['Week_passed'] = test['predict_Week'] - test['base_Week']\ntest.set_index('Patient_Week', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train[['Patient', TARGET]].copy()\nFold = GroupKFold(n_splits=N_FOLD)\ngroups = folds['Patient'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[TARGET], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MODEL","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#===========================================================\n# model\n#===========================================================\ndef run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):\n    \n    trn_idx = folds[folds.fold!=fold_num].index\n    val_idx = folds[folds.fold==fold_num].index\n    \n    y_tr = target.iloc[trn_idx].values\n    X_tr = train_df.iloc[trn_idx][features].values\n    y_val = target.iloc[val_idx].values\n    X_val = train_df.iloc[val_idx][features].values\n    \n    oof = np.zeros(len(train_df))\n    predictions = np.zeros(len(test_df))\n    clf.fit(X_tr, y_tr)\n    \n    oof[val_idx] = clf.predict(X_val)\n    predictions += clf.predict(test_df[features])\n    return oof, predictions\n\n\ndef run_kfold_model(clf, train, test, folds, features, target, n_fold=7):\n    \n    oof = np.zeros(len(train))\n    predictions = np.zeros(len(test))\n    feature_importance_df = pd.DataFrame()\n\n    for fold_ in range(n_fold):\n\n        _oof, _predictions = run_single_model(clf,\n                                              train, \n                                              test,\n                                              folds,  \n                                              features,\n                                              target, \n                                              fold_num=fold_)\n        oof += _oof\n        predictions += _predictions/n_fold\n    \n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## predict FVC","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"target = train[TARGET]\ntest[TARGET] = np.nan\n\n# features\ncat_features = ['Sex', 'SmokingStatus']\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)]\nfeatures = num_features + cat_features\ndrop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']\nfeatures = [c for c in features if c not in drop_features]\n\nif cat_features:\n    ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')\n    ce_oe.fit(train)\n    train = ce_oe.transform(train)\n    test = ce_oe.transform(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model Trials","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''for alpha1 in (1,0.3,0.1,0.03,0.01):\n    for l1s in (0.01,0.03,0.1,0.2,0.5,0.8,0.9,0.97,0.99):\n        \n        print(\" For alpha:\",alpha1,\"& l1_ratio:\",l1s)\n        clf = ElasticNet(alpha=alpha1, l1_ratio = l1s)\n        oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)\n\n        train['FVC_pred'] = oof\n        test['FVC_pred'] = predictions\n\n        # baseline score\n        train['Confidence'] = 100\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)\n\n        def loss_func(weight, row):\n            confidence = weight\n            sigma_clipped = max(confidence, 70)\n            diff = abs(row['FVC'] - row['FVC_pred'])\n            delta = min(diff, 1000)\n            score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n            return -score\n\n        results = []\n        tk0 = tqdm(train.iterrows(), total=len(train))\n        for _, row in tk0:\n            loss_partial = partial(loss_func, row=row)\n            weight = [100]\n            result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n            x = result['x']\n            results.append(x[0])\n\n        # optimized score\n        train['Confidence'] = results\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for alpha1 in [0.3]:\n    for l1s in [0.8]:\n        \n        print(\" For alpha:\",alpha1,\"& l1_ratio:\",l1s)\n        clf = ElasticNet(alpha=alpha1, l1_ratio = l1s)\n        oof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)\n\n        train['FVC_pred'] = oof\n        test['FVC_pred'] = predictions\n\n        # baseline score\n        train['Confidence'] = 100\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)\n\n        def loss_func(weight, row):\n            confidence = weight\n            sigma_clipped = max(confidence, 70)\n            diff = abs(row['FVC'] - row['FVC_pred'])\n            delta = min(diff, 1000)\n            score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n            return -score\n\n        results = []\n        tk0 = tqdm(train.iterrows(), total=len(train))\n        for _, row in tk0:\n            loss_partial = partial(loss_func, row=row)\n            weight = [100]\n            result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n            x = result['x']\n            results.append(x[0])\n\n        # optimized score\n        train['Confidence'] = results\n        train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n        train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n        train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n        train['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\n        score = train['score'].mean()\n        print(score)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict Confidence","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = 'Confidence'\n\ntarget = train[TARGET]\ntest[TARGET] = np.nan\n\n# features\ncat_features = ['Sex', 'SmokingStatus']\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)]\nfeatures = num_features + cat_features\ndrop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']\nfeatures = [c for c in features if c not in drop_features]\n\noof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Confidence'] = oof\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['Confidence'] = predictions\ntest = test.reset_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = submission[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')\nsub = sub.rename(columns={'FVC_pred': 'FVC'})\n\nfor i in range(len(otest)):\n    sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    sub.loc[sub['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1\n    \nsub[sub.Confidence<1]\n\nsub.to_csv('submission.csv', index=False, float_format='%.1f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}