{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# data preprocessing origins from https://www.kaggle.com/ulrich07/osic-multiple-quantile-regression-starter\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold\nfrom lightgbm import LGBMRegressor","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in ['Sex', 'SmokingStatus']:\n    data[col] = data[col].astype('category').cat.codes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_list = ['Age', 'Sex', 'SmokingStatus', 'Percent', 'base_week', 'min_FVC']\ncat_feat = ['Sex', 'SmokingStatus']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data\n\ntr.shape, chunk.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics"},{"metadata":{},"cell_type":"markdown","source":"$\\sigma_{clipped} = max(\\sigma, 70),$\n\n$\\Delta = min ( |FVC_{true} - FVC_{predicted}|, 1000 ),$\n\n$metric = -   \\frac{\\sqrt{2} \\Delta}{\\sigma_{clipped}} - \\ln ( \\sqrt{2} \\sigma_{clipped} ).$"},{"metadata":{},"cell_type":"markdown","source":"## LGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_params = {\n    'n_jobs': 1,\n    'max_depth': 4,\n    'min_data_in_leaf': 16,\n    'subsample': 0.9,\n    'n_estimators': 500,\n    'learning_rate': 0.02,\n    'colsample_bytree': 0.9,\n    'boosting_type': 'gbdt',\n    'metric': ['quantile', 'rmse']\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = tr['FVC']#.values\nz = tr[feature_list]#.values\nze = sub[feature_list]#.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD = 5\nkf = KFold(n_splits=NFOLD)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = np.zeros((z.shape[0], 3))\npe = np.zeros((ze.shape[0], 3))\n\nquantiles = [0.2, 0.5, 0.8]\ncnt = 0\nfor tr_idx, val_idx in kf.split(z):\n    cnt += 1\n    for i in range(len(quantiles)): \n        q = quantiles[i]\n        print(f\"FOLD {cnt}, quantile {q}\")\n        lgb = LGBMRegressor(objective='quantile', alpha=q, **lgb_params)\n        lgb.fit(X=z.loc[tr_idx], y=y[tr_idx], eval_set=[[z.loc[val_idx], y[val_idx]]], \n                categorical_feature=cat_feat, early_stopping_rounds=10, verbose=0)\n        \n        pred[val_idx, i] = lgb.predict(z.loc[val_idx])\n        pe[:, i] += lgb.predict(ze)/NFOLD","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"err = mean_absolute_error(y, pred[:, 1])\nunc = np.mean(pred[:, 2] - pred[:, 0])\nprint(err, unc)a","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save sub"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_submission(sub, pe):\n    sub['FVC1'] = pe[:, 1]*0.996\n    sub['Confidence1'] = pe[:, 2] - pe[:, 0]\n    \n    subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()\n    subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']\n    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']\n    \n    print(\"fill in prediction that already exists\")\n    otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\n    for i in range(len(otest)):\n        subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n        subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1\n    subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission.csv\", index=False)\n    print(\"sub file saved\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_submission(sub, pe)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}