{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.model_selection import KFold, GroupKFold","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom plotly.subplots import make_subplots\nimport plotly.express as px\nimport plotly.graph_objects as go","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\nBATCH_SIZE= 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tr.shape, chunk.shape, sub.shape, data.shape)\nprint(tr.Patient.nunique(), chunk.Patient.nunique(), sub.Patient.nunique(), \n      data.Patient.nunique())\n#","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, sub.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.5, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model():\n    z = L.Input((9,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.775), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = make_model()\nprint(net.summary())\nprint(net.count_params())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = tr['FVC'].values\nz = tr[FE].values\nze = sub[FE].values\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))\ndelta = np.zeros((z.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD =5\nkf = GroupKFold(n_splits=NFOLD)\nsplit_groups = tr['Patient']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\n\n\n\nfor tr_idx, val_idx in kf.split(z,groups=split_groups):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    \n    \n    net = make_model()\n    \n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    \n    \n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD\n    \n    delta += net.predict(z) / NFOLD\n    \n#==============","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scoring\n\no_clipped = np.maximum(delta[:,2] - delta[:,0], 70)\ndelta = np.minimum(np.abs(delta[:, 1] - y), 1000)\nsqrt = (np.sqrt((2)))\nscore = (-(sqrt * (delta))/(o_clipped)) - tf.math.log(sqrt * o_clipped)\n\nprint(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(unc.min(), unc.mean(), unc.max(), (unc>=0).mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idxs = np.random.randint(0, y.shape[0], 100)\nplt.plot(y[idxs], label=\"ground truth\")\nplt.plot(pred[idxs, 0], label=\"q25\")\nplt.plot(pred[idxs, 1], label=\"q50\")\nplt.plot(pred[idxs, 2], label=\"q75\")\nplt.legend(loc=\"best\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['FVC1'] = pe[:, 1]\nsub['Confidence1'] = pe[:, 2] - pe[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull()].head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    subm['Confidence'] = sigma_opt\nelse:\n    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission_Quantile_Regression.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# QUANTILE REGRESSION "},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsubmission = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsubmission['Patient'] = submission['Patient_Week'].apply(lambda x:x.split('_')[0])\nsubmission['Weeks'] = submission['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsubmission =  submission[['Patient','Weeks','Confidence','Patient_Week']]\nsubmission = submission.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsubmission['WHERE'] = 'test'\ndata = tr.append([chunk, submission])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n\nbase = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsubmission = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, submission.shape\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.5, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model():\n    z = L.Input((9,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"elu\", name=\"d2\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d3\")(z)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.8), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = tr['FVC'].values\nz = tr[FE].values\nze = submission[FE].values\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))\ndelta = np.zeros((z.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD =5\nkf = GroupKFold(n_splits=NFOLD)\nsplit_groups = tr['Patient']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\n\n\n\nfor tr_idx, val_idx in kf.split(z,groups=split_groups):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    \n    \n    net = make_model()\n    \n    \n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    \n    \n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD\n    \n    delta += net.predict(z) / NFOLD\n    \n#==============","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scoring\n\no_clipped = np.maximum(delta[:,2] - delta[:,0], 70)\ndelta = np.minimum(np.abs(delta[:, 1] - y), 1000)\nsqrt = (np.sqrt((2)))\nscore = (-(sqrt * (delta))/(o_clipped)) - tf.math.log(sqrt * o_clipped)\n\nprint(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(unc.min(), unc.mean(), unc.max(), (unc>=0).mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['FVC1'] = pe[:, 1]\nsubmission['Confidence1'] = pe[:, 2] - pe[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1 = submission[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1.loc[~submission1.FVC1.isnull(),'FVC'] = submission1.loc[~subm.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    submission1['Confidence'] = sigma_opt\nelse:\n    submission1.loc[~submission1.FVC1.isnull(),'Confidence'] = submission1.loc[~submission1.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    submission1.loc[submission1['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    submission1.loc[submission1['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission_Quantile_Regression_V2.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# QUANTILE REGRESSION MODEL 3"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f\"{ROOT}/train.csv\")\ntrain.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\ntest = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsubmission2 = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsubmission2['Patient'] = submission2['Patient_Week'].apply(lambda x:x.split('_')[0])\nsubmission2['Weeks'] = submission2['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsubmission2 =  submission2[['Patient','Weeks','Confidence','Patient_Week']]\nsubmission2 = submission2.merge(test.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['WHERE'] = 'train'\ntest['WHERE'] = 'val'\nsubmission2['WHERE'] = 'test'\ndata = train.append([test, submission2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = data.loc[data.WHERE=='train']\ntest = data.loc[data.WHERE=='val']\nsubmission2 = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.50, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model():\n    z = L.Input((9,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.11, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = make_model()\nprint(net.summary())\nprint(net.count_params())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = tr['FVC'].values\nz = tr[FE].values\nze = sub[FE].values\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))\ndelta = np.zeros((z.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD =5\nkf = GroupKFold(n_splits=NFOLD)\nsplit_groups = tr['Patient']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\n\n\n\nfor tr_idx, val_idx in kf.split(z,groups=split_groups):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    \n    \n    net = make_model()\n    \n    \n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    \n    \n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD\n    \n    delta += net.predict(z) / NFOLD\n    \n#==============","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scoring\n\no_clipped = np.maximum(delta[:,2] - delta[:,0], 70)\ndelta = np.minimum(np.abs(delta[:, 1] - y), 1000)\nsqrt = (np.sqrt((2)))\nscore = (-(sqrt * (delta))/(o_clipped)) - tf.math.log(sqrt * o_clipped)\n\nprint(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission2['FVC1'] = pe[:, 1]\nsubmission2['Confidence1'] = pe[:, 2] - pe[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission2 = submission2[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission2.loc[~submission2.FVC1.isnull(),'FVC'] = submission2.loc[~submission2.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    submission2['Confidence'] = sigma_opt\nelse:\n    submission2.loc[~submission2.FVC1.isnull(),'Confidence'] = submission2.loc[~submission2.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    submission2.loc[submission2['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    submission2.loc[submission2['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission2[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission_Quantile_Regression_V3.csv\", index=False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# QUANTILE REGRESSION v4"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsubmission3 = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsubmission3['Patient'] = submission3['Patient_Week'].apply(lambda x:x.split('_')[0])\nsubmission3['Weeks'] = submission3['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsubmission3 =  submission3[['Patient','Weeks','Confidence','Patient_Week']]\nsubmission3 = submission3.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsubmission3['WHERE'] = 'test'\ndata = tr.append([chunk, submission3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\ndata = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']\ntr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsubmission3 = data.loc[data.WHERE=='test']\ndel data\ntr.shape, chunk.shape, submission3.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.5, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model():\n    z = L.Input((9,), name=\"Patient\")\n    x = L.Dense(200, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(200, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.775), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = make_model()\nprint(net.summary())\nprint(net.count_params())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = tr['FVC'].values\nz = tr[FE].values\nze = sub[FE].values\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))\ndelta = np.zeros((z.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"NFOLD =5\nkf = GroupKFold(n_splits=NFOLD)\nsplit_groups = tr['Patient']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\n\n\n\nfor tr_idx, val_idx in kf.split(z,groups=split_groups):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    \n    \n    net = make_model()\n    \n    \n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    \n    \n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD\n    \n    delta += net.predict(z) / NFOLD\n    \n#==============","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scoring\n\no_clipped = np.maximum(delta[:,2] - delta[:,0], 70)\ndelta = np.minimum(np.abs(delta[:, 1] - y), 1000)\nsqrt = (np.sqrt((2)))\nscore = (-(sqrt * (delta))/(o_clipped)) - tf.math.log(sqrt * o_clipped)\n\nprint(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission3['FVC1'] = pe[:, 1]\nsubmission3['Confidence1'] = pe[:, 2] - pe[:, 0]\nsubmission3 = submission3[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission3.loc[~submission3.FVC1.isnull(),'FVC'] = submission3.loc[~submission3.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    submission3['Confidence'] = sigma_opt\nelse:\n    submission3.loc[~submission3.FVC1.isnull(),'Confidence'] = submission3.loc[~submission3.FVC1.isnull(),'Confidence1']\notest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    submission3.loc[submission3['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    submission3.loc[submission3['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1\nsubmission3[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission_Quantile_Regression_v4.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# QUANTILE REGRESSION V5"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\nsubmission4 = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\n\nsubmission4['Patient'] = submission4['Patient_Week'].apply(lambda x:x.split('_')[0])\nsubmission4['Weeks'] = submission4['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsubmission4 =  submission4[['Patient','Weeks','Confidence','Patient_Week']]\nsubmission4 = submission4.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsubmission4['WHERE'] = 'test'\ndata = tr.append([chunk, submission4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')\nbase = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\ndata = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base\nCOLS = ['Sex','SmokingStatus']\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']\ntr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsubmission4 = data.loc[data.WHERE=='test']\ndel data\ntr.shape, chunk.shape, sub.shape\n((1535, 22), (5, 22), (730, 22))\nC1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.5, 0.8]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n#=================\ndef make_model():\n    z = L.Input((9,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    #x = L.Dense(100, activation=\"relu\", name=\"d3\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    \n    model = M.Model(z, preds, name=\"CNN\")\n    #model.compile(loss=qloss, optimizer=\"adam\", metrics=[score])\n    model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = tr['FVC'].values\nz = tr[FE].values\nze = sub[FE].values\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))\ndelta = np.zeros((z.shape[0], 3))\nNFOLD =5\nkf = GroupKFold(n_splits=NFOLD)\nsplit_groups = tr['Patient']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\n\n\n\nfor tr_idx, val_idx in kf.split(z,groups=split_groups):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    \n    \n    net = make_model()\n    \n    \n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=800, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    \n    \n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD\n    \n    delta += net.predict(z) / NFOLD\n    \n#==============","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Scoring\n\no_clipped = np.maximum(delta[:,2] - delta[:,0], 70)\ndelta = np.minimum(np.abs(delta[:, 1] - y), 1000)\nsqrt = (np.sqrt((2)))\nscore = (-(sqrt * (delta))/(o_clipped)) - tf.math.log(sqrt * o_clipped)\n\nprint(np.mean(score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission4['FVC1'] = pe[:, 1]\nsubmission4['Confidence1'] = pe[:, 2] - pe[:, 0]\nsubmission4 = submission4[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()\n\nsubmission4.loc[~submission4.FVC1.isnull(),'FVC'] = submission4.loc[~submission4.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    submission4['Confidence'] = sigma_opt\nelse:\n    submission4.loc[~submission4.FVC1.isnull(),'Confidence'] = submission4.loc[~submission4.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    submission4.loc[submission4['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    submission4.loc[submission4['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission4[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission_Quantile_Regression_v5.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BAYESIAN RIDGE"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ndf_test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nsub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\n\nprint('Train shape: ', df_train.shape)\nprint('Number of unique customers in train: {}'.format(df_train['Patient'].nunique()))\nprint('Test shape:', df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_base = df_train.drop_duplicates(subset='Patient', keep='first')\ndf_base = df_base[['Patient', 'Weeks', 'FVC', \n                   'Percent', 'Age']].rename(columns={'Weeks': 'base_week',\n                                                      'Percent': 'base_percent',\n                                                      'Age': 'base_age',\n                                                      'FVC': 'base_FVC'})\ndf_base.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['visit'] = 1\ndf_train['visit'] = df_train[['Patient', 'visit']].groupby('Patient').cumsum()\ndf_train = df_train.loc[df_train['visit'] > 0, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge with base info\ndf_train = pd.merge(df_train,\n                    df_base,\n                    on='Patient',\n                    how='left')\nprint(df_train.shape)\ndf_train.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['weeks_passed'] = df_train['Weeks'] - df_train['base_week']\ndf_train = pd.get_dummies(df_train, columns=['Sex', 'SmokingStatus'])\nsub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.rename(columns={'Weeks': 'base_week', \n                                  'Percent': 'base_percent',\n                                  'Age': 'base_age',\n                                  'FVC': 'base_FVC'})\ndf_test = pd.merge(sub,\n                   df_test,\n                   on='Patient',\n                   how='right')\ndf_test = pd.get_dummies(df_test, columns=['Sex', 'SmokingStatus'])\ndf_test['weeks_passed'] = df_test['Weeks'] - df_test['base_week']\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_columns = np.setdiff1d(df_train.drop(['Patient', 'FVC', 'Percent', 'Age', 'visit'], axis = 1).columns, df_test.columns)\nif len(missing_columns) > 0:\n    print('/!\\ Missing columns in test: ', missing_columns)\n    for col in missing_columns:\n        df_test[col] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def OSIC_metric(y_true, y_pred, y_pred_std):\n    delta = np.clip(abs(y_true - y_pred), 0, 1000)\n    std_clipped = np.clip(y_pred_std, 70, np.inf)\n    return np.mean(-(np.sqrt(2)*delta/std_clipped) - np.log(np.sqrt(2)*std_clipped))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import BayesianRidge\n\nclass Model():\n    def __init__(self, model=BayesianRidge(alpha_1=0.1,alpha_2=0.1,lambda_1=0.03,lambda_2=0.01), n_splits=2):\n        self.regressor = model\n        self.n_splits = n_splits\n        self.gkf = GroupKFold(n_splits=n_splits)\n        self.train_cols = ['Weeks', 'base_week', 'base_FVC', \n                           'base_percent', 'base_age', 'weeks_passed', 'Sex_Female',\n                           'Sex_Male', 'SmokingStatus_Currently smokes', \n                           'SmokingStatus_Ex-smoker', 'SmokingStatus_Never smoked']\n    \n    def fit(self, X, y):\n        self.regressor.fit(X, y)\n            \n    def predict(self, X):\n        pred = self.regressor.predict(X, return_std=True)        \n        return pred\n    \n    def fit_predict_cv(self, df, df_test=pd.DataFrame()):\n        \n        scores = np.zeros((self.n_splits, ))\n        oof = np.zeros((len(df), ))\n        oof_std = np.zeros_like(oof)\n        \n        if len(df_test) > 0:\n            pred_sub = np.zeros((len(df_test), self.n_splits))\n            pred_sub_std = np.zeros_like(pred_sub)\n        \n        target = 'FVC'\n        \n        for i, (train_idx, val_idx) in enumerate(self.gkf.split(df, groups=df['Patient'])):\n            X_train = df.loc[train_idx, self.train_cols]\n            y_train = df.loc[train_idx, target]\n            X_val = df.loc[val_idx, self.train_cols]\n            y_val = df.loc[val_idx, target]\n            \n            self.fit(X_train, y_train)\n            \n            pred_train, pred_train_std = self.predict(X_train)\n            pred_val, pred_val_std = self.predict(X_val)\n            \n            if len(df_test) > 0:\n                pred_sub[:, i], pred_sub_std[:, i] = self.predict(df_test[self.train_cols])\n            \n            oof[val_idx] = pred_val\n            oof_std[val_idx] = pred_val_std\n            print('Train score: {0:.2f} | Test score: {1:.2f}'.format(OSIC_metric(y_train, pred_train, pred_train_std),\n                                                                    OSIC_metric(y_val, pred_val, pred_val_std)))\n        print('OOF score: {0:.4f}'.format(OSIC_metric(df[target], oof, oof_std)))\n        res = dict()\n        res['oof'] = oof\n        res['oof_std'] = oof_std\n        \n        if len(df_test) > 0:\n            res['pred_sub'] = pred_sub.mean(axis=1)\n            res['pred_sub_std'] = pred_sub_std.mean(axis=1)\n        \n        return res\nfvc_model = Model()\nres = fvc_model.fit_predict_cv(df_train, df_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['FVC'] = res['pred_sub']\ndf_test['Confidence'] = res['pred_sub_std']\n\nsubmission5 = sub[['Patient_Week']]\nsubmission5 = pd.merge(submission5,\n                      df_test[['Patient_Week', 'FVC', 'Confidence']],\n                      on='Patient_Week',\n                      how='left')\nsubmission5.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# RIDGE"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\n\nfrom tqdm.notebook import tqdm\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.metrics import mean_squared_error\nimport category_encoders as ce\n\nfrom sklearn.linear_model import Ridge\nfrom functools import partial\nimport scipy as sp\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=777):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DICT = './'\n\nID = 'Patient_Week'\nTARGET = 'FVC'\nSEED = 37\nseed_everything(seed=SEED)\n\nN_FOLD = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\notest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct train input\ntrain = pd.concat([train,otest])\noutput = pd.DataFrame()\ngb = train.groupby('Patient')\ntk0 = tqdm(gb, total=len(gb))\nfor _, usr_df in tk0:\n    usr_output = pd.DataFrame()\n    for week, tmp in usr_df.groupby('Weeks'):\n        rename_cols = {'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'}\n        tmp = tmp.rename(columns=rename_cols)\n        drop_cols = ['Age', 'Sex', 'SmokingStatus', 'Percent']\n        _usr_output = usr_df.drop(columns=drop_cols).rename(columns={'Weeks': 'predict_Week'}).merge(tmp, on='Patient')\n        _usr_output['Week_passed'] = _usr_output['predict_Week'] - _usr_output['base_Week']\n        usr_output = pd.concat([usr_output, _usr_output])\n    output = pd.concat([output, usr_output])\n    \ntrain = output[output['Week_passed']!=0].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct test input\ntest = otest.rename(columns={'Weeks': 'base_Week', 'FVC': 'base_FVC', 'Age': 'base_Age'})\nsubmission6 = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubmission6['Patient'] = submission6['Patient_Week'].apply(lambda x: x.split('_')[0])\nsubmission6['predict_Week'] = submission6['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\ntest = submission6.drop(columns=['FVC', 'Confidence']).merge(test, on='Patient')\ntest['Week_passed'] = test['predict_Week'] - test['base_Week']\ntest.set_index('Patient_Week', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"folds = train[['Patient', TARGET]].copy()\nFold = GroupKFold(n_splits=N_FOLD)\ngroups = folds['Patient'].values\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[TARGET], groups)):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#===========================================================\n# model\n#===========================================================\ndef run_single_model(clf, train_df, test_df, folds, features, target, fold_num=0):\n    \n    trn_idx = folds[folds.fold!=fold_num].index\n    val_idx = folds[folds.fold==fold_num].index\n    \n    y_tr = target.iloc[trn_idx].values\n    X_tr = train_df.iloc[trn_idx][features].values\n    y_val = target.iloc[val_idx].values\n    X_val = train_df.iloc[val_idx][features].values\n    \n    oof = np.zeros(len(train_df))\n    predictions = np.zeros(len(test_df))\n    clf.fit(X_tr, y_tr)\n    \n    oof[val_idx] = clf.predict(X_val)\n    predictions += clf.predict(test_df[features])\n    return oof, predictions\n\n\ndef run_kfold_model(clf, train, test, folds, features, target, n_fold=5):\n    \n    oof = np.zeros(len(train))\n    predictions = np.zeros(len(test))\n    feature_importance_df = pd.DataFrame()\n\n    for fold_ in range(n_fold):\n\n        _oof, _predictions = run_single_model(clf,\n                                              train, \n                                              test,\n                                              folds,  \n                                              features,\n                                              target, \n                                              fold_num=fold_)\n        oof += _oof\n        predictions += _predictions/n_fold\n    \n    return oof, predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = train[TARGET]\ntest[TARGET] = np.nan\n\n# features\ncat_features = ['Sex', 'SmokingStatus']\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)]\nfeatures = num_features + cat_features\ndrop_features = [TARGET, 'predict_Week', 'Percent', 'base_Week']\nfeatures = [c for c in features if c not in drop_features]\n\nif cat_features:\n    ce_oe = ce.OrdinalEncoder(cols=cat_features, handle_unknown='impute')\n    ce_oe.fit(train)\n    train = ce_oe.transform(train)\n    test = ce_oe.transform(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = Ridge(alpha=0.1)\noof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)\n\ntrain['FVC_pred'] = oof\ntest['FVC_pred'] = predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# baseline score\ntrain['Confidence'] = 100\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_func(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = abs(row['FVC'] - row['FVC_pred'])\n    delta = min(diff, 1000)\n    score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n    return -score\n\nresults = []\ntk0 = tqdm(train.iterrows(), total=len(train))\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [100]\n    result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n    x = result['x']\n    results.append(x[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# optimized score\ntrain['Confidence'] = results\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET = 'Confidence'\n\ntarget = train[TARGET]\ntest[TARGET] = np.nan\n\n# features\ncat_features = ['Sex', 'SmokingStatus']\nnum_features = [c for c in test.columns if (test.dtypes[c] != 'object') & (c not in cat_features)]\nfeatures = num_features + cat_features\ndrop_features = [ID, TARGET, 'predict_Week', 'base_Week', 'FVC', 'FVC_pred']\nfeatures = [c for c in features if c not in drop_features]\n\noof, predictions = run_kfold_model(clf, train, test, folds, features, target, n_fold=N_FOLD)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Confidence'] = oof\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['Confidence'] = predictions\ntest = test.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission6 = submission6[['Patient_Week']].merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], on='Patient_Week')\nsubmission6 = submission6.rename(columns={'FVC_pred': 'FVC'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(otest)):\n    submission6.loc[submission6['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    submission6.loc[submission6['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission6.to_csv('submission_Ridge.csv', index=False, float_format='%.1f')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final = (subm.drop(['Patient_Week'], axis=1) + submission1.drop(['Patient_Week'], axis=1) + submission2.drop(['Patient_Week'], axis=1) + submission3.drop(['Patient_Week'], axis=1) + submission4.drop(['Patient_Week'], axis=1))/5\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final['Patient_Week'] = subm['Patient_Week']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = [submission_final.columns[-1]] + list(submission_final.columns[: -1])\nsubmission_final = submission_final[cols]\nsubmission_final.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final.drop(['FVC1','Confidence1'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final=pd.merge(submission_final,submission5,on='Patient_Week',how='inner').merge(submission6,on='Patient_Week',how='inner').rename(columns={'FVC':'FVC1','Confidence':'Confidence1'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final['FVC']=(submission_final['FVC_x']+submission_final['FVC_y']+submission_final['FVC1'])/3\n\nsubmission_final['Confidence'] = (submission_final['Confidence1']+submission_final['Confidence_x']+submission_final['Confidence_y'])/3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_final[['Patient_Week','FVC','Confidence']].to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}