{"cells":[{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:16:13.690309Z","iopub.status.busy":"2020-09-09T15:16:13.689251Z","iopub.status.idle":"2020-09-09T15:16:13.691944Z","shell.execute_reply":"2020-09-09T15:16:13.692558Z"},"papermill":{"duration":0.023224,"end_time":"2020-09-09T15:16:13.692712","exception":false,"start_time":"2020-09-09T15:16:13.669488","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"Dropout_model = 0.65\nFVC_weight = 0.12\nConfidence_weight = 0.25","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-09-09T15:16:14.145745Z","iopub.status.busy":"2020-09-09T15:16:14.144834Z","iopub.status.idle":"2020-09-09T15:16:33.130287Z","shell.execute_reply":"2020-09-09T15:16:33.12969Z"},"papermill":{"duration":19.008446,"end_time":"2020-09-09T15:16:33.130397","exception":false,"start_time":"2020-09-09T15:16:14.121951","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"!pip install ../input/kerasapplications/keras-team-keras-applications-3b180cb -f ./ --no-index\n!pip install ../input/efficientnet/efficientnet-1.1.0/ -f ./ --no-index","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-input":false,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.execute_input":"2020-09-09T15:16:33.17378Z","iopub.status.busy":"2020-09-09T15:16:33.172353Z","iopub.status.idle":"2020-09-09T15:16:39.345905Z","shell.execute_reply":"2020-09-09T15:16:39.345143Z"},"papermill":{"duration":6.199515,"end_time":"2020-09-09T15:16:39.346049","exception":false,"start_time":"2020-09-09T15:16:33.146534","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport pydicom\nimport pandas as pd\nimport numpy as np \nimport tensorflow as tf \nimport matplotlib.pyplot as plt \nimport random\nfrom tqdm.notebook import tqdm \nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.metrics import mean_absolute_error\nfrom tensorflow_addons.optimizers import RectifiedAdam\nfrom tensorflow.keras import Model\nimport tensorflow.keras.backend as K\nimport tensorflow.keras.layers as L\nimport tensorflow.keras.models as M\nfrom tensorflow.keras.optimizers import Nadam\nimport seaborn as sns\nfrom PIL import Image\n\ndef seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(42)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-09T15:16:39.382826Z","iopub.status.busy":"2020-09-09T15:16:39.382071Z","iopub.status.idle":"2020-09-09T15:16:42.341662Z","shell.execute_reply":"2020-09-09T15:16:42.340591Z"},"papermill":{"duration":2.979967,"end_time":"2020-09-09T15:16:42.341836","exception":false,"start_time":"2020-09-09T15:16:39.361869","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"config = tf.compat.v1.ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = tf.compat.v1.Session(config=config)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.020329,"end_time":"2020-09-09T15:16:42.383049","exception":false,"start_time":"2020-09-09T15:16:42.36272","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 3. Download data, auxiliary functions and model tuning <a class=\"anchor\" id=\"3\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:16:42.440653Z","iopub.status.busy":"2020-09-09T15:16:42.434985Z","iopub.status.idle":"2020-09-09T15:16:42.448866Z","shell.execute_reply":"2020-09-09T15:16:42.450002Z"},"papermill":{"duration":0.044289,"end_time":"2020-09-09T15:16:42.450207","exception":false,"start_time":"2020-09-09T15:16:42.405918","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv') ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-09T15:16:42.505407Z","iopub.status.busy":"2020-09-09T15:16:42.504643Z","iopub.status.idle":"2020-09-09T15:16:42.509835Z","shell.execute_reply":"2020-09-09T15:16:42.511034Z"},"papermill":{"duration":0.037885,"end_time":"2020-09-09T15:16:42.511209","exception":false,"start_time":"2020-09-09T15:16:42.473324","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_tab(df):\n    vector = [(df.Age.values[0] - 30) / 30] \n    \n    if df.Sex.values[0] == 'male':\n       vector.append(0)\n    else:\n       vector.append(1)\n    \n    if df.SmokingStatus.values[0] == 'Never smoked':\n        vector.extend([0,0])\n    elif df.SmokingStatus.values[0] == 'Ex-smoker':\n        vector.extend([1,1])\n    elif df.SmokingStatus.values[0] == 'Currently smokes':\n        vector.extend([0,1])\n    else:\n        vector.extend([1,0])\n    return np.array(vector) ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:16:42.565236Z","iopub.status.busy":"2020-09-09T15:16:42.564242Z","iopub.status.idle":"2020-09-09T15:16:43.042289Z","shell.execute_reply":"2020-09-09T15:16:43.043171Z"},"papermill":{"duration":0.511539,"end_time":"2020-09-09T15:16:43.043364","exception":false,"start_time":"2020-09-09T15:16:42.531825","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"A = {} \nTAB = {} \nP = [] \nfor i, p in tqdm(enumerate(train.Patient.unique())):\n    sub = train.loc[train.Patient == p, :] \n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a, b = np.linalg.lstsq(c, fvc)[0]\n    \n    A[p] = a\n    TAB[p] = get_tab(sub)\n    P.append(p)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-09T15:16:43.082263Z","iopub.status.busy":"2020-09-09T15:16:43.081647Z","iopub.status.idle":"2020-09-09T15:16:43.086195Z","shell.execute_reply":"2020-09-09T15:16:43.08549Z"},"papermill":{"duration":0.025525,"end_time":"2020-09-09T15:16:43.086307","exception":false,"start_time":"2020-09-09T15:16:43.060782","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def get_img(path):\n    d = pydicom.dcmread(path)\n    return cv2.resize(d.pixel_array / 2**11, (512, 512))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-09T15:16:43.133441Z","iopub.status.busy":"2020-09-09T15:16:43.132752Z","iopub.status.idle":"2020-09-09T15:16:43.137475Z","shell.execute_reply":"2020-09-09T15:16:43.136938Z"},"papermill":{"duration":0.035078,"end_time":"2020-09-09T15:16:43.137572","exception":false,"start_time":"2020-09-09T15:16:43.102494","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from tensorflow.keras.utils import Sequence\n\nclass IGenerator(Sequence):\n    BAD_ID = ['ID00011637202177653955184', 'ID00052637202186188008618']\n    def __init__(self, keys, a, tab, batch_size=32):\n        self.keys = [k for k in keys if k not in self.BAD_ID]\n        self.a = a\n        self.tab = tab\n        self.batch_size = batch_size\n        \n        self.train_data = {}\n        for p in train.Patient.values:\n            self.train_data[p] = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n    \n    def __len__(self):\n        return 1000\n    \n    def __getitem__(self, idx):\n        x = []\n        a, tab = [], [] \n        keys = np.random.choice(self.keys, size = self.batch_size)\n        for k in keys:\n            try:\n                i = np.random.choice(self.train_data[k], size=1)[0]\n                img = get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{k}/{i}')\n                x.append(img)\n                a.append(self.a[k])\n                tab.append(self.tab[k])\n            except:\n                print(k, i)\n       \n        x,a,tab = np.array(x), np.array(a), np.array(tab)\n        x = np.expand_dims(x, axis=-1)\n        return [x, tab] , a","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:16:43.189855Z","iopub.status.busy":"2020-09-09T15:16:43.188903Z","iopub.status.idle":"2020-09-09T15:17:43.104079Z","shell.execute_reply":"2020-09-09T15:17:43.104838Z"},"papermill":{"duration":59.951435,"end_time":"2020-09-09T15:17:43.105031","exception":false,"start_time":"2020-09-09T15:16:43.153596","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import (\n    Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n    LeakyReLU, Concatenate \n)\nimport efficientnet.tfkeras as efn\n\ndef get_efficientnet(model, shape):\n    models_dict = {\n        'b0': efn.EfficientNetB0(input_shape=shape,weights=None,include_top=False),\n        'b1': efn.EfficientNetB1(input_shape=shape,weights=None,include_top=False),\n        'b2': efn.EfficientNetB2(input_shape=shape,weights=None,include_top=False),\n        'b3': efn.EfficientNetB3(input_shape=shape,weights=None,include_top=False),\n        'b4': efn.EfficientNetB4(input_shape=shape,weights=None,include_top=False),\n        'b5': efn.EfficientNetB5(input_shape=shape,weights=None,include_top=False),\n        'b6': efn.EfficientNetB6(input_shape=shape,weights=None,include_top=False),\n        'b7': efn.EfficientNetB7(input_shape=shape,weights=None,include_top=False)\n    }\n    return models_dict[model]\n\ndef build_model(shape=(512, 512, 1), model_class=None):\n    inp = Input(shape=shape)\n    base = get_efficientnet(model_class, shape)\n    x = base(inp)\n    x = GlobalAveragePooling2D()(x)\n    inp2 = Input(shape=(4,))\n    x2 = tf.keras.layers.GaussianNoise(0.2)(inp2)\n    x = Concatenate()([x, x2]) \n    x = Dropout(Dropout_model)(x)\n    x = Dense(1)(x)\n    model = Model([inp, inp2] , x)\n    \n    weights = [w for w in os.listdir('../input/osic-model-weights') if model_class in w][0]\n    model.load_weights('../input/osic-model-weights/' + weights)\n    return model\n\nmodel_classes = ['b5'] #['b0','b1','b2','b3',b4','b5','b6','b7']\nmodels = [build_model(shape=(512, 512, 1), model_class=m) for m in model_classes]\nprint('Number of models: ' + str(len(models)))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:17:43.144687Z","iopub.status.busy":"2020-09-09T15:17:43.142698Z","iopub.status.idle":"2020-09-09T15:17:43.145535Z","shell.execute_reply":"2020-09-09T15:17:43.146036Z"},"papermill":{"duration":0.024791,"end_time":"2020-09-09T15:17:43.146155","exception":false,"start_time":"2020-09-09T15:17:43.121364","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tr_p, vl_p = train_test_split(P, shuffle=True, train_size = 0.8) ","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2020-09-09T15:17:43.18611Z","iopub.status.busy":"2020-09-09T15:17:43.185133Z","iopub.status.idle":"2020-09-09T15:17:43.187912Z","shell.execute_reply":"2020-09-09T15:17:43.188489Z"},"papermill":{"duration":0.026113,"end_time":"2020-09-09T15:17:43.188619","exception":false,"start_time":"2020-09-09T15:17:43.162506","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def score(fvc_true, fvc_pred, sigma):\n    sigma_clip = np.maximum(sigma, 70) # changed from 70, trie 66.7 too\n    delta = np.abs(fvc_true - fvc_pred)\n    delta = np.minimum(delta, 1000)\n    sq2 = np.sqrt(2)\n    metric = (delta / sigma_clip)*sq2 + np.log(sigma_clip* sq2)\n    return np.mean(metric)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:17:43.245999Z","iopub.status.busy":"2020-09-09T15:17:43.235673Z","iopub.status.idle":"2020-09-09T15:33:57.317546Z","shell.execute_reply":"2020-09-09T15:33:57.316613Z"},"papermill":{"duration":974.111575,"end_time":"2020-09-09T15:33:57.317677","exception":false,"start_time":"2020-09-09T15:17:43.206102","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"subs = []\nfor model in models:\n    metric = []\n    for q in tqdm(range(1, 10)):\n        m = []\n        for p in vl_p:\n            x = [] \n            tab = [] \n\n            if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n                continue\n\n            ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/')\n            for i in ldir:\n                if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:\n                    x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/train/{p}/{i}')) \n                    tab.append(get_tab(train.loc[train.Patient == p, :])) \n            if len(x) < 1:\n                continue\n            tab = np.array(tab) \n\n            x = np.expand_dims(x, axis=-1) \n            _a = model.predict([x, tab]) \n            a = np.quantile(_a, q / 10)\n\n            percent_true = train.Percent.values[train.Patient == p]\n            fvc_true = train.FVC.values[train.Patient == p]\n            weeks_true = train.Weeks.values[train.Patient == p]\n\n            fvc = a * (weeks_true - weeks_true[0]) + fvc_true[0]\n            percent = percent_true[0] - a * abs(weeks_true - weeks_true[0])\n            m.append(score(fvc_true, fvc, percent))\n        print(np.mean(m))\n        metric.append(np.mean(m))\n\n    q = (np.argmin(metric) + 1)/ 10\n\n    sub = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv') \n    test = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv') \n    A_test, B_test, P_test,W, FVC= {}, {}, {},{},{} \n    STD, WEEK = {}, {} \n    for p in test.Patient.unique():\n        x = [] \n        tab = [] \n        ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/')\n        for i in ldir:\n            if int(i[:-4]) / len(ldir) < 0.8 and int(i[:-4]) / len(ldir) > 0.15:\n                x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/test/{p}/{i}')) \n                tab.append(get_tab(test.loc[test.Patient == p, :])) \n        if len(x) <= 1:\n            continue\n        tab = np.array(tab) \n\n        x = np.expand_dims(x, axis=-1) \n        _a = model.predict([x, tab]) \n        a = np.quantile(_a, q)\n        A_test[p] = a\n        B_test[p] = test.FVC.values[test.Patient == p] - a*test.Weeks.values[test.Patient == p]\n        P_test[p] = test.Percent.values[test.Patient == p] \n        WEEK[p] = test.Weeks.values[test.Patient == p]\n\n    for k in sub.Patient_Week.values:\n        p, w = k.split('_')\n        w = int(w) \n\n        fvc = A_test[p] * w + B_test[p]\n        sub.loc[sub.Patient_Week == k, 'FVC'] = fvc\n        sub.loc[sub.Patient_Week == k, 'Confidence'] = (\n            P_test[p] - A_test[p] * abs(WEEK[p] - w) \n    ) \n\n    _sub = sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]].copy()\n    subs.append(_sub)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.016215,"end_time":"2020-09-09T15:33:57.350183","exception":false,"start_time":"2020-09-09T15:33:57.333968","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 4. Prediction and submission <a class=\"anchor\" id=\"4\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"papermill":{"duration":0.015432,"end_time":"2020-09-09T15:33:57.381436","exception":false,"start_time":"2020-09-09T15:33:57.366004","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 4.1 Average prediction <a class=\"anchor\" id=\"4.1\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.424395Z","iopub.status.busy":"2020-09-09T15:33:57.423588Z","iopub.status.idle":"2020-09-09T15:33:57.44178Z","shell.execute_reply":"2020-09-09T15:33:57.44083Z"},"papermill":{"duration":0.044466,"end_time":"2020-09-09T15:33:57.441894","exception":false,"start_time":"2020-09-09T15:33:57.397428","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"N = len(subs)\nsub = subs[0].copy() # ref\nsub[\"FVC\"] = 0\nsub[\"Confidence\"] = 0\nfor i in range(N):\n    sub[\"FVC\"] += subs[0][\"FVC\"] * (1/N)\n    sub[\"Confidence\"] += subs[0][\"Confidence\"] * (1/N)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.483233Z","iopub.status.busy":"2020-09-09T15:33:57.482327Z","iopub.status.idle":"2020-09-09T15:33:57.491779Z","shell.execute_reply":"2020-09-09T15:33:57.492328Z"},"papermill":{"duration":0.034184,"end_time":"2020-09-09T15:33:57.492473","exception":false,"start_time":"2020-09-09T15:33:57.458289","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.534006Z","iopub.status.busy":"2020-09-09T15:33:57.533086Z","iopub.status.idle":"2020-09-09T15:33:57.68704Z","shell.execute_reply":"2020-09-09T15:33:57.685988Z"},"papermill":{"duration":0.177509,"end_time":"2020-09-09T15:33:57.687159","exception":false,"start_time":"2020-09-09T15:33:57.50965","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission_img.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.727001Z","iopub.status.busy":"2020-09-09T15:33:57.726207Z","iopub.status.idle":"2020-09-09T15:33:57.730234Z","shell.execute_reply":"2020-09-09T15:33:57.729727Z"},"papermill":{"duration":0.026227,"end_time":"2020-09-09T15:33:57.730351","exception":false,"start_time":"2020-09-09T15:33:57.704124","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"img_sub = sub[[\"Patient_Week\",\"FVC\",\"Confidence\"]].copy()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.016023,"end_time":"2020-09-09T15:33:57.763218","exception":false,"start_time":"2020-09-09T15:33:57.747195","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 4.2 Osic-Multiple-Quantile-Regression <a class=\"anchor\" id=\"4.2\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.808732Z","iopub.status.busy":"2020-09-09T15:33:57.80787Z","iopub.status.idle":"2020-09-09T15:33:57.848452Z","shell.execute_reply":"2020-09-09T15:33:57.847712Z"},"papermill":{"duration":0.068855,"end_time":"2020-09-09T15:33:57.848585","exception":false,"start_time":"2020-09-09T15:33:57.77973","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\nBATCH_SIZE=128\n\ntr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']]\nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.890863Z","iopub.status.busy":"2020-09-09T15:33:57.889992Z","iopub.status.idle":"2020-09-09T15:33:57.898206Z","shell.execute_reply":"2020-09-09T15:33:57.897739Z"},"papermill":{"duration":0.032679,"end_time":"2020-09-09T15:33:57.898302","exception":false,"start_time":"2020-09-09T15:33:57.865623","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.937683Z","iopub.status.busy":"2020-09-09T15:33:57.936813Z","iopub.status.idle":"2020-09-09T15:33:57.944254Z","shell.execute_reply":"2020-09-09T15:33:57.943763Z"},"papermill":{"duration":0.029629,"end_time":"2020-09-09T15:33:57.944355","exception":false,"start_time":"2020-09-09T15:33:57.914726","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(tr.shape, chunk.shape, sub.shape, data.shape)\nprint(tr.Patient.nunique(), chunk.Patient.nunique(), sub.Patient.nunique(), \n      data.Patient.nunique())","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:57.986591Z","iopub.status.busy":"2020-09-09T15:33:57.985739Z","iopub.status.idle":"2020-09-09T15:33:57.998705Z","shell.execute_reply":"2020-09-09T15:33:57.999198Z"},"papermill":{"duration":0.038253,"end_time":"2020-09-09T15:33:57.999326","exception":false,"start_time":"2020-09-09T15:33:57.961073","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.043557Z","iopub.status.busy":"2020-09-09T15:33:58.042839Z","iopub.status.idle":"2020-09-09T15:33:58.050611Z","shell.execute_reply":"2020-09-09T15:33:58.050128Z"},"papermill":{"duration":0.034686,"end_time":"2020-09-09T15:33:58.050715","exception":false,"start_time":"2020-09-09T15:33:58.016029","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.092704Z","iopub.status.busy":"2020-09-09T15:33:58.091803Z","iopub.status.idle":"2020-09-09T15:33:58.09986Z","shell.execute_reply":"2020-09-09T15:33:58.100384Z"},"papermill":{"duration":0.032953,"end_time":"2020-09-09T15:33:58.100517","exception":false,"start_time":"2020-09-09T15:33:58.067564","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"data = data.merge(base, on='Patient', how='left')\ndata['base_week'] = data['Weeks'] - data['min_week']\ndel base","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.143866Z","iopub.status.busy":"2020-09-09T15:33:58.142995Z","iopub.status.idle":"2020-09-09T15:33:58.149561Z","shell.execute_reply":"2020-09-09T15:33:58.150014Z"},"papermill":{"duration":0.032215,"end_time":"2020-09-09T15:33:58.150128","exception":false,"start_time":"2020-09-09T15:33:58.117913","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"COLS = ['Sex','SmokingStatus'] #,'Age'\nFE = []\nfor col in COLS:\n    for mod in data[col].unique():\n        FE.append(mod)\n        data[mod] = (data[col] == mod).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.200703Z","iopub.status.busy":"2020-09-09T15:33:58.199771Z","iopub.status.idle":"2020-09-09T15:33:58.202238Z","shell.execute_reply":"2020-09-09T15:33:58.202689Z"},"papermill":{"duration":0.035712,"end_time":"2020-09-09T15:33:58.202814","exception":false,"start_time":"2020-09-09T15:33:58.167102","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"#\ndata['age'] = (data['Age'] - data['Age'].min() ) / ( data['Age'].max() - data['Age'].min() )\ndata['BASE'] = (data['min_FVC'] - data['min_FVC'].min() ) / ( data['min_FVC'].max() - data['min_FVC'].min() )\ndata['week'] = (data['base_week'] - data['base_week'].min() ) / ( data['base_week'].max() - data['base_week'].min() )\ndata['percent'] = (data['Percent'] - data['Percent'].min() ) / ( data['Percent'].max() - data['Percent'].min() )\nFE += ['age','percent','week','BASE']","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.24364Z","iopub.status.busy":"2020-09-09T15:33:58.242666Z","iopub.status.idle":"2020-09-09T15:33:58.2506Z","shell.execute_reply":"2020-09-09T15:33:58.250109Z"},"papermill":{"duration":0.03062,"end_time":"2020-09-09T15:33:58.2507","exception":false,"start_time":"2020-09-09T15:33:58.22008","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tr = data.loc[data.WHERE=='train']\nchunk = data.loc[data.WHERE=='val']\nsub = data.loc[data.WHERE=='test']\ndel data","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.290512Z","iopub.status.busy":"2020-09-09T15:33:58.289724Z","iopub.status.idle":"2020-09-09T15:33:58.293727Z","shell.execute_reply":"2020-09-09T15:33:58.293197Z"},"papermill":{"duration":0.026156,"end_time":"2020-09-09T15:33:58.293824","exception":false,"start_time":"2020-09-09T15:33:58.267668","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"tr.shape, chunk.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.017677,"end_time":"2020-09-09T15:33:58.329576","exception":false,"start_time":"2020-09-09T15:33:58.311899","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 4.3 The change of mloss <a class=\"anchor\" id=\"4.3\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.385334Z","iopub.status.busy":"2020-09-09T15:33:58.38292Z","iopub.status.idle":"2020-09-09T15:33:58.387646Z","shell.execute_reply":"2020-09-09T15:33:58.38808Z"},"papermill":{"duration":0.041431,"end_time":"2020-09-09T15:33:58.388203","exception":false,"start_time":"2020-09-09T15:33:58.346772","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    return K.mean(metric)\n\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    qs = [0.2, 0.60, 0.85]\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda)*score(y_true, y_pred)\n    return loss\n\ndef make_model(nh):\n    z = L.Input((nh,), name=\"Patient\")\n    x = L.Dense(100, activation=\"relu\", name=\"d1\")(z)\n    x = L.Dense(100, activation=\"relu\", name=\"d2\")(x)\n    p1 = L.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = L.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = L.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n    \n    model = M.Model(z, preds, name=\"CNN\")\n    model.compile(loss=mloss(0.65), optimizer=tf.keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.01, amsgrad=False), metrics=[score])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.432176Z","iopub.status.busy":"2020-09-09T15:33:58.431458Z","iopub.status.idle":"2020-09-09T15:33:58.434657Z","shell.execute_reply":"2020-09-09T15:33:58.434137Z"},"papermill":{"duration":0.028948,"end_time":"2020-09-09T15:33:58.434752","exception":false,"start_time":"2020-09-09T15:33:58.405804","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"y = tr['FVC'].values\nz = tr[FE].values\nze = sub[FE].values\nnh = z.shape[1]\npe = np.zeros((ze.shape[0], 3))\npred = np.zeros((z.shape[0], 3))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.478075Z","iopub.status.busy":"2020-09-09T15:33:58.477223Z","iopub.status.idle":"2020-09-09T15:33:58.550323Z","shell.execute_reply":"2020-09-09T15:33:58.549809Z"},"papermill":{"duration":0.09873,"end_time":"2020-09-09T15:33:58.550457","exception":false,"start_time":"2020-09-09T15:33:58.451727","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"net = make_model(nh)\nprint(net.summary())\nprint(net.count_params())","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.590191Z","iopub.status.busy":"2020-09-09T15:33:58.589435Z","iopub.status.idle":"2020-09-09T15:33:58.592837Z","shell.execute_reply":"2020-09-09T15:33:58.593313Z"},"papermill":{"duration":0.025041,"end_time":"2020-09-09T15:33:58.593506","exception":false,"start_time":"2020-09-09T15:33:58.568465","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"NFOLD = 5 # originally 5\nkf = KFold(n_splits=NFOLD)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:33:58.637596Z","iopub.status.busy":"2020-09-09T15:33:58.635728Z","iopub.status.idle":"2020-09-09T15:37:43.827541Z","shell.execute_reply":"2020-09-09T15:37:43.828396Z"},"papermill":{"duration":225.217242,"end_time":"2020-09-09T15:37:43.828631","exception":false,"start_time":"2020-09-09T15:33:58.611389","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"%%time\ncnt = 0\nEPOCHS = 800\nfor tr_idx, val_idx in kf.split(z):\n    cnt += 1\n    print(f\"FOLD {cnt}\")\n    net = make_model(nh)\n    net.fit(z[tr_idx], y[tr_idx], batch_size=BATCH_SIZE, epochs=EPOCHS, \n            validation_data=(z[val_idx], y[val_idx]), verbose=0) #\n    print(\"train\", net.evaluate(z[tr_idx], y[tr_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"val\", net.evaluate(z[val_idx], y[val_idx], verbose=0, batch_size=BATCH_SIZE))\n    print(\"predict val...\")\n    pred[val_idx] = net.predict(z[val_idx], batch_size=BATCH_SIZE, verbose=0)\n    print(\"predict test...\")\n    pe += net.predict(ze, batch_size=BATCH_SIZE, verbose=0) / NFOLD","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:43.876893Z","iopub.status.busy":"2020-09-09T15:37:43.876098Z","iopub.status.idle":"2020-09-09T15:37:43.880497Z","shell.execute_reply":"2020-09-09T15:37:43.881115Z"},"papermill":{"duration":0.031032,"end_time":"2020-09-09T15:37:43.881252","exception":false,"start_time":"2020-09-09T15:37:43.85022","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sigma_opt = mean_absolute_error(y, pred[:, 1])\nunc = pred[:,2] - pred[:, 0]\nsigma_mean = np.mean(unc)\nprint(sigma_opt, sigma_mean)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:43.927349Z","iopub.status.busy":"2020-09-09T15:37:43.926657Z","iopub.status.idle":"2020-09-09T15:37:44.179783Z","shell.execute_reply":"2020-09-09T15:37:44.180312Z"},"papermill":{"duration":0.280478,"end_time":"2020-09-09T15:37:44.180475","exception":false,"start_time":"2020-09-09T15:37:43.899997","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"idxs = np.random.randint(0, y.shape[0], 100)\nplt.plot(y[idxs], label=\"ground truth\")\nplt.plot(pred[idxs, 0], label=\"q25\")\nplt.plot(pred[idxs, 1], label=\"q50\")\nplt.plot(pred[idxs, 2], label=\"q75\")\nplt.legend(loc=\"best\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.225008Z","iopub.status.busy":"2020-09-09T15:37:44.224216Z","iopub.status.idle":"2020-09-09T15:37:44.228818Z","shell.execute_reply":"2020-09-09T15:37:44.229268Z"},"papermill":{"duration":0.029174,"end_time":"2020-09-09T15:37:44.229388","exception":false,"start_time":"2020-09-09T15:37:44.200214","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(unc.min(), unc.mean(), unc.max(), (unc>=0).mean())","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.276547Z","iopub.status.busy":"2020-09-09T15:37:44.271497Z","iopub.status.idle":"2020-09-09T15:37:44.448497Z","shell.execute_reply":"2020-09-09T15:37:44.449032Z"},"papermill":{"duration":0.20131,"end_time":"2020-09-09T15:37:44.449178","exception":false,"start_time":"2020-09-09T15:37:44.247868","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"plt.hist(unc)\nplt.title(\"uncertainty in prediction\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.52282Z","iopub.status.busy":"2020-09-09T15:37:44.508587Z","iopub.status.idle":"2020-09-09T15:37:44.527331Z","shell.execute_reply":"2020-09-09T15:37:44.527915Z"},"papermill":{"duration":0.054641,"end_time":"2020-09-09T15:37:44.528047","exception":false,"start_time":"2020-09-09T15:37:44.473406","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.578508Z","iopub.status.busy":"2020-09-09T15:37:44.575252Z","iopub.status.idle":"2020-09-09T15:37:44.593009Z","shell.execute_reply":"2020-09-09T15:37:44.592543Z"},"papermill":{"duration":0.045087,"end_time":"2020-09-09T15:37:44.593108","exception":false,"start_time":"2020-09-09T15:37:44.548021","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# PREDICTION\nsub['FVC1'] = 1.*pe[:, 1]\nsub['Confidence1'] = pe[:, 2] - pe[:, 0]\nsubm = sub[['Patient_Week','FVC','Confidence','FVC1','Confidence1']].copy()\nsubm.loc[~subm.FVC1.isnull()].head(10)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.648588Z","iopub.status.busy":"2020-09-09T15:37:44.646861Z","iopub.status.idle":"2020-09-09T15:37:44.649448Z","shell.execute_reply":"2020-09-09T15:37:44.649951Z"},"papermill":{"duration":0.036273,"end_time":"2020-09-09T15:37:44.650064","exception":false,"start_time":"2020-09-09T15:37:44.613791","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"subm.loc[~subm.FVC1.isnull(),'FVC'] = subm.loc[~subm.FVC1.isnull(),'FVC1']\nif sigma_mean<70:\n    subm['Confidence'] = sigma_opt\nelse:\n    subm.loc[~subm.FVC1.isnull(),'Confidence'] = subm.loc[~subm.FVC1.isnull(),'Confidence1']","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.700954Z","iopub.status.busy":"2020-09-09T15:37:44.700081Z","iopub.status.idle":"2020-09-09T15:37:44.705493Z","shell.execute_reply":"2020-09-09T15:37:44.70611Z"},"papermill":{"duration":0.03616,"end_time":"2020-09-09T15:37:44.706222","exception":false,"start_time":"2020-09-09T15:37:44.670062","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"subm.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.757802Z","iopub.status.busy":"2020-09-09T15:37:44.75485Z","iopub.status.idle":"2020-09-09T15:37:44.778182Z","shell.execute_reply":"2020-09-09T15:37:44.77874Z"},"papermill":{"duration":0.051596,"end_time":"2020-09-09T15:37:44.778868","exception":false,"start_time":"2020-09-09T15:37:44.727272","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"subm.describe().T","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.857795Z","iopub.status.busy":"2020-09-09T15:37:44.856869Z","iopub.status.idle":"2020-09-09T15:37:44.860815Z","shell.execute_reply":"2020-09-09T15:37:44.861406Z"},"papermill":{"duration":0.061563,"end_time":"2020-09-09T15:37:44.86161","exception":false,"start_time":"2020-09-09T15:37:44.800047","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"otest = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\nfor i in range(len(otest)):\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'FVC'] = otest.FVC[i]\n    subm.loc[subm['Patient_Week']==otest.Patient[i]+'_'+str(otest.Weeks[i]), 'Confidence'] = 0.1","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.936645Z","iopub.status.busy":"2020-09-09T15:37:44.935656Z","iopub.status.idle":"2020-09-09T15:37:44.943063Z","shell.execute_reply":"2020-09-09T15:37:44.944228Z"},"papermill":{"duration":0.053268,"end_time":"2020-09-09T15:37:44.944468","exception":false,"start_time":"2020-09-09T15:37:44.8912","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].to_csv(\"submission_regression.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:44.999975Z","iopub.status.busy":"2020-09-09T15:37:44.998054Z","iopub.status.idle":"2020-09-09T15:37:45.001233Z","shell.execute_reply":"2020-09-09T15:37:45.001964Z"},"papermill":{"duration":0.031337,"end_time":"2020-09-09T15:37:45.002131","exception":false,"start_time":"2020-09-09T15:37:44.970794","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"reg_sub = subm[[\"Patient_Week\",\"FVC\",\"Confidence\"]].copy()","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.019964,"end_time":"2020-09-09T15:37:45.045229","exception":false,"start_time":"2020-09-09T15:37:45.025265","status":"completed"},"tags":[]},"cell_type":"markdown","source":"## 4.4 Ensemble and blending <a class=\"anchor\" id=\"4.4\"></a>\n\n[Back to Table of Contents](#0.1)"},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:45.09488Z","iopub.status.busy":"2020-09-09T15:37:45.094059Z","iopub.status.idle":"2020-09-09T15:37:45.097692Z","shell.execute_reply":"2020-09-09T15:37:45.098127Z"},"papermill":{"duration":0.032769,"end_time":"2020-09-09T15:37:45.098243","exception":false,"start_time":"2020-09-09T15:37:45.065474","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"df1 = img_sub.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)\ndf2 = reg_sub.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:45.154927Z","iopub.status.busy":"2020-09-09T15:37:45.154101Z","iopub.status.idle":"2020-09-09T15:37:45.159806Z","shell.execute_reply":"2020-09-09T15:37:45.159316Z"},"papermill":{"duration":0.040974,"end_time":"2020-09-09T15:37:45.159903","exception":false,"start_time":"2020-09-09T15:37:45.118929","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"df = df1[['Patient_Week']].copy()\ndf['FVC'] = FVC_weight*df1['FVC'] + (1-FVC_weight)*df2['FVC']\ndf['Confidence'] = Confidence_weight*df1['Confidence'] + (1-Confidence_weight)*df2['Confidence']\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-09-09T15:37:45.208975Z","iopub.status.busy":"2020-09-09T15:37:45.207621Z","iopub.status.idle":"2020-09-09T15:37:45.217626Z","shell.execute_reply":"2020-09-09T15:37:45.217012Z"},"papermill":{"duration":0.036503,"end_time":"2020-09-09T15:37:45.217726","exception":false,"start_time":"2020-09-09T15:37:45.181223","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.020534,"end_time":"2020-09-09T15:37:45.258601","exception":false,"start_time":"2020-09-09T15:37:45.238067","status":"completed"},"tags":[]},"cell_type":"markdown","source":"[Go to Top](#0)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}