{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"#%cd ../input/python3gdcm\n!dpkg -i ../input/python3gdcm/build_1-1_amd64.deb\n!apt-get install -f","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp /usr/local/lib/gdcm.py /opt/conda/lib/python3.7/site-packages/.\n!cp /usr/local/lib/gdcmswig.py /opt/conda/lib/python3.7/site-packages/.\n!cp /usr/local/lib/_gdcmswig.so /opt/conda/lib/python3.7/site-packages/.\n!cp /usr/local/lib/libgdcm* /opt/conda/lib/python3.7/site-packages/.\n!ldconfig","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import cv2\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport glob\nfrom pathlib import Path\nimport os\nimport math\nimport random\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom PIL import Image\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn import decomposition\nfrom typing import List, NoReturn, Union, Tuple, Optional, Text, Generic, Callable, Dict\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder, QuantileTransformer\nfrom sklearn.model_selection import KFold, StratifiedKFold, TimeSeriesSplit\nimport itertools\nfrom functools import partial\nimport umap\nimport gdcm\nimport scipy as sp\n\nimport pydicom\nfrom pydicom.tag import Tag\n\nimport scipy.ndimage as ndimage\nfrom scipy.ndimage import zoom\nfrom scipy.stats import kurtosis\nfrom scipy.stats import skew\n\nfrom sklearn.cluster import KMeans\nfrom skimage import measure, morphology, segmentation\n\nfrom sklearn.svm import SVR\nfrom sklearn.linear_model import Ridge, Lasso, BayesianRidge, ElasticNet\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostRegressor\n\n# visualize\nimport matplotlib.pyplot as plt\nimport matplotlib.style as style\nimport seaborn as sns\nfrom matplotlib_venn import venn2\nfrom matplotlib import pyplot\nfrom matplotlib.ticker import ScalarFormatter\nsns.set_context(\"talk\")\nstyle.use('seaborn-colorblind')\npd.options.display.max_columns = None\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# keras\nimport tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import models\nfrom tensorflow.keras import losses\nfrom tensorflow.keras import utils\nfrom tensorflow.keras import layers\nimport tensorflow_addons as tfa\nfrom tensorflow.keras.utils import Sequence\nimport tensorflow.keras.backend as K \n\ndef seed_everything(seed : int) -> NoReturn :    \n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(64)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Config"},{"metadata":{"trusted":true},"cell_type":"code","source":"# CONFIG\nINPUT_DIR = \"../input/osic-pulmonary-fibrosis-progression\"\nSEED = 42\nNFOLD = 5\nSCALER = 'MinMax'\nSAVE_BEST = True\nSA = 3\nFOLD_TYPE = 'StratifiedGroupKFold'\nEARLY_STOP = 80\nverbosity = 1000\nVAL_STRATEGY = 'cv'\nEPOCHS = 16\nBATCH_SIZE = 32\nLR = 0.001\nw = [1, 1]\nMAGIC = 1\nDEBUG = False\n\n# ---------------------------\n# MLP ---------------------\n# ---------------------------\nnn_params = {\n        'input_dropout': 0.0,\n        'hidden_layers': 3,\n        'hidden_units': 128,\n        'embedding_out_dim': 4,\n        'hidden_activation': 'Mish', \n        'hidden_dropout': 0.32,\n        'gauss_noise': 0.0001,\n        'norm_type': 'none', # layer\n        'optimizer': {'type': 'radam', 'lr': 1e-1},\n        'batch_size': 128,\n        'epochs': 80\n        }\n\n# ---------------------------\n# small MLP ---------------------\n# ---------------------------\nsnn_params = {\n        'input_dropout': 0.0,\n        'hidden_layers': 2,\n        'hidden_units': 64,\n        'embedding_out_dim': 4,\n        'hidden_activation': 'relu', \n        'hidden_dropout': 0.16,\n        'gauss_noise': 0.0001,\n        'norm_type': 'none', # layer\n        'optimizer': {'type': 'adam', 'lr': 1e-1},\n        'batch_size': 128,\n        'epochs': 80\n        }\n\n# -----------------------\n# XGB -------------------\n# -----------------------\nxgb_params = {\n            'colsample_bytree': 0.32,                 \n            'learning_rate': 0.04,\n            'max_depth': 4,\n            'subsample': 1,\n            'min_child_weight': 4,\n            'gamma': 0.24,\n            'alpha': 0,\n            'lambda': 1,\n            'seed': SEED,\n            'n_estimators': 240000\n        }\nxgb_params[\"objective\"] = 'reg:squarederror'\nxgb_params[\"eval_metric\"] = \"rmse\"\n\n# -----------------------\n# LGB -------------------\n# -----------------------\nlgb_params = {\n            'n_estimators': 240000,\n            'boosting_type': 'gbdt',\n            'max_depth': 3,\n            'learning_rate': 0.08,\n            'subsample': 0.72,\n            'subsample_freq': 4,\n            'feature_fraction': 0.24,\n            'lambda_l1': 1,\n            'lambda_l2': 1,\n            'seed': SEED,\n            'early_stopping_rounds': EARLY_STOP,\n            }    \nlgb_params[\"objective\"] = 'huber'\nlgb_params[\"metric\"] = \"huber\"\n\n# -----------------------\n# CATB -------------------\n# -----------------------\ncatb_params = { 'task_type': \"CPU\",\n            'learning_rate': 0.04, \n            'iterations': 240000,\n            'colsample_bylevel': 0.01,\n            'random_seed': SEED,\n            'use_best_model': True,\n            'early_stopping_rounds': EARLY_STOP\n            }\ncatb_params[\"loss_function\"] = \"MAE\"\ncatb_params[\"eval_metric\"] = \"MAE\"\n\n# ---------------------------\n# Linear ---------------------\n# ---------------------------\nlin_params = {\n        'alpha': 8, \n        'fit_intercept': True,\n        'max_iter': 8000, \n        'tol': 1e-06,\n        'random_state': SEED,\n}\n\n# ---------------------------\n# BayesianRidge ---------------------\n# ---------------------------\nbr_params = {\n        'n_iter': 8000, \n        'fit_intercept': True,\n        'tol': 1e-06,\n}\n\n# ---------------------------\n# SVR ---------------------\n# ---------------------------\nsvm_params = {\n        'C': 8,\n        'cache_size': 2400.0,\n        'max_iter': 8000,\n        'verbose': True\n} \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_tabular():\n    train = pd.read_csv(INPUT_DIR + '/train.csv')\n    test = pd.read_csv(INPUT_DIR + '/test.csv')\n    sub = pd.read_csv(INPUT_DIR + '/sample_submission.csv')\n    return train, test, sub\ntrain, test, sub = read_tabular()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nprint(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nprint(sub.shape)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.merge(sub[['Patient','Weeks','Confidence','Patient_Week']], test.drop(columns=['Weeks']), on='Patient')\ntrain['where'] = 'train'\ntest['where'] = 'test'\nsub['where'] = 'sub'\ndata = pd.concat([train, test, sub], ignore_index=True)\nprint(data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct train input\ndef fe(data):\n    data['min_week'] = data['Weeks']\n    data.loc[data['where'] == 'test','min_week'] = np.nan\n    data['min_week'] = data.groupby('Patient')['min_week'].transform('min')\n    \n    base = data.loc[data.Weeks == data.min_week]\n    base = base[['Patient','FVC', 'Percent']].copy()\n    base.columns = ['Patient','base_FVC', 'base_Percent']\n    base['nb'] = 1\n    base['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\n    base = base[base.nb==1]\n    base.drop('nb', axis=1, inplace=True)\n\n    data = data.merge(base, on='Patient', how='left')\n    data['base_week'] = data['Weeks'] - data['min_week']\n    del base\n    \n    train = data.loc[data['where'] == 'train', :].reset_index(drop=True)\n    test = data.loc[data['where'] == 'test', :].reset_index(drop=True)\n    sub = data.loc[data['where'] == 'sub', :].reset_index(drop=True)\n\n    return train, test, sub\ntrain, test, sub = fe(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = sub\nprint(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image(path):\n    ds = pydicom.dcmread(path)\n    img = cv2.resize(ds.pixel_array / 2**11, (512, 512))\n#     img = ds.pixel_array                                  # Now, img is pixel_array. it is input of our demo code\n                                                          # Convert pixel_array (img) to -> gray image (img_2d_scaled)\n    img_2d = img.astype(float)                            # Step 1. Convert to float to avoid overflow or underflow losses.\n    img = (np.maximum(img_2d,0) / img_2d.max()) * 255.0   # Step 2. Rescaling grey scale between 0-255\n    \n    return img\n\ndef find_contour(img_thr, find_max_con=False):\n    contours,hierarchy,=cv2.findContours(img_thr,cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)\n    \n    # Max area contour\n    if find_max_con:\n        contours,hierarchy,=cv2.findContours(img_thr,cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)\n        max_con= max(contours,key=cv2.contourArea)\n        \n        mask=np.zeros(img_thr.shape)\n        mask= cv2.fillConvexPoly(mask, max_con, 1.0)\n        \n        return img_thr * mask\n    \n    # EXTERNAL CONTOUR\n    ext_contours=np.zeros(img_thr.shape)\n    for i in range(len(contours)):\n        if hierarchy[0][i][3]==-1:  # checking for ext. contour (-1)  else 'specific no' for diff. internal contours \n            cv2.drawContours(ext_contours,contours,i,255,-1)\n    \n    # INTERNAL CONTOUR\n    int_contours=np.zeros(img_thr.shape)\n    for i in range(len(contours)):\n        if hierarchy[0][i][3]!= -1:  # checking for ext. contour (-1)  else internal contour\n            cv2.drawContours(int_contours,contours,i,255,-1)\n    return ext_contours, int_contours\n\ndef seg_lung(org_img, in_contours_img):\n    kernel= cv2.getStructuringElement(cv2.MORPH_RECT,(5,5))\n    \n    clr_noise_img=cv2.morphologyEx(in_contours_img, cv2.MORPH_OPEN, kernel)\n    forg_img= cv2.dilate(clr_noise_img, kernel,iterations = 2)\n    \n    forg_img= cv2.bitwise_not(forg_img.astype('uint8'))\n    return cv2.bitwise_or(org_img.astype('uint8'), forg_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['a'] = 0\ntrain['b'] = 0\nfor i, p in tqdm(enumerate(train.Patient.unique())):\n    sub = train.loc[train.Patient == p, :] \n    fvc = sub.FVC.values\n    weeks = sub.Weeks.values\n    c = np.vstack([weeks, np.ones(len(weeks))]).T\n    a, b = np.linalg.lstsq(c, fvc)[0]\n    \n    train.loc[train['Patient'] == p, 'a'] = a\n    train.loc[train['Patient'] == p, 'b'] = b","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_imgs(p, data='train'):\n    x = [] \n\n#     if p in ['ID00011637202177653955184', 'ID00052637202186188008618']:\n#         return x\n\n    ldir = os.listdir(f'../input/osic-pulmonary-fibrosis-progression/{data}/{p}/')\n    for i in ldir:\n        if int(i[:-4]) / len(ldir) < 0.7 and int(i[:-4]) / len(ldir) > 0.3:\n#             x.append(get_img(f'../input/osic-pulmonary-fibrosis-progression/{data}/{p}/{i}')) \n#             x.append(get_img(p, i[:-4])) \n                \n            x.append(load_image(f'../input/osic-pulmonary-fibrosis-progression/{data}/{p}/{i}'))\n        if len(x) < 1:\n            continue\n    x = np.expand_dims(x, axis=-1)\n\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(shape=(512, 512, 1)):\n    def res_block(x, n_features):\n        _x = x\n        x = layers.BatchNormalization()(x)\n        x = layers.LeakyReLU(0.05)(x)\n    \n        x = layers.Conv2D(n_features, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n        x = layers.Add()([_x, x])\n        return x\n    \n    inp = layers.Input(shape=shape)\n    \n    # 512\n    x = layers.Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same', input_shape=shape)(inp)\n    x = layers.BatchNormalization()(x)\n    x = layers.LeakyReLU(0.05)(x)\n    \n    x = layers.Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    x = layers.BatchNormalization()(x)\n    x = layers.LeakyReLU(0.05)(x)\n    \n    x = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 256\n    x = layers.Conv2D(8, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 8)\n    x = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 128\n    x = layers.Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(2):\n        x = res_block(x, 16)\n    x = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 64\n    x = layers.Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 32)\n    x = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)\n    \n    # 32\n    x = layers.Conv2D(64, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 64)\n    x = layers.AveragePooling2D(pool_size=(2, 2), strides=(2, 2))(x)    \n    \n    # 16\n    x = layers.Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same')(x)\n    for _ in range(3):\n        x = res_block(x, 128)\n        \n    # pool\n    x = layers.GlobalAveragePooling2D(name='hidden')(x)\n    x = layers.Dropout(0.4)(x) \n    out = layers.Dense(1)(x)\n    \n    # compile\n    model = models.Model(inputs=inp, outputs=out)\n    opt = tfa.optimizers.RectifiedAdam(lr=LR)\n    opt = tfa.optimizers.SWA(opt)\n    model.compile(optimizer=opt, loss=\"mae\")\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kf = KFold(n_splits=NFOLD, random_state=42, shuffle=True)\nndim = 128\ntr_vec = {\n    'average': pd.DataFrame(np.nan * np.ones((train['Patient'].nunique(), ndim))),\n    'max': pd.DataFrame(np.nan * np.ones((train['Patient'].nunique(), ndim)))\n         }\nte_vec = {\n    'average': pd.DataFrame(np.zeros((test['Patient'].nunique(), ndim))),\n    'max': pd.DataFrame(np.zeros((test['Patient'].nunique(), ndim)))\n    }\nfor t in ['average', 'max']:\n    tr_vec[t]['Patient'] = train['Patient'].unique()\n    te_vec[t]['Patient'] = test['Patient'].unique()\n    \ntrain['cnn'] = 0\ntest['cnn'] = 0\n\nfor fold, (tr_idx, val_idx) in enumerate(kf.split(train['Patient'].unique())):\n    print('#####################')\n    print('####### Fold %i ######'%fold)\n    print('#####################')\n    print('Training...')\n    \n    er = tf.keras.callbacks.EarlyStopping(\n        monitor=\"val_loss\",\n        min_delta=1e-3,\n        patience=10,\n        verbose=1,\n        mode=\"auto\",\n        baseline=None,\n        restore_best_weights=True,\n    )\n\n    cpt = tf.keras.callbacks.ModelCheckpoint(\n        filepath='fold-%i.h5'%fold,\n        monitor='val_loss', \n        verbose=1, \n        save_best_only=SAVE_BEST,\n        mode='auto'\n    )\n\n    rlp = tf.keras.callbacks.ReduceLROnPlateau(\n        monitor='val_loss', \n        factor=0.5,\n        patience=5, \n        verbose=1, \n        min_lr=1e-8\n    )\n    model = get_model()\n    \n    # laod pretrained weight\n    model.load_weights(f'../input/osic-2dcnn/fold-{fold}.h5')\n        \n    # extract CNN features\n    hidden_model = models.Model(inputs=model.input,\n                         outputs=model.get_layer('hidden').output)\n    \n    # val\n    for p in tqdm(train['Patient'].unique()[val_idx]):\n        x = load_imgs(p, data='train')\n        \n        # CNN features\n        try:\n            cnn_vec = hidden_model.predict(x)\n            me = np.mean(cnn_vec, axis=0)\n            ma = np.max(cnn_vec, axis=0)\n            for k in range(ndim):\n                tr_vec['average'].loc[tr_vec['average']['Patient'] == p, k] = me[k]\n                tr_vec['max'].loc[tr_vec['max']['Patient'] == p, k] = ma[k]\n        except:\n            print(p, ' cnn feats error')\n            continue\n            \n        # true model\n        try:\n            a = model.predict(x)\n            cond = train['Patient'] == p\n            train.loc[cond, 'cnn'] = np.median(a) * (train.loc[cond, 'Weeks'] - train.loc[cond, 'min_week']) + train.loc[cond, 'base_FVC']\n                    \n        except:\n            print(p, ' cnn pred error')\n            continue            \n            \n    # test\n    for p in tqdm(test['Patient'].unique()):\n        x = load_imgs(p, data='test')\n        \n        # CNN features\n        cnn_vec = hidden_model.predict(x)\n        me = np.mean(cnn_vec, axis=0)\n        ma = np.max(cnn_vec, axis=0)\n        for k in range(ndim):\n            te_vec['average'].loc[te_vec['average']['Patient'] == p, k] += me[k] / NFOLD\n            te_vec['max'].loc[te_vec['max']['Patient'] == p, k] += ma[k] / NFOLD\n        \n        # true model\n        a = model.predict(x)\n        cond = test['Patient'] == p\n        test.loc[cond, 'cnn'] += (np.median(a) * (test.loc[cond, 'Weeks'] - test.loc[cond, 'min_week']) + test.loc[cond, 'base_FVC']) / NFOLD\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# CNN feats\nndim = 8\ntr_cnn_df = pd.DataFrame()\nte_cnn_df = pd.DataFrame()\ntr_cnn_df['Patient'] = tr_vec['average']['Patient'].values\nte_cnn_df['Patient'] = te_vec['average']['Patient'].values\n\n# pca\nfor t in ['average', 'max']:\n    # fillna\n    te_vec[t] = te_vec[t].fillna(tr_vec[t].median())\n    tr_vec[t] = tr_vec[t].fillna(tr_vec[t].median())\n    \n    # scaling\n    feats = [f for f in range(128)]\n    scaler = StandardScaler()\n    tr_vec[t][feats] = scaler.fit_transform(tr_vec[t][feats].values)\n    te_vec[t][feats] = scaler.transform(te_vec[t][feats].values)\n    \n    # dimensionality reduction\n    trans = decomposition.PCA(n_components=80)\n    train_dist = trans.fit_transform(tr_vec[t][feats].values)\n    test_dist = trans.transform(te_vec[t][feats].values)\n    \n    trans = umap.UMAP(n_components=ndim, random_state=SEED)\n    train_dist2 = trans.fit_transform(train_dist)\n    test_dist2 = trans.transform(test_dist)\n    \n    # add to df\n    for k in range(ndim):\n        tr_cnn_df['pca_' + t + str(k)] = train_dist[:, k]\n        te_cnn_df['pca_' + t + str(k)] = test_dist[:, k]\n        tr_cnn_df['umap_' + t + str(k)] = train_dist2[:, k]\n        te_cnn_df['umap_' + t + str(k)] = test_dist2[:, k]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add cnn feats\ntrain = pd.merge(train, tr_cnn_df, how='left', on='Patient')\ntest = pd.merge(test, te_cnn_df, how='left', on='Patient')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final FE"},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_lasts(train):\n    train['validation'] = 0\n    for p in train['Patient'].unique():\n        for i, w in enumerate(train.loc[train['Patient'] == p, 'Weeks'].values[-3:]):\n            train.loc[(train['Patient'] == p) & (train['Weeks'] == w), 'validation'] = i+1\n    return train\n\ntrain = add_lasts(train)\n\ndef calculate_height(row):\n    if row['Sex'] == 'Male':\n        return row['base_FVC'] / (27.63 - 0.112 * row['Age'])\n    else:\n        return row['base_FVC'] / (21.78 - 0.101 * row['Age'])\n\ndef fe2(data):    \n    data['height'] = data.apply(calculate_height, axis=1)\n    data['is_exsmoker'] = data['SmokingStatus'].apply(lambda x : 1 if 'Ex-smoker' in x else 0)\n    data['is_neversmoker'] = data['SmokingStatus'].apply(lambda x : 1 if 'Never smoked' in x else 0)\n    data['is_nowsmoker'] = data['SmokingStatus'].apply(lambda x : 1 if 'Currently smokes' in x else 0)\n    data['is_smoker'] = data['is_exsmoker'] + data['is_nowsmoker']\n    data['Sex'] = data['Sex'].map({'Male': 1, 'Female': 0})\n    \n    # binning\n#     percent_bin = [71.824968, 76.672493, 79.258903]\n#     fvc_bin = [2739, 2925, 3020]\n#     def binner(x, bins):\n#         if x < bins[0]:\n#             return 1\n#         elif (x >= bins[0]) & (x < bins[1]):\n#             return 2\n#         elif (x >= bins[1]) & (x < bins[2]):\n#             return 3\n#         elif x >= bins[2]:\n#             return 4\n    \n#     data['bin_Age'] = data['Age'].apply(lambda x : 1 if x >= 70 else 0)\n#     data['bin_FVC'] = data['base_FVC'].apply(lambda x : binner(x, fvc_bin))\n#     data['bin_min_week'] = data['min_week'].apply(lambda x : 0 if x <=0 else 1)\n#     data['bin_Percent'] = data['base_Percent'].apply(lambda x : binner(x, percent_bin))\n#     data['bin_FVCxPercent'] =  data['bin_Percent'] * data['bin_FVC']\n#     data['bin_FVCwPercent'] =  data['bin_FVC'] / data['bin_Percent']\n    data['base_FVCxPercent'] =  data['base_Percent'] * data['base_FVC']\n    data['base_FVCwPercent'] =  data['base_Percent'] / data['base_FVC']\n#     data['pred_FVC'] = data['base_FVCwPercent'] * data['base_Percent']\n\n#     if 'Volume' in data.columns.values.tolist():\n#         for f in ['Volume', 'Mean', 'Skew', 'Kurtosis']:\n#             data[f'base_FVCw{f}'] = data[f] / data['base_FVC'] \n#             data[f'base_FVCx{f}'] = data['base_FVC'] * data[f]\n                \n    # interaction\n    for f in ['is_exsmoker', 'is_neversmoker', 'is_nowsmoker', 'is_smoker', 'base_FVC', 'base_Percent',]:\n        data['Sex_' + f] = data['Sex'] * data[f]\n    for f in ['base_FVC', 'Sex', 'is_exsmoker', 'is_neversmoker', 'is_nowsmoker', 'is_smoker','base_Percent',]:\n        data['Age_' + f] = data['Age'] * data[f]\n    for f in ['base_FVC', 'is_exsmoker', 'is_neversmoker', 'is_nowsmoker', 'is_smoker','base_Percent',]:\n        data['Age_Sex_' + f] = data['Age'] * data['Sex'] * data[f]\n    \n#     # magic\n#     magic = [0.81039892, 0.92171452, 1.00464756]\n#     for f in ['base_FVC']:\n#         for i, m in enumerate(magic):\n#             data[f + f'_magic{i}'] = data[f] * m\n#     for v in itertools.permutations([0, 1, 2], 2):\n#         data['base_FVCxPercent_{}{}'.format(v[0], v[1])] = data['base_Percent_magic{}'.format(v[0])] * data['base_FVC_magic{}'.format(v[1])]\n#         data['base_FVCwPercent_{}{}'.format(v[0], v[1])] = data['base_FVC_magic{}'.format(v[1])] / data['base_Percent_magic{}'.format(v[0])]\n# #         data['pred_FVC_{}{}'.format(v[0], v[1])] = data['base_FVCwPercent_{}{}'.format(v[0], v[1])] * data['base_Percent']\n    drops = ['SmokingStatus']\n#     for f in ['base_Percent', 'base_FVC']:\n#         for i, m in enumerate(magic):\n#             drops.append(f + f'_magic{i}')\n        \n    # drops\n    data.drop(columns=drops, inplace=True)\n    \n    return data\n\ntrain = fe2(train)\ntest = fe2(test)\n\ndef fe3(train, test):\n    # agg by patient\n    train.groupby('Patient')\n    \n    # mean for category\n    target_feats = ['base_FVC']\n    cat_feats = ['Sex', 'is_smoker']\n    for r in range(len(cat_feats)-1):\n        comb = list(itertools.combinations(cat_feats, r+1))\n        for c in comb:\n            for t in target_feats:\n                suf = '_'.join(list(c))               \n                tmp = train.groupby(list(c))[t].agg(['mean', 'max', 'min', ]).reset_index().rename(columns={'mean': t + '_mean_' + suf,\n                                                                                                          'max': t + '_max_' + suf,\n                                                                                                          'min': t + '_min_' + suf})\n                train = pd.merge(train, tmp, how='left', on=list(c)) \n                test = pd.merge(test, tmp, how='left', on=list(c))\n                \n                # diff\n                for m in ['mean', 'max', 'min']:\n                    train[t + '_' + m + '_' + suf] = train[t] - train[t + '_' + m + '_' + suf]\n                    test[t + '_' + m + '_' + suf] = test[t] - test[t + '_' + m + '_' + suf]\n    return train, test\n\ntrain, test = fe3(train, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# min      -28.182575\n# 25%       -7.588524\n# 50%       -3.909110\n# 75%       -0.950083\n# max       14.682612\n# group by a\ntrain['group_a'] = 0\ntrain.loc[(-7.58 <= train['a']) & (train['a'] < -3.9), 'group_a'] = 1\ntrain.loc[(-3.9 <= train['a']) & (train['a'] < -0.95), 'group_a'] = 2\ntrain.loc[-0.95 < train['a'], 'group_a'] = 3\ntrain['group_a'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nfrom collections import Counter, defaultdict\nfrom sklearn import model_selection\n\n# ---- GroupKFold ----\nclass GroupKFold(object):\n    \"\"\"\n    GroupKFold with random shuffle with a sklearn-like structure\n    \"\"\"\n\n    def __init__(self, n_splits=4, shuffle=True, random_state=42):\n        self.n_splits = n_splits\n        self.shuffle = shuffle\n        self.random_state = random_state\n\n    def get_n_splits(self, X=None, y=None, group=None):\n        return self.n_splits\n\n    def split(self, X, y, group):\n        kf = model_selection.KFold(n_splits=self.n_splits, shuffle=self.shuffle, random_state=self.random_state)\n        unique_ids = X[group].unique()\n        for fold, (tr_group_idx, va_group_idx) in enumerate(kf.split(unique_ids)):\n            # split group\n            tr_group, va_group = unique_ids[tr_group_idx], unique_ids[va_group_idx]\n            train_idx = np.where(X[group].isin(tr_group))[0]\n            val_idx = np.where(X[group].isin(va_group))[0]\n            yield train_idx, val_idx\n            \n# ---- StratifiedGroupKFold ----\nclass StratifiedGroupKFold(object):\n    \"\"\"\n    StratifiedGroupKFold with random shuffle with a sklearn-like structure\n    \"\"\"\n\n    def __init__(self, n_splits=4, shuffle=True, random_state=42):\n        self.n_splits = n_splits\n        self.shuffle = shuffle\n        self.random_state = random_state\n\n    def get_n_splits(self, X=None, y=None, group=None):\n        return self.n_splits\n\n    def split(self, X, y, group):\n        labels_num = np.max(y) + 1\n        y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))\n        y_distr = Counter()\n        groups = X[group].values\n        for label, g in zip(y, groups):\n            y_counts_per_group[g][label] += 1\n            y_distr[label] += 1\n\n        y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))\n        groups_per_fold = defaultdict(set)\n\n        def eval_y_counts_per_fold(y_counts, fold):\n            y_counts_per_fold[fold] += y_counts\n            std_per_label = []\n            for label in range(labels_num):\n                label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(self.n_splits)])\n                std_per_label.append(label_std)\n            y_counts_per_fold[fold] -= y_counts\n            return np.mean(std_per_label)\n        \n        groups_and_y_counts = list(y_counts_per_group.items())\n        random.Random(self.random_state).shuffle(groups_and_y_counts)\n\n        for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])):\n            best_fold = None\n            min_eval = None\n            for i in range(self.n_splits):\n                fold_eval = eval_y_counts_per_fold(y_counts, i)\n                if min_eval is None or fold_eval < min_eval:\n                    min_eval = fold_eval\n                    best_fold = i\n            y_counts_per_fold[best_fold] += y_counts\n            groups_per_fold[best_fold].add(g)\n\n        all_groups = set(groups)\n        for i in range(self.n_splits):\n            train_groups = all_groups - groups_per_fold[i]\n            test_groups = groups_per_fold[i]\n\n            train_idx = [i for i, g in enumerate(groups) if g in train_groups]\n            test_idx = [i for i, g in enumerate(groups) if g in test_groups]\n\n            yield train_idx, test_idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# to normal\nID = 'Patient_Week'\ntarget = 'FVC'\ngroup = 'Patient'\ndropcols = [ID, 'test_patient', 'where', 'Confidence', 'validation', 'a', 'b', 'group_a']\ndropcols += [target, group] + ['Percent', 'cnn']\nfeatures = [f for f in train.columns.values.tolist() if f not in dropcols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(features))\nfeatures","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Linear decay"},{"metadata":{"trusted":true},"cell_type":"code","source":"C1, C2 = tf.constant(70, dtype='float32'), tf.constant(1000, dtype=\"float32\")\n\nfrom tensorflow.keras.layers import Activation\nfrom tensorflow.keras.utils import get_custom_objects\n\n# mish\nclass Mish(Activation):\n    '''\n    Mish Activation Function.\n    .. math::\n        mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + e^{x}))\n    Shape:\n        - Input: Arbitrary. Use the keyword argument `input_shape`\n        (tuple of integers, does not include the samples axis)\n        when using this layer as the first layer in a model.\n        - Output: Same shape as the input.\n    Examples:\n        >>> X = Activation('Mish', name=\"conv1_act\")(X_input)\n    '''\n\n    def __init__(self, activation, **kwargs):\n        super(Mish, self).__init__(activation, **kwargs)\n        self.__name__ = 'Mish'\n\ndef mish(inputs):\n    return inputs * tf.math.tanh(tf.math.softplus(inputs))\n\nget_custom_objects().update({'Mish': Mish(mish)})\n\n#=============================#\ndef score(y_true, y_pred):\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    sigma = y_pred[:, 2] - y_pred[:, 0]\n    fvc_pred = y_pred[:, 1]\n    \n    #sigma_clip = sigma + C1\n    sigma_clip = tf.maximum(sigma, C1)\n    delta = tf.abs(y_true[:, 0] - fvc_pred)\n    delta = tf.minimum(delta, C2)\n    sq2 = tf.sqrt( tf.dtypes.cast(2, dtype=tf.float32) )\n    metric = (delta / sigma_clip)*sq2 + tf.math.log(sigma_clip* sq2)\n    \n    return K.mean(metric)\n\n#============================#\ndef qloss(y_true, y_pred):\n    # Pinball loss for multiple quantiles\n    tf.dtypes.cast(y_true, tf.float32)\n    tf.dtypes.cast(y_pred, tf.float32)\n    q = tf.constant(np.array([qs]), dtype=tf.float32)\n    e = y_true - y_pred\n    v = tf.maximum(q*e, (q-1)*e)\n    return K.mean(v)\n\n#=============================#\ndef mloss(_lambda):\n    def loss(y_true, y_pred):\n        y_true = tf.dtypes.cast(y_true, tf.float32)\n        y_pred = tf.dtypes.cast(y_pred, tf.float32)\n        return _lambda * qloss(y_true, y_pred) + (1 - _lambda) * score(y_true, y_pred)\n    return loss\n\n\ndef nn_model(L, params, obj='regression', mloss_param=0.8):\n    inputs = layers.Input((L, ), name='Patient')\n    x = layers.Dense(params['hidden_units'], activation=params['hidden_activation'])(inputs)\n    x = layers.Dropout(params['hidden_dropout'])(x)\n    if params['norm_type'] == 'batch':\n        x = layers.BatchNormalization()(x)\n    elif params['norm_type'] == 'layer':\n        x = layers.LayerNormalization()(x)\n    else:\n        pass\n    \n    # more layers\n    for i in np.arange(params['hidden_layers'] - 1):\n        x = layers.Dense(params['hidden_units'] // (2 * (i+1)), activation=params['hidden_activation'])(x)\n        x = layers.Dropout(params['hidden_dropout'])(x)\n        if params['norm_type'] == 'batch':\n            x = layers.BatchNormalization()(x)\n        elif params['norm_type'] == 'layer':\n            x = layers.LayerNormalization()(x)\n        else:\n            pass\n            \n    if obj == 'quantile':\n        p1 = layers.Dense(3, activation=\"linear\", name=\"p1\")(x)\n        p2 = layers.Dense(3, activation=\"relu\", name=\"p2\")(x)\n        preds = layers.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                         name=\"preds\")([p1, p2])\n    elif obj == 'regression':\n        preds = layers.Dense(1, activation=\"linear\", name=\"p1\")(x)\n        \n    model = models.Model(inputs, preds, name=\"NN\")\n    if params['optimizer']['type'] == 'adam':\n        opt = optimizers.Adam(lr=params['optimizer']['lr'])\n    elif params['optimizer']['type'] == 'sgd':\n        opt = optimizers.SGD(lr=params['optimizer']['lr'], decay=1e-6, momentum=0.9)\n    elif params['optimizer']['type'] == 'radam':\n        opt = tfa.optimizers.RectifiedAdam(lr=params['optimizer']['lr'])\n    opt = tfa.optimizers.SWA(opt)\n\n    if obj == 'quantile':\n        model.compile(loss=mloss(mloss_param), optimizer=opt, metrics=[score])\n    elif obj == 'regression':\n        model.compile(loss=tf.keras.losses.MeanAbsoluteError(), optimizer=opt)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nn_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n\n    # scaling\n    if SCALER == \"MinMax\":\n        scaler = MinMaxScaler()\n    elif SCALER == \"Standard\":\n        scaler = StandardScaler()\n#     trs = scaler.fit_transform(train[features])\n#     tes = scaler.transform(test[features])\n    df = pd.concat([train[features], test[features]])\n    df[features] = scaler.fit_transform(df[features])\n    trs = df.iloc[:train.shape[0]].values\n    tes = df.iloc[train.shape[0]:].values\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # MLP ----------------------\n        # --------------------------\n        model = nn_model(len(features), nn_params, 'regression')\n        early_stop = callbacks.EarlyStopping(patience=8, restore_best_weights=True, monitor='val_loss')\n        lr_schedule = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, verbose=2, mode='min')\n        history = model.fit(trs[tr_idx, :], train[target].values[tr_idx], callbacks=[early_stop, lr_schedule],\n                        epochs=nn_params['epochs'], batch_size=nn_params['batch_size'],\n                        validation_data=(trs[val_idx, :], train[target].values[val_idx]), verbose=0)\n\n        oof[val_idx] = model.predict(trs[val_idx, :]).ravel()\n        ypred += model.predict(tes).ravel() / NFOLD\n            \n    return oof, ypred, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def snn_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n\n    # scaling\n    if SCALER == \"MinMax\":\n        scaler = MinMaxScaler()\n    elif SCALER == \"Standard\":\n        scaler = StandardScaler()\n#     trs = scaler.fit_transform(train[features])\n#     tes = scaler.transform(test[features])\n    df = pd.concat([train[features], test[features]])\n    df[features] = scaler.fit_transform(df[features])\n    trs = df.iloc[:train.shape[0]].values\n    tes = df.iloc[train.shape[0]:].values\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # MLP ----------------------\n        # --------------------------\n        model = nn_model(len(features), snn_params, 'regression')\n        early_stop = callbacks.EarlyStopping(patience=8, restore_best_weights=True, monitor='val_loss')\n        lr_schedule = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, verbose=2, mode='min')\n        history = model.fit(trs[tr_idx, :], train[target].values[tr_idx], callbacks=[early_stop, lr_schedule],\n                        epochs=nn_params['epochs'], batch_size=nn_params['batch_size'],\n                        validation_data=(trs[val_idx, :], train[target].values[val_idx]), verbose=0)\n\n        oof[val_idx] = model.predict(trs[val_idx, :]).ravel()\n        ypred += model.predict(tes).ravel() / NFOLD\n            \n    return oof, ypred, history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def xgb_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # XGB ----------------------\n        # --------------------------\n        xgb_params['seed'] = seed\n        model = xgb.XGBRegressor(**xgb_params)\n        model.fit(train[features].iloc[tr_idx], train[target].iloc[tr_idx], eval_set=[(train[features].iloc[val_idx], train[target].iloc[val_idx])],\n                            early_stopping_rounds=EARLY_STOP, verbose=1000)\n        \n        oof[val_idx] = model.predict(train[features].iloc[val_idx])\n        ypred += model.predict(test[features]) / NFOLD\n            \n    return oof, ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lgb_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # CatB ----------------------\n        # --------------------------\n        lgb_params['seed'] = seed\n        model = lgb.LGBMRegressor(**lgb_params)\n        model.fit(train[features].iloc[tr_idx], train[target].iloc[tr_idx], eval_set=[(train[features].iloc[val_idx], train[target].iloc[val_idx])],\n            verbose=1000)\n        fi['importance'] += model.booster_.feature_importance(importance_type=\"gain\") / NFOLD\n\n        oof[val_idx] = model.predict(train[features].iloc[val_idx])\n        ypred += model.predict(test[features]) / NFOLD\n            \n    return oof, ypred, fi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def catb_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # CatB ----------------------\n        # --------------------------\n        catb_params['random_seed'] = seed\n        model = CatBoostRegressor(**catb_params)\n        model.fit(train[features].iloc[tr_idx], train[target].iloc[tr_idx], \n                  eval_set=(train[features].iloc[val_idx], train[target].iloc[val_idx]),\n                  verbose=1000, cat_features=[])\n        fi['importance'] += model.get_feature_importance() / NFOLD\n        oof[val_idx] = model.predict(train[features].iloc[val_idx])\n        ypred += model.predict(test[features]) / NFOLD\n            \n    return oof, ypred, fi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lin_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n\n    # scaling\n    if SCALER == \"MinMax\":\n        scaler = MinMaxScaler()\n    elif SCALER == \"Standard\":\n        scaler = StandardScaler()\n#     trs = scaler.fit_transform(train[features])\n#     tes = scaler.transform(test[features])\n    df = pd.concat([train[features], test[features]])\n    df[features] = scaler.fit_transform(df[features])\n    trs = df.iloc[:train.shape[0]].values\n    tes = df.iloc[train.shape[0]:].values\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # Linear ----------------------\n        # --------------------------\n        lin_params['random_state'] = seed\n        model = Ridge(**lin_params)\n        model.fit(trs[tr_idx, :], train[target].iloc[tr_idx])\n        fi['importance'] += model.coef_.ravel()\n\n        oof[val_idx] = model.predict(trs[val_idx, :])\n        ypred += model.predict(tes) / NFOLD\n            \n    return oof, ypred, fi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lasso_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n\n    # scaling\n    if SCALER == \"MinMax\":\n        scaler = MinMaxScaler()\n    elif SCALER == \"Standard\":\n        scaler = StandardScaler()\n#     trs = scaler.fit_transform(train[features])\n#     tes = scaler.transform(test[features])\n    df = pd.concat([train[features], test[features]])\n    df[features] = scaler.fit_transform(df[features])\n    trs = df.iloc[:train.shape[0]].values\n    tes = df.iloc[train.shape[0]:].values\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # Linear ----------------------\n        # --------------------------\n        lin_params['random_state'] = seed\n        model = Lasso(**lin_params)\n        model.fit(trs[tr_idx, :], train[target].iloc[tr_idx])\n\n        oof[val_idx] = model.predict(trs[val_idx, :])\n        ypred += model.predict(tes) / NFOLD\n            \n    return oof, ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def br_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n\n    # scaling\n    if SCALER == \"MinMax\":\n        scaler = MinMaxScaler()\n    elif SCALER == \"Standard\":\n        scaler = StandardScaler()\n#     trs = scaler.fit_transform(train[features])\n#     tes = scaler.transform(test[features])\n    df = pd.concat([train[features], test[features]])\n    df[features] = scaler.fit_transform(df[features])\n    trs = df.iloc[:train.shape[0]].values\n    tes = df.iloc[train.shape[0]:].values\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # Linear ----------------------\n        # --------------------------\n        model = BayesianRidge(**br_params)\n        model.fit(trs[tr_idx, :], train[target].iloc[tr_idx])\n        fi['importance'] += model.coef_.ravel()\n\n        oof[val_idx] = model.predict(trs[val_idx, :])\n        ypred += model.predict(tes) / NFOLD\n            \n    return oof, ypred, fi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def svr_mae(train, test, features, target, seed=SEED):\n    ypred = np.zeros(test.shape[0])\n    oof = np.zeros(train.shape[0])\n    fi = pd.DataFrame()\n    fi['features'] = features\n    fi['importance'] = 0\n    if FOLD_TYPE == 'GroupKFold':\n        kf = GroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train[target], group)\n    elif FOLD_TYPE == 'KFold':\n        kf = KFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train)\n    elif FOLD_TYPE == 'StratifiedKFold':\n        kf = StratifiedKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['test_patient'].astype(int))\n    elif FOLD_TYPE == 'StratifiedGroupKFold':\n        kf = StratifiedGroupKFold(n_splits=NFOLD, shuffle=True, random_state=seed)\n        kf = kf.split(train, train['group_a'].astype(int), group)\n\n    # scaling\n    if SCALER == \"MinMax\":\n        scaler = MinMaxScaler()\n    elif SCALER == \"Standard\":\n        scaler = StandardScaler()\n#     trs = scaler.fit_transform(train[features])\n#     tes = scaler.transform(test[features])\n    df = pd.concat([train[features], test[features]])\n    df[features] = scaler.fit_transform(df[features])\n    trs = df.iloc[:train.shape[0]].values\n    tes = df.iloc[train.shape[0]:].values\n    \n    for cnt, (tr_idx, val_idx) in enumerate(kf):\n        print(f\"FOLD {cnt}\")\n\n        # --------------------------\n        # Linear ----------------------\n        # --------------------------\n        model = SVR(**svm_params)\n        model.fit(trs[tr_idx, :], train[target].iloc[tr_idx])\n#         fi['importance'] += model.coef_.ravel()\n\n        oof[val_idx] = model.predict(trs[val_idx, :])\n        ypred += model.predict(tes) / NFOLD\n            \n    return oof, ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NN\ntarget = 'FVC'\noof_df = pd.DataFrame()\nypred_df = pd.DataFrame()\nif 'cnn' in test.columns.values.tolist():\n    oof_df['cnn'] = train['cnn'].values\n    ypred_df['cnn'] = test['cnn'].values\n\nypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, history = nn_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['nn'] = oof\nypred_df['nn'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot training & validation loss values\ndef plot_history(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('Model loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Val'], loc='upper right', frameon=False)\n    plt.show()\n    \nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Small NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NN\nypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, history = snn_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['snn'] = oof\nypred_df['snn'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## XGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_ = xgb_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['xgb'] = oof\nypred_df['xgb'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = lgb_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['lgb'] = oof\nypred_df['lgb'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 12))\nsns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CatB"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = catb_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['catb'] = oof\nypred_df['catb'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 12))\nsns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Linear"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = lin_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['lin'] = oof\nypred_df['lin'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 12))\nsns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lasso"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_ = lasso_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['lasso'] = oof\nypred_df['lasso'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## BayesianRidge"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = br_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['br'] = oof\nypred_df['br'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SVR"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ypred = np.zeros(test.shape[0])\n# oof = np.zeros(train.shape[0])\n# for s in range(SA): # seed average\n#     oof_, ypred_ = svr_mae(train, test, features, target, seed=SEED+s)\n#     oof += oof_ / SA\n#     ypred += ypred_ / SA\n# oof_df['svm'] = oof\n# ypred_df['svm'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"# CV\nidx = train['validation'] > 0\nfor m in oof_df.columns.values.tolist():\n    print('{}: MAE = {:.3f}, MAE (last 3) = {:.3f}'.format(m, \n          mean_absolute_error(train[target], oof_df[m]),\n          mean_absolute_error(train.loc[idx, target], oof_df.loc[idx, m])))\n    plt.hist(oof_df[m].values, alpha=0.4, label=m)\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(12, 12))\nsns.heatmap(oof_df.corr(), annot=True, ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# stacking\nfor f in [target, 'group_a', 'Patient']:\n    oof_df[f] = train[f].values\n    \nypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, history = snn_mae(oof_df, ypred_df, ypred_df.columns.values.tolist(), target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA \n# oof, ypred, history = snn_mae(oof_df, ypred_df, ypred_df.columns.values.tolist(), target, seed=SEED)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fig, ax = plt.subplots(1, 1, figsize=(8, 12))\n# sns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# FVC CV score"},{"metadata":{"trusted":true},"cell_type":"code","source":"# FVC mae CV\nscore = mean_absolute_error(train['FVC'], oof)\nprint(f'MAE all = {score}')\n\nidx = train['validation'] > 0\nscore = mean_absolute_error(train.loc[idx, 'FVC'], oof[idx])\nprint(f'MAE last 3 = {score}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optimize Confidence"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['FVC_pred'] = oof\ntest['FVC_pred'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy as sp\nfrom functools import partial\n\ndef loss_func(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = abs(row['FVC'] - row['FVC_pred'])\n    delta = min(diff, 1000)\n    score = -math.sqrt(2)*delta/sigma_clipped - np.log(math.sqrt(2)*sigma_clipped)\n    return -score\n\nresults = []\ntk0 = tqdm(train.iterrows(), total=len(train))\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [281]\n#     bounds = [(70, 100)]\n#     result = sp.optimize.minimize(loss_partial, weight, method='SLSQP', bounds=bounds)\n    result = sp.optimize.minimize(loss_partial, weight, method='Nelder-Mead')\n    x = result['x']\n    results.append(x[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# optimized score\ntrain['Confidence'] = results\ntrain['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\ntrain['diff'] = abs(train['FVC'] - train['FVC_pred'])\ntrain['delta'] = train['diff'].apply(lambda x: min(x, 1000))\ntrain['score'] = -math.sqrt(2)*train['delta']/train['sigma_clipped'] - np.log(math.sqrt(2)*train['sigma_clipped'])\nscore = train['score'].mean()\nscore2 = train.loc[train['validation'] > 0, 'score'].mean()\nprint(score, score2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Confidence'].hist(alpha=0.4)\ntrain['sigma_clipped'].hist(alpha=0.4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# all last 3\nnp.random.seed(SEED)\nfor p in tqdm(train['Patient'].unique()):\n    lasts = train.loc[(train['Patient'] == p) & (train['validation'] > 0), 'Confidence'].values\n    cond = (train['Patient'] == p) & (train['validation'] == 0)\n    train.loc[cond, 'Confidence'] = np.random.choice(lasts, size=np.sum(cond))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Confidence'].hist(alpha=0.4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Confidence to normal distribution\n# train['normal_Confidence'] = train['sigma_clipped'].values\n# train.loc[train['normal_Confidence'] <= 0, 'normal_Confidence'] = 0\n# train['normal_Confidence'] = np.log1p(train['normal_Confidence'].values)\n\n# train['normal_Confidence'].hist(alpha=0.4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NN\ntarget = 'Confidence'\noof_df = pd.DataFrame()\nypred_df = pd.DataFrame()\n\nypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, history = nn_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['nn'] = oof\nypred_df['nn'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot training & validation loss values\nplot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Small NN"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NN\nypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, history = snn_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['snn'] = oof\nypred_df['snn'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## XGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_ = xgb_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['xgb'] = oof\nypred_df['xgb'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LGB"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = lgb_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['lgb'] = oof\nypred_df['lgb'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 12))\nsns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CatB"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = catb_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['catb'] = oof\nypred_df['catb'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 12))\nsns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Linear"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = lin_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['lin'] = oof\nypred_df['lin'] = ypred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(8, 12))\nsns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lasso"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_ = lasso_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['lasso'] = oof\nypred_df['lasso'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## BayesianRidge"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, fi = br_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['br'] = oof\nypred_df['br'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SVR"},{"metadata":{"trusted":true},"cell_type":"code","source":"ypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_ = svr_mae(train, test, features, target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA\noof_df['svm'] = oof\nypred_df['svm'] = ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"# CV\nidx = train['validation'] > 0\nfor m in oof_df.columns.values.tolist():\n    print('{}: MAE = {:.3f}, MAE (last 3) = {:.3f}'.format(m, \n          mean_absolute_error(train[target], oof_df[m]),\n          mean_absolute_error(train.loc[idx, target], oof_df.loc[idx, m])))\n    plt.hist(oof_df[m].values, alpha=0.4, label=m)\nplt.legend(frameon=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(12, 12))\nsns.heatmap(oof_df.corr(), annot=True, ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# stacking\nfor f in [target, 'group_a', 'Patient']:\n    oof_df[f] = train[f].values\n    \nypred = np.zeros(test.shape[0])\noof = np.zeros(train.shape[0])\nfor s in range(SA): # seed average\n    oof_, ypred_, history = snn_mae(oof_df, ypred_df, ypred_df.columns.values.tolist(), target, seed=SEED+s)\n    oof += oof_ / SA\n    ypred += ypred_ / SA \n# oof, ypred, history = snn_mae(oof_df, ypred_df, ypred_df.columns.values.tolist(), target, seed=SEED)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fig, ax = plt.subplots(1, 1, figsize=(8, 12))\n# sns.barplot(x='importance', y='features', data=fi.sort_values(by='importance', ascending=False).iloc[:20], ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# oof = oof_df.median(axis=1)\n# ypred = ypred_df.median(axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Final CV score"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['Confidence'] = oof\ntest['Confidence'] = ypred\n\n# train['Confidence'] = np.expm1(oof)\n# test['Confidence'] = np.expm1(ypred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lb_metric(data):\n    data['sigma_clipped'] = data['Confidence'].apply(lambda x: max(x, 70))\n    data['diff'] = abs(data['FVC'] - data['FVC_pred'])\n    data['delta'] = data['diff'].apply(lambda x: min(x, 1000))\n    data['score'] = -math.sqrt(2)*data['delta']/data['sigma_clipped'] - np.log(math.sqrt(2)*data['sigma_clipped'])\n    score = data['score'].mean()\n    score2 = data.loc[data['validation'] > 0, 'score'].mean()\n    return score, score2\n\nscore, score2 = lb_metric(train)\nprint(f'Overall CV = {score}, {score2}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"def prediction_example(train, patient='ID00419637202311204720264'):\n    fig, ax = plt.subplots(1, 1, figsize=(8, 5))\n    idx = train['Patient'] == patient\n    labels = ['upper confidence', 'FVC prediction', 'lower confidence']\n    ax.plot(train.loc[idx, 'Weeks'].values, train.loc[idx, 'FVC_pred'].values, color='k', lw=4, alpha=0.4, label=labels[1])\n    ax.plot(train.loc[idx, 'Weeks'].values, train.loc[idx, 'FVC_pred'].values + train.loc[idx, 'Confidence'].values, color='r', lw=4, alpha=0.4, label=labels[0])\n    ax.plot(train.loc[idx, 'Weeks'].values, train.loc[idx, 'FVC_pred'].values - train.loc[idx, 'Confidence'].values, color='r', lw=4, alpha=0.4, label=labels[-1])\n    ax.plot(train.loc[idx, 'Weeks'].values, train.loc[idx, 'FVC'].values, '-og', label='True')\n    _, score = lb_metric(train.loc[idx, :])\n    ax.legend(frameon=False, loc='center left', bbox_to_anchor=(1, 0.5))\n    ax.set_xlabel('weeks')\n    ax.set_ylabel('FVC')\n    ax.set_title('{}\\nscore={}'.format(patient, score))\nprediction_example(train, patient=test['Patient'].unique()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_example(train, patient=test['Patient'].unique()[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_example(train, patient=test['Patient'].unique()[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_example(train, patient=test['Patient'].unique()[3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction_example(train, patient=test['Patient'].unique()[4])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(INPUT_DIR + '/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[[ID, 'FVC_pred', 'Confidence']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[['FVC_pred', 'Confidence']].describe().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = submission.drop(columns=['FVC', 'Confidence']).merge(test[['Patient_Week', 'FVC_pred', 'Confidence']], \n                                                           on='Patient_Week')\nsub.loc[sub['Confidence'] < 70, 'Confidence'] = 70\nsub.columns = submission.columns\nsub.to_csv('submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}