{"cells":[{"metadata":{"_uuid":"2bf08ec9-3798-4c06-b639-340ae8340a73","_cell_guid":"667a6523-56c1-4dca-a74e-87b94972cfc6","trusted":true},"cell_type":"markdown","source":"# My best solution (-6.8348)\n\nAlthouth I sadly didn't select this one for submission, if I had it would have placed me right into the 5th place. I wrote this one after realising two things:\n\n1. I couldn't trust public LB at all and cross-validation was the key.\n\n2. Image data wasn't giving me any advantage, so relying solely on tabular data would enable me faster iteration and better results.\n\nHowever, at this stage I hadn't improved my internal testing was still poorly made and that's exactly why I didn't trust my internal results at the moment. I will update this kernel as soon as I can so that I can include all my findings.\n\n## How I envisioned it\n\nThe very first thing I did was inspecting the data, so I was also keen to rely on linear regression (and my submitted versions relied on that). Nevertheless, in this version I gave a try to a direct prediction method, where I tried to fit the whole function with a randomized ensemble of multilayer perceptrons.\n\n## What I learned in this competition\n\n1. The value of visualization: visualize everything you can in order to properly understand it and, if you think you can't visualize it, think of why you can't and how you could solve it.\n\n2. The value of a good test and a cross-validation strategy. This turned out to be very important in this competition, but even in ones in which you can trust the public LB you don't want to do all your testing on that. A good CV lets you try all sort of things.\n\n3. How to compute uncertainty in a trustable manner. I already was familiar with MC-dropout, a technique developed by [Yarin Gal around 2016](https://arxiv.org/abs/1506.02142), but then I read [this scientific article](https://arxiv.org/abs/1709.01907) written by researchers at Uber, in which they compute 95% confidence intervals for time-series prediction. They manage some good concepts there.\n\n4. Try first your own ideas and approaches. Better don't contaminate yourself with others' solutions before you've done this, or you're doomed to be mundane ;)"},{"metadata":{"_uuid":"5d3f0886-7247-48e7-8f1e-43b8768e3b11","_cell_guid":"c40d6769-ee5a-459f-82c1-51248be75ef0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\n\nINTERNAL_TEST = True\n\nrandom.seed(0)\nnp.random.seed(0)\n\ndef metric(fvc_pred, fvc_true, sigma):\n    sigma_clip = np.maximum(sigma, 70)\n    err = np.minimum(np.abs(fvc_true - fvc_pred), 1000)\n    twosqr = np.sqrt(2.0)\n    metric = - (twosqr*err)/sigma_clip - np.log(twosqr*sigma_clip)\n    return np.mean(metric)\n\ndef get_stats(sex):\n    mean = np.zeros(len(sex), dtype=np.float32)\n    std  = mean.copy()\n    for s in (0,1):\n        k = sex==s\n        mean[k] = FVC_MEAN[s]\n        std[k]  = FVC_STD[s]\n    return mean, std\n\ndef normalize(values, sex):\n    mean, std = get_stats(sex)\n    return (values - mean) / std\n\ndef denormalize(values, sex, only_std=False):\n    mean, std = get_stats(sex)\n    return values * std if only_std else (values * std) + mean\n\ndef random_state_generator():\n    s = 0\n    while True:\n        yield s\n        s += 1\nrsit = random_state_generator()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56b6485f-e8d1-47dc-aa78-39701cbf45cb","_cell_guid":"7622c898-1b2a-487d-b012-39c9f19df8e9","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/train.csv')\ntest_data  = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/test.csv')\n\n# Normalize with respect to the Patients' first measure\nall_data = pd.concat([train_data, test_data])\nfemales  = pd.concat([ pd.iloc[:1] for _, pd in all_data[all_data['Sex']=='Female'].groupby('Patient') ])\nmales    = pd.concat([ pd.iloc[:1] for _, pd in all_data[all_data['Sex']=='Male'].groupby('Patient') ])\nFVC_MEAN = (females['FVC'].mean(), males['FVC'].mean())\nFVC_STD  = (females['FVC'].std(),  males['FVC'].std())\n\nsex_map = {'Female':0, 'Male':1}\n\ntrain_data['Sex'] = train_data['Sex'].map(sex_map)\ntrain_data = train_data.join(pd.get_dummies(train_data['SmokingStatus'], prefix=''))\ntrain_data.drop(columns=['SmokingStatus'], inplace=True)\ntrain_data['FVC'] = normalize(train_data['FVC'], train_data['Sex'])\n\ntest_data['Sex'] = test_data['Sex'].map(sex_map)\ntest_data = test_data.join(pd.get_dummies(test_data['SmokingStatus'], prefix=''))\ntest_data.drop(columns=['SmokingStatus'], inplace=True)\ntest_data['FVC'] = normalize(test_data['FVC'], test_data['Sex'])\nfor c in train_data.columns:\n    if c[0] == '_' and c not in test_data.columns:\n        test_data[c] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Cross validation\n# 8-fold (round number of patients per fold)\n# 10x repetition\n# Sex-balanced folds\n# Augmenting and (random) weeks/sex-balancing is done in-fold\n\npatient_ids = sorted(train_data['Patient'].unique().tolist())\nrandom.shuffle(patient_ids)\n# 176 patients\n\ndef split(seq, n):\n    fold_size = len(seq)/n\n    splits = [0] + [ int(fold_size*(i+1)) for i in range(n) ]\n    folds  = [ seq[splits[i]:splits[i+1]] for i in range(n) ]\n    return folds\n\ndef group(seqs, groups):\n    assert len(seqs) == sum(groups)\n    glist = list()\n    it = iter(seqs)\n    for g in groups:\n        gr = list()\n        for i in range(g):\n            gr.extend(next(it))\n        glist.append(gr)\n    return glist\n\ncv_size = 8\nfem_ids = train_data[train_data['Sex']==0]['Patient'].unique().tolist()\nmal_ids = train_data[train_data['Sex']==1]['Patient'].unique().tolist()\nfemales_folds = split(fem_ids, cv_size)\nmales_folds   = split(mal_ids, cv_size)\npat_id_folds = [ f+m for f,m in zip(females_folds, males_folds) ]\nfinal_conf_ids = pat_id_folds[-1]\npat_id_folds = pat_id_folds[:-1]\ncv_size -= 1\n    \ndef augment(data, only_extremes=False):\n    data_list = list()\n    for pat_id, df in data.groupby('Patient'):\n        df = df.copy()\n        df['FirstFVC']     = 0.0\n        df['FirstPercent'] = 0.0\n        df['FirstWeeks']   = 0\n        for i in [0] if only_extremes else range(len(df)-1):\n            df = df.copy()\n            row = df.iloc[i]\n            idx = df['Patient']==pat_id\n            df.loc[idx, 'FirstFVC']     = row['FVC']\n            df.loc[idx, 'FirstPercent'] = row['Percent']\n            df.loc[idx, 'FirstWeeks']   = row['Weeks']\n            if only_extremes:\n                data_list.append(df.iloc[-3:])\n            else:\n                data_list.append(df.iloc[i+1:])\n    aug = pd.concat(data_list)\n    aug['DiffWeeks'] = aug['Weeks'] - aug['FirstWeeks']\n    return aug\n\ndef balance_sex(data):\n    males, females = list(), list()\n    for pat_id, df in data.groupby('Patient'):\n        if df['Sex'].values[0] < 0.5:\n            females.append(pat_id)\n        else:\n            males.append(pat_id)\n    random.shuffle(males)\n    random.shuffle(females)\n    min_pats = min(len(males), len(females))\n    assert min_pats > 0\n    males    = males[:min_pats]\n    females  = females[:min_pats]\n    return data[data['Patient'].isin(males+females)]\n\ndef balance_weeks(data):\n    min_weeks = min([ len(df) for _, df in data.groupby('Patient') ])\n    pats = list()\n    for _, df in data.groupby('Patient'):\n        if len(df) > min_weeks:\n            i = random.randint(0, len(df)-min_weeks-1)\n            df = df.iloc[i:i+min_weeks]\n            pats.append(df)\n    return pd.concat(pats)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(regr_list, scaler, in_tab, out_tab):\n    cv_regr_list   = regr_list if isinstance(regr_list, list) else [regr_list]\n    cv_scaler_list = scaler if isinstance(scaler, list) else [scaler]\n    cols = list()\n    i = 0\n    for regr_list, scaler in zip(cv_regr_list, cv_scaler_list):\n        for reg in regr_list['TargetFVC']:\n            c = f'PredTargetFVC{i}'\n            out_tab[c] = reg.predict(scaler.transform(in_tab[reg_columns])) * reg.target_fvc_std + in_tab['FirstFVC'] + reg.target_fvc_avg\n            cols.append(c)\n            i += 1\n    preds = out_tab[cols].values\n    out_tab['PredFVC']    = np.mean(preds, axis=1)\n    out_tab['PredFVCVar'] = np.var(preds, axis=1)\n    return cols\n\ndef compute_inherent_noise(regr_list, scaler, data, pat_ids):\n    data = data[data['Patient'].isin(pat_ids)].copy()\n    data = balance_sex(data)\n    data = augment(data, only_extremes=True)\n    predict(regr_list, scaler, data, data)\n    return (data['FVC'] - data['PredFVC']).var()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae94d67c-b8a5-4892-ad4a-a6032cfac000","_cell_guid":"5b3a25b8-a618-4b0f-801c-6e1fbff7da43","trusted":true},"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.neural_network import MLPRegressor\nfrom sklearn.preprocessing import StandardScaler\n\nreg_columns  = ['FirstFVC','FirstPercent','DiffWeeks','FirstWeeks','Weeks','Age','Sex','_Currently smokes','_Ex-smoker','_Never smoked']\npred_columns = ['TargetFVC']\n\ninstantiators = [ lambda: MLPRegressor(solver='sgd', activation='relu', random_state=next(rsit),\n                                       learning_rate_init=0.001, learning_rate='adaptive', batch_size=16, alpha=1e-1,\n                                       hidden_layer_sizes=(512,256,128,64), validation_fraction=0.3, n_iter_no_change=5,\n                                       max_iter=int(1e6), max_fun=int(1e6), early_stopping=True, verbose=False) ]\n\ntrain_data_orig = train_data.copy()\n\ninh_noise_list = list()\nloss_list = list()\ncv_regr_list = list()\ncv_scaler_list = list()\n\nfor rep in range(1 if INTERNAL_TEST else 1):\n    for cv_i in range(cv_size if INTERNAL_TEST else 1):\n\n        if INTERNAL_TEST:\n            test_ids  = pat_id_folds[(cv_i - 1) % cv_size]\n            conf_ids  = pat_id_folds[(cv_i - 2) % cv_size]\n        train_ids = list()\n        for i in range(cv_size-2 if INTERNAL_TEST else cv_size):\n            train_ids.extend(pat_id_folds[(cv_i + i) % cv_size])\n\n        train_data = train_data_orig.copy()\n        data    = train_data[train_data['Patient'].isin(train_ids)]\n        fem_ids = data[data['Sex']==0]['Patient'].unique().tolist()\n        mal_ids = data[data['Sex']==1]['Patient'].unique().tolist()\n\n        n_folds  = 1\n        fem_folds   = split(fem_ids, n_folds)\n        mal_folds   = split(mal_ids, n_folds)\n        #id_folds = [ f+m for f,m in zip(fem_folds, mal_folds) ]\n        sample_len = int(1.0*min(len(fem_ids), len(mal_ids)))\n        id_folds = [ random.sample(fem_ids, sample_len) + random.sample(mal_ids, sample_len) for i in range(n_folds) ]\n        if cv_i == 0:\n            print('Fold size:', len(id_folds[0]))\n\n        # Fit data scaler\n        data   = train_data[train_data['Patient'].isin(train_ids)].copy()\n        data   = balance_sex(data)\n        data   = augment(data)\n        scaler = StandardScaler().fit(data[reg_columns])\n        cv_scaler_list.append(scaler)\n\n        # Prediction\n        regr_list = { k:list() for k in pred_columns }\n        for inst_i, inst in enumerate(instantiators):\n            for fold_i in range(n_folds):\n                i = inst_i*n_folds + fold_i\n                print('Member', i)\n\n                id_list = id_folds[fold_i]\n                data = train_data[train_data['Patient'].isin(id_list)].copy()\n                data = balance_sex(data)\n                #data = balance_weeks(data)\n                data = augment(data)\n                data['TargetFVC'] = data['FVC'] - data['FirstFVC']\n\n                X = data[reg_columns]\n                X = scaler.transform(X)\n                for k in pred_columns:\n                    reg = inst()\n                    avg, std = data['TargetFVC'].mean(), data['TargetFVC'].std()\n                    reg.target_fvc_avg = avg\n                    reg.target_fvc_std = std\n                    Y = (data[k] - avg) / std\n                    reg.fit(X,Y)\n                    regr_list[k].append(reg)\n        cv_regr_list.append(regr_list)\n\n        if INTERNAL_TEST:\n            inh_noise = compute_inherent_noise(regr_list, scaler, train_data, conf_ids)\n            inh_noise_list.append(inh_noise)\n            print('Fold', cv_i)\n            print('Inh noise:', inh_noise * FVC_STD[data['Sex'].values[0]])\n\n            data = train_data[train_data['Patient'].isin(test_ids)].copy()\n            data = balance_sex(data)\n            data = augment(data, only_extremes=True)\n            predict(regr_list, scaler, data, data)\n            ens_pred = denormalize(data['PredFVC'], data['Sex'])\n            conf = np.sqrt(data['PredFVCVar'] + inh_noise)\n            conf = denormalize(conf, data['Sex'], only_std=True)\n            fvc  = denormalize(data['FVC'], data['Sex'])\n            loss = metric(ens_pred, fvc, conf)\n            print('Loss:', loss)\n            loss_list.append(loss)\n    \nif len(loss_list) > 0:\n    print('Expected loss', np.mean(loss_list), np.var(loss_list))\n    inh_noise = np.mean(inh_noise_list)\n    print('Mean inh noise', inh_noise * FVC_STD[data['Sex'].values[0]])\n    \ninh_noise = compute_inherent_noise(cv_regr_list, cv_scaler_list, train_data_orig, final_conf_ids)\nprint('Final inherent noise', inh_noise)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e06fadef-932b-4b64-9b69-dc2309c149cd","_cell_guid":"81244cc2-2823-4e66-a0a0-3b7bb427ce93","trusted":true},"cell_type":"code","source":"subdf = pd.read_csv('../input/osic-pulmonary-fibrosis-progression/sample_submission.csv')\nsubdf['Sex']        = 0\nsubdf['FirstWeeks'] = 0\nsubdf['FirstFVC']   = 0.0\nsubdf['Patient'] = subdf['Patient_Week'].str.split('_').str[0]\nsubdf['Weeks']   = subdf['Patient_Week'].str.split('_').str[1].astype(int)\n\nfvc_columns = list()\ni = 0\nfor regr_list in cv_regr_list:\n    for reg in regr_list['TargetFVC']:\n        c = f'PredTargetFVC{i}'\n        subdf[c] = 0.0\n        fvc_columns.append(c)\n        i += 1\n\ntest_data['FirstFVC']     = test_data['FVC']\ntest_data['FirstPercent'] = test_data['Percent']\ntest_data['FirstWeeks']   = test_data['Weeks']\n\ncopy_cols = ['FirstFVC','FirstPercent','FirstWeeks','Age','Sex','_Currently smokes','_Ex-smoker','_Never smoked']\nfor c in copy_cols:\n    subdf[c] = 0.0\n    \nfor pat_id, df in test_data.groupby('Patient'):\n    row = df.iloc[0]\n    idx = subdf['Patient']==pat_id\n    for c in copy_cols:\n        subdf.loc[idx, c] = row[c]\nsubdf['DiffWeeks'] = subdf['Weeks'] - subdf['FirstWeeks']\n\ni = 0\nfor regr_list, scaler in zip(cv_regr_list, cv_scaler_list):\n    X = subdf[reg_columns]\n    X = scaler.transform(X)\n    for reg in regr_list['TargetFVC']:\n        subdf[f'PredTargetFVC{i}'] = reg.predict(X) * reg.target_fvc_std + reg.target_fvc_avg\n        i += 1\n        \npreds = subdf[fvc_columns].values + subdf['FirstFVC'].values.reshape(-1,1)\nsubdf['FVC']    = np.mean(preds, axis=1)\nsubdf['FVCVar'] = np.var(preds, axis=1)\n        \nsubdf['FVC']        = denormalize(subdf['FVC'], subdf['Sex'])\nsubdf['Confidence'] = np.sqrt(subdf['FVCVar'] + inh_noise)\nsubdf['Confidence'] = denormalize(subdf['Confidence'], subdf['Sex'], only_std=True)\n\nidx = subdf['Weeks'] == subdf['FirstWeeks']\nsubdf.loc[idx, 'FVC']        = denormalize(subdf['FirstFVC'], subdf['Sex'])\nsubdf.loc[idx, 'Confidence'] = 0\n\nif True:\n    for i, (pat_id, df) in enumerate(subdf.groupby('Patient')):\n        plt.figure()\n        w    = df['Weeks']\n        fvc  = df['FVC']\n        conf = df['Confidence']\n        plt.plot(w, fvc, label='pred')\n        plt.plot(w, fvc + conf, linestyle='dashed', color='grey')\n        plt.plot(w, fvc - conf, linestyle='dashed', color='grey')\n        plt.show()\n        if i==4:\n            break\n\nsubdf = subdf[['Patient_Week','FVC','Confidence']]\nsubdf.to_csv('submission.csv', index=False)\nsubdf.sort_values(by=['Confidence']).head(20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}