{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Objective\n\nMake a baseline script and understand the problem\n\n# Comments\n\n* We are asked to predict forced vital capacity (FVC) and the confidence of the prediction. \n* We are only have the FVC in the train set so a smart thing to do is to calculate the confidence for each observation\n* We can use the metric formula to determine which is the best confidence for each observation in the train set\n* If you check the format of the test set, we only have the initial values of the forced vital capacity measurement (FVC), we need to find a way to transform our train data and align the structured of the test set\n* A nice way to deal with the previous point is to use the initial values and fill the other observations with this values.\n* We are not using image date, in the next script i will try to incorporate somehow\n\nA lot of this ideas come from this scipt, https://www.kaggle.com/yasufuminakama/osic-lgb-baseline. Good work.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport random\nimport math\nfrom tqdm import tqdm_notebook as tqdm\nimport lightgbm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import GroupKFold\nimport scipy as sp\nfrom functools import partial\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define a seed\nSEED = 222\n\n## function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\n# function to read and transform our training data\ndef read_and_transform_train():\n    # read the train set\n    train = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/train.csv')\n    # get the id (patient, week)\n    train['Patient_Week'] = train['Patient'].astype(str) + '_' + train['Weeks'].astype(str)\n    # for each user, we are going to cross each observation with each other\n    # in other words we are using the given values for each user, to fill the other weeks\n    # create a dataframe to store our final train set\n    train_expanded = pd.DataFrame()\n    # group by patient id\n    patients = train.groupby('Patient')\n    # iterate over each patient\n    for _, user in tqdm(patients, total = len(patients)):\n        # create a dataframe to store user data\n        user_data = pd.DataFrame()\n        # iterate over each week\n        for week, week_data in user.groupby('Weeks'):\n            # rename columns to usea initial values as features\n            rename_cols = {\n                'Weeks': 'base_Week', \n                'FVC': 'base_FVC', \n                'Percent': 'base_Percent', \n                'Age': 'base_Age'\n            }\n            week_data = week_data.drop(['Patient_Week'], axis = 1).rename(columns = rename_cols)\n            # drop original values\n            drop_cols = ['Percent', 'Age', 'Sex', 'SmokingStatus']\n            user_ = user.drop(drop_cols, axis = 1).rename(columns = {'Weeks': 'predict_Week'})\n            # merge all the user data with this specific week\n            user_ = user_.merge(week_data, on = 'Patient')\n            # calculate the week of difference between the initial values and the predicted week\n            user_['diff_Week']  = user_['predict_Week'] - user_['base_Week']\n            # add the data to the usar data\n            user_data = pd.concat([user_data, user_], axis = 0)\n        # add the user data to our final expanded dataset\n        train_expanded = pd.concat([train_expanded, user_data])\n    \n    # filter out the same week\n    train_expanded = train_expanded[train_expanded['diff_Week']!=0].reset_index(drop = True)\n    return train_expanded\n\n# function to read and transform our test data\ndef read_and_transform_test():\n    # read the test csv\n    test = pd.read_csv('/kaggle/input/osic-pulmonary-fibrosis-progression/test.csv')\n    # read the submission csv\n    sub = pd.read_csv(\"/kaggle/input/osic-pulmonary-fibrosis-progression/sample_submission.csv\")\n    # rename the test column to be align with the train set\n    rename_cols = {\n                'Weeks': 'base_Week', \n                'FVC': 'base_FVC', \n                'Percent': 'base_Percent', \n                'Age': 'base_Age'\n            }\n    test.rename(columns = rename_cols, inplace = True)\n    # get the patient id of the submissions\n    sub['Patient'] = sub['Patient_Week'].apply(lambda x: x.split('_')[0])\n    # get the prediction week\n    sub['predict_Week'] = sub['Patient_Week'].apply(lambda x: x.split('_')[1]).astype(int)\n    # merge the submission data with the test data (using the initial values)\n    test = sub.drop(['FVC', 'Confidence'], axis = 1).merge(test, on = 'Patient', how = 'left')\n    test['diff_Week'] = test['predict_Week'] - test['base_Week']\n    return test\n\n# function to preprocess data for lightgbm prediction\ndef preprocess_lgbm(train, test):\n    for col in ['Sex', 'SmokingStatus']:\n        encoder = preprocessing.LabelEncoder()\n        train[col] = encoder.fit_transform(train[col])\n        test[col] = encoder.transform(test[col])\n    return train, test\n\n# function to make a regresion using groupkfold because we are surely predicting unknown patients\ndef train_and_evaluate_lgbm(train, test, target, notarget):\n    \n    # define some random initial paramters\n    params = {\n        'boosting_type': 'rf',\n        'metric': 'rmse',\n        'objective': 'regression',\n        'n_jobs': -1,\n        'seed': SEED,\n        'learning_rate': 0.1,\n        'bagging_fraction': 0.8,\n        'bagging_freq': 1,\n    }\n    \n    # GroupKFold by Patient\n    kf = GroupKFold(n_splits = 5)\n    oof_pred = np.zeros(len(train))\n    y_pred = np.zeros(len(test))\n    features = [col for col in train.columns if col not in ['predict_Week', 'base_Week', 'Patient', target, 'Patient_Week', notarget, 'FVC_pred']]\n    for fold, (tr_ind, val_ind) in enumerate(kf.split(train, groups = train['Patient'])):\n        print(f'Training fold {fold + 1}')\n        x_train, x_val = train[features].iloc[tr_ind], train[features].iloc[val_ind]\n        y_train, y_val = train[target][tr_ind], train[target][val_ind]\n        train_set = lgb.Dataset(x_train, y_train)\n        val_set = lgb.Dataset(x_val, y_val)\n        model = lgb.train(params, train_set, num_boost_round = 10000, early_stopping_rounds = 50, \n                          valid_sets = [train_set, val_set], verbose_eval = 50)\n        oof_pred[val_ind] = model.predict(x_val)\n        \n        y_pred += model.predict(test[features]) / kf.n_splits\n        \n    return oof_pred, y_pred\n\n# function to create the confidence label\ndef make_confidence_labels(train, oof_pred):\n    # add oof predictions to the train set\n    train['FVC_pred'] = oof_pred\n    # define a optimization function to extract the optimal confidence\n    def loss_func(weight, row):\n        confidence = weight\n        sigma_clipped = max(confidence, 70)\n        diff = abs(row['FVC']- row['FVC_pred'])\n        delta = min(diff, 1000)\n        score = (-math.sqrt(2) * delta / sigma_clipped) - (np.log(math.sqrt(2) * sigma_clipped))\n        return - score \n    \n    # make a list to store our result\n    results = []\n    for ind, row in tqdm(train.iterrows(), total = len(train)):\n        loss_partial = partial(loss_func, row = row)\n        weight = [100]\n        result = sp.optimize.minimize(loss_partial, weight, method = 'SLSQP')\n        x = result['x']\n        results.append(x[0])\n        \n    # add confidence to the train data\n    train['Confidence'] = results\n    train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n    train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n    train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n    train['score'] = (-math.sqrt(2) * train['delta'] / train['sigma_clipped']) - (np.log(math.sqrt(2) * train['sigma_clipped']))\n    score = train['score'].mean()\n    print(f'With our optimal confidence the laplace log likelihood is {score}')\n    train.drop(['sigma_clipped', 'diff', 'delta', 'score'], axis = 1, inplace = True)\n    return train\n\n# function to calculate our out of folds laplace log likelihood\ndef calculate_out_of_folds(train, oof_pred):\n    train['Confidence'] = oof_pred\n    train['sigma_clipped'] = train['Confidence'].apply(lambda x: max(x, 70))\n    train['diff'] = abs(train['FVC'] - train['FVC_pred'])\n    train['delta'] = train['diff'].apply(lambda x: min(x, 1000))\n    train['score'] = (-math.sqrt(2) * train['delta'] / train['sigma_clipped']) - (np.log(math.sqrt(2) * train['sigma_clipped']))\n    score = train['score'].mean()\n    print(f'Our out of folds laplace log likelihood is {score}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# seed everything for deterministic results\nseed_everything(SEED)\n# read and transform our train data\ntrain = read_and_transform_train()\n# read and transform our tetst data\ntest = read_and_transform_test()\n# preprocess our train and test data\ntrain, test = preprocess_lgbm(train, test)\n# train FVC and get out of folds and test predictions\noof_pred, y_pred = train_and_evaluate_lgbm(train, test, 'FVC', 'Confidence')\n# save FVC predictions\ntest['FVC'] = y_pred\n# now that we have FVC prediction we can calculate our confidence\nprint('-'* 50)\nprint('\\n')\ntrain = make_confidence_labels(train, oof_pred)\nprint('-'* 50)\nprint('\\n')\n# train Confidence and get out of folds and test predictions\noof_pred, y_pred = train_and_evaluate_lgbm(train, test, 'Confidence', 'FVC')\nprint('-'* 50)\nprint('\\n')\n# calculate our out of folds metric\ncalculate_out_of_folds(train, oof_pred)\nprint('-'* 50)\nprint('\\n')\n# add Confidence to the test set\ntest['Confidence'] = y_pred\n# save submissions\ntest[['Patient_Week', 'FVC', 'Confidence']].to_csv('submission.csv', index = False)\ntest[['Patient_Week', 'FVC', 'Confidence']].head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}