{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport random\nnp.random.seed(12)\n\nimport matplotlib.pyplot as plt\nimport glob\nimport cv2\n\nimport time\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras import layers, models, optimizers, regularizers\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport scipy.ndimage as ndimage\nfrom skimage import measure, morphology, segmentation, color\nimport pydicom\nimport imageio\nfrom joblib import parallel_backend, Parallel, delayed\nimport PIL\nfrom tensorflow.keras.utils import Sequence\nfrom tensorflow.keras.applications import ResNet50\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom functools import partial\nimport xgboost as xgb\nimport scipy as sp\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"path = '../input/osic-pulmonary-fibrosis-progression/'\n\npath_imgs_train = path + '/train/'\npath_imgs_test = path + '/test/'\n\n\npath_train_masks = path + '/train_masks_fast_masks/'\npath_test_masks = path + '/test_masks_fast_masks/'\n\npath_scans_train = path + 'train/'\npath_scans_test = path + 'test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(path + 'train.csv')\ndf_test = pd.read_csv(path + 'test.csv')\n\nprint(f'1.1 -> There are {df_train.Patient.unique().shape[0]} train unique patients')\nprint(f'1.2 -> There are {df_test.Patient.unique().shape[0]} test unique patients')\n\ntrain_paths = glob.glob(path_imgs_train + '*')\ntest_paths = glob.glob(path_imgs_test + '*')\n      \nprint(f'No. of Train Images : {len(train_paths)}')\nprint(f'No. of Test Images : {len(test_paths)}')\n      \nunique_train_patients = df_train.Patient.unique()\nunique_test_patients = df_test.Patient.unique()\n\ndict_train_patients_paths = {patient: path_imgs_train + patient + '/' for patient in unique_train_patients}\ndict_test_patients_paths = {patient: path_imgs_test + patient + '/' for patient in unique_test_patients}\n\ndict_train_patients_masks_paths = {patient: path_train_masks + patient + '/' for patient in unique_train_patients}\ndict_test_patients_masks_paths = {patient: path_test_masks + patient + '/' for patient in unique_test_patients}\n\n\nfor patient in tqdm(dict_train_patients_paths):\n    list_files = os.listdir(dict_train_patients_paths[patient])\n    list_files = [dict_train_patients_paths[patient] + file for file in list_files]\n    dict_train_patients_paths[patient] = list_files\n    \nfor patient in tqdm(dict_test_patients_paths):\n    list_files = os.listdir(dict_test_patients_paths[patient])\n    list_files = [dict_test_patients_paths[patient] + file for file in list_files]\n    dict_test_patients_paths[patient] = list_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def customLossFunction(y_true, y_pred, std=70):\n    std_clipped = tf.cast(tf.maximum(std, 70), dtype=tf.float32)\n    delta = tf.cast(tf.minimum(tf.abs(y_true - y_pred), 1_000), dtype=tf.float32)\n    sq2 = tf.sqrt(2.)\n    loss = (delta/std_clipped) * sq2 + tf.math.log(sq2 * std_clipped)\n    loss = tf.reduce_mean(loss)\n    return loss\n\n\ndef quantileLoss(quantiles, y_true, y_pred):\n    e = y_true - y_pred\n    v = tf.maximum(quantiles * e, (quantiles-1) * e)\n    return tf.reduce_mean(v)\n\n\ndef negloglik(y, p_y): \n    return -p_y.log_prob(y)\n\n\ndef scale(x, mean_, std_):\n    return (x - mean_) / std_\n\n\ndef unscale(x, mean_, std_):\n    return (x * std_) + mean_\n\n\ndef normalize(x, min_, max_):\n    return (x - min_) / (max_ - min_)\n\n\ndef unormalize(x, min_, max_):\n    return x * (max_ - min_) + min_\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv( path + 'train.csv')\ndf_test = pd.read_csv(path + 'test.csv')\n\nprint(f'1.1 -> There are {df_train.Patient.unique().shape[0]} train unique patients')\nprint(f'1.2 -> There are {df_test.Patient.unique().shape[0]} test unique patients')\n\n\n\n\n      \nunique_train_patients = df_train.Patient.unique()\nunique_test_patients = df_test.Patient.unique()\n\n\n\ndict_train_patients_scans_paths = {patient: path_scans_train + patient + '/' for patient in unique_train_patients}\ndict_test_patients_scans_paths = {patient: path_scans_test + patient + '/' for patient in unique_test_patients}\n\n\n    \n\nfor patient in tqdm(dict_train_patients_scans_paths):\n    list_files = os.listdir(dict_train_patients_scans_paths[patient])\n    list_files = [dict_train_patients_scans_paths[patient] + file for file in list_files]\n    dict_train_patients_scans_paths[patient] = list_files\n    \nfor patient in tqdm(dict_test_patients_scans_paths):\n    list_files = os.listdir(dict_test_patients_scans_paths[patient])\n    list_files = [dict_test_patients_scans_paths[patient] + file for file in list_files]\n    dict_test_patients_scans_paths[patient] = list_files\n    \n# Preprocessing:\n\ndf_train = df_train.groupby(['Patient', 'Weeks']).agg({\n    'FVC': np.mean,\n    'Percent': np.mean,\n    'Age': np.max,\n    'Sex': np.max,\n    'SmokingStatus': np.max \n}).reset_index()\n\ndf_train['FVC_Percent'] = (df_train['FVC'] / df_train['Percent']) * 100\ndf_test['FVC_Percent'] = (df_test['FVC'] / df_test['Percent']) * 100\n\n\n# Standarize data\n\nmean_fvc, std_fvc = df_train.FVC.mean(), df_train.FVC.std()\nmean_perc, std_perc = df_train.Percent.mean(), df_train.Percent.std()\nmean_age, std_age = df_train.Age.mean(), df_train.Age.std()\n\ndf_train['Age'] = df_train['Age'].apply(lambda x: (x-mean_age)/std_age)\ndf_test['Age'] = df_test['Age'].apply(lambda x: (x-mean_age)/std_age)\n\ndf_train['FVC'] = df_train['FVC'].apply(lambda x: (x-mean_fvc)/std_fvc)\ndf_test['FVC'] = df_test['FVC'].apply(lambda x: (x-mean_fvc)/std_fvc)\ndf_train['FVC_Percent'] = df_train['FVC_Percent'].apply(lambda x: (x-mean_fvc)/std_fvc)\ndf_test['FVC_Percent'] = df_test['FVC_Percent'].apply(lambda x: (x-mean_fvc)/std_fvc)\n\ndf_train['Percent'] = df_train['Percent'].apply(lambda x: (x-mean_perc)/std_perc)\ndf_test['Percent'] = df_test['Percent'].apply(lambda x: (x-mean_perc)/std_perc)\n\n# Mapping categories dictionaries \n\ndict_sex = {'Male': 0, 'Female': 1}\ndict_sex_inv = {0: 'Male', 1: 'Female'}\n\ndict_smoke = {'Ex-smoker': 0, 'Never smoked': 1, 'Currently smokes': 2}\ndict_smoke_inv = {0: 'Ex-smoker', 1:'Never smoked', 2:'Currently smokes'}\n\ndict_kind_patient = {'decreased': 0, 'regular': 1, 'increased': 2}\ndict_kind_patient_inv = {0: 'decreased', 1: 'regular', 2: 'increased'}\n\ndf_train.Sex = df_train.Sex.apply(lambda x: dict_sex[x])\ndf_train.SmokingStatus = df_train.SmokingStatus.apply(lambda x: dict_smoke[x])\n\ndf_test.Sex = df_test.Sex.apply(lambda x: dict_sex[x])\ndf_test.SmokingStatus = df_test.SmokingStatus.apply(lambda x: dict_smoke[x])\n\n# Build WeeksSinceLastVisit feature\n\ndf_train['ElapsedWeeks'] = df_train['Weeks']\ndf_test['ElapsedWeeks'] = df_test['Weeks']\n\ntrain_weeks_elapsed = df_train.set_index(['Patient', 'Weeks'])['ElapsedWeeks'].diff().reset_index()\ntest_weeks_elapsed = df_test.set_index(['Patient', 'Weeks'])['ElapsedWeeks'].diff().reset_index()\n\ndf_train = df_train.drop('ElapsedWeeks', axis=1)\ndf_test = df_test.drop('ElapsedWeeks', axis=1)\n\ntrain_weeks_elapsed['ElapsedWeeks'] = train_weeks_elapsed['ElapsedWeeks'].fillna(0).astype(int)\ntest_weeks_elapsed['ElapsedWeeks'] = test_weeks_elapsed['ElapsedWeeks'].fillna(0).astype(int)\n\ndf_train = df_train.merge(train_weeks_elapsed, how='inner', on=['Patient', 'Weeks'])\ndf_test = df_test.merge(test_weeks_elapsed, how='inner', on=['Patient', 'Weeks'])\n\ndf_train['patient_row'] = df_train.sort_values(['Patient', 'Weeks'], ascending=[True, True]) \\\n             .groupby(['Patient']) \\\n             .cumcount() + 1\n\ndf_test['patient_row'] = df_test.sort_values(['Patient', 'Weeks'], ascending=[True, True]) \\\n             .groupby(['Patient']) \\\n             .cumcount() + 1\n\ndf_train['WeeksSinceLastVisit'] = df_train.apply(lambda x: x['Weeks'] if x['patient_row']==1 else x['ElapsedWeeks'], axis=1)\ndf_test['WeeksSinceLastVisit'] = df_test.apply(lambda x: x['Weeks'] if x['patient_row']==1 else x['ElapsedWeeks'], axis=1)\n\n# Norm Weeks\n\nmean_weeks, std_weeks = df_train.Weeks.mean(), df_train.Weeks.std()\n\ndf_train['WeeksSinceLastVisit'] = df_train['WeeksSinceLastVisit'].apply(lambda x: (x-mean_weeks)/std_weeks)\ndf_test['WeeksSinceLastVisit'] = df_test['WeeksSinceLastVisit'].apply(lambda x: (x-mean_weeks)/std_weeks)\n\n\ndf_train['Weeks'] = df_train['Weeks'].apply(lambda x: (x-mean_weeks)/std_weeks)\ndf_test['Weeks'] = df_test['Weeks'].apply(lambda x: (x-mean_weeks)/std_weeks)\n\n# Ini dictionaries\n\ncolumns = ['FVC', 'Age', 'Sex', 'SmokingStatus', 'WeeksSinceLastVisit', 'Percent']\ndict_patients_train_ini_features, dict_patients_test_ini_features = {}, {}\ndict_patients_train_kind_patient, dict_patients_test_kind_patient = {}, {}\ndf_train_patients, df_test_patients = df_train.set_index('Patient'), df_test.set_index('Patient')\n\nfor patient in unique_train_patients:\n    dict_patients_train_ini_features[patient] = df_train_patients[columns][df_train_patients.index==patient].\\\n                                                                    to_dict('records')[0]\n    std = np.std(unscale(df_train_patients['FVC'][df_train_patients.index==patient], mean_fvc, std_fvc).values)\n    mean_first_1 = np.mean(unscale(df_train_patients['FVC'][df_train_patients.index==patient], mean_fvc, std_fvc).values[:1])\n    mean_last_1 = np.mean(unscale(df_train_patients['FVC'][df_train_patients.index==patient], mean_fvc, std_fvc).values[-1:])\n    if std<=100:\n        dict_patients_train_kind_patient[patient] = 'regular'\n    elif std>100 and mean_last_1 > mean_first_1 :\n        dict_patients_train_kind_patient[patient] = 'increased'\n    elif std>100 and mean_last_1 <= mean_first_1 :\n        dict_patients_train_kind_patient[patient] = 'decreased'\n    dict_patients_train_ini_features[patient]['kind'] = dict_kind_patient[dict_patients_train_kind_patient[patient]]\n        \n    \nfor patient in unique_test_patients:\n    dict_patients_test_ini_features[patient] = df_test_patients[columns][df_test_patients.index==patient].\\\n                                                                    to_dict('records')[0]\n    std = np.std(unscale(df_train_patients['FVC'][df_train_patients.index==patient], mean_fvc, std_fvc).values)\n    mean_first_1 = np.mean(unscale(df_train_patients['FVC'][df_train_patients.index==patient], mean_fvc, std_fvc).values[:1])\n    mean_last_1 = np.mean(unscale(df_train_patients['FVC'][df_train_patients.index==patient], mean_fvc, std_fvc).values[-1:])\n    if std<=100:\n        dict_patients_test_kind_patient[patient] = 'regular'\n    elif std>100 and mean_last_1 > mean_first_1 :\n        dict_patients_test_kind_patient[patient] = 'increased'\n    elif std>100 and mean_last_1 <= mean_first_1 :\n        dict_patients_test_kind_patient[patient] = 'decreased'\n    dict_patients_test_ini_features[patient]['kind'] = dict_kind_patient[dict_patients_test_kind_patient[patient]]\n\n# Decoder inputs\n\ndict_train_sequence_fvc, dict_train_sequence_weekssincelastvisit = {}, {}\ndict_train_sequence_cumweeks = {}\nfor patient in unique_train_patients:\n    dict_train_sequence_fvc[patient] = list(df_train_patients['FVC'].loc[patient].values[1:])\n    dict_train_sequence_weekssincelastvisit[patient] = list(df_train_patients['WeeksSinceLastVisit'].loc[patient].values[1:])\n    dict_train_sequence_cumweeks[patient] = list(df_train_patients['Weeks'].loc[patient].values[1:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def buildDataSet(list_patients, dict_ini_features, dict_seq_weeks, dict_seq_cumweeks, \n                 training=True, predictions=None):\n    \n    dict_to_tree = {\n        'Patient' : [],\n        'Weeks_Elapsed_since_firstVisit': [],\n        'Base_Percent' : [],\n        'Age' : [],\n        'Sex' : [],\n        'Base_Week' : [],\n        'Base_FVC' : [],\n        'Curr_Smokes' : [],\n        'Ex_Smoker' : [],\n        'Never_Smoked' : []\n    }\n\n    if training:\n        dict_to_tree['fvc_real'] = []\n        dict_to_tree['kind'] = []\n    \n\n    for patient in tqdm(list_patients, position=0):\n        \n        dict_to_tree['Weeks_Elapsed_since_firstVisit'].extend([dict_seq_cumweeks[patient][i] \\\n                                            for i in range(len(dict_seq_cumweeks[patient]))])\n        \n        for i in range(len(dict_seq_weeks[patient])):\n            dict_to_tree['Patient'].extend([patient])\n\n            dict_to_tree['Base_Percent'].extend([dict_ini_features[patient]['Percent']])\n\n            dict_to_tree['Age'].extend([dict_ini_features[patient]['Age']])\n\n            dict_to_tree['Sex'].extend([dict_ini_features[patient]['Sex']])\n\n            dict_to_tree['Base_Week'].extend([dict_ini_features[patient]['WeeksSinceLastVisit']])\n\n            dict_to_tree['Base_FVC'].extend([dict_ini_features[patient]['FVC']])\n\n            dict_to_tree['Curr_Smokes'].extend([1 if dict_ini_features[patient]['SmokingStatus']==2 else 0])\n\n            dict_to_tree['Ex_Smoker'].extend([1 if dict_ini_features[patient]['SmokingStatus']==0 else 0])\n\n            dict_to_tree['Never_Smoked'].extend([1 if dict_ini_features[patient]['SmokingStatus']==1 else 0])\n            \n            if training:\n                dict_to_tree['kind'].extend([dict_ini_features[patient]['kind']])\n\n        list_weeks_elapsed = list(dict_seq_weeks[patient])\n        list_weeks_cum = list(dict_seq_cumweeks[patient])\n\n        if training:\n            dict_to_tree['fvc_real'].extend(dict_train_sequence_fvc[patient])\n\n    df_tree = pd.DataFrame.from_dict(dict_to_tree, orient='columns')\n    \n    return df_tree\n\n\ndef buildTrainModel(dict_params, features, df_train, df_val, epochs, verbose_eval=10):\n    X_train, y_train = df_train[features], df_train['Confidence']\n    X_val, y_val = df_val[features], df_val['Confidence']\n    \n    xgb_data = [(xgb.DMatrix(X_train, y_train), 'train'), (xgb.DMatrix(X_val, y_val), 'valid')]\n   \n    xgb_model = xgb.train(\n                        params=dict_params,\n                        dtrain=xgb.DMatrix(X_train, y_train),\n                        num_boost_round=epochs,\n                        evals=xgb_data,\n                        verbose_eval=verbose_eval,\n                        early_stopping_rounds=100\n                        \n    )\n\n    return xgb_model\n\n\ndef lossFuncWeights(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = np.abs(row['fvc_real'] - row['fvc_pred'])\n    delta = min(diff, 1000)\n    score = -np.sqrt(2)*delta/sigma_clipped - np.log(np.sqrt(2)*sigma_clipped)\n    return -score\n\n\ndef getConfidenceWeights(df):\n    results = []\n    tk0 = tqdm(df.iterrows(), total=len(df), position=0)\n    for _, row in tk0:\n        loss_partial = partial(lossFuncWeights, row=row)\n        weight = [100]\n        result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n        x = result['x']\n        results.append(x[0])\n        \n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mloss(_lambda):\n    def loss(y_true, y_pred):\n        y_true = unscale(y_true, mean_fvc, std_fvc)\n        y_pred = unscale(y_pred, mean_fvc, std_fvc)\n        return _lambda * quantileLoss(tf.constant([0.2, 0.5, 0.8]), y_true, y_pred) + (1 - _lambda)*customLossFunction(y_true, y_pred)\n    return loss\n\ndef buildModel(num_inputs, lambda_factor):\n    z = layers.Input((num_inputs,), name=\"Patient\")\n    x = layers.Dense(64, activation=\"relu\", name=\"d1\")(z)\n    x = layers.Dropout(0.2)(x)\n    x = layers.Dense(32, activation=\"relu\",  name=\"d2\")(x)\n    x = layers.Dropout(0.2)(x)\n    p1 = layers.Dense(3, activation=\"linear\", name=\"p1\")(x)\n    p2 = layers.Dense(3, activation=\"relu\", name=\"p2\")(x)\n    preds = layers.Lambda(lambda x: x[0] + tf.cumsum(x[1], axis=1), \n                     name=\"preds\")([p1, p2])\n\n    model = models.Model(z, p1, name=\"CNN\")\n    model_loss = mloss(lambda_factor)\n    model.compile(loss=model_loss, \n                  optimizer=tf.keras.optimizers.Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, epsilon=None,\n                                                     amsgrad=False, clipvalue=10), \n                  metrics=['mae'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgb_inputs = {\n    'objective': 'reg:squarederror', \n    'eta': 0.01, \n    'max_depth': 8,\n    'subsample': 0.8,\n    'colsample_bytree': 0.9, \n    'gamma': 0.4, \n    'booster' : 'gbtree',\n    'eval_metric': 'rmse', \n    'seed': 20 \n}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(n_splits = 7, random_state = 12, shuffle = True)\nlist_models, list_history, list_final_metric = [], [], []\n\nfor num_fold, (train_index, val_index) in enumerate(skf.split(unique_train_patients, \n                                                              np.zeros(unique_train_patients.shape[0]))):\n\n    x_train_patients = list(unique_train_patients[train_index])\n    x_val_patients = list(unique_train_patients[val_index])\n    \n    print(f'Num Fold: {num_fold + 1}')\n    print(f'Train patients: {len(x_train_patients)}, Test patients: {len(x_val_patients)}')  \n\n    df_train_weights = buildDataSet(x_train_patients, \n                 dict_ini_features=dict_patients_train_ini_features, \n                 dict_seq_weeks=dict_train_sequence_weekssincelastvisit, \n                 dict_seq_cumweeks=dict_train_sequence_cumweeks, \n                 training=True, \n                 predictions=None)\n\n    df_val_weights = buildDataSet(x_val_patients,\n                 dict_ini_features=dict_patients_train_ini_features, \n                 dict_seq_weeks=dict_train_sequence_weekssincelastvisit, \n                 dict_seq_cumweeks=dict_train_sequence_cumweeks, \n                 training=True, \n                 predictions=None)\n                                               \n    features = list(col for col in df_train_weights.columns if col not in ['Patient', 'fvc_real', 'kind'])   \n    y_train = df_train_weights['fvc_real'].astype(float)\n    y_val = df_val_weights['fvc_real'].astype(float)\n\n    X_train =  df_train_weights[features]\n    X_val =  df_val_weights[features]\n\n    model_weights = buildModel(len(features), lambda_factor=0.8)\n\n    model_weights.fit(X_train, y_train, shuffle=True, batch_size=16, epochs=40, \n                validation_data=(X_val, y_val), verbose=0)\n    \n    list_models.append(model_weights)\n\n    y_val_pred = model_weights.predict(X_val)\n    y_val_pred_median = unscale(y_val_pred[:, 1], mean_fvc, std_fvc)\n    y_val_pred_std = unscale(y_val_pred[:, 2], mean_fvc, std_fvc) - unscale(y_val_pred[:, 0], mean_fvc, std_fvc)\n\n    \n    metric = customLossFunction(unscale(y_val, mean_fvc, std_fvc),\n                                      y_val_pred_median,\n                                     y_val_pred_std).numpy()\n    \n    list_history.append({'metric' : metric})\n    print(f'Metric base model: {metric}')\n    \n    ### Confidence ###\n    \n    df_all_weights = pd.concat([df_train_weights, df_val_weights], axis=0)\n    df_all_weights = df_all_weights[features + ['fvc_real', 'Patient']]\n    \n    predictions = model_weights.predict(df_all_weights[features])\n    df_all_weights['fvc_real'] = unscale(df_all_weights['fvc_real'], mean_fvc, std_fvc)\n    df_all_weights['fvc_pred'] = unscale(predictions[:, 1], mean_fvc, std_fvc)\n    df_all_weights['Confidence'] = unscale(predictions[:, 2], mean_fvc, std_fvc) - unscale(predictions[:, 0], mean_fvc, std_fvc)\n    df_all_weights['sigma_clipped'] = df_all_weights['Confidence'].apply(lambda x: max(x, 70))\n    df_all_weights['diff'] = np.abs(df_all_weights['fvc_real'] - df_all_weights['fvc_pred'])\n    df_all_weights['delta'] = df_all_weights['diff'].apply(lambda x: min(x, 1_000))\n    df_all_weights['score'] = -np.sqrt(2)*df_all_weights['delta']/df_all_weights['sigma_clipped'] - np.log(np.sqrt(2)*df_all_weights['sigma_clipped'])\n    \n    score = customLossFunction(df_all_weights['fvc_real'],\n                                 df_all_weights['fvc_pred'],\n                                 df_all_weights['Confidence']).numpy()\n    print(f'Metric train+val, before confidence weights: {score}') \n    \n    confidence_weights = getConfidenceWeights(df_all_weights)\n    \n    df_all_weights['Confidence'] = confidence_weights\n    df_all_weights['sigma_clipped'] = df_all_weights['Confidence'].apply(lambda x: max(x, 70))\n    df_all_weights['diff'] = np.abs(df_all_weights['fvc_real'] - df_all_weights['fvc_pred'])\n    df_all_weights['delta'] = df_all_weights['diff'].apply(lambda x: min(x, 1_000))\n    df_all_weights['score'] = -np.sqrt(2)*df_all_weights['delta']/df_all_weights['sigma_clipped'] - np.log(np.sqrt(2)*df_all_weights['sigma_clipped'])\n    score = customLossFunction(df_all_weights['fvc_real'],\n                                 df_all_weights['fvc_pred'],\n                                 df_all_weights['Confidence']).numpy()\n    print(f'Metric train+val, confidence weights: {score}') \n    \n    # xgboost\n    \n    df_tmp_train = df_all_weights[df_all_weights['Patient'].isin(x_train_patients)]\n    df_tmp_val = df_all_weights[df_all_weights['Patient'].isin(x_val_patients)]\n    \n    xgb_model = buildTrainModel(xgb_inputs, features, \\\n                                df_train=df_tmp_train, df_val=df_tmp_val, epochs=900, verbose_eval=50)\n    \n    pred_confidence = xgb_model.predict(xgb.DMatrix(df_tmp_val[features]))\n    final_metric = customLossFunction(y_true=df_tmp_val['fvc_real'],\n                             y_pred=df_tmp_val['fvc_pred'],\n                             std=pred_confidence)\n    \n    print('***'*20)\n    print(f'Validation Weights predicted: {final_metric}')\n    print('***'*20)\n    list_final_metric.append(final_metric)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_metric = np.mean([history['metric'] for history in list_history])\n\nprint(val_metric)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_confidence = buildDataSet(unique_train_patients,\n                                 dict_ini_features=dict_patients_train_ini_features, \n                                 dict_seq_weeks=dict_train_sequence_weekssincelastvisit, \n                                 dict_seq_cumweeks=dict_train_sequence_cumweeks, \n                                 training=True, \n                                 predictions=None)\n\npredictions = np.mean([model.predict(df_train_confidence[features]) for model in list_models], axis=0)\ndf_train_confidence['fvc_real'] = unscale(df_train_confidence['fvc_real'], mean_fvc, std_fvc)\ndf_train_confidence['fvc_pred'] = unscale(predictions[:, 0], mean_fvc, std_fvc)\ndf_train_confidence['Confidence'] = unscale(predictions[:, 2], mean_fvc, std_fvc) - unscale(predictions[:, 0], mean_fvc, std_fvc)\ndf_train_confidence['sigma_clipped'] = df_train_confidence['Confidence'].apply(lambda x: max(x, 70))\ndf_train_confidence['diff'] = np.abs(df_train_confidence['fvc_real'] - df_train_confidence['fvc_pred'])\ndf_train_confidence['delta'] = df_train_confidence['diff'].apply(lambda x: min(x, 1_000))\ndf_train_confidence['score'] = -np.sqrt(2)*df_train_confidence['delta']/df_train_confidence['sigma_clipped'] - np.log(np.sqrt(2)*df_train_confidence['sigma_clipped'])\nscore = df_train_confidence['score'].mean()\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy as sp\n\ndef loss_func(weight, row):\n    confidence = weight\n    sigma_clipped = max(confidence, 70)\n    diff = abs(row['fvc_real'] - row['fvc_pred'])\n    delta = min(diff, 1000)\n    score = -np.sqrt(2)*delta/sigma_clipped - np.log(np.sqrt(2)*sigma_clipped)\n    return -score\n\nresults = []\ntk0 = tqdm(df_train_confidence.iterrows(), total=len(df_train_confidence), position=0)\nfor _, row in tk0:\n    loss_partial = partial(loss_func, row=row)\n    weight = [100]\n    result = sp.optimize.minimize(loss_partial, weight, method='SLSQP')\n    x = result['x']\n    results.append(x[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef seed_everything(seed=2020):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nseed_everything(509)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ROOT = \"../input/osic-pulmonary-fibrosis-progression\"\ntr = pd.read_csv(f\"{ROOT}/train.csv\")\ntr.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr[\"Patient\"].unique().shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ntr.drop_duplicates(keep=False, inplace=True, subset=['Patient','Weeks'])\nchunk = pd.read_csv(f\"{ROOT}/test.csv\")\n\nprint(\"add infos\")\nsub = pd.read_csv(f\"{ROOT}/sample_submission.csv\")\nsub['Patient'] = sub['Patient_Week'].apply(lambda x:x.split('_')[0])\nsub['Weeks'] = sub['Patient_Week'].apply(lambda x: int(x.split('_')[-1]))\nsub =  sub[['Patient','Weeks','Confidence','Patient_Week']] \nsub = sub.merge(chunk.drop('Weeks', axis=1), on=\"Patient\")\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['WHERE'] = 'train'\nchunk['WHERE'] = 'val'\nsub['WHERE'] = 'test'\ndata = tr.append([chunk, sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tr.shape, chunk.shape, sub.shape, data.shape)\nprint(tr.Patient.nunique(), chunk.Patient.nunique(), sub.Patient.nunique(), \n      data.Patient.nunique())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['min_week'] = data['Weeks']\ndata.loc[data.WHERE=='test','min_week'] = np.nan\ndata['min_week'] = data.groupby('Patient')['min_week'].transform('min')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"base = data.loc[data.Weeks == data.min_week]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nbase = base[['Patient','FVC']].copy()\nbase.columns = ['Patient','min_FVC']\nbase['nb'] = 1\nbase['nb'] = base.groupby('Patient')['nb'].transform('cumsum')\nbase = base[base.nb==1]\nbase.drop('nb', axis=1, inplace=True)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_confidence['Confidence'] = results\ndf_train_confidence['sigma_clipped'] = df_train_confidence['Confidence'].apply(lambda x: max(x, 70))\ndf_train_confidence['diff'] = np.abs(df_train_confidence['fvc_real'] - df_train_confidence['fvc_pred'])\ndf_train_confidence['delta'] = df_train_confidence['diff'].apply(lambda x: min(x, 1_000))\ndf_train_confidence['score'] = -np.sqrt(2)*df_train_confidence['delta']/df_train_confidence['sigma_clipped'] - np.log(np.sqrt(2)*df_train_confidence['sigma_clipped'])\nscore = df_train_confidence['score'].mean()\nprint(score)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}