{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Update:\n    \n* Add another efficientnet model predictions to the stacking.","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\nimport random\nimport os\nimport warnings\nwarnings.filterwarnings('ignore')\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold, KFold\nfrom sklearn import metrics\nfrom sklearn import preprocessing\nfrom bayes_opt import BayesianOptimization\nfrom catboost import CatBoostClassifier\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"## function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n# let´s start seeding everything\nseed_everything(42)\n\n# function to read data and image data models predictions\ndef read_data():\n    train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\n    test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\n    sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\n    tr_ef1 = pd.read_csv('/kaggle/input/eff-b3/EfficientNetB3_256.csv')\n    te_ef1 = pd.read_csv('/kaggle/input/eff-b3/sub_EfficientNetB3_256.csv')\n    tr_ef2 = pd.read_csv('/kaggle/input/effb3-512/EfficientNetB3_512.csv')\n    te_ef2 = pd.read_csv('/kaggle/input/effb3-512/sub_EfficientNetB3_512.csv')\n    tr_ef3 = pd.read_csv('/kaggle/input/effb3-384/EfficientNetB3_384.csv')\n    te_ef3 = pd.read_csv('/kaggle/input/effb3-384/sub_EfficientNetB3_384.csv')\n    tr_ef4 = pd.read_csv('/kaggle/input/eff-256v2/EfficientNetB0_256.csv')\n    te_ef4 = pd.read_csv('/kaggle/input/eff-256v2/sub_EfficientNetB0_256.csv')\n    roc_auc_256 = metrics.roc_auc_score(tr_ef1['target'], tr_ef1['predictions'])\n    roc_auc_512 = metrics.roc_auc_score(tr_ef2['target'], tr_ef2['predictions'])\n    roc_auc_384 = metrics.roc_auc_score(tr_ef3['target'], tr_ef3['predictions'])\n    roc_auc_256_B0 = metrics.roc_auc_score(tr_ef4['target'], tr_ef4['predictions'])\n    print(f'EfficientnetB3 model for 256 x 256 images out of folds roc auc score is {roc_auc_256}')\n    print(f'EfficientnetB3 model for 512 x 512 images out of folds roc auc score is {roc_auc_512}')\n    print(f'EfficientnetB3 model for 384 x 384 images out of folds roc auc score is {roc_auc_384}, this model includes meta data')\n    print(f'EfficientnetB0 model for 256 x 256 images out of folds roc auc score is {roc_auc_256_B0}, this model includes meta data')\n    te_ef1.columns = ['image_name', 'predictions']\n    tr_ef1 = tr_ef1[['image_name', 'predictions']]\n    te_ef2.columns = ['image_name', 'predictions_1']\n    tr_ef2 = tr_ef2[['image_name', 'predictions']]\n    tr_ef2.columns = ['image_name', 'predictions_1']\n    te_ef3.columns = ['image_name', 'predictions_2']\n    tr_ef3 = tr_ef3[['image_name', 'predictions']]\n    tr_ef3.columns = ['image_name', 'predictions_2']\n    te_ef4.columns = ['image_name', 'predictions_3']\n    tr_ef4 = tr_ef4[['image_name', 'predictions']]\n    tr_ef4.columns = ['image_name', 'predictions_3']\n    train = train.merge(tr_ef1, on = 'image_name').merge(tr_ef2, on = 'image_name').merge(tr_ef3, on = 'image_name').merge(tr_ef4, on = 'image_name')\n    test = test.merge(te_ef1, on = 'image_name').merge(te_ef2, on = 'image_name').merge(te_ef3, on = 'image_name').merge(te_ef4, on = 'image_name')\n    return train, test, sub\n\ndef feature_engineering(train, test):\n    # size of images\n    trn_images = train['image_name'].values\n    trn_sizes = np.zeros((trn_images.shape[0], 2))\n    for i, img_path in enumerate(tqdm(trn_images)):\n        img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/', f'{img_path}.jpg'))\n        trn_sizes[i] = np.array([img.size[0], img.size[1]])\n    test_images = test['image_name'].values\n    test_sizes = np.zeros((test_images.shape[0],2))\n    for i, img_path in enumerate(tqdm(test_images)):\n        img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/test/', f'{img_path}.jpg'))\n        test_sizes[i] = np.array([img.size[0],img.size[1]])\n    train['w'] = trn_sizes[:,0]\n    train['h'] = trn_sizes[:,1]\n    test['w'] = test_sizes[:,0]\n    test['h'] = test_sizes[:,1]\n    \n    return train, test\n\ndef encode_categorical(train, test):\n    for col in ['sex', 'anatom_site_general_challenge']:\n        encoder = preprocessing.LabelEncoder()\n        train[col].fillna('unknown', inplace = True)\n        test[col].fillna('unknown', inplace = True)\n        train[col] = encoder.fit_transform(train[col])\n        test[col] = encoder.transform(test[col])\n    age_approx = np.nanmean(np.concatenate([np.array(train['age_approx']), np.array(test['age_approx'])]))\n    train['age_approx'].fillna(age_approx, inplace = True)\n    test['age_approx'].fillna(age_approx, inplace = True)\n    train['patient_id'].fillna('unknown', inplace = True)\n    return train, test\n\ndef train_and_evaluate_cat(train, test, cat_params, verbose_eval, folds = 5):\n    \n    # define usefull features\n    features = [col for col in train.columns if col not in ['image_name', 'patient_id', 'diagnosis', 'benign_malignant', 'target', 'source']]\n    if verbose_eval != False:\n        print('Training with features: ', features)\n    \n    \n    # groupkfolds to predict evaluate unknown clients (just like the test set)\n    kf = GroupKFold(n_splits = folds)\n    target = 'target'\n    \n    oof_pred = np.zeros(len(train))\n    y_pred = np.zeros(len(test))\n     \n    for fold, (tr_ind, val_ind) in enumerate(kf.split(train, groups = train['patient_id'])):\n        if verbose_eval != False:\n            print('\\n')\n            print('-'*50)\n            print(f'Training fold {fold + 1}\"')\n        x_train, x_val = train[features].iloc[tr_ind], train[features].iloc[val_ind]\n        y_train, y_val = train[target][tr_ind], train[target][val_ind]\n        \n        model = CatBoostClassifier(**cat_params)\n        model.fit(x_train, y_train, eval_set = (x_val, y_val), cat_features = ['sex', 'anatom_site_general_challenge'], \n                      use_best_model = True, early_stopping_rounds = 50, verbose_eval = verbose_eval)\n        \n        oof_pred[val_ind] = model.predict_proba(x_val)[:, 1]\n#         oof_pred[val_ind] = (cat_pred - cat_pred.min())/(cat_pred.max() - cat_pred.min())\n        \n        y_pred += model.predict_proba(test[features])[:, 1] / kf.n_splits\n        \n    rauc = metrics.roc_auc_score(train['target'], oof_pred)\n    if verbose_eval != False:\n        print(f'Our oof roc auc score for our cat boost model is {rauc}')\n        \n    gc.collect()\n    \n    return rauc, y_pred\n\ntrain, test, sub = read_data()\ntrain, test = feature_engineering(train, test)\ntrain, test = encode_categorical(train, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function to perform bayesian optimization search\ndef run_cat_bayesian(learning_rate, depth, bagging_temperature, colsample_bylevel):\n    \n    params = {\n        'learning_rate': learning_rate,\n        'eval_metric': 'AUC',\n        'loss_function': 'Logloss',\n        'random_seed': 42,\n        'task_type': 'CPU',\n        'depth': int(depth),\n        'bagging_temperature': bagging_temperature,\n        'colsample_bylevel': colsample_bylevel,\n        \n    } \n    \n    roc_auc, y_pred = train_and_evaluate_cat(train, test, params, False)\n    return roc_auc\n\n# run bayesian optimization with optimal features\nbounds_cat = {\n    'learning_rate': (0.1, 0.5),\n    'depth': (2, 12),\n    'bagging_temperature': (0.0, 2.0),\n    'colsample_bylevel': (0.5, 1.0)\n}\n\ncat_bo = BayesianOptimization(run_cat_bayesian, bounds_cat, random_state = 42)\ncat_bo.maximize(init_points = 100, n_iter = 100, acq = 'ucb', xi = 0.0, alpha = 1e-6)\n\n# get new hyperparameters\nparams = {\n    'learning_rate': cat_bo.max['params']['learning_rate'],\n    'eval_metric': 'AUC',\n    'loss_function': 'Logloss',\n    'random_seed': 42,\n    'task_type': 'CPU',\n    'depth': int(cat_bo.max['params']['depth']),\n    'bagging_temperature': cat_bo.max['params']['bagging_temperature'],\n    'colsample_bylevel': cat_bo.max['params']['colsample_bylevel']\n}\n\n# train with new hyperparameters\nroc_auc, y_pred = train_and_evaluate_cat(train, test, params, 50)\n\n# predict\ntest['target'] = y_pred\nsub = test[['image_name', 'target']]\nsub.to_csv('cat_baseline_sub.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can make a lot of image models and use their predictions as features for out stacked model. This is just a tutorial, it is not intended to get a high score.\n\nAnother thing to mention is that there maybee more meta data features!.","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}