{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comments\n\n* I used Chris datasets (tf records 256, 384, 512 and 512 with external data) to make efficientnet models.The model is posted here https://www.kaggle.com/ragnar123/efficientnet-x-384. In this public notebook the cv is 0.919 and the lb is 0.937. I made another experiment with a different seed and it got lb 0.925. This suggest that my cv is not align with the public test set and i should not trust on that score, so making a good cv strategy is very important.\n\n* Here im stacking 19 models predictions, and used groupkfold with patient id as the group (test set is formed by patients that are not in the train set so this sounds a good strategy)\n\n* For experimentation purpose i will run this script 5 times changing the seed and check the cv and lb, i hope this is more stable than my publick cv.\n\nHere are the results\n\nCV1: 0.9439, LB1: 0.940, SEED: 42\n\nCV2: 0.9423, LB2: 0.939, SEED: 100\n\nCV3: 0.9456, LB3: 0.940, SEED: 999\n\nCV4: 0.9446, LB4: 0.941, SEED: 6\n\nLet's add the picture size.\n\nCV5: Look the end of the script, LB5: Look at the end of the script, SEED: 22\n\nThis ensembles with different seed are more stable than our initial model (lb 0.937)\n\nI hope this experiment helps you in finding the best cv strategy, make confident predictions and dont overfitt the public leadearboard","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nfrom tqdm import tqdm\nfrom PIL import Image\nimport random\nimport os\nimport warnings\nwarnings.filterwarnings('ignore')\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold, KFold\nfrom sklearn import metrics\nfrom sklearn import preprocessing\nfrom bayes_opt import BayesianOptimization\nfrom catboost import CatBoostClassifier\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"## function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \n# define a seed\nSEED = 22\n    \n# let´s start seeding everything\nseed_everything(SEED)\n\n# function to read data and image data models predictions\ndef read_data():\n    train = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\n    test = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\n    sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\n    train1 = pd.read_csv('../input/melanoma-subs/EfficientNetB0_256.csv')\n    train2 = pd.read_csv('../input/melanoma-subs/EfficientNetB0_512EX.csv')\n    train3 = pd.read_csv('../input/melanoma-subs/EfficientNetB1_256.csv')\n    train4 = pd.read_csv('../input/melanoma-subs/EfficientNetB1_384.csv')\n    train5 = pd.read_csv('../input/melanoma-subs/EfficientNetB1_512.csv')\n    train6 = pd.read_csv('../input/melanoma-subs/EfficientNetB1_512EX.csv')\n    train7 = pd.read_csv('../input/melanoma-subs/EfficientNetB2_256.csv')\n    train8 = pd.read_csv('../input/melanoma-subs/EfficientNetB2_384.csv')\n    train9 = pd.read_csv('../input/melanoma-subs/EfficientNetB2_512.csv')\n    train10 = pd.read_csv('../input/melanoma-subs/EfficientNetB2_512EX.csv')\n    train11 = pd.read_csv('../input/melanoma-subs/EfficientNetB3_256.csv')\n    train12 = pd.read_csv('../input/melanoma-subs/EfficientNetB3_384.csv')\n    train13 = pd.read_csv('../input/melanoma-subs/EfficientNetB3_512.csv')\n    train14 = pd.read_csv('../input/melanoma-subs/EfficientNetB4_256.csv')\n    train15 = pd.read_csv('../input/melanoma-subs/EfficientNetB4_384.csv')\n    train16 = pd.read_csv('../input/melanoma-subs/EfficientNetB4_512.csv')\n    train17 = pd.read_csv('../input/melanoma-subs/EfficientNetB5_256.csv')\n    train18 = pd.read_csv('../input/melanoma-subs/EfficientNetB5_384.csv')\n    train19 = pd.read_csv('../input/melanoma-subs/EfficientNetB6_384.csv')\n    test1 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB0_256.csv')\n    test2 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB0_512EX.csv')\n    test3 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB1_256.csv')\n    test4 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB1_384.csv')\n    test5 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB1_512.csv')\n    test6 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB1_512EX.csv')\n    test7 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB2_256.csv')\n    test8 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB2_384.csv')\n    test9 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB2_512.csv')\n    test10 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB2_512EX.csv')\n    test11 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB3_256.csv')\n    test12 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB3_384.csv')\n    test13 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB3_512.csv')\n    test14 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB4_256.csv')\n    test15 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB4_384.csv')\n    test16 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB4_512.csv')\n    test17 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB5_256.csv')\n    test18 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB5_384.csv')\n    test19 = pd.read_csv('../input/melanoma-subs/sub_EfficientNetB6_384.csv')\n\n    \n    def print_roc_auc(df, model):\n        roc_auc = metrics.roc_auc_score(df['target'], df['predictions'])\n        print(f'Our model {model} out of folds roc auc score is {roc_auc}')\n        print('-'*50)\n        print('\\n')\n        \n    print_roc_auc(train1, 1)\n    print_roc_auc(train2, 2)\n    print_roc_auc(train3, 3)\n    print_roc_auc(train4, 4)\n    print_roc_auc(train5, 5)\n    print_roc_auc(train6, 6)\n    print_roc_auc(train7, 7)\n    print_roc_auc(train8, 8)\n    print_roc_auc(train9, 9)\n    print_roc_auc(train10, 10)\n    print_roc_auc(train11, 11)\n    print_roc_auc(train12, 12)\n    print_roc_auc(train13, 13)\n    print_roc_auc(train14, 14)\n    print_roc_auc(train15, 15)\n    print_roc_auc(train16, 16)\n    print_roc_auc(train17, 17)\n    print_roc_auc(train18, 18)\n    print_roc_auc(train19, 19)\n    \n    def fix_predictions(train, test, model):\n        test.columns = ['image_name', 'predictions_{}'.format(model)]\n        train = train[['image_name', 'predictions']]\n        train.columns = ['image_name', 'predictions_{}'.format(model)]\n        return train, test\n    \n    train1, test1 = fix_predictions(train1, test1, 1)\n    train2, test2 = fix_predictions(train2, test2, 2)\n    train3, test3 = fix_predictions(train3, test3, 3)\n    train4, test4 = fix_predictions(train4, test4, 4)\n    train5, test5 = fix_predictions(train5, test5, 5)\n    train6, test6 = fix_predictions(train6, test6, 6)\n    train7, test7 = fix_predictions(train7, test7, 7)\n    train8, test8 = fix_predictions(train8, test8, 8)\n    train9, test9 = fix_predictions(train9, test9, 9)\n    train10, test10 = fix_predictions(train10, test10, 10)\n    train11, test11 = fix_predictions(train11, test11, 11)\n    train12, test12 = fix_predictions(train12, test12, 12)\n    train13, test13 = fix_predictions(train13, test13, 13)\n    train14, test14 = fix_predictions(train14, test14, 14)\n    train15, test15 = fix_predictions(train15, test15, 15)\n    train16, test16 = fix_predictions(train16, test16, 16)\n    train17, test17 = fix_predictions(train17, test17, 17)\n    train18, test18 = fix_predictions(train18, test18, 18)\n    train19, test19 = fix_predictions(train19, test19, 19)\n\n    train = train.merge(train1, on = 'image_name').merge(train2, on = 'image_name').merge(train3, on = 'image_name').merge(train4, on = 'image_name').merge(train5, on = 'image_name').merge(train6, on = 'image_name').merge(train7, on = 'image_name').merge(train8, on = 'image_name').merge(train9, on = 'image_name').merge(train10, on = 'image_name').merge(train11, on = 'image_name').merge(train12, on = 'image_name').merge(train13, on = 'image_name').merge(train14, on = 'image_name').merge(train15, on = 'image_name').merge(train16, on = 'image_name').merge(train17, on = 'image_name').merge(train18, on = 'image_name').merge(train19, on = 'image_name')\n    test = test.merge(test1, on = 'image_name').merge(test2, on = 'image_name').merge(test3, on = 'image_name').merge(test4, on = 'image_name').merge(test5, on = 'image_name').merge(test6, on = 'image_name').merge(test7, on = 'image_name').merge(test8, on = 'image_name').merge(test9, on = 'image_name').merge(test10, on = 'image_name').merge(test11, on = 'image_name').merge(test12, on = 'image_name').merge(test13, on = 'image_name').merge(test14, on = 'image_name').merge(test15, on = 'image_name').merge(test16, on = 'image_name').merge(test17, on = 'image_name').merge(test18, on = 'image_name').merge(test19, on = 'image_name')\n    return train, test, sub\n\ndef feature_engineering(train, test):\n    # size of images\n    trn_images = train['image_name'].values\n    trn_sizes = np.zeros((trn_images.shape[0], 2))\n    for i, img_path in enumerate(tqdm(trn_images)):\n        img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/train/', f'{img_path}.jpg'))\n        trn_sizes[i] = np.array([img.size[0], img.size[1]])\n    test_images = test['image_name'].values\n    test_sizes = np.zeros((test_images.shape[0],2))\n    for i, img_path in enumerate(tqdm(test_images)):\n        img = Image.open(os.path.join('/kaggle/input/siim-isic-melanoma-classification/jpeg/test/', f'{img_path}.jpg'))\n        test_sizes[i] = np.array([img.size[0],img.size[1]])\n    train['w'] = trn_sizes[:,0]\n    train['h'] = trn_sizes[:,1]\n    test['w'] = test_sizes[:,0]\n    test['h'] = test_sizes[:,1]\n    \n    return train, test\n\ndef encode_categorical(train, test):\n    for col in ['sex', 'anatom_site_general_challenge']:\n        encoder = preprocessing.LabelEncoder()\n        train[col].fillna('unknown', inplace = True)\n        test[col].fillna('unknown', inplace = True)\n        train[col] = encoder.fit_transform(train[col])\n        test[col] = encoder.transform(test[col])\n      # dont impute age, let light gradient boosting handle this\n#     age_approx = np.nanmean(np.concatenate([np.array(train['age_approx']), np.array(test['age_approx'])]))\n#     train['age_approx'].fillna(age_approx, inplace = True)\n#     test['age_approx'].fillna(age_approx, inplace = True)\n    train['patient_id'].fillna('unknown', inplace = True)\n    return train, test\n\ndef train_and_evaluate_lgbm(train, test, params, verbose_eval, folds = 5):\n    \n    # define usefull features\n    features = [col for col in train.columns if col not in ['image_name', 'patient_id', 'diagnosis', 'benign_malignant', 'target', 'source']]\n    if verbose_eval != False:\n        print('Training with features: ', features)\n    \n    \n    # groupkfolds to predict evaluate unknown clients (just like the test set)\n    kf = GroupKFold(n_splits = folds)\n    target = 'target'\n    \n    oof_pred = np.zeros(len(train))\n    y_pred = np.zeros(len(test))\n     \n    for fold, (tr_ind, val_ind) in enumerate(kf.split(train, groups = train['patient_id'])):\n        if verbose_eval != False:\n            print('\\n')\n            print('-'*50)\n            print(f'Training fold {fold + 1}\"')\n        x_train, x_val = train[features].iloc[tr_ind], train[features].iloc[val_ind]\n        y_train, y_val = train[target][tr_ind], train[target][val_ind]\n        train_set = lgb.Dataset(x_train, y_train)\n        val_set = lgb.Dataset(x_val, y_val)\n        \n        model = lgb.train(params, train_set, num_boost_round = 10000, early_stopping_rounds = 50, \n                         valid_sets = [train_set, val_set], verbose_eval = verbose_eval)\n        \n        \n        oof_pred[val_ind] = model.predict(x_val)\n        \n        y_pred += model.predict(test[features]) / kf.n_splits\n        \n    rauc = metrics.roc_auc_score(train['target'], oof_pred)\n    if verbose_eval != False:\n        print(f'Our oof roc auc score for our lgbm model is {rauc}')\n        \n    gc.collect()\n    \n    return rauc, y_pred\n\ntrain, test, sub = read_data()\ntrain, test = feature_engineering(train, test)\ntrain, test = encode_categorical(train, test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run_lgb_bayesian(num_leaves, learning_rate, max_depth, lambda_l1, lambda_l2, bagging_fraction, bagging_freq, colsample_bytree, colsample_bynode, min_data_per_leaf, min_sum_hessian_per_leaf):\n    \n    params = {\n        'boosting_type': 'gbdt',\n        'metric': 'auc',\n        'objective': 'binary',\n        'n_jobs': -1,\n        'seed': SEED,\n        'num_leaves': int(num_leaves),\n        'learning_rate': learning_rate,\n        'max_depth': int(max_depth),\n        'lambda_l1': lambda_l1,\n        'lambda_l2': lambda_l2,\n        'bagging_fraction': bagging_fraction,\n        'bagging_freq': int(bagging_freq),\n        'colsample_bytree': colsample_bytree,\n        'colsample_bynode': colsample_bynode,\n        'min_data_per_leaf': int(min_data_per_leaf),\n        'min_sum_hessian_per_leaf': min_sum_hessian_per_leaf,\n        'verbose': 0\n    }\n    \n    rauc, y_pred = train_and_evaluate_lgbm(train, test, params, False)\n    return rauc\n\n\n# run bayesian optimization with optimal features\nbounds_lgb = {\n    'num_leaves': (20, 500),\n    'learning_rate': (0.01, 0.2),\n    'max_depth': (8, 250),\n    'lambda_l1': (0, 3),\n    'lambda_l2': (0, 3),\n    'bagging_fraction': (0.4, 1),\n    'bagging_freq': (1, 10),\n    'colsample_bytree': (0.4, 1),\n    'colsample_bynode': (0.4, 1),\n    'min_data_per_leaf': (10, 100),\n    'min_sum_hessian_per_leaf': (0.0001, 0.01)\n}\n\nlgb_bo = BayesianOptimization(run_lgb_bayesian, bounds_lgb, random_state = SEED)\nlgb_bo.maximize(init_points = 300, n_iter = 300, acq = 'ucb', xi = 0.0, alpha = 1e-6)\n\nparams = {\n    'boosting_type': 'gbdt',\n    'metric': 'auc',\n    'objective': 'binary',\n    'n_jobs': -1,\n    'seed': SEED,\n    'num_leaves': int(lgb_bo.max['params']['num_leaves']),\n    'learning_rate': lgb_bo.max['params']['learning_rate'],\n    'max_depth': int(lgb_bo.max['params']['max_depth']),\n    'lambda_l1': lgb_bo.max['params']['lambda_l1'],\n    'lambda_l2': lgb_bo.max['params']['lambda_l2'],\n    'bagging_fraction': lgb_bo.max['params']['bagging_fraction'],\n    'bagging_freq': int(lgb_bo.max['params']['bagging_freq']),\n    'colsample_bytree': lgb_bo.max['params']['colsample_bytree'],\n    'colsample_bynode': lgb_bo.max['params']['colsample_bynode'],\n    'min_data_per_leaf': int(lgb_bo.max['params']['min_data_per_leaf']),\n    'min_sum_hessian_per_leaf': lgb_bo.max['params']['min_sum_hessian_per_leaf']\n}\n\n\n# train with new hyperparameters\nroc_auc, y_pred = train_and_evaluate_lgbm(train, test, params, 50)\n\n# predict\ntest['target'] = y_pred\nsub = test[['image_name', 'target']]\nsub.to_csv('lgbm_baseline_sub.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}