{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nfrom sklearn import metrics\n\nfrom scipy.stats import rankdata\nfrom bayes_opt import BayesianOptimization\n\n\n\ntrain = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\ntest = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load OOF and Test predictions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Our single best models are:\n* https://www.kaggle.com/cdeotte/triple-stratified-kfold-with-tfrecords\n* https://www.kaggle.com/hiramcho/melanoma-efficientnetb6-with-attention-mechanism\n* https://www.kaggle.com/digvijayyadav/getting-started-with-tfrecords\n\nit is important that models use the same fold cv strategy to not overfit the ensemble model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [ \"getting-started-with-tfrecords\", \"melanoma-efficientnetb6-with-attention-mechanism\", \"triple-stratified-kfold-with-tfrecords\"]\n\nfor model in models:\n    dirname = \"/kaggle/input/\" + model\n    _oof = pd.read_csv(os.path.join(dirname, \"oof.csv\"))\n    score = metrics.roc_auc_score(_oof['target'], _oof['pred'])\n    print(f\"{model}: OOF auc:{score:.4}\")\n\n    _oof = _oof.rename(columns={\"pred\":model}).drop([\"target\"],axis=1)\n    if \"fold\" in _oof.columns:\n        _oof = _oof.drop([\"fold\"],axis=1)\n\n    train = train.merge(_oof, on=\"image_name\")   \n\n\n    _sub = pd.read_csv(os.path.join(dirname, \"submission.csv\"))\n    _sub.columns = [\"image_name\",model]    \n    test = test.merge(_sub, on=\"image_name\")   \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# OOF Ensembling","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Avg, Rank, Pow Avg","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"pred_rank\"] = 0\ntrain[\"pred_power\"] = 0\ntrain[\"pred_avg\"] = 0\n\nfor c in models:\n    train[\"pred_rank\"] += train[c].rank() / train[c].rank().max()\n    train[\"pred_power\"] += np.power(train[c],2)/np.power(train[c],2).max()\n    train[\"pred_avg\"] += train [c]/train [c].max()\n    \ntrain[\"pred_rank\"] = train[\"pred_rank\"]/len(models)\ntrain[\"pred_power\"] = train[\"pred_power\"]/len(models)\ntrain[\"pred_avg\"] = train[\"pred_avg\"]/len(models)\n\n\nscore = metrics.roc_auc_score(train['target'], train[\"pred_avg\"])\nprint(f'OOF avg_auc:{score}')\n   \n    \nscore = metrics.roc_auc_score(train['target'], train[\"pred_rank\"])\nprint(f'OOF rank_auc:{score}')\n\nscore = metrics.roc_auc_score(train['target'], train[\"pred_power\"])\nprint(f'OOF pow_auc:{score}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submissions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = 0.0\nfor c in models:\n    test[\"target\"] += test[c].rank() / test[c].rank().max()\ntest[\"target\"] = test[\"target\"]/len(models) \n    \nsub = test[[\"image_name\",\"target\"]]\nsub.to_csv(\"submission_rank.csv\",index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = 0.0\nfor c in models:\n    test[\"target\"] += np.power(test[c],2)/np.power(test[c],2).max()\ntest[\"target\"] = test[\"target\"]/len(models) \n    \nsub = test[[\"image_name\",\"target\"]]\nsub.to_csv(\"submission_pow.csv\",index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = 0.0\nfor c in models:\n    test[\"target\"] += test[c]/test[c].max()\ntest[\"target\"] = test[\"target\"]/len(models) \n    \nsub = test[[\"image_name\",\"target\"]]\nsub.to_csv(\"submission_avg.csv\",index=False)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Weighted Avg (Bayesian Optimization)\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def dim_optimizer (df_oof, features, init_points = 20, n_iter = 30  ):\n    pbounds = {'c0': (0.0, 1.0), 'c1': (0.0, 1.0), 'c2': (0.0, 1.0)}\n    \n    features = features\n\n    def dim_opt (df_oof, c0,c1,c2):\n\n        x = c0*df_oof[  features[0] ] + c1*df_oof[ features[1]] + c2*df_oof[ features[2]]\n        return metrics.roc_auc_score(df_oof['target'], x)\n\n\n\n    def q (c0, c1,c2):\n        return dim_opt  ( df_oof,  c0, c1,c2 )\n\n    optimizer = BayesianOptimization(\n        f=q,\n        pbounds=pbounds,\n        random_state=42,\n    )\n\n\n    optimizer.maximize(\n        init_points=init_points,\n        n_iter=n_iter,\n    )\n\n    c0 = optimizer.max[\"params\"][\"c0\"]\n    c1 = optimizer.max[\"params\"][\"c1\"]\n    c2= optimizer.max[\"params\"][\"c2\"]\n    \n    t = optimizer.max[\"target\"]\n    print ( f'bo auc:{t}, c0:{c0}, c1:{c1}, c2:{c2}' )\n    \n    return c0, c1, c2\n\n\nc0, c1, c2 = dim_optimizer (train, models, init_points = 40, n_iter = 40  )\nprint (models[0],c0)\nprint (models[1],c1)\nprint (models[2],c2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def bo_pred (df):\n    x = c0*df[  models[0] ] + c1*df[ models[1]] + c2*df[ models[2]]\n    return x\n\ntrain[\"pred\"] = bo_pred (train)\nscore = metrics.roc_auc_score(train['target'], train['pred'])\nprint(f\"auc bo:{score}\")\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"target\"] = bo_pred (test)\n    \nsub = test[[\"image_name\",\"target\"]]\nsub.to_csv(\"submission_bo.csv\",index=False)\nsub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}