{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np, os\nimport matplotlib.pyplot as plt\nfrom scipy.optimize import minimize, fmin\nfrom sklearn.metrics import roc_auc_score\nfrom bayes_opt import BayesianOptimization\nfrom functools import partial\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### OptimizeAUC\n\n- This function is from [Abhishek Thakur](https://www.kaggle.com/abhishek) book: [Approaching (Almost) Any Machine Learning Problem](https://www.amazon.in/dp/B089P13QHT/ref=dp-kindle-redirect?_encoding=UTF8&btkr=1#:~:text=This%20book%20is%20for%20people,learning%20and%20deep%20learning%20problems.)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class OptimizeAUC:\n    \"\"\"\n    Class for optimizing AUC\n    This class is all you need to find best weights for\n    any model and for any metric and for any type of predictions\n    With very small changes, this class can be used for optimization of\n    weights in ensemble models for _any_ type of predictions\n    \"\"\"\n    def __init__(self):\n        self.coef_ = 0\n    \n    def _auc(self, coef, X, y):\n        \"\"\"\n        This functions calculates and returns AUC\n        :param coef: coef list, of the same length as number of models\n        :param X: predictions, in this case a 2d array\n        :param y: targets, in our case binary 1d array\n        \"\"\"\n\n        # multiply coefficients with every column of the array\n        # with predictions.\n        # this means: element 1 of coef is multiplied by column 1\n        # of the prediction array, element 2 of coef is multiplied\n        # by column 2 of the prediction array and so on!\n        x_coef = X * coef\n        # create predictions by taking row wise sum\n        predictions = np.sum(x_coef, axis=1)\n\n        # calculate auc score\n        auc_score = metrics.roc_auc_score(y, predictions)\n\n        #return negative auc\n        return -1.0 * auc_score\n    \n    def fit(self, X, y):\n        # remember partial from hypeparamer optimization chapter?\n        loss_partial = partial(self._auc, X=X, y=y)\n\n        # dirichlet distribution. you can use any distribution you want\n        # to initialize the coefficients\n        # we want the coefficients to sum to 1\n        initial_coef = np.random.dirichlet(np.ones(X.shape[1]), size=1)\n\n        # use scipy fmin to minimize the loss function, in our case auc\n        self.coef_ = fmin(loss_partial, initial_coef, disp=True)\n\n    def predict(self, X):\n        # this is similar to _auc function\n        x_coef = X * self.coef_\n        predictions = np.sum(x_coef, axis=1)\n        return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/siim-isic-melanoma-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SUB_DIR = \"../input/siimsubmitionfiles/Final_Submissions/Final_Submissions\"\nTF_0OFS = \"../input/siimsubmitionfiles/TF-OOFS/TF-OOFS\"\nTF_SUBS = \"../input/siimsubmitionfiles/TF-SUBS/TF-SUBS\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pytorch Models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"offs_b5_df = pd.read_csv(f\"{SUB_DIR}/offs_b5_df.csv\")\noffs_b5_ns_df = pd.read_csv(f\"{SUB_DIR}/offs_b5_ns_df.csv\")\noffs_b5_ds_df = pd.read_csv(f\"{SUB_DIR}/offs_b5_512_ds_df.csv\")\noffs_b5_10_ds_df = pd.read_csv(f\"{SUB_DIR}/offs_b5_10_ds_df.csv\")\noffs_b5_f_ds_df = pd.read_csv(f\"{SUB_DIR}/offs_b5_f_ds_df.csv\")\noffs_tf_b5_df = pd.read_csv(f\"{SUB_DIR}/offs_tf_b5_df.csv\")\noffs_tf_b5_v1_df = pd.read_csv(f\"{SUB_DIR}/offs_tf_b5_v1_df.csv\")\n\noffs_b6_df = pd.read_csv(f\"{SUB_DIR}/offs_b6_df.csv\")\noffs_b6_2_df = pd.read_csv(f\"{SUB_DIR}/offs_b6_2_ds_df.csv\")\noffs_b6_10_df = pd.read_csv(f\"{SUB_DIR}/offs_b6_10_ds_df.csv\")\noffs_tf_b6_df = pd.read_csv(f\"{SUB_DIR}/offs_tf_b6_df.csv\")\n\noffs_d201_df = pd.read_csv(f\"{SUB_DIR}/offs_d201_df.csv\")\noffs_d201_ns_df = pd.read_csv(f\"{SUB_DIR}/offs_d201_ns_df.csv\")\noffs_d201_512_df = pd.read_csv(f\"{SUB_DIR}/offs_d201_512_df.csv\")\noffs_d201_512_ds_df = pd.read_csv(f\"{SUB_DIR}/offs_d201_512_ds_df.csv\")\noffs_tf_d201_df = pd.read_csv(f\"{SUB_DIR}/offs_tf_d201_df.csv\")\n\noffs_b4_df = pd.read_csv(f\"{SUB_DIR}/offs_b4_df.csv\")\noffs_b4_f_df = pd.read_csv(f\"{SUB_DIR}/offs_b4_f_5_ds_df.csv\")\n\noffs_b7_224_df = pd.read_csv(f\"{SUB_DIR}/offs_b7_224_ds_df.csv\")\noffs_tf_b7_df = pd.read_csv(f\"{SUB_DIR}/offs_tf_b7_df.csv\")\noffs_b7_512_ds_df = pd.read_csv(f\"{SUB_DIR}/offs_b7_512_ds_df.csv\")\n\noffs_dpn92_ds_df = pd.read_csv(f\"{SUB_DIR}/offs_dpn92_ds_df.csv\")\n\noffs_sk50_ds_df = pd.read_csv(f\"{SUB_DIR}/offs_sk50_5_ds_df.csv\")\n\noffs_meta_df = pd.read_csv(f\"{SUB_DIR}/tabular_oof_df.csv\").sort_values(\"image_name\")\noffs_meta_df = offs_meta_df[offs_meta_df.image_name.isin(offs_dpn92_ds_df.image_name)].reset_index(drop=True)\n\noffs_meta_2_df = pd.read_csv(f\"{SUB_DIR}/tabular2_oof_df .csv\").sort_values(\"image_name\")\noffs_meta_2_df = offs_meta_2_df[offs_meta_2_df.image_name.isin(offs_dpn92_ds_df.image_name)].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"offs_final_df = pd.DataFrame({\n    \"image_name\" : offs_b5_df.image_name.values,\n    \"target\" : offs_b5_df.target.values,\n\n    \"b5\" : offs_b5_df.prediction.values,\n    \"b5_ns\": offs_b5_ns_df.prediction.values,\n    \"b5_ds\": offs_b5_ds_df.prediction.values,\n    \"b5_ds_10\": offs_b5_10_ds_df.prediction.values,\n    \"b5_f_ds\" : offs_b5_f_ds_df.prediction.values,\n    \"tf_b5\" : offs_tf_b5_df.pred.values,\n    \"tf_b5_v1\": offs_tf_b5_v1_df.pred.values,\n\n    \"b6\" : offs_b6_df.prediction.values,\n    \"b6_2\" : offs_b6_2_df.prediction.values,\n    \"b6_10\" : offs_b6_10_df.prediction,\n    \"tf_b6\": offs_tf_b6_df.pred.values,\n\n    \"b4\" : offs_b4_df.prediction.values,\n    \"b4_f\" : offs_b4_f_df.prediction.values,\n\n    \"d201\" :offs_d201_df.prediction.values,\n    \"d201_ns\" : offs_d201_ns_df.prediction.values,\n    \"d201_512\" : offs_d201_512_df.prediction.values,\n    \"d201_512_ds\" : offs_d201_512_ds_df.prediction.values,\n    \"tf_d201\" : offs_tf_d201_df.pred.values,\n\n    \"b7_224_ds\" : offs_b7_224_df.prediction.values,\n    \"tf_b7\" : offs_tf_b7_df.pred.values,\n    \"b7_512_ds\" : offs_b7_512_ds_df.prediction.values,\n\n    \"dpn92_ds\" : offs_dpn92_ds_df.prediction.values,\n\n    \"sk50_ds\" : offs_sk50_ds_df.prediction.values,\n\n    \"meta_sub\" : offs_meta_df.prediction.values,\n    \"meta_sub_2\" : offs_meta_2_df.prediction.values\n})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = offs_final_df\ntrain_cols = [\n    'b5', 'b5_ns', 'b5_ds', 'b5_ds_10', 'b5_f_ds','tf_b5', 'tf_b5_v1', \n    'b6', 'b6_2', 'b6_10', 'tf_b6', \n    'b4', 'b4_f',\n    'd201', 'd201_ns', 'd201_512', 'd201_512_ds', 'tf_d201', \n    'b7_224_ds','tf_b7', 'b7_512_ds', \n    'dpn92_ds', \n    'sk50_ds', \n    'meta_sub', 'meta_sub_2' \n]\n\nX = train_df[train_cols]\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = OptimizeAUC()\nopt.fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt.coef_","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pytorch SUB","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# b5\nb5_512_5    = pd.read_csv(f\"{SUB_DIR}/Final_B5_512_456_5.csv\")[\"target\"]\nb5_512_5_ns = pd.read_csv(f\"{SUB_DIR}/Final_NS_E5_512_456_5.csv\")[\"target\"]\nb5_512_5_ds = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_5.csv\")[\"target\"]\nb5_512_10_ds = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_10.csv\")[\"target\"]\nb5_512_f_ds = pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B5_512_456_5.csv\")[\"target\"]\nb5_512_5_tf = pd.read_csv(f\"{SUB_DIR}/TF-B5-512-V.csv\")[\"target\"]\nb5_512_5_v1 = pd.read_csv(f\"{SUB_DIR}/TF-B5-512.csv\")[\"target\"]\n\n# b6\nb6_528_5    = pd.read_csv(f\"{SUB_DIR}/Final_B6_768_528_5.csv\")[\"target\"]\nb6_512_2    = pd.read_csv(f\"{SUB_DIR}/Final_DSO_B6_768_528_2.csv\")[\"target\"]\nb6_512_10   = pd.read_csv(f\"{SUB_DIR}/Final_DS_B6_768_528_10.csv\")[\"target\"]\nb6_512_5_tf = pd.read_csv(f\"{SUB_DIR}/TF-B6-512-5.csv\")[\"target\"]\n\n#b4\nb4_300_10      = pd.read_csv(f\"{SUB_DIR}/Final_B4_512_380_10.csv\")[\"target\"]\nb4_f           = pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B4_512_380_5.csv\")[\"target\"]\n\n# d201\nd201_224_5     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_224_5.csv\")[\"target\"]\nd201_224_5_ns  = pd.read_csv(f\"{SUB_DIR}/Final_NS_D201_512_224_5.csv\")[\"target\"]\nd201_512_5     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_512_5.csv\")[\"target\"]\nd201_512_5_ds  = pd.read_csv(f\"{SUB_DIR}/Final_DS_D201_512_512_5.csv\")[\"target\"]\nd201_512_5_tf  = pd.read_csv(f\"{SUB_DIR}/TF-D201-512-5.csv\")[\"target\"]\n\n# b7\nb7_224_ds      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_224_5.csv\")[\"target\"]\nb7_512_ds      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_512_5.csv\")[\"target\"]\nb7_512_tf      = pd.read_csv(f\"{SUB_DIR}/TF-B7-512-5.csv\")[\"target\"]\n\n# dpn92\ndpn92_ds       = pd.read_csv(f\"{SUB_DIR}/Final_DS_dpn92_512_512_5.csv\")[\"target\"]\n\n# sk50\nsk50_ds        = pd.read_csv(f\"{SUB_DIR}/Final_DS_sk50_512_512_5.csv\")[\"target\"]\n\n# meta\nmeta_1 = pd.read_csv(f\"{SUB_DIR}/tabular_test_prediction.csv\")[\"target\"]\nmeta_2 = pd.read_csv(f\"{SUB_DIR}/tabular2_test_prediction .csv\")[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"W = opt.coef_\n\npytorch_sub = np.mean([\n         \n    # b5\n   W[0] * b5_512_5, #    = pd.read_csv(f\"{SUB_DIR}/Final_B5_512_456_5.csv\")[\"target\"]\n   W[1] * b5_512_5_ns,# = pd.read_csv(f\"{SUB_DIR}/Final_NS_E5_512_456_5.csv\")[\"target\"]\n   W[2] * b5_512_5_ds,# = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_5.csv\")[\"target\"]\n   W[3] * b5_512_10_ds,# = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_10.csv\")[\"target\"]\n   W[4] * b5_512_f_ds, #= pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B5_512_456_5.csv\")[\"target\"]\n   W[5] * b5_512_5_tf, #= pd.read_csv(f\"{SUB_DIR}/TF-B5-512-V.csv\")[\"target\"]\n   W[6] * b5_512_5_v1, #= pd.read_csv(f\"{SUB_DIR}/TF-B5-512.csv\")[\"target\"]\n\n    # b6\n   W[7] * b6_528_5,#    = pd.read_csv(f\"{SUB_DIR}/Final_B6_768_528_5.csv\")[\"target\"]\n   W[8] * b6_512_2,#    = pd.read_csv(f\"{SUB_DIR}/Final_DSO_B6_768_528_2.csv\")[\"target\"]\n   W[9] * b6_512_10,#   = pd.read_csv(f\"{SUB_DIR}/Final_DS_B6_768_528_10.csv\")[\"target\"]\n   W[10] * b6_512_5_tf,# = pd.read_csv(f\"{SUB_DIR}/TF-B6-512-5.csv\")[\"target\"]\n\n    #b4\n   W[11] * b4_300_10,#      = pd.read_csv(f\"{SUB_DIR}/Final_B4_512_380_10.csv\")[\"target\"]\n   W[12] * b4_f,#           = pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B4_512_380_5.csv\")[\"target\"]\n\n    # d201\n   W[13] * d201_224_5,#     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_224_5.csv\")[\"target\"]\n   W[14] * d201_224_5_ns,#  = pd.read_csv(f\"{SUB_DIR}/Final_NS_D201_512_224_5.csv\")[\"target\"]\n   W[15] * d201_512_5,#     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_512_5.csv\")[\"target\"]\n   W[16] * d201_512_5_ds,#  = pd.read_csv(f\"{SUB_DIR}/Final_DS_D201_512_512_5.csv\")[\"target\"]\n   W[17] * d201_512_5_tf,#  = pd.read_csv(f\"{SUB_DIR}/TF-D201-512-5.csv\")[\"target\"]\n\n    # b7\n   W[18] * b7_224_ds,#      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_224_5.csv\")[\"target\"]\n   W[19] * b7_512_ds,#      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_512_5.csv\")[\"target\"]\n   W[20] * b7_512_tf,#      = pd.read_csv(f\"{SUB_DIR}/TF-B7-512-5.csv\")[\"target\"]\n\n\n    # dpn92\n   W[21] * dpn92_ds,#       = pd.read_csv(f\"{SUB_DIR}/Final_DS_dpn92_512_512_5.csv\")[\"target\"]\n\n    # sk50\n   W[22] * sk50_ds,#        = pd.read_csv(f\"{SUB_DIR}/Final_DS_sk50_512_512_5.csv\")[\"target\"]\n\n    # meta\n  W[23] *  meta_1,# = pd.read_csv(f\"{SUB_DIR}/tabular_test_prediction.csv\")[\"target\"]\n  W[24] *  meta_2,# = pd.read_csv(f\"{SUB_DIR}/tabular2_test_prediction .csv\")[\"target\"]\n\n], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission[\"target\"] = pytorch_sub\nsample_submission.to_csv(\"Final_Pytorch.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TF models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"offs_b7_512_15 = pd.read_csv(f\"{TF_0OFS}/offs_b7_512_15.csv\")\noffs_b6_512_15 = pd.read_csv(f\"{TF_0OFS}/offs_b6_512_15.csv\")\noffs_b5_512_15 = pd.read_csv(f\"{TF_0OFS}/offs_b5_512_15.csv\")\n\noffs_b7_512_10 = pd.read_csv(f\"{TF_0OFS}/offs_b7_512_10.csv\")\noffs_b6_512_10 = pd.read_csv(f\"{TF_0OFS}/offs_b6_512_10.csv\")\noffs_b5_512_10 = pd.read_csv(f\"{TF_0OFS}/offs_b5_512_10.csv\")\noffs_b4_512_10 = pd.read_csv(f\"{TF_0OFS}/offs_b4_512_10.csv\")\noffs_b3_512_10 = pd.read_csv(f\"{TF_0OFS}/offs_b3_512_10.csv\")\noffs_b2_512_10 = pd.read_csv(f\"{TF_0OFS}/offs_b2_512_10.csv\")\noffs_b1_512_10 = pd.read_csv(f\"{TF_0OFS}/offs_b1_512_10.csv\")\n\noffs_b7_768_5 = pd.read_csv(f\"{TF_0OFS}/offs_b7_768_5.csv\")\noffs_b6_768_5 = pd.read_csv(f\"{TF_0OFS}/offs_b6_768_5.csv\")\noffs_b5_768_5 = pd.read_csv(f\"{TF_0OFS}/offs_b5_768_5.csv\")\noffs_b4_768_5 = pd.read_csv(f\"{TF_0OFS}/offs_b4_768_5.csv\")\noffs_b3_768_5 = pd.read_csv(f\"{TF_0OFS}/offs_b3_768_5.csv\")\noffs_b2_768_5 = pd.read_csv(f\"{TF_0OFS}/offs_b2_768_5.csv\")\noffs_b1_768_5 = pd.read_csv(f\"{TF_0OFS}/offs_b1_768_5.csv\")\n\noffs_b7_384_15 = pd.read_csv(f\"{TF_0OFS}/offs_b7_384_15.csv\")\noffs_b6_384_15 = pd.read_csv(f\"{TF_0OFS}/offs_b6_384_15.csv\")\noffs_b5_384_15 = pd.read_csv(f\"{TF_0OFS}/offs_b5_384_15.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"offs_final_tf_df = pd.DataFrame({\n\n    \"target\" : offs_b7_512_15.target.values,\n    \"image_name\" : offs_b7_512_15.image_name.values,\n\n    \"b7_512_15\" : offs_b7_512_15.prediction.values,\n    \"b6_512_15\" : offs_b6_512_15.prediction.values,\n    \"b5_512_15\" : offs_b5_512_15.prediction.values,\n\n    \"b7_512_10\" : offs_b7_512_10.prediction.values,\n    \"b6_512_10\" : offs_b6_512_10.prediction.values,\n    \"b5_512_10\" : offs_b5_512_10.prediction.values,\n    \"b4_512_10\" : offs_b4_512_10.prediction.values,\n    \"b3_512_10\" : offs_b3_512_10.prediction.values,\n    \"b2_512_10\" : offs_b2_512_10.prediction.values,\n    \"b1_512_10\" : offs_b1_512_10.prediction.values,\n\n    \"b7_768_5\" : offs_b7_768_5.prediction.values,\n    \"b6_768_5\" : offs_b6_768_5.prediction.values,\n    \"b5_768_5\" : offs_b5_768_5.prediction.values,\n    \"b4_768_5\" : offs_b4_768_5.prediction.values,\n    \"b3_768_5\" : offs_b3_768_5.prediction.values,\n    \"b2_768_5\" : offs_b2_768_5.prediction.values,\n    \"b1_768_5\" : offs_b1_768_5.prediction.values,\n\n    \"b7_384_15\" : offs_b7_384_15.prediction.values,\n    \"b6_384_15\" : offs_b6_384_15.prediction.values,\n    \"b5_384_15\" : offs_b5_384_15.prediction.values,\n\n})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = offs_final_tf_df\n\ntrain_cols = [\n    'b7_512_15', 'b6_512_15', 'b5_512_15',\n    'b7_512_10', 'b6_512_10', 'b5_512_10', 'b4_512_10', 'b3_512_10',\n    'b2_512_10', 'b1_512_10', 'b7_768_5', 'b6_768_5', 'b5_768_5',\n    'b4_768_5', 'b3_768_5', 'b2_768_5', 'b1_768_5', 'b7_384_15',\n    'b6_384_15', 'b5_384_15'\n]\n\nX = train_df[train_cols]\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt.coef_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"B7_512_15 = pd.read_csv(f\"{TF_SUBS}/B7_512_15.csv\")[\"target\"]\nB6_512_15 = pd.read_csv(f\"{TF_SUBS}/B6_512_15.csv\")[\"target\"]\nB5_512_15 = pd.read_csv(f\"{TF_SUBS}/B5_512_15.csv\")[\"target\"]\n\nB7_512_10 = pd.read_csv(f\"{TF_SUBS}/B7_512_10.csv\")[\"target\"]\nB6_512_10 = pd.read_csv(f\"{TF_SUBS}/B6_512_10.csv\")[\"target\"]\nB5_512_10 = pd.read_csv(f\"{TF_SUBS}/B5_512_10.csv\")[\"target\"]\nB4_512_10 = pd.read_csv(f\"{TF_SUBS}/B4_512_10.csv\")[\"target\"]\nB3_512_10 = pd.read_csv(f\"{TF_SUBS}/B3_512_10.csv\")[\"target\"]\nB2_512_10 = pd.read_csv(f\"{TF_SUBS}/B2_512_10.csv\")[\"target\"]\nB1_512_10 = pd.read_csv(f\"{TF_SUBS}/B1_512_10.csv\")[\"target\"]\n\nB7_768_5 = pd.read_csv(f\"{TF_SUBS}/B7_768_5.csv\")[\"target\"]\nB6_768_5 = pd.read_csv(f\"{TF_SUBS}/B6_768_5.csv\")[\"target\"]\nB5_768_5 = pd.read_csv(f\"{TF_SUBS}/B5_768_5.csv\")[\"target\"]\nB4_768_5 = pd.read_csv(f\"{TF_SUBS}/B4_768_5.csv\")[\"target\"]\nB3_768_5 = pd.read_csv(f\"{TF_SUBS}/B3_768_5.csv\")[\"target\"]\nB2_768_5 = pd.read_csv(f\"{TF_SUBS}/B2_768_5.csv\")[\"target\"]\nB1_768_5 = pd.read_csv(f\"{TF_SUBS}/B1_768_5.csv\")[\"target\"]\n\nB7_384_15 = pd.read_csv(f\"{TF_SUBS}/B7_384_15.csv\")[\"target\"]\nB6_384_15 = pd.read_csv(f\"{TF_SUBS}/B6_384_15.csv\")[\"target\"]\nB5_384_15 = pd.read_csv(f\"{TF_SUBS}/B5_384_15.csv\")[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"W = opt.coef_\n\ntf_sub = np.mean([\n    \n    W[0] * B7_512_15,#= pd.read_csv(f\"{TF_SUBS}/B7_512_15.csv\")[\"target\"]\n    W[1] * B6_512_15,# = pd.read_csv(f\"{TF_SUBS}/B6_512_15.csv\")[\"target\"]\n    W[2] * B5_512_15,# = pd.read_csv(f\"{TF_SUBS}/B5_512_15.csv\")[\"target\"]\n\n    W[3] * B7_512_10,# = pd.read_csv(f\"{TF_SUBS}/B7_512_10.csv\")[\"target\"]\n    W[4] * B6_512_10,# = pd.read_csv(f\"{TF_SUBS}/B6_512_10.csv\")[\"target\"]\n    W[5] * B5_512_10,# = pd.read_csv(f\"{TF_SUBS}/B5_512_10.csv\")[\"target\"]\n    W[6] * B4_512_10,# = pd.read_csv(f\"{TF_SUBS}/B4_512_10.csv\")[\"target\"]\n    W[7] * B3_512_10,# = pd.read_csv(f\"{TF_SUBS}/B3_512_10.csv\")[\"target\"]\n    W[8] * B2_512_10,# = pd.read_csv(f\"{TF_SUBS}/B2_512_10.csv\")[\"target\"]\n    W[9] * B1_512_10,# = pd.read_csv(f\"{TF_SUBS}/B1_512_10.csv\")[\"target\"]\n\n    W[10] * B7_768_5,# = pd.read_csv(f\"{TF_SUBS}/B7_768_5.csv\")[\"target\"]\n    W[11] * B6_768_5,# = pd.read_csv(f\"{TF_SUBS}/B6_768_5.csv\")[\"target\"]\n    W[12] * B5_768_5,# = pd.read_csv(f\"{TF_SUBS}/B5_768_5.csv\")[\"target\"]\n    W[13] * B4_768_5,# = pd.read_csv(f\"{TF_SUBS}/B4_768_5.csv\")[\"target\"]\n    W[14] * B3_768_5,# = pd.read_csv(f\"{TF_SUBS}/B3_768_5.csv\")[\"target\"]\n    W[15] * B2_768_5,# = pd.read_csv(f\"{TF_SUBS}/B2_768_5.csv\")[\"target\"]\n    W[16] * B1_768_5,# = pd.read_csv(f\"{TF_SUBS}/B1_768_5.csv\")[\"target\"]\n\n    W[17] * B7_384_15,# = pd.read_csv(f\"{TF_SUBS}/B7_384_15.csv\")[\"target\"]\n    W[18] * B6_384_15,# = pd.read_csv(f\"{TF_SUBS}/B6_384_15.csv\")[\"target\"]\n    W[19] * B5_384_15,# = pd.read_csv(f\"{TF_SUBS}/B5_384_15.csv\")[\"target\"]\n\n], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission[\"target\"] = tf_sub\nsample_submission.to_csv(\"Final_TF.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pytorch + TF","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.merge(offs_final_df, offs_final_tf_df, how=\"left\", left_on=[\"image_name\", \"target\"], right_on=[\"image_name\", \"target\"])\n\ntrain_cols = [\n    'b5', 'b5_ns', 'b5_ds', 'b5_ds_10', 'b5_f_ds',\n       'tf_b5', 'tf_b5_v1', 'b6', 'b6_2', 'b6_10', 'tf_b6', 'b4', 'b4_f',\n       'd201', 'd201_ns', 'd201_512', 'd201_512_ds', 'tf_d201', 'b7_224_ds',\n       'tf_b7', 'b7_512_ds', 'dpn92_ds', 'sk50_ds', \n       #'meta_sub', 'meta_sub_2',\n       'b7_512_15', 'b6_512_15', 'b5_512_15', 'b7_512_10', 'b6_512_10',\n       'b5_512_10', 'b4_512_10', 'b3_512_10', 'b2_512_10', 'b1_512_10',\n       'b7_768_5', 'b6_768_5', 'b5_768_5', 'b4_768_5', 'b3_768_5', 'b2_768_5',\n       'b1_768_5', 'b7_384_15', 'b6_384_15', 'b5_384_15'\n]\n\nX = train_df[train_cols]\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt = OptimizeAUC()\nopt.fit(X, y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt.coef_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"W = opt.coef_\n\npytorch_tf_sub = np.mean([\n         \n        # b5\n    W[0] * b5_512_5, #    = pd.read_csv(f\"{SUB_DIR}/Final_B5_512_456_5.csv\")[\"target\"]\n    W[1] * b5_512_5_ns,# = pd.read_csv(f\"{SUB_DIR}/Final_NS_E5_512_456_5.csv\")[\"target\"]\n    W[2] * b5_512_5_ds,# = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_5.csv\")[\"target\"]\n    W[3] * b5_512_10_ds,# = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_10.csv\")[\"target\"]\n    W[4] * b5_512_f_ds, #= pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B5_512_456_5.csv\")[\"target\"]\n    W[5] * b5_512_5_tf, #= pd.read_csv(f\"{SUB_DIR}/TF-B5-512-V.csv\")[\"target\"]\n    W[6] * b5_512_5_v1, #= pd.read_csv(f\"{SUB_DIR}/TF-B5-512.csv\")[\"target\"]\n\n        # b6\n    W[7] * b6_528_5,#    = pd.read_csv(f\"{SUB_DIR}/Final_B6_768_528_5.csv\")[\"target\"]\n    W[8] * b6_512_2,#    = pd.read_csv(f\"{SUB_DIR}/Final_DSO_B6_768_528_2.csv\")[\"target\"]\n    W[9] * b6_512_10,#   = pd.read_csv(f\"{SUB_DIR}/Final_DS_B6_768_528_10.csv\")[\"target\"]\n    W[10] * b6_512_5_tf,# = pd.read_csv(f\"{SUB_DIR}/TF-B6-512-5.csv\")[\"target\"]\n\n        #b4\n    W[11] * b4_300_10,#      = pd.read_csv(f\"{SUB_DIR}/Final_B4_512_380_10.csv\")[\"target\"]\n    W[12] * b4_f,#           = pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B4_512_380_5.csv\")[\"target\"]\n\n        # d201\n    W[13] * d201_224_5,#     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_224_5.csv\")[\"target\"]\n    W[14] * d201_224_5_ns,#  = pd.read_csv(f\"{SUB_DIR}/Final_NS_D201_512_224_5.csv\")[\"target\"]\n    W[15] * d201_512_5,#     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_512_5.csv\")[\"target\"]\n    W[16] * d201_512_5_ds,#  = pd.read_csv(f\"{SUB_DIR}/Final_DS_D201_512_512_5.csv\")[\"target\"]\n    W[17] * d201_512_5_tf,#  = pd.read_csv(f\"{SUB_DIR}/TF-D201-512-5.csv\")[\"target\"]\n\n        # b7\n    W[18] * b7_224_ds,#      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_224_5.csv\")[\"target\"]\n    W[19] * b7_512_ds,#      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_512_5.csv\")[\"target\"]\n    W[20] * b7_512_tf,#      = pd.read_csv(f\"{SUB_DIR}/TF-B7-512-5.csv\")[\"target\"]\n\n\n        # dpn92\n    W[21] * dpn92_ds,#       = pd.read_csv(f\"{SUB_DIR}/Final_DS_dpn92_512_512_5.csv\")[\"target\"]\n\n        # sk50\n    W[22] * sk50_ds,#        = pd.read_csv(f\"{SUB_DIR}/Final_DS_sk50_512_512_5.csv\")[\"target\"]\n\n        # meta\n    #W[23] *  meta_1,# = pd.read_csv(f\"{SUB_DIR}/tabular_test_prediction.csv\")[\"target\"]\n    #W[24] *  meta_2,# = pd.read_csv(f\"{SUB_DIR}/tabular2_test_prediction .csv\")[\"target\"]\n\n\n    W[23] * B7_512_15,#= pd.read_csv(f\"{TF_SUBS}/B7_512_15.csv\")[\"target\"]\n    W[24] * B6_512_15,# = pd.read_csv(f\"{TF_SUBS}/B6_512_15.csv\")[\"target\"]\n    W[25] * B5_512_15,# = pd.read_csv(f\"{TF_SUBS}/B5_512_15.csv\")[\"target\"]\n\n    W[26] * B7_512_10,# = pd.read_csv(f\"{TF_SUBS}/B7_512_10.csv\")[\"target\"]\n    W[27] * B6_512_10,# = pd.read_csv(f\"{TF_SUBS}/B6_512_10.csv\")[\"target\"]\n    W[38] * B5_512_10,# = pd.read_csv(f\"{TF_SUBS}/B5_512_10.csv\")[\"target\"]\n    W[39] * B4_512_10,# = pd.read_csv(f\"{TF_SUBS}/B4_512_10.csv\")[\"target\"]\n    W[30] * B3_512_10,# = pd.read_csv(f\"{TF_SUBS}/B3_512_10.csv\")[\"target\"]\n    W[31] * B2_512_10,# = pd.read_csv(f\"{TF_SUBS}/B2_512_10.csv\")[\"target\"]\n    W[32] * B1_512_10,# = pd.read_csv(f\"{TF_SUBS}/B1_512_10.csv\")[\"target\"]\n\n    W[33] * B7_768_5,# = pd.read_csv(f\"{TF_SUBS}/B7_768_5.csv\")[\"target\"]\n    W[34] * B6_768_5,# = pd.read_csv(f\"{TF_SUBS}/B6_768_5.csv\")[\"target\"]\n    W[35] * B5_768_5,# = pd.read_csv(f\"{TF_SUBS}/B5_768_5.csv\")[\"target\"]\n    W[36] * B4_768_5,# = pd.read_csv(f\"{TF_SUBS}/B4_768_5.csv\")[\"target\"]\n    W[37] * B3_768_5,# = pd.read_csv(f\"{TF_SUBS}/B3_768_5.csv\")[\"target\"]\n    W[38] * B2_768_5,# = pd.read_csv(f\"{TF_SUBS}/B2_768_5.csv\")[\"target\"]\n    W[39] * B1_768_5,# = pd.read_csv(f\"{TF_SUBS}/B1_768_5.csv\")[\"target\"]\n\n    W[40] * B7_384_15,# = pd.read_csv(f\"{TF_SUBS}/B7_384_15.csv\")[\"target\"]\n    W[41] * B6_384_15,# = pd.read_csv(f\"{TF_SUBS}/B6_384_15.csv\")[\"target\"]\n    W[42] * B5_384_15,# = pd.read_csv(f\"{TF_SUBS}/B5_384_15.csv\")[\"target\"]\n\n\n], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission[\"target\"] = pytorch_tf_sub\nsample_submission.to_csv(\"Final_Pytorch_TF.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Simple Average","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"W = [1] * 50\n\nsimple_avg = np.mean([\n         \n        # b5\n    W[0] * b5_512_5, #    = pd.read_csv(f\"{SUB_DIR}/Final_B5_512_456_5.csv\")[\"target\"]\n    W[1] * b5_512_5_ns,# = pd.read_csv(f\"{SUB_DIR}/Final_NS_E5_512_456_5.csv\")[\"target\"]\n    W[2] * b5_512_5_ds,# = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_5.csv\")[\"target\"]\n    W[3] * b5_512_10_ds,# = pd.read_csv(f\"{SUB_DIR}/Final_DS_B5_512_456_10.csv\")[\"target\"]\n    W[4] * b5_512_f_ds, #= pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B5_512_456_5.csv\")[\"target\"]\n    W[5] * b5_512_5_tf, #= pd.read_csv(f\"{SUB_DIR}/TF-B5-512-V.csv\")[\"target\"]\n    W[6] * b5_512_5_v1, #= pd.read_csv(f\"{SUB_DIR}/TF-B5-512.csv\")[\"target\"]\n\n        # b6\n    W[7] * b6_528_5,#    = pd.read_csv(f\"{SUB_DIR}/Final_B6_768_528_5.csv\")[\"target\"]\n    W[8] * b6_512_2,#    = pd.read_csv(f\"{SUB_DIR}/Final_DSO_B6_768_528_2.csv\")[\"target\"]\n    W[9] * b6_512_10,#   = pd.read_csv(f\"{SUB_DIR}/Final_DS_B6_768_528_10.csv\")[\"target\"]\n    W[10] * b6_512_5_tf,# = pd.read_csv(f\"{SUB_DIR}/TF-B6-512-5.csv\")[\"target\"]\n\n        #b4\n    W[11] * b4_300_10,#      = pd.read_csv(f\"{SUB_DIR}/Final_B4_512_380_10.csv\")[\"target\"]\n    W[12] * b4_f,#           = pd.read_csv(f\"{SUB_DIR}/F_Final_DS_B4_512_380_5.csv\")[\"target\"]\n\n        # d201\n    W[13] * d201_224_5,#     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_224_5.csv\")[\"target\"]\n    W[14] * d201_224_5_ns,#  = pd.read_csv(f\"{SUB_DIR}/Final_NS_D201_512_224_5.csv\")[\"target\"]\n    W[15] * d201_512_5,#     = pd.read_csv(f\"{SUB_DIR}/Final_D201_512_512_5.csv\")[\"target\"]\n    W[16] * d201_512_5_ds,#  = pd.read_csv(f\"{SUB_DIR}/Final_DS_D201_512_512_5.csv\")[\"target\"]\n    W[17] * d201_512_5_tf,#  = pd.read_csv(f\"{SUB_DIR}/TF-D201-512-5.csv\")[\"target\"]\n\n        # b7\n    W[18] * b7_224_ds,#      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_224_5.csv\")[\"target\"]\n    W[19] * b7_512_ds,#      = pd.read_csv(f\"{SUB_DIR}/Final_DS_B7_512_512_5.csv\")[\"target\"]\n    W[20] * b7_512_tf,#      = pd.read_csv(f\"{SUB_DIR}/TF-B7-512-5.csv\")[\"target\"]\n\n\n        # dpn92\n    W[21] * dpn92_ds,#       = pd.read_csv(f\"{SUB_DIR}/Final_DS_dpn92_512_512_5.csv\")[\"target\"]\n\n        # sk50\n    W[22] * sk50_ds,#        = pd.read_csv(f\"{SUB_DIR}/Final_DS_sk50_512_512_5.csv\")[\"target\"]\n\n        # meta\n    #W[23] *  meta_1,# = pd.read_csv(f\"{SUB_DIR}/tabular_test_prediction.csv\")[\"target\"]\n    #W[24] *  meta_2,# = pd.read_csv(f\"{SUB_DIR}/tabular2_test_prediction .csv\")[\"target\"]\n\n\n    W[23] * B7_512_15,#= pd.read_csv(f\"{TF_SUBS}/B7_512_15.csv\")[\"target\"]\n    W[24] * B6_512_15,# = pd.read_csv(f\"{TF_SUBS}/B6_512_15.csv\")[\"target\"]\n    W[25] * B5_512_15,# = pd.read_csv(f\"{TF_SUBS}/B5_512_15.csv\")[\"target\"]\n\n    W[26] * B7_512_10,# = pd.read_csv(f\"{TF_SUBS}/B7_512_10.csv\")[\"target\"]\n    W[27] * B6_512_10,# = pd.read_csv(f\"{TF_SUBS}/B6_512_10.csv\")[\"target\"]\n    W[38] * B5_512_10,# = pd.read_csv(f\"{TF_SUBS}/B5_512_10.csv\")[\"target\"]\n    W[39] * B4_512_10,# = pd.read_csv(f\"{TF_SUBS}/B4_512_10.csv\")[\"target\"]\n    W[30] * B3_512_10,# = pd.read_csv(f\"{TF_SUBS}/B3_512_10.csv\")[\"target\"]\n    W[31] * B2_512_10,# = pd.read_csv(f\"{TF_SUBS}/B2_512_10.csv\")[\"target\"]\n    W[32] * B1_512_10,# = pd.read_csv(f\"{TF_SUBS}/B1_512_10.csv\")[\"target\"]\n\n    W[33] * B7_768_5,# = pd.read_csv(f\"{TF_SUBS}/B7_768_5.csv\")[\"target\"]\n    W[34] * B6_768_5,# = pd.read_csv(f\"{TF_SUBS}/B6_768_5.csv\")[\"target\"]\n    W[35] * B5_768_5,# = pd.read_csv(f\"{TF_SUBS}/B5_768_5.csv\")[\"target\"]\n    W[36] * B4_768_5,# = pd.read_csv(f\"{TF_SUBS}/B4_768_5.csv\")[\"target\"]\n    W[37] * B3_768_5,# = pd.read_csv(f\"{TF_SUBS}/B3_768_5.csv\")[\"target\"]\n    W[38] * B2_768_5,# = pd.read_csv(f\"{TF_SUBS}/B2_768_5.csv\")[\"target\"]\n    W[39] * B1_768_5,# = pd.read_csv(f\"{TF_SUBS}/B1_768_5.csv\")[\"target\"]\n\n    W[40] * B7_384_15,# = pd.read_csv(f\"{TF_SUBS}/B7_384_15.csv\")[\"target\"]\n    W[41] * B6_384_15,# = pd.read_csv(f\"{TF_SUBS}/B6_384_15.csv\")[\"target\"]\n    W[42] * B5_384_15,# = pd.read_csv(f\"{TF_SUBS}/B5_384_15.csv\")[\"target\"]\n\n\n], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission[\"target\"] = simple_avg\nsample_submission.to_csv(\"Final_simple_avg.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}