{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## This notebook forked from [Optimise Blending Weights with Bonus :0](https://www.kaggle.com/gogo827jz/optimise-blending-weights-with-bonus-0) by [Yirun Zhang](https://www.kaggle.com/gogo827jz).","metadata":{}},{"cell_type":"code","source":"import datetime\nimport pandas as pd\nfrom time import time\n\nimport numpy as np\nfrom scipy.optimize import minimize, fsolve\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import log_loss","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:25:45.272090Z","iopub.execute_input":"2021-08-14T03:25:45.272474Z","iopub.status.idle":"2021-08-14T03:25:46.154868Z","shell.execute_reply.started":"2021-08-14T03:25:45.272389Z","shell.execute_reply":"2021-08-14T03:25:46.154038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define major metrics","metadata":{}},{"cell_type":"code","source":"def log_loss_numpy(y_pred):\n    loss = 0\n    y_pred_clip = np.clip(y_pred, 1e-15, 1 - 1e-15)\n    for i in range(y_pred.shape[1]):\n        loss += - np.mean(y_true[:, i] * np.log(y_pred_clip[:, i]) + (1 - y_true[:, i]) * np.log(1 - y_pred_clip[:, i]))\n    return loss / y_pred.shape[1]\n\ndef calc_auc(y_pred):\n    auc = 0\n    for task_id in range(y_pred.shape[1]):\n        auc -= roc_auc_score(y_true=y_true[:, task_id], y_score=y_pred[:, task_id])\n    return auc / y_pred.shape[1]\n\ndef func_numpy_metric2(weights):\n    oof_blend = np.tensordot(weights, oof, axes = ((0), (0)))\n    score = calc_auc(oof_blend)\n    return score\n\ndef func_numpy_metric(weights):\n    oof_blend = np.tensordot(weights, oof, axes = ((0), (0)))\n    score = log_loss_numpy(oof_blend)\n    return score\n\ndef grad_func(weights):\n    oof_clip = np.clip(oof, 1e-15, 1 - 1e-15)\n    gradients = np.zeros(oof.shape[0])\n    for i in range(oof.shape[0]):\n        a, b, c = y_true, oof_clip[i], 0\n        for j in range(oof.shape[0]):\n            if j != i:\n                c += weights[j] * oof_clip[j]\n        gradients[i] = -np.mean((-a*b+(b**2)*weights[i]+b*c)/((b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))\n    return gradients","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:25:46.156274Z","iopub.execute_input":"2021-08-14T03:25:46.156595Z","iopub.status.idle":"2021-08-14T03:25:46.167791Z","shell.execute_reply.started":"2021-08-14T03:25:46.156570Z","shell.execute_reply":"2021-08-14T03:25:46.166892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### open oof csv files","metadata":{}},{"cell_type":"code","source":"train_targets_scored = pd.read_csv('../input/oof-weight-optimizer-public/model1.csv')\ntarget_columns = [c for c in train_targets_scored.columns if c not in ['id']]\n\ny_true = train_targets_scored['target'].values\ny_true = y_true.reshape(-1,1)\noof_dict = {\n    'model1':\"../input/oof-weight-optimizer-public/model1.csv\",\n    'model2':\"../input/oof-weight-optimizer-public/model2.csv\",\n    'model3':\"../input/oof-weight-optimizer-public/model3.csv\",   \n    'model4':\"../input/oof-weight-optimizer-public/model4.csv\",   \n    'model5':\"../input/oof-weight-optimizer-public/model5.csv\",   \n    'model6':\"../input/oof-weight-optimizer-public/model6.csv\",   \n}\n\ntarget_columns  = ['preds']\noof = np.zeros((len(oof_dict), y_true.shape[0], len(target_columns) ))\n\nfor i in range(oof.shape[0]):\n    valid = pd.read_csv( list(oof_dict.values())[i] )\n    valid = train_targets_scored.drop(columns=target_columns).merge(valid[['id']+target_columns], on='id', how='left').fillna(0)\n    oof[i] = valid[target_columns].values ","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:25:46.169435Z","iopub.execute_input":"2021-08-14T03:25:46.169754Z","iopub.status.idle":"2021-08-14T03:25:58.643750Z","shell.execute_reply.started":"2021-08-14T03:25:46.169725Z","shell.execute_reply":"2021-08-14T03:25:58.642774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nlog_loss_scores = {}\nfor n, key in enumerate(oof_dict.keys()):\n    score_oof = calc_auc(oof[n])\n    log_loss_scores[key] = score_oof\n    print(f'{key:40s} CV:', score_oof)\n    \nprint('-' * 60)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:25:58.645355Z","iopub.execute_input":"2021-08-14T03:25:58.645801Z","iopub.status.idle":"2021-08-14T03:25:59.851347Z","shell.execute_reply.started":"2021-08-14T03:25:58.645762Z","shell.execute_reply":"2021-08-14T03:25:59.850177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observe correlation","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.style as style\nimport seaborn as sns\nfrom matplotlib import pyplot\nfrom matplotlib.ticker import ScalarFormatter\nsns.set_context(\"talk\")\nstyle.use('fivethirtyeight')\n\nsubmit = pd.read_csv(\"../input/g2net-gravitational-wave-detection/training_labels.csv\")\nsubs = np.zeros((len(oof_dict), y_true.shape[0], len(target_columns) ))\n\nfor i, p in enumerate(oof_dict.keys()):\n    print(i,p)\n    tmp = pd.read_csv(oof_dict[p])\n    valid = train_targets_scored.drop(columns=target_columns).merge(tmp[['id']+target_columns], on='id', how='left').fillna(0)\n    subs[i,:,:] = valid[target_columns].values \n\ncorr = np.corrcoef(subs.reshape(len(oof_dict), -1))\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(15, 12))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, cmap=cmap, annot=True, fmt=\"g\",\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5}, ax=ax)\nax.set_ylim(corr.shape[0], 0)\nplt.yticks(rotation=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:25:59.853058Z","iopub.execute_input":"2021-08-14T03:25:59.853414Z","iopub.status.idle":"2021-08-14T03:26:08.841617Z","shell.execute_reply.started":"2021-08-14T03:25:59.853375Z","shell.execute_reply":"2021-08-14T03:26:08.840781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Blending Weights Optimize","metadata":{}},{"cell_type":"code","source":"tol = 1e-10\ninit_guess = [1 / oof.shape[0]] * oof.shape[0]\nbnds = [(0, 1) for _ in range(oof.shape[0])]\ncons = {'type': 'eq', \n        'fun': lambda x: np.sum(x) - 1, \n        'jac': lambda x: [1] * len(x)}\n\nprint('Inital Blend OOF:', func_numpy_metric2(init_guess))\nstart_time = time()\n\nres_scipy = minimize(fun = func_numpy_metric2, \n                     x0 = init_guess, \n                     method = 'Nelder-Mead', \n                     #method='SLSQP',\n                     jac = grad_func, \n                     bounds = bnds, \n                     constraints = cons, \n                     tol = tol)\n\nprint(f'[{str(datetime.timedelta(seconds = time() - start_time))[2:7]}] Optimised Blend OOF:', res_scipy.fun)\nprint('Optimised Weights:', res_scipy.x)\nprint('-' * 70)\n\nfor n, key in enumerate(oof_dict.keys()):\n    print(f'{key:40s} Optimised Weights:', res_scipy.x[n])","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:26:08.842845Z","iopub.execute_input":"2021-08-14T03:26:08.843109Z","iopub.status.idle":"2021-08-14T03:28:21.732795Z","shell.execute_reply.started":"2021-08-14T03:26:08.843081Z","shell.execute_reply":"2021-08-14T03:28:21.731585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The nelder-mead method requires normalization because the sum of the weights does not equal 1.","metadata":{}},{"cell_type":"code","source":"ws = [ res_scipy.x[i] for i in range(len(oof_dict.keys()))]\nws/np.sum(ws)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:21.733982Z","iopub.execute_input":"2021-08-14T03:28:21.734210Z","iopub.status.idle":"2021-08-14T03:28:21.742145Z","shell.execute_reply.started":"2021-08-14T03:28:21.734185Z","shell.execute_reply":"2021-08-14T03:28:21.740978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Rank Averaging from [tips: rank averaging](https://www.kaggle.com/c/ranzcr-clip-catheter-line-classification/discussion/205564) by [Tawara](https://www.kaggle.com/ttahara).","metadata":{}},{"cell_type":"code","source":"oof = np.zeros((len(oof_dict), y_true.shape[0], len(target_columns) ))\n\nfor i in range(oof.shape[0]):\n    valid = pd.read_csv( list(oof_dict.values())[i] )\n    valid = train_targets_scored.drop(columns=target_columns).merge(valid[['id']+target_columns], on='id', how='left').fillna(0)\n    oof[i] = valid[target_columns].values","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:21.744037Z","iopub.execute_input":"2021-08-14T03:28:21.744254Z","iopub.status.idle":"2021-08-14T03:28:29.555845Z","shell.execute_reply.started":"2021-08-14T03:28:21.744231Z","shell.execute_reply":"2021-08-14T03:28:29.554650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import rankdata\nfor i in range(oof.shape[0]):\n    for j in range(len(target_columns)):\n        oof[i,:,j] = rankdata(oof[i,:,j],method='average')","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:29.557634Z","iopub.execute_input":"2021-08-14T03:28:29.557907Z","iopub.status.idle":"2021-08-14T03:28:30.075575Z","shell.execute_reply.started":"2021-08-14T03:28:29.557884Z","shell.execute_reply":"2021-08-14T03:28:30.074266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"overall_oof = np.mean(oof, axis=0)/np.max(oof)\nprint(calc_auc(overall_oof))","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:30.076763Z","iopub.execute_input":"2021-08-14T03:28:30.076992Z","iopub.status.idle":"2021-08-14T03:28:30.275215Z","shell.execute_reply.started":"2021-08-14T03:28:30.076969Z","shell.execute_reply":"2021-08-14T03:28:30.274071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making submission","metadata":{}},{"cell_type":"code","source":"predictions = []\ntarget_columns = ['target']\nlabel_cols = target_columns","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:30.276281Z","iopub.execute_input":"2021-08-14T03:28:30.276497Z","iopub.status.idle":"2021-08-14T03:28:30.281283Z","shell.execute_reply.started":"2021-08-14T03:28:30.276475Z","shell.execute_reply":"2021-08-14T03:28:30.280183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_dict = {\n    'model1':\"../input/oof-weight-optimizer-public/submission1.csv\",\n    'model2':\"../input/oof-weight-optimizer-public/submission2.csv\",\n    'model3':\"../input/oof-weight-optimizer-public/submission3.csv\",\n    'model4':'../input/oof-weight-optimizer-public/submission4.csv',\n    'model5':'../input/oof-weight-optimizer-public/submission5.csv',\n    'model6':'../input/oof-weight-optimizer-public/submission6.csv',\n}","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:30.282400Z","iopub.execute_input":"2021-08-14T03:28:30.282682Z","iopub.status.idle":"2021-08-14T03:28:30.295398Z","shell.execute_reply.started":"2021-08-14T03:28:30.282656Z","shell.execute_reply":"2021-08-14T03:28:30.294164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(preds_dict)):\n    sub = pd.read_csv(list(preds_dict.values())[i])\n    predictions.append(sub)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:30.296661Z","iopub.execute_input":"2021-08-14T03:28:30.296901Z","iopub.status.idle":"2021-08-14T03:28:31.547428Z","shell.execute_reply.started":"2021-08-14T03:28:30.296873Z","shell.execute_reply":"2021-08-14T03:28:31.546601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = predictions[0]['target'].values\ntrain_targets_scored = predictions[0]\npreds = np.zeros((len(preds_dict), y_pred.shape[0], len(target_columns) ))","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:31.548348Z","iopub.execute_input":"2021-08-14T03:28:31.548691Z","iopub.status.idle":"2021-08-14T03:28:31.562501Z","shell.execute_reply.started":"2021-08-14T03:28:31.548659Z","shell.execute_reply":"2021-08-14T03:28:31.561429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Using optimized weight","metadata":{}},{"cell_type":"code","source":"weights = ws\nweights = weights/np.sum(weights)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:31.564004Z","iopub.execute_input":"2021-08-14T03:28:31.564236Z","iopub.status.idle":"2021-08-14T03:28:31.579392Z","shell.execute_reply.started":"2021-08-14T03:28:31.564214Z","shell.execute_reply":"2021-08-14T03:28:31.578288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weighted_y_pred = pd.DataFrame()\nweighted_y_pred['id'] = predictions[0]['id']\nfor column in label_cols:\n    column_data = []\n    for i in range(len(preds_dict)):\n        column_data.append(predictions[i][column] * weights[i])\n    weighted_y_pred[column] = np.sum(column_data, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:31.580488Z","iopub.execute_input":"2021-08-14T03:28:31.580749Z","iopub.status.idle":"2021-08-14T03:28:31.631912Z","shell.execute_reply.started":"2021-08-14T03:28:31.580726Z","shell.execute_reply":"2021-08-14T03:28:31.630985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = weighted_y_pred\nsubmission.to_csv('submission_optimized.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:31.633039Z","iopub.execute_input":"2021-08-14T03:28:31.633321Z","iopub.status.idle":"2021-08-14T03:28:32.123447Z","shell.execute_reply.started":"2021-08-14T03:28:31.633293Z","shell.execute_reply":"2021-08-14T03:28:32.122774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Simple averaged","metadata":{}},{"cell_type":"code","source":"weighted_y_pred = pd.DataFrame()\nweighted_y_pred['id'] = predictions[0]['id']\nfor column in label_cols:\n    column_data = []\n    for i in range(len(preds_dict)):\n        column_data.append(predictions[i][column] / len(preds_dict))\n    weighted_y_pred[column] = np.sum(column_data, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:32.124413Z","iopub.execute_input":"2021-08-14T03:28:32.124626Z","iopub.status.idle":"2021-08-14T03:28:32.153630Z","shell.execute_reply.started":"2021-08-14T03:28:32.124603Z","shell.execute_reply":"2021-08-14T03:28:32.152163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = weighted_y_pred\nsubmission.to_csv('submission_mean.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:32.154811Z","iopub.execute_input":"2021-08-14T03:28:32.155117Z","iopub.status.idle":"2021-08-14T03:28:32.631836Z","shell.execute_reply.started":"2021-08-14T03:28:32.155085Z","shell.execute_reply":"2021-08-14T03:28:32.630745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Rank averaging","metadata":{}},{"cell_type":"code","source":"preds = np.zeros((len(preds_dict), y_pred.shape[0], len(target_columns) ))\n#print(preds.shape)\nweighted_y_pred = pd.DataFrame()\nweighted_y_pred['id'] = predictions[0]['id']\nfor i in range(preds.shape[0]):\n    valid = pd.read_csv( list(preds_dict.values())[i] )\n    valid = train_targets_scored.drop(columns=target_columns).merge(valid[['id']+target_columns], on='id', how='left').fillna(0)\n    preds[i] = valid[target_columns].values\n    \nfrom scipy.stats import rankdata\n\nfor i in range(preds.shape[0]):\n    for j in range(len(target_columns)):\n        preds[i,:,j] = rankdata(preds[i,:,j],method='average')\n        \nweighted_y_pred[label_cols] = np.mean(preds,axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:32.633498Z","iopub.execute_input":"2021-08-14T03:28:32.633858Z","iopub.status.idle":"2021-08-14T03:28:34.839236Z","shell.execute_reply.started":"2021-08-14T03:28:32.633821Z","shell.execute_reply":"2021-08-14T03:28:34.838131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = weighted_y_pred\nsubmission.to_csv('submission_rank.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-14T03:28:34.840265Z","iopub.execute_input":"2021-08-14T03:28:34.840496Z","iopub.status.idle":"2021-08-14T03:28:35.294805Z","shell.execute_reply.started":"2021-08-14T03:28:34.840469Z","shell.execute_reply":"2021-08-14T03:28:35.293972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}