{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Acknowledgements\n\n- Built-upon https://www.kaggle.com/hamditarek/ensemble by @Tarek\n- Added in my own research work","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Ensemble","execution_count":null},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stacking","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_path = \"../input/jigsaw-multilingual-submission-files\"\nall_files = os.listdir(sub_path)\nall_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# no 0.9361 led from LB 0.9471 to 0.9472\n# no 0.9361 and 0.9383 led from LB 0.9472 to 0.9475\n# no 0.9361 and 0.9383 and 0.9416 led from LB 0.9475 to 0.9468\n# no 0.9361 and 0.9383 and 0.9459b led from LB 0.9475 to 0.9474\n# no 0.9361 and 0.9383 and 0.9459 led from LB 0.9475 to 0.9473\n# no 0.9361 and 0.9383, and added in 0.9422 from MLM (mixed language models) led from LB 0.9475 to 0.9477\n# no 0.9361 and 0.9383, and added in 0.9422 and 0.9432 from MLM (mixed language models) led from LB 0.9477 to 0.9478\n# no 0.9361 and 0.9383, and added in 0.9432 from MLM (mixed language models) and removing 0.9422 from MLM led from LB 0.9478 to 0.9479\n# (seems like removing very correlated lower LB results improves LB!)\n# ok removed 0.9459 which is very correlated to the others but LB dropped..\n# found 0.9428 for MLM also, added in and no 0.9361 and 0.9383, and added in 0.9432 & 0.9428 from MLM (mixed language models) led from LB 0.9479 to 0.9480\n# added 0.9431 MLM but LB dropped from 0.9480 to 0.9479\n# added 0.9404 parcor regularization (https://www.kaggle.com/aiaiooas/parcor-regularised-classification) which led to 0.9482\n# added 0.9366 wide and shallow CNN but dropped from 0.9482 to 0.9481\n# removed 0.9322 but dropped from 0.9482 to 0.9481\n# added 0.9426 MLM submission but dropped from 0.9482 to 0.9481\n# added second parcor sub with 0.9414 but remained at 0.9482\n# added 0.9409 parcor sub and LB increased from 0.9482 to 0.9485\nall_files = [f for f in all_files if '0.9361' not in f and '0.9383' not in f and '0.9422' not in f] #+ _all_files\nall_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0) for f in all_files]\nconcat_sub = pd.concat(outs, axis=1)\ncols = list(map(lambda x: \"jigsaw\" + str(x), range(len(concat_sub.columns))))\nconcat_sub.columns = cols\nconcat_sub.reset_index(inplace=True)\nconcat_sub.head()\nncol = concat_sub.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check correlation\nconcat_sub.iloc[:,1:ncol].corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corr = concat_sub.iloc[:,1:].corr()\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get the data fields ready for stacking\nconcat_sub['jigsaw_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)\nconcat_sub['jigsaw_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)\nconcat_sub['jigsaw_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)\nconcat_sub['jigsaw_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cutoff_lo = 0.3\ncutoff_hi = 0.7","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['toxic'] = concat_sub['jigsaw_mean']\nconcat_sub[['id', 'toxic']].to_csv('submission1.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub[['toxic']].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(concat_sub['jigsaw_mean'],bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['toxic'] = concat_sub['jigsaw_median']\nconcat_sub[['id', 'toxic']].to_csv('submission2.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub[['toxic']].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(concat_sub['jigsaw_median'],bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1, \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             0, concat_sub['jigsaw_median']))\nconcat_sub[['id', 'toxic']].to_csv('submission3.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(concat_sub['toxic'],bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), \n                                    concat_sub['jigsaw_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             concat_sub['jigsaw_min'], \n                                             concat_sub['jigsaw_mean']))\nconcat_sub[['id', 'toxic']].to_csv('submission.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(concat_sub['toxic'],bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['toxic'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), \n                                    concat_sub['jigsaw_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             concat_sub['jigsaw_min'], \n                                             concat_sub['jigsaw_median']))\nconcat_sub[['id', 'toxic']].to_csv('submission5.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(concat_sub['toxic'],bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Blending","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"all_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = [f for f in all_files if '0.9462' in f][0]\ndf2 = [f for f in all_files if '0.9459b' in f][0]\ndf3 = [f for f in all_files if '0.9459' in f and '0.9459b' not in f][0]\ndf4 = [f for f in all_files if '0.9450' in f][0]\ndf5 = [f for f in all_files if '0.9432' in f][0]\ndf6 = [f for f in all_files if '0.9428' in f][0]\ndf7 = [f for f in all_files if '0.9428b' in f][0]\ndf8 = [f for f in all_files if '0.9416' in f][0]\ndf9 = [f for f in all_files if '0.9414' in f and '0.9414b' not in f][0]\ndf10 = [f for f in all_files if '0.9414b' in f][0]\ndf11 = [f for f in all_files if '0.9409' in f][0]\ndf12 = [f for f in all_files if '0.9322' in f][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = pd.read_csv(os.path.join(sub_path, df1))\ndf2 = pd.read_csv(os.path.join(sub_path, df2))\ndf3 = pd.read_csv(os.path.join(sub_path, df3))\ndf4 = pd.read_csv(os.path.join(sub_path, df4))\ndf5 = pd.read_csv(os.path.join(sub_path, df5))\ndf6 = pd.read_csv(os.path.join(sub_path, df6))\ndf7 = pd.read_csv(os.path.join(sub_path, df7))\ndf8 = pd.read_csv(os.path.join(sub_path, df8))\ndf9 = pd.read_csv(os.path.join(sub_path, df9))\ndf10 = pd.read_csv(os.path.join(sub_path, df10))\ndf11 = pd.read_csv(os.path.join(sub_path, df11))\ndf12 = pd.read_csv(os.path.join(sub_path, df12))\ndf1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = df1.copy()[['id']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"W = 0.9462 + 2*0.9459 + 0.9450 + 2*0.9428 + 0.9422 + 0.9416 + 2*0.9414 + 0.9409 + 0.9322\nw1 = 0.9462/W\nw2 = 0.9459/W\nw3 = 0.9459/W\nw4 = 0.9450/W\nw5 = 0.9432/W\nw6 = 0.9428/W\nw7 = 0.9428/W\nw8 = 0.9416/W\nw9 = 0.9414/W\nw10 = 0.9414/W\nw11 = 0.9409/W\nw12 = 0.9322/W","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = (w1*df1['toxic'] + \n                w2*df2['toxic'] + \n                w3*df3['toxic'] + \n                w4*df4['toxic'] + \n                w5*df5['toxic'] + \n                w6*df6['toxic'] + \n                w7*df7['toxic'] + \n                w8*df8['toxic'] + \n                w9*df9['toxic'] + \n                w10*df10['toxic'] + \n                w11*df11['toxic'] + \n                w12*df12['toxic'])\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(sub.toxic,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission_blend.csv', index=False)\n#sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DEBoost","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.stats import gaussian_kde\nfrom math import sqrt\n\nclass DistanceMetrics:\n    '''\n    - non-built-in distance metrics are found here\n    - work in progress\n    '''\n\n    @staticmethod\n    def get_density(x, cov_factor=0.1):\n        #Produces a continuous density function for the data in 'x'. Some benefit may be gained from adjusting the cov_factor.\n        density = gaussian_kde(x)\n        density.covariance_factor = lambda:cov_factor\n        density._compute_covariance()\n        return density\n    \n    @classmethod\n    def battacharyya(cls, X1, X2, method='continuous'):\n        '''\n        Original Author: Eric Williamson (ericpaulwill@gmail.com)\n        Obtained from: https://github.com/EricPWilliamson/bhattacharyya-distance/blob/master/bhatta_dist.py\n        - This calculates the Bhattacharyya distance between vectors X1 and X2. X1 and X2 should be 1D numpy arrays representing the same\n          feature in two separate classes.\n        '''\n        #Combine X1 and X2, we'll use it later:\n        cX = np.concatenate((X1,X2))\n        if method == 'noiseless':\n            ###This method works well when the feature is qualitative (rather than quantitative). Each unique value is\n            ### treated as an individual bin.\n            uX = np.unique(cX)\n            A1 = len(X1) * (max(cX)-min(cX)) / len(uX)\n            A2 = len(X2) * (max(cX)-min(cX)) / len(uX)\n            bht = 0\n            for x in uX:\n                p1 = (X1==x).sum() / A1\n                p2 = (X2==x).sum() / A2\n                bht += sqrt(p1*p2) * (max(cX)-min(cX))/len(uX)\n\n        elif method == 'hist':\n            ###Bin the values into a hardcoded number of bins (This is sensitive to N_BINS)\n            N_BINS = int(len(X1) * 2)\n            #Bin the values:\n            h1 = np.histogram(X1,bins=N_BINS,range=(min(cX),max(cX)), density=True)[0]\n            h2 = np.histogram(X2,bins=N_BINS,range=(min(cX),max(cX)), density=True)[0]\n            #Calc coeff from bin densities:\n            bht = 0\n            for i in range(N_BINS):\n                p1 = h1[i]\n                p2 = h2[i]\n                bht += sqrt(p1*p2) * (max(cX)-min(cX))/N_BINS\n\n        elif method == 'autohist':\n            ###Bin the values into bins automatically set by np.histogram:\n            #Create bins from the combined sets:\n            # bins = np.histogram(cX, bins='fd')[1]\n            bins = np.histogram(cX, bins='doane')[1] #Seems to work best\n            # bins = np.histogram(cX, bins='auto')[1]\n\n            h1 = np.histogram(X1,bins=bins, density=True)[0]\n            h2 = np.histogram(X2,bins=bins, density=True)[0]\n\n            #Calc coeff from bin densities:\n            bht = 0\n            for i in range(len(h1)):\n                p1 = h1[i]\n                p2 = h2[i]\n                bht += sqrt(p1*p2) * (max(cX)-min(cX))/len(h1)\n\n        elif method == 'continuous':\n            ###Use a continuous density function to calculate the coefficient (This is the most consistent, but also slightly slow):\n            N_STEPS = int(len(X1) * 20)\n            #Get density functions:\n            d1 = cls.get_density(X1)\n            d2 = cls.get_density(X2)\n            #Calc coeff:\n            xs = np.linspace(min(cX),max(cX),N_STEPS)\n            bht = 0\n            for x in xs:\n                p1 = d1(x)\n                p2 = d2(x)\n                bht += sqrt(p1*p2)*(max(cX)-min(cX))/N_STEPS\n\n        else:\n            raise ValueError(\"The value of the 'method' parameter does not match any known method\")\n\n        ###Lastly, convert the coefficient into distance:\n        if bht==0:\n            return float('Inf')\n        else:\n            return -np.log(bht)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.spatial.distance import euclidean, cosine, jaccard, chebyshev, correlation, cityblock, canberra, braycurtis, hamming\n\ndef get_dist_preds(predictions, metric):\n    new_preds = []\n    for j, pred in enumerate(predictions):\n        distances = []\n        remaining_preds = predictions[:j] + predictions[j+1:]\n        for pred_ in remaining_preds:\n            if metric == 'euclid':\n                distances += [euclidean(pred_, pred)]\n            elif metric == 'cosine':\n                distances += [cosine(pred_, pred)]\n            elif metric == 'jaccard': # i think this is only for boolean\n                distances += [jaccard(pred_, pred)]\n            elif metric == 'chebyshev':\n                distances += [chebyshev(pred_, pred)]\n            elif metric == 'correlation':\n                distances += [correlation(pred_, pred)]\n            elif metric == 'cityblock':\n                distances += [cityblock(pred_, pred)]\n            elif metric == 'canberra':\n                distances += [canberra(pred_, pred)]\n            elif metric == 'braycurtis':\n                distances += [braycurtis(pred_, pred)]\n            elif metric == 'hamming': # i think this is only for boolean\n                distances += [hamming(pred_, pred)]\n            elif metric == 'battacharyya':\n                distances += [DistanceMetrics.battacharyya(pred_, pred, method='continuous')]\n        new_preds += [(pred, sum(distances))] # (precdictions, weight)\n\n    weights = [tup[1] for tup in new_preds]\n    W = sum(weights) # total weight\n    # those with lower distances have higher weight\n    # sort in ascending order of aggregated distances\n    preds_ascending_dist = sorted(new_preds, key=lambda x: x[1])\n    weights_descending = sorted(weights, reverse=True)\n    weighted_pred = sum([pred_tup[0]*(weights_descending[k]/W) for k, pred_tup in enumerate(preds_ascending_dist)])\n    return weighted_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs = [df1, df2, df3, df4, df5, df6, df7, df8, df9, df10, df11, df12]\npreds = [np.array(df['toxic']) for df in dfs]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_deboost = get_dist_preds(preds, 'correlation')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_sub = df1.copy()[['id']]\n_sub['toxic'] = df_deboost\n_sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(_sub.toxic,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_sub.to_csv('submission_deboost.csv', index=False)\n#_sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ranking Weights Ensemble","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_folds = len(dfs)\nn_folds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nrow = len(dfs[0])\nnrow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create fold partition indices\nnum_row_per_df = nrow//n_folds\nnum_row_per_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"remainder_rows = nrow - n_folds * num_row_per_df\nremainder_rows","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"end_indices = [num_row_per_df*i for i in range(1,n_folds)] + [nrow]\nend_indices","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"partitions = [[[]]*n_folds]*n_folds\npartitions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = {}\n\nfor i, df in enumerate(dfs):\n    data[str(i)] = df\n\nranks = pd.DataFrame(columns=data.keys())\nfor key in data.keys():\n    ranks[key] = data[key].toxic.rank(method='min')\nranks['Average'] = ranks.mean(axis=1)\nranks['Scaled Rank'] = (ranks['Average'] - ranks['Average'].min()) / (ranks['Average'].max() - ranks['Average'].min())\nranks.corr()[:1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/jigsaw-multilingual-toxic-comment-classification","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weights = [0.15, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.05, 0.05, 0.05, 0.05, 0.05]\nranks['Score'] = ranks[[str(i) for i in range(n_folds)]].mul(weights).sum(1) / ranks.shape[0]\nsubmission_lb = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsubmission_lb['toxic'] = ranks['Score']\nsubmission_lb.to_csv(\"WEIGHT_AVERAGE_RANK.csv\", index=False)\n#submission_lb.to_csv(\"submission.csv\", index=False)\nsubmission_lb.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission_lb.toxic,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bi-Weighted Correlations","execution_count":null},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"'''\nRecall\n------\ndf1 = [f for f in all_files if '0.9462' in f][0]\ndf2 = [f for f in all_files if '0.9459b' in f][0]\ndf3 = [f for f in all_files if '0.9459' in f and '0.9459b' not in f][0]\ndf4 = [f for f in all_files if '0.9450' in f][0]\ndf5 = [f for f in all_files if '0.9432' in f][0]\ndf6 = [f for f in all_files if '0.9428' in f][0]\ndf7 = [f for f in all_files if '0.9416' in f][0]\ndf8 = [f for f in all_files if '0.9322' in f][0]\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = [\"toxic\"]\nfor label in labels:\n    print(label)\n    print(np.round(np.corrcoef([df[label].rank(pct=True) for df in dfs]), 4))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The strategy is something like blending high correlations with low correlations and higher weights are assigned to higher ones. Thereafter, higher weights can be assigned to pairs with greater combined correlation.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Rough work\n- 0 & 5: 0.9177\n- 1 & 5: 0.9127\n- 2 & 7: 0.9133\n- 3 & 7: 0.9109\n- 4 & 5: 0.8829\n- 5 & 4: 0.8829\n- 6 & 4: 0.9018\n- 7 & 5: 0.8934\n\nMore weights to higher scoring results? E.g. in `0.2*(0.8*dfs[2][label] + 0.2*dfs[7][label])` more weight is assigned to dfs[2] which has much higher LB score than dfs[7].","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = dfs[0][['id']]\nfor label in labels:\n    preds = []\n    submission[label] = (\n        0.15*(0.9*dfs[0][label] + 0.1*dfs[5][label]) + \n        0.15*(0.9*dfs[1][label] + 0.1*dfs[5][label]) + \n        0.15*(0.9*dfs[2][label] + 0.1*dfs[7][label]) + \n        0.15*(0.9*dfs[3][label] + 0.1*dfs[7][label]) + \n        0.1*(0.5*dfs[4][label] + 0.5*dfs[5][label]) + \n        0.1*(0.5*dfs[5][label] + 0.5*dfs[4][label]) + \n        0.1*(0.1*dfs[6][label] + 0.9*dfs[4][label]) + \n        0.1*(0.5*dfs[7][label] + 0.5*dfs[5][label])\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(submission.toxic,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission_correlations.csv', index=False)\n#submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Power-Weighted Blend (Work In Progress)\n\n- https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/discussion/100661\n- https://medium.com/data-design/reaching-the-depths-of-power-geometric-ensembling-when-targeting-the-auc-metric-2f356ea3250e","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nall_files = \n['submission_0.9450.csv',\n 'submission_0.9459.csv',\n 'submission_0.9462.csv',\n 'submission_0.9416.csv',\n 'submission_0.9432.csv',\n 'submission_0.9428b.csv',\n 'submission_0.9414.csv',\n 'submission_0.9428.csv',\n 'submission_0.9459b.csv',\n 'submission_0.9322.csv']\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1 = [f for f in all_files if '0.9462' in f][0]\ndf2 = [f for f in all_files if '0.9459b' in f][0]\ndf3 = [f for f in all_files if '0.9459' in f and '0.9459b' not in f][0]\ndf4 = [f for f in all_files if '0.9450' in f][0]\ndf5 = [f for f in all_files if '0.9432' in f][0]\ndf6 = [f for f in all_files if '0.9428b' in f][0]\ndf7 = [f for f in all_files if '0.9428' in f][0]\ndf8 = [f for f in all_files if '0.9416' in f][0]\ndf9 = [f for f in all_files if '0.9414' in f][0]\ndf10 = [f for f in all_files if '0.9322' in f][0]\ndf1 = pd.read_csv(os.path.join(sub_path, df1))\ndf2 = pd.read_csv(os.path.join(sub_path, df2))\ndf3 = pd.read_csv(os.path.join(sub_path, df3))\ndf4 = pd.read_csv(os.path.join(sub_path, df4))\ndf5 = pd.read_csv(os.path.join(sub_path, df5))\ndf6 = pd.read_csv(os.path.join(sub_path, df6))\ndf7 = pd.read_csv(os.path.join(sub_path, df7))\ndf8 = pd.read_csv(os.path.join(sub_path, df8))\ndf9 = pd.read_csv(os.path.join(sub_path, df9))\ndf10 = pd.read_csv(os.path.join(sub_path, df10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs = [df1, df2, df3, df4, df5, df6, df7, df8, df9, df10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = 1.5 # power","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = [\"toxic\"]\nfor label in labels:\n    print(label)\n    print(np.round(np.corrcoef([df[label].rank(pct=True) for df in dfs]), 4))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_power = submission[['id']].copy()\nsub_power['toxic'] = (df1['toxic']**p * 0.2 + \n                      df2['toxic']**p * 0.1 + \n                      df3['toxic']**p * 0.1 + \n                      df4['toxic']**p * 0.15 + \n                      df5['toxic']**p * 0.25 + \n                      df6['toxic']**p * 0.025 + \n                      df7['toxic']**p * 0.025 + \n                      df8['toxic']**p * 0.05 + \n                      df9['toxic']**p * 0.05 + \n                      df10['toxic']**p * 0.05)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_power[['toxic']].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_power.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(sub_power.toxic,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_power.to_csv('submission_power_ensemble.csv', index=False)\n#sub_power.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pair-wise Blend","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = submission[['id']].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = 0.1*df10['toxic'] + 0.9*df9['toxic']\nsub['toxic'] = 0.51*df9['toxic'] + 0.49*sub['toxic']\n\nsub['toxic_temp'] = 0.1*df8['toxic'] + 0.9*df7['toxic']\nsub['toxic_temp'] = 0.51*df7['toxic'] + 0.49*sub['toxic_temp']\nsub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']\n\nsub['toxic_temp'] = 0.1*df6['toxic'] + 0.9*df5['toxic']\nsub['toxic_temp'] = 0.51*df5['toxic'] + 0.49*sub['toxic_temp']\nsub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']\n\nsub['toxic_temp'] = 0.1*df6['toxic'] + 0.9*df5['toxic']\nsub['toxic_temp'] = 0.51*df5['toxic'] + 0.49*sub['toxic_temp']\nsub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']\n\nsub['toxic_temp'] = 0.1*df4['toxic'] + 0.9*df3['toxic']\nsub['toxic_temp'] = 0.51*df3['toxic'] + 0.49*sub['toxic_temp']\nsub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']\n\nsub['toxic_temp'] = 0.1*df2['toxic'] + 0.9*df1['toxic']\nsub['toxic_temp'] = 0.51*df1['toxic'] + 0.49*sub['toxic_temp']\nsub['toxic'] = 0.5*sub['toxic_temp'] + 0.5*sub['toxic']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(sub.toxic,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[['id', 'toxic']].to_csv('submission_pairwise_blend_ensemble.csv', index=False)\n#sub[['id', 'toxic']].to_csv('submission.csv', index=False)\nsub[['id', 'toxic']].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Min-Max Power Scaling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = sub[['id']].copy()\nsub_final = sub[['id']].copy()\nsub['toxic'] = concat_sub['jigsaw_mean']\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"upper = 0.95\nlower = 0.05\nlp = 0.8 # lower power\nup = 1.5 # upper power","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[(sub['toxic'] <= lower)] = sub[(sub['toxic'] <= lower)]**lp\n#sub[(sub['toxic'] >= upper)] = sub[(sub['toxic'] >= upper)]**up","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_final['toxic'] = sub['toxic']\nsub_final.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(sub_final.toxic,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_final.to_csv('submission_minmax_power_scaling_ensemble.csv', index=False)\n#sub_final.to_csv('submission.csv', index=False)\nsub_final.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}