{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Stacking the Best Models\n<pre><b>\nThis Kernel shows how the scores can be improved using Stacking Method.\nCredit Goes to the following kernels\nref:\n1. https://www.kaggle.com/zaharch/keras-model-boosted-with-plates-leak\n2. https://www.kaggle.com/xhlulu/recursion-2-headed-efficientnet-2-stage-training\n3. https://www.kaggle.com/antgoldbloom/doing-inference-using-google-automl\n</b></pre>"},{"metadata":{},"cell_type":"markdown","source":"## Stat Stack"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from scipy.stats import rankdata\nimport glob\nLABELS = [\"sirna\"]\nall_files = glob.glob(\"../input/cellstack/*.csv\")\nall_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"outs = [pd.read_csv(f, index_col=0) for f in all_files]\nconcat_sub = pd.concat(outs, axis=1)\ncols = list(map(lambda x: \"m\" + str(x), range(len(concat_sub.columns))))\nconcat_sub.columns = cols\nconcat_sub.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"rank = np.tril(concat_sub.iloc[:,1:].corr().values,-1)\nm = (rank>0).sum()\nm_gmean, s = 0, 0\nfor n in range(min(rank.shape[0],m)):\n    mx = np.unravel_index(rank.argmin(), rank.shape)\n    w = (m-n)/(m+n/10)\n    print(w)\n    m_gmean += w*(np.log(concat_sub.iloc[:,mx[0]+1])+np.log(concat_sub.iloc[:,mx[1]+1]))/2\n    s += w\n    rank[mx] = 1\nm_gmean = np.exp(m_gmean/s).clip(0.0,1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"predict_list = []\npredict_list.append(pd.read_csv(\"../input/cellstack/submission-174.csv\")[LABELS].values)\npredict_list.append(pd.read_csv(\"../input/cellstack/submission-201.csv\")[LABELS].values)\npredict_list.append(pd.read_csv(\"../input/cellstack/submission-231.csv\")[LABELS].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nprint(\"Rank averaging on \", len(predict_list), \" files\")\npredictions = np.zeros_like(predict_list[0])\nfor predict in predict_list:\n    for i in range(1):\n        predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])  \n\npredictions = predictions /len(predict_list)\n\nsubmission = pd.read_csv('../input/recursion-cellular-image-classification/sample_submission.csv')\nsubmission[LABELS] = predictions\nsubmission.to_csv('AggStacker.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sub_path = \"../input/cellstack\"\nall_files = os.listdir(sub_path)\nall_files","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nouts = [pd.read_csv(os.path.join(sub_path, f), index_col=0) for f in all_files]\nconcat_sub = pd.concat(outs, axis=1)\ncols = list(map(lambda x: \"var\" + str(x), range(len(concat_sub.columns))))\nconcat_sub.columns = cols\nconcat_sub.reset_index(inplace=True)\nconcat_sub.head()\nncol = concat_sub.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check correlation\nconcat_sub.iloc[:,1:ncol].corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"corr = concat_sub.iloc[:,1:7].corr()\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# get the data fields ready for stacking\nconcat_sub['m_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)\nconcat_sub['m_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)\nconcat_sub['m_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"cutoff_lo = 0.8\ncutoff_hi = 0.2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mean Stacking"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna'] = m_gmean.astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_mean.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Median Stacking"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna']  = concat_sub['m_median'].astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pushout + Median Stacking\n>* Pushout strategy is bit aggresive"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna']  = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1, \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             0, concat_sub['m_median']))\nconcat_sub[['id_code','sirna']].to_csv('stack_pushout_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MinMax + Mean Stacking\n>* MinMax seems more gentle and it outperforms the previous one"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['m_mean'] = m_gmean.astype(int)\nconcat_sub['sirna']  = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), \n                                    concat_sub['m_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             concat_sub['m_min'], \n                                             concat_sub['m_mean'])).astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_minmax_mean.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MinMax + Median Stacking"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), \n                                    concat_sub['m_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             concat_sub['m_min'], \n                                             concat_sub['m_median'])).astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_minmax_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}