{"cells":[{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":"## Stat Stack"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from scipy.stats import rankdata\n\nLABELS = [\"sirna\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"predict_list = []\npredict_list.append(pd.read_csv(\"../input/cellstack/submission (231).csv\")[LABELS].values)\npredict_list.append(pd.read_csv(\"../input/cellstack/submission(174).csv\")[LABELS].values)\npredict_list.append(pd.read_csv(\"../input/cellstack/submission(201).csv\")[LABELS].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nprint(\"Rank averaging on \", len(predict_list), \" files\")\npredictions = np.zeros_like(predict_list[0])\nfor predict in predict_list:\n    for i in range(1):\n        predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])  \n\npredictions = predictions /len(predict_list)\n\nsubmission = pd.read_csv('../input/recursion-cellular-image-classification/sample_submission.csv')\nsubmission[LABELS] = predictions\nsubmission.to_csv('AggStacker.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sub_path = \"../input/cellstack\"\nall_files = os.listdir(sub_path)\nall_files","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nouts = [pd.read_csv(os.path.join(sub_path, f), index_col=0) for f in all_files]\nconcat_sub = pd.concat(outs, axis=1)\ncols = list(map(lambda x: \"var\" + str(x), range(len(concat_sub.columns))))\nconcat_sub.columns = cols\nconcat_sub.reset_index(inplace=True)\nconcat_sub.head()\nncol = concat_sub.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check correlation\nconcat_sub.iloc[:,1:ncol].corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"corr = concat_sub.iloc[:,1:7].corr()\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# get the data fields ready for stacking\nconcat_sub['m_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)\nconcat_sub['m_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)\nconcat_sub['m_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"cutoff_lo = 0.8\ncutoff_hi = 0.2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mean Stacking"},{"metadata":{"trusted":false},"cell_type":"code","source":"rank = np.tril(concat_sub.iloc[:,1:ncol].corr().values,-1)\nm_gmean = 0\nm = 14\nfor n in range(m):\n    mx = np.unravel_index(rank.argmin(), rank.shape)\n    m_gmean += (m-n)*(np.log(concat_sub.iloc[:, mx[0]+1]) + np.log(concat_sub.iloc[:, mx[1]+1]))/2\n    rank[mx] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"m_gmean = np.exp(m_gmean/(n-1)**2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna'] = m_gmean.astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_mean.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Median Stacking"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna']  = concat_sub['m_median'].astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pushout + Median Stacking\n>* Pushout strategy is bit aggresive"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna']  = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1, \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             0, concat_sub['m_median']))\nconcat_sub[['id_code','sirna']].to_csv('stack_pushout_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MinMax + Mean Stacking\n>* MinMax seems more gentle and it outperforms the previous one"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['m_mean'] = m_gmean.astype(int)\nconcat_sub['sirna']  = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), \n                                    concat_sub['m_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             concat_sub['m_min'], \n                                             concat_sub['m_mean'])).astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_minmax_mean.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# MinMax + Median Stacking"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), \n                                    concat_sub['m_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),\n                                             concat_sub['m_min'], \n                                             concat_sub['m_median'])).astype(int)\nconcat_sub[['id_code','sirna']].to_csv('stack_minmax_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Averaging Ranks"},{"metadata":{"trusted":false},"cell_type":"code","source":"concat_sub['sirna'] = concat_sub['var0'].rank(method ='min') + concat_sub['var1'].rank(method ='min') + concat_sub['var2'].rank(method ='min') \nconcat_sub['sirna'] = (concat_sub['sirna']-concat_sub['sirna'].min())/(concat_sub['sirna'].max() - concat_sub['sirna'].min()).astype(int)\nconcat_sub.describe()\nconcat_sub[['id_code','sirna']].to_csv('stack_rank.csv', index=False, float_format='%.8f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}