{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np, os\nfrom sklearn.metrics import roc_auc_score\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = '../input/ranzcr-oof-and-subs/'\nFILES = os.listdir(PATH)\nFILES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# removed subs and oofs with potential data leakage\n#FILES = [fn for fn in FILES if '9' not in fn and '10' not in fn]\nFILES = [fn for fn in FILES if '9' not in fn and '10' not in fn and '4' not in fn]\nFILES","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OOF = np.sort( [f for f in FILES if 'oof' in f] )\nOOF_CSV = [pd.read_csv(PATH+k) for k in OOF]\n\nprint('We have %i oof files...'%len(OOF))\nprint(); print(OOF)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OOF_CSV[0].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OOF_CSV[0].columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OOF_CSV[0].iloc[:, 1:12].columns.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_cols = OOF_CSV[0].iloc[:, 1:12].columns.tolist()\npred_cols = OOF_CSV[0].iloc[:, 15:].columns.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def macro_multilabel_auc(label, pred):\n    aucs = []\n    for i in range(len(target_cols)):\n        aucs.append(roc_auc_score(label[:, i], pred[:, i]))\n    #print(np.round(aucs, 4))\n    return np.mean(aucs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = np.zeros(( len(OOF_CSV[0]), len(OOF)*len(pred_cols)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k in range(len(OOF)):\n    x[:, int(k*len(pred_cols)):int((k+1)*len(pred_cols))] = OOF_CSV[k][pred_cols].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRUE = OOF_CSV[0][target_cols].values\nTRUE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all = []\nfor k in range(len(OOF)):\n    #auc = roc_auc_score(OOF_CSV[0].target,x[:,k])\n    auc = macro_multilabel_auc(OOF_CSV[0][target_cols].values, x[:, int(k*len(pred_cols)):int((k+1)*len(pred_cols))])\n    all.append(auc)\n    print('Model %i has OOF AUC = %.4f'%(k,auc))\n    \nm = [np.argmax(all)]; w = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build OOF Ensemble. Maximize CV Score"},{"metadata":{"trusted":true},"cell_type":"code","source":"old = np.max(all); \n\nRES = 500;#200; \nPATIENCE = 30;#10; \nTOL = 0.00005#0.0003\n'''\nRES = 200;#200; \nPATIENCE = 10;#10; \nTOL = 0.0003#0.0003\n'''\nDUPLICATES = False\n\nprint('Ensemble AUC = %.4f by beginning with model %i'%(old,m[0]))\nprint()\n\nfor kk in range(len(OOF)):\n    \n    # BUILD CURRENT ENSEMBLE\n    md = x[:,int(m[0]*len(pred_cols)):int((m[0]+1)*len(pred_cols))]\n    for i, k in enumerate(m[1:]):\n        md = w[i]*x[:, int(k*len(pred_cols)):int((k+1)*len(pred_cols))] + (1-w[i])*md\n        \n    # FIND MODEL TO ADD\n    mx = 0; mx_k = 0; mx_w = 0\n    print('Searching for best model to add... ')\n    \n    # TRY ADDING EACH MODEL\n    for k in range(len(OOF)):\n        print(k,', ',end='')\n        if not DUPLICATES and (k in m): continue\n            \n        # EVALUATE ADDING MODEL K WITH WEIGHTS W\n        bst_j = 0; bst = 0; ct = 0\n        for j in range(RES):\n            tmp = j/RES*x[:, int(k*len(pred_cols)):int((k+1)*len(pred_cols))] + (1-j/RES)*md\n            auc = macro_multilabel_auc(TRUE,tmp)\n            if auc>bst:\n                bst = auc\n                bst_j = j/RES\n            else: ct += 1\n            if ct>PATIENCE: break\n        if bst>mx:\n            mx = bst\n            mx_k = k\n            mx_w = bst_j\n            \n    # STOP IF INCREASE IS LESS THAN TOL\n    inc = mx-old\n    if inc<=TOL: \n        print(); print('No increase. Stopping.')\n        break\n        \n    # DISPLAY RESULTS\n    print(); #print(kk,mx,mx_k,mx_w,'%.5f'%inc)\n    print('Ensemble AUC = %.4f after adding model %i with weight %.3f. Increase of %.4f'%(mx,mx_k,mx_w,inc))\n    print()\n    \n    old = mx; m.append(mx_k); w.append(mx_w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('We are using models',m)\nprint('with weights',w)\nprint('and achieve ensemble AUC = %.4f'%old)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"md = x[:, int(m[0]*len(pred_cols)):int((m[0]+1)*len(pred_cols))]\nfor i, k in enumerate(m[1:]):\n    md = w[i]*x[:, int(k*len(pred_cols)):int((k+1)*len(pred_cols))] + (1-w[i])*md\nplt.hist(md,bins=100)\nplt.title('Ensemble OOF predictions')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = OOF_CSV[0].copy()\ndf.pred = md\ndf.to_csv('ensemble_oof.csv',index=False)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load SUB Files"},{"metadata":{"trusted":true},"cell_type":"code","source":"SUB = np.sort( [f for f in FILES if 'sub' in f] )\nSUB_CSV = [pd.read_csv(PATH+k) for k in SUB]\n\nprint('We have %i submission files...'%len(SUB))\nprint(); print(SUB)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# VERFIY THAT SUBMISSION FILES MATCH OOF FILES\na = np.array( [ int( x.split('_')[1].split('.')[0]) for x in SUB ] )\nb = np.array( [ int( x.split('_')[1].split('.')[0]) for x in OOF ] )\nif len(a)!=len(b):\n    print('ERROR submission files dont match oof files')\nelse:\n    for k in range(len(a)):\n        if a[k]!=b[k]: print('ERROR submission files dont match oof files')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = np.zeros(( len(SUB_CSV[0]), len(SUB)*len(pred_cols)))\nfor k in range(len(SUB)):\n    y[:, int(k*len(pred_cols)):int((k+1)*len(pred_cols))] = SUB_CSV[k][target_cols].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build SUB Ensemble"},{"metadata":{"trusted":true},"cell_type":"code","source":"md2 = y[:, int(m[0]*len(pred_cols)):int((m[0]+1)*len(pred_cols))]\nfor i, k in enumerate(m[1:]):\n    md2 = w[i]*y[:, int(k*len(pred_cols)):int((k+1)*len(pred_cols))] + (1-w[i])*md2\nplt.hist(md2,bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = SUB_CSV[0].copy()\ndf[target_cols] = md2\ndf.to_csv('ensemble_sub.csv',index=False)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('../input/ranzcr-clip-catheter-line-classification/sample_submission.csv',usecols=[0],index_col=0).join(pd.read_csv('ensemble_sub.csv').set_index('StudyInstanceUID')).fillna(0).to_csv('submission.csv')   ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}