{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Here is the 5th solution (Ensemble). \n\nThe idea of normalization comes from VLADIMIR SLAYKOVSKIY\n\nThanks for sharing!","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport glob\nfrom tqdm.notebook import tqdm\nimport os","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def std(x):\n    return (x - np.mean(x)) / np.std(x)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSIONS = {\n    '../input/cite-submissionz/NN1_submission.csv':1,\n    \n    '../input/cite-submissionz/NN2_submission.csv':1,\n    \n    '../input/cite-submissionz/NN3_submission.csv':1,\n    \n    '../input/cite-submissionz/NN4_submission.csv':1,\n    \n    '../input/cite-submissionz/LGBM1submission.csv':1,\n    \n    '../input/cite-submissionz/LGBM2submission.csv':1,\n    \n    \n    \n    '../input/last-multi/Multi_l2_gelu_dayanddonor.csv':1,\n    \n    '../input/last-multi/Multi_l2_silu_dayanddonor.csv':1,\n    \n    '../input/last-multi/Multi_max_hardswish_dayanddonor.csv':1,\n    \n    '../input/last-multi/Multi_max_swish_dayanddonor.csv':1\n}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_ids = pd.read_parquet('../input/multimodal-single-cell-as-sparse-matrix/evaluation.parquet').cell_id","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_std_submission(path, cell_ids):\n    \"\"\"\n    Standardize submission per cell_id\n    \"\"\"\n    df = pd.read_csv(path)\n    df['cell_id'] = cell_ids    \n    vals = []\n    for idx, g in tqdm(df.groupby('cell_id', sort=False), desc=f'Standardizing {path}', miniters=1000):\n        vals.append(std(g.target).values)\n    vals = np.concatenate(vals)\n    return vals","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_ensemble(technology):\n    ensemble = None\n    for path in tqdm([path for path in SUBMISSIONS.keys() if technology in path], desc='Process submission'):\n        weight = SUBMISSIONS[path]\n        if ensemble is None:\n            ensemble = gen_std_submission(path, cell_ids) * weight\n        else:\n            ensemble += gen_std_submission(path, cell_ids) * weight\n    return ensemble","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PRED_SEGMENTS = [(0, 6812820), (6812820, 65744180)]\nensemble = []\nfor tech, (from_idx, to_idx) in tqdm(list(zip(['cite', 'multi'], PRED_SEGMENTS)), desc='Technology'):    \n    ensemble.append(gen_ensemble(tech)[from_idx: to_idx])\n    \n    \nensemble = np.concatenate(ensemble)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit = pd.read_parquet('../input/multimodal-single-cell-as-sparse-matrix/sample_submission.parquet')\ndf_submit['target'] = ensemble\ndf_submit.to_csv('submission.csv', index=False)\ndf_submit","metadata":{},"execution_count":null,"outputs":[]}]}