{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:21:53.350411Z","iopub.execute_input":"2022-11-18T09:21:53.350837Z","iopub.status.idle":"2022-11-18T09:21:53.356757Z","shell.execute_reply.started":"2022-11-18T09:21:53.350805Z","shell.execute_reply":"2022-11-18T09:21:53.355641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CITE_SUBMISSIONS = {\n    \n    # NN_online 0.89580\n    \"../input/cite-submissions/submission_nn_online.csv\":0.7,\n\n    # CNN 0.8953\n    \"../input/cite-submissions/submission_cnn/submission\":0.7,\n\n    # NN 0.89585\n    \"../input/cite-submissions/submission_nn/submission\":1.2,\n\n    # stacking 0.901\n    \"../input/cite-submissions/submission_stacking/submission\":1.5,\n\n    # kernel_rigde 0.893258\n    \"../input/cite-submissions/submission_krr/submission\":0.5,\n\n    # LGBM 0.8927\n    \"../input/cite-submissions/submission_lgbm/submission\":0.1,\n \n    # Catboost 0.8910\n    \"../input/cite-submissions/submission_catboost/submission\":0.1,\n\n}","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-18T09:28:25.857860Z","iopub.execute_input":"2022-11-18T09:28:25.858345Z","iopub.status.idle":"2022-11-18T09:28:25.865134Z","shell.execute_reply.started":"2022-11-18T09:28:25.858310Z","shell.execute_reply":"2022-11-18T09:28:25.863750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MULTI_SUBMISSIONS = {\n    # stacking \n    \"../input/multi-submissions/submission_stacking/submission\":1.5,\n\n    # NN 0.6691\n    \"../input/multi-submissions/submission_nn_1/submission\":1,\n\n     # NN 0.67256\n    \"../input/multi-submissions/submission_nn_2/submission\":1,\n\n     # NN \n    \"../input/multi-submissions/submission_nn_3/submission\":0.5,\n    \n    # Catboost\n    \"../input/multi-submissions/submission_catboost.csv\": 0.3,\n\n    # LGBM\n    \"../input/multi-submissions/submission_lgbm/submission\":0.1,\n    \n    # best online\n    \"../input/multi-submissions/submission_online.csv\":0.5\n\n}","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:25:42.692916Z","iopub.execute_input":"2022-11-18T09:25:42.693389Z","iopub.status.idle":"2022-11-18T09:25:42.699898Z","shell.execute_reply.started":"2022-11-18T09:25:42.693355Z","shell.execute_reply":"2022-11-18T09:25:42.698560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_df = pd.read_csv(\"../input/cite-submissions/submission_catboost/submission\",index_col =0)\nbest_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:15:11.383354Z","iopub.execute_input":"2022-11-18T09:15:11.383755Z","iopub.status.idle":"2022-11-18T09:16:38.470865Z","shell.execute_reply.started":"2022-11-18T09:15:11.383723Z","shell.execute_reply":"2022-11-18T09:16:38.468908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for path, weight in CITE_SUBMISSIONS.items():\n    break\nbest_df = pd.read_csv(path,index_col =0)\nbest_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:13:27.720559Z","iopub.execute_input":"2022-11-18T09:13:27.721021Z","iopub.status.idle":"2022-11-18T09:14:52.599471Z","shell.execute_reply.started":"2022-11-18T09:13:27.720977Z","shell.execute_reply":"2022-11-18T09:14:52.598192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"part = [j for i,j in CITE_SUBMISSIONS.items()]\ns_part = sum(part)\npart = [i/s_part for i in part]\npart","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:11:20.779104Z","iopub.execute_input":"2022-11-18T09:11:20.780160Z","iopub.status.idle":"2022-11-18T09:11:20.791482Z","shell.execute_reply.started":"2022-11-18T09:11:20.780110Z","shell.execute_reply":"2022-11-18T09:11:20.790381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"part = [j for i,j in MULTI_SUBMISSIONS.items()]\ns_part = sum(part)\npart = [i/s_part for i in part]\npart","metadata":{"execution":{"iopub.status.busy":"2022-11-18T10:17:29.655432Z","iopub.execute_input":"2022-11-18T10:17:29.655954Z","iopub.status.idle":"2022-11-18T10:17:29.663609Z","shell.execute_reply.started":"2022-11-18T10:17:29.655916Z","shell.execute_reply":"2022-11-18T10:17:29.662744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_ids = pd.read_csv(r'../input/open-problems-multimodal/evaluation_ids.csv').cell_id","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:12:08.256514Z","iopub.execute_input":"2022-11-18T09:12:08.257070Z","iopub.status.idle":"2022-11-18T09:13:27.718551Z","shell.execute_reply.started":"2022-11-18T09:12:08.257024Z","shell.execute_reply":"2022-11-18T09:13:27.717488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def std(x):\n    return (x - np.mean(x)) / np.std(x)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:25:47.187804Z","iopub.execute_input":"2022-11-18T09:25:47.188246Z","iopub.status.idle":"2022-11-18T09:25:47.194328Z","shell.execute_reply.started":"2022-11-18T09:25:47.188214Z","shell.execute_reply":"2022-11-18T09:25:47.193176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_std_submission(path, cell_ids):\n    \"\"\"\n    Standardize submission per cell_id\n    \"\"\"\n    df = pd.read_csv(path)\n    df['cell_id'] = cell_ids    \n    vals = []\n    for idx, g in tqdm(df.groupby('cell_id', sort=False), desc=f'Standardizing {path}', miniters=1000):\n        vals.append(std(g.target).values)\n    del df\n    gc.collect()\n    vals = np.concatenate(vals)\n    return vals","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:25:47.692754Z","iopub.execute_input":"2022-11-18T09:25:47.693169Z","iopub.status.idle":"2022-11-18T09:25:47.700509Z","shell.execute_reply.started":"2022-11-18T09:25:47.693136Z","shell.execute_reply":"2022-11-18T09:25:47.699604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gen_ensemble(SUBMISSIONS):\n    ensemble = None\n    for path in tqdm([path for path in SUBMISSIONS.keys()], desc='Process submission'):\n        weight = SUBMISSIONS[path]\n        if ensemble is None:\n            ensemble = gen_std_submission(path, cell_ids) * weight\n        else:\n            ensemble += gen_std_submission(path, cell_ids) * weight\n    gc.collect()\n    return ensemble","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:25:48.337409Z","iopub.execute_input":"2022-11-18T09:25:48.337843Z","iopub.status.idle":"2022-11-18T09:25:48.344968Z","shell.execute_reply.started":"2022-11-18T09:25:48.337807Z","shell.execute_reply":"2022-11-18T09:25:48.343597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nPRED_SEGMENTS = [(0, 6812820),(6812820, 65744180) ] #\nSUBMISSIONS = [CITE_SUBMISSIONS,MULTI_SUBMISSIONS]\nensemble = []\nfor sub, (from_idx, to_idx) in tqdm(list(zip(SUBMISSIONS, PRED_SEGMENTS)), desc='Technology'):    \n    ensemble.append(gen_ensemble(sub)[from_idx: to_idx])\n    \n    \nensemble = np.concatenate(ensemble)","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:28:30.462245Z","iopub.execute_input":"2022-11-18T09:28:30.462631Z","iopub.status.idle":"2022-11-18T09:51:41.741483Z","shell.execute_reply.started":"2022-11-18T09:28:30.462601Z","shell.execute_reply":"2022-11-18T09:51:41.740136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit = pd.read_csv('../input/open-problems-multimodal/sample_submission.csv')\ndf_submit['target'] = ensemble\nprint(\"ensemble -> submission.zip\")\ndf_submit.to_csv('submission.zip', index=False)\ndf_submit","metadata":{"execution":{"iopub.status.busy":"2022-11-18T09:52:10.908109Z","iopub.execute_input":"2022-11-18T09:52:10.908544Z","iopub.status.idle":"2022-11-18T09:58:49.317883Z","shell.execute_reply.started":"2022-11-18T09:52:10.908511Z","shell.execute_reply":"2022-11-18T09:58:49.316588Z"},"trusted":true},"execution_count":null,"outputs":[]}]}