{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc, pickle, datetime, scipy.sparse\nimport pandas as pd\nimport numpy as np","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-11-08T07:42:23.807306Z","iopub.execute_input":"2022-11-08T07:42:23.807843Z","iopub.status.idle":"2022-11-08T07:42:23.813529Z","shell.execute_reply.started":"2022-11-08T07:42:23.807778Z","shell.execute_reply":"2022-11-08T07:42:23.812415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Greetings!  \n\n### This is a very simple analysis of weights of CITE part and MULTI parts in public score and private score\n\n### It was already diccussed [here](https://www.kaggle.com/competitions/open-problems-multimodal/discussion/349591) and [here](https://www.kaggle.com/competitions/open-problems-multimodal/discussion/359222)\n\n### Now I want to finish the discussion","metadata":{}},{"cell_type":"markdown","source":"#### Here is length of CITE part in submission file","metadata":{}},{"cell_type":"code","source":"cite_size = 48663 * 140\ncite_size","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:23.829099Z","iopub.execute_input":"2022-11-08T07:42:23.829779Z","iopub.status.idle":"2022-11-08T07:42:23.835638Z","shell.execute_reply.started":"2022-11-08T07:42:23.829742Z","shell.execute_reply":"2022-11-08T07:42:23.834814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### This is the length of DATA LEAK","metadata":{}},{"cell_type":"code","source":"leak_size = 7476 * 140\nleak_size","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:23.855806Z","iopub.execute_input":"2022-11-08T07:42:23.856663Z","iopub.status.idle":"2022-11-08T07:42:23.865478Z","shell.execute_reply.started":"2022-11-08T07:42:23.856616Z","shell.execute_reply":"2022-11-08T07:42:23.864113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Reading metadata file","metadata":{}},{"cell_type":"code","source":"metadata = pd.read_csv('../input/open-problems-multimodal/metadata.csv')","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:23.882101Z","iopub.execute_input":"2022-11-08T07:42:23.883250Z","iopub.status.idle":"2022-11-08T07:42:24.180095Z","shell.execute_reply.started":"2022-11-08T07:42:23.883197Z","shell.execute_reply":"2022-11-08T07:42:24.178787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Values of 'donor' variable. Donor 27678 is not presented in train data","metadata":{}},{"cell_type":"code","source":"metadata.donor.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:24.184191Z","iopub.execute_input":"2022-11-08T07:42:24.184837Z","iopub.status.idle":"2022-11-08T07:42:24.196384Z","shell.execute_reply.started":"2022-11-08T07:42:24.184799Z","shell.execute_reply":"2022-11-08T07:42:24.194789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CITE part of metadata","metadata":{}},{"cell_type":"code","source":"metadata_cite = metadata[metadata.technology == 'citeseq']","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:24.198115Z","iopub.execute_input":"2022-11-08T07:42:24.198593Z","iopub.status.idle":"2022-11-08T07:42:24.239353Z","shell.execute_reply.started":"2022-11-08T07:42:24.198557Z","shell.execute_reply":"2022-11-08T07:42:24.237745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Values of 'day' variable in CITE part. Day 7 is used for private test only","metadata":{}},{"cell_type":"code","source":"metadata_cite.day.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:24.241833Z","iopub.execute_input":"2022-11-08T07:42:24.242217Z","iopub.status.idle":"2022-11-08T07:42:24.264477Z","shell.execute_reply.started":"2022-11-08T07:42:24.242175Z","shell.execute_reply":"2022-11-08T07:42:24.263050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### MULTI part of metadata","metadata":{}},{"cell_type":"code","source":"metadata_multi = metadata[metadata.technology == 'multiome']","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:24.266289Z","iopub.execute_input":"2022-11-08T07:42:24.267034Z","iopub.status.idle":"2022-11-08T07:42:24.313548Z","shell.execute_reply.started":"2022-11-08T07:42:24.266983Z","shell.execute_reply":"2022-11-08T07:42:24.312190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Values of 'day' variable in MULTI part. Day 10 is used for private test only","metadata":{}},{"cell_type":"code","source":"metadata_multi.day.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:24.314764Z","iopub.execute_input":"2022-11-08T07:42:24.315346Z","iopub.status.idle":"2022-11-08T07:42:24.326262Z","shell.execute_reply.started":"2022-11-08T07:42:24.315309Z","shell.execute_reply":"2022-11-08T07:42:24.325101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Reading evaluation ids","metadata":{}},{"cell_type":"code","source":"evaluation_ids = pd.read_csv('../input/open-problems-multimodal/evaluation_ids.csv', index_col = 'row_id')","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:42:24.327610Z","iopub.execute_input":"2022-11-08T07:42:24.328786Z","iopub.status.idle":"2022-11-08T07:43:56.023050Z","shell.execute_reply.started":"2022-11-08T07:42:24.328728Z","shell.execute_reply":"2022-11-08T07:43:56.021902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Here we make set of cell_ids that were in leak and not used in submission","metadata":{}},{"cell_type":"code","source":"cite2exclude = set(evaluation_ids.cell_id.iloc[:leak_size])\nlen(cite2exclude)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:43:56.024599Z","iopub.execute_input":"2022-11-08T07:43:56.025429Z","iopub.status.idle":"2022-11-08T07:43:56.127949Z","shell.execute_reply.started":"2022-11-08T07:43:56.025393Z","shell.execute_reply":"2022-11-08T07:43:56.126479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Set of cell_ids used in submission. Leak ids are still here","metadata":{}},{"cell_type":"code","source":"\neval_ids = set(evaluation_ids.cell_id)\nlen(eval_ids)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:43:56.129676Z","iopub.execute_input":"2022-11-08T07:43:56.130058Z","iopub.status.idle":"2022-11-08T07:44:01.669811Z","shell.execute_reply.started":"2022-11-08T07:43:56.130016Z","shell.execute_reply":"2022-11-08T07:44:01.669013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of cell_ids without leak","metadata":{}},{"cell_type":"code","source":"len(eval_ids - cite2exclude)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.672869Z","iopub.execute_input":"2022-11-08T07:44:01.673993Z","iopub.status.idle":"2022-11-08T07:44:01.688977Z","shell.execute_reply.started":"2022-11-08T07:44:01.673955Z","shell.execute_reply":"2022-11-08T07:44:01.687440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of CITE cells used for PUBLIC score WITH leak cells","metadata":{}},{"cell_type":"code","source":"public_cite_cells = set(metadata_cite.cell_id[(metadata_cite.donor == 27678) & (metadata_cite.day != 7)])\nlen(public_cite_cells)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.690536Z","iopub.execute_input":"2022-11-08T07:44:01.690904Z","iopub.status.idle":"2022-11-08T07:44:01.709925Z","shell.execute_reply.started":"2022-11-08T07:44:01.690871Z","shell.execute_reply":"2022-11-08T07:44:01.709071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of CITE cells used for PUBLIC score WITHOUT leak cells. All 7476 leak cells are in PUBLIC part of submission","metadata":{}},{"cell_type":"code","source":"public_cite_cells = set(metadata_cite.cell_id[(metadata_cite.donor == 27678) & (metadata_cite.day != 7)]) - cite2exclude\nlen(public_cite_cells)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.711796Z","iopub.execute_input":"2022-11-08T07:44:01.712644Z","iopub.status.idle":"2022-11-08T07:44:01.732334Z","shell.execute_reply.started":"2022-11-08T07:44:01.712595Z","shell.execute_reply":"2022-11-08T07:44:01.731071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of CITE cells used for PRIVATE score","metadata":{}},{"cell_type":"code","source":"private_cite_cells = set(metadata_cite.cell_id[metadata_cite.day == 7]) - cite2exclude\nlen(private_cite_cells)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.734543Z","iopub.execute_input":"2022-11-08T07:44:01.735039Z","iopub.status.idle":"2022-11-08T07:44:01.755481Z","shell.execute_reply.started":"2022-11-08T07:44:01.734992Z","shell.execute_reply":"2022-11-08T07:44:01.754047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of MULTI cells used for PUBLIC score","metadata":{}},{"cell_type":"code","source":"public_multi_cells = set(metadata_multi.cell_id[(metadata_multi.donor == 27678) & (metadata_multi.day != 10)])\npublic_multi_cells = set.intersection(public_multi_cells, eval_ids)\nlen(public_multi_cells)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.756858Z","iopub.execute_input":"2022-11-08T07:44:01.757195Z","iopub.status.idle":"2022-11-08T07:44:01.776637Z","shell.execute_reply.started":"2022-11-08T07:44:01.757165Z","shell.execute_reply":"2022-11-08T07:44:01.775599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Weight of CITE part in PUBLIC score after leak","metadata":{}},{"cell_type":"code","source":"len(public_cite_cells) / (len(public_cite_cells) + len(public_multi_cells))","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.778354Z","iopub.execute_input":"2022-11-08T07:44:01.778691Z","iopub.status.idle":"2022-11-08T07:44:01.785885Z","shell.execute_reply.started":"2022-11-08T07:44:01.778662Z","shell.execute_reply":"2022-11-08T07:44:01.784481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Weight of MULTI part in PUBLIC score after leak","metadata":{}},{"cell_type":"code","source":"len(public_multi_cells) / (len(public_cite_cells) + len(public_multi_cells))","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.787594Z","iopub.execute_input":"2022-11-08T07:44:01.788007Z","iopub.status.idle":"2022-11-08T07:44:01.797472Z","shell.execute_reply.started":"2022-11-08T07:44:01.787971Z","shell.execute_reply":"2022-11-08T07:44:01.796625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Weight of CITE part in PUBLIC score before leak","metadata":{}},{"cell_type":"code","source":"21796 / (21796 + 7319)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.798726Z","iopub.execute_input":"2022-11-08T07:44:01.799226Z","iopub.status.idle":"2022-11-08T07:44:01.811928Z","shell.execute_reply.started":"2022-11-08T07:44:01.799194Z","shell.execute_reply":"2022-11-08T07:44:01.810886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Weight of MULTI part in PUBLIC score before leak","metadata":{}},{"cell_type":"code","source":"7319 / (21796 + 7319)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.813541Z","iopub.execute_input":"2022-11-08T07:44:01.814698Z","iopub.status.idle":"2022-11-08T07:44:01.824162Z","shell.execute_reply.started":"2022-11-08T07:44:01.814606Z","shell.execute_reply":"2022-11-08T07:44:01.823091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of MULTI cells used for PRIVATE score","metadata":{}},{"cell_type":"code","source":"private_multi_cells = set(metadata_multi.cell_id[metadata_multi.day == 10])\nprivate_multi_cells = set.intersection(private_multi_cells, eval_ids)\nlen(private_multi_cells)","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.825743Z","iopub.execute_input":"2022-11-08T07:44:01.826141Z","iopub.status.idle":"2022-11-08T07:44:01.853060Z","shell.execute_reply.started":"2022-11-08T07:44:01.826108Z","shell.execute_reply":"2022-11-08T07:44:01.852065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Weight of CITE part in PRIVATE score","metadata":{}},{"cell_type":"code","source":"len(private_cite_cells) / (len(private_cite_cells) + len(private_multi_cells))","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.855635Z","iopub.execute_input":"2022-11-08T07:44:01.856683Z","iopub.status.idle":"2022-11-08T07:44:01.863803Z","shell.execute_reply.started":"2022-11-08T07:44:01.856644Z","shell.execute_reply":"2022-11-08T07:44:01.862639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Weight of MULTI part in PRIVATE score","metadata":{}},{"cell_type":"code","source":"len(private_multi_cells) / (len(private_cite_cells) + len(private_multi_cells))","metadata":{"execution":{"iopub.status.busy":"2022-11-08T07:44:01.865094Z","iopub.execute_input":"2022-11-08T07:44:01.865467Z","iopub.status.idle":"2022-11-08T07:44:01.875792Z","shell.execute_reply.started":"2022-11-08T07:44:01.865434Z","shell.execute_reply":"2022-11-08T07:44:01.874496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusion\n\n#### Weights of CITE and MULTI parts initially were 3:1 for both PUBLIC & PRIVATE scores\n\n#### But after leak was deleted PUBLIC weights are 2:1","metadata":{}}]}