{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7712331,"sourceType":"datasetVersion","datasetId":4441094},{"sourceId":168702463,"sourceType":"kernelVersion"}],"dockerImageVersionId":30674,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"scape","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.8"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install gdown\n!gdown 10EI3VgW1_iclzl1J08iGvFbpWjamv9Xj -O data.zip\n!unzip data.zip\n!pip install git+https://github.com/scapeML/scape.git","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:34:18.749974Z","iopub.status.busy":"2024-03-30T21:34:18.749041Z","iopub.status.idle":"2024-03-30T21:35:10.042995Z","shell.execute_reply":"2024-03-30T21:35:10.042007Z","shell.execute_reply.started":"2024-03-30T21:34:18.749932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import scape\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\n\nscape.__version__, pd.__version__, np.__version__, tf.__version__","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:10.045346Z","iopub.status.busy":"2024-03-30T21:35:10.045025Z","iopub.status.idle":"2024-03-30T21:35:23.093217Z","shell.execute_reply":"2024-03-30T21:35:23.092239Z","shell.execute_reply.started":"2024-03-30T21:35:10.045315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_de = scape.io.load_slogpvals(\"_data/de_train.parquet\")\ndf_lfc = scape.io.load_lfc(\"_data/lfc_train.parquet\")\n\n# Make sure rows/columns are in the same order\ndf_lfc = df_lfc.loc[df_de.index, df_de.columns]\ndf_de.shape, df_lfc.shape","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:23.094807Z","iopub.status.busy":"2024-03-30T21:35:23.094261Z","iopub.status.idle":"2024-03-30T21:35:28.850366Z","shell.execute_reply":"2024-03-30T21:35:28.849383Z","shell.execute_reply.started":"2024-03-30T21:35:23.094781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_de","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_de = scape.io.load_slogpvals(\"_data/de_train.parquet\")\ndf_lfc = scape.io.load_lfc(\"_data/lfc_train.parquet\")\n\n# Make sure rows/columns are in the same order\ndf_lfc = df_lfc.loc[df_de.index, df_de.columns]\ndf_de.shape, df_lfc.shape","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:28.852160Z","iopub.status.busy":"2024-03-30T21:35:28.851826Z","iopub.status.idle":"2024-03-30T21:35:34.160128Z","shell.execute_reply":"2024-03-30T21:35:34.159065Z","shell.execute_reply.started":"2024-03-30T21:35:28.852132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell = \"NK cells\"\ndrugs = df_de.loc[df_de.index.get_level_values(\"cell_type\") == cell].index.get_level_values(\"sm_name\").unique().tolist()\nlen(drugs)","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:34.164294Z","iopub.status.busy":"2024-03-30T21:35:34.163992Z","iopub.status.idle":"2024-03-30T21:35:34.182373Z","shell.execute_reply":"2024-03-30T21:35:34.181420Z","shell.execute_reply.started":"2024-03-30T21:35:34.164268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_id_map = pd.read_csv(\"_data/id_map.zip\")\ndf_sub = pd.read_csv(\"/kaggle/input/top-16-base/submission.csv\", index_col = 0)\n\ndf_sub_ix = df_id_map.set_index([\"cell_type\", \"sm_name\"])\ndf_sub_ix","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:34.184227Z","iopub.status.busy":"2024-03-30T21:35:34.183812Z","iopub.status.idle":"2024-03-30T21:35:37.756696Z","shell.execute_reply":"2024-03-30T21:35:37.755655Z","shell.execute_reply.started":"2024-03-30T21:35:34.184165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Enhanced predictions\n\nWe selected a subset of the dataset consisting of the top 256 genes and top 60 drugs (sorted by variance). We trained the same model as before on this subset of the data, and used the 146 models to generate a median prediction on this subset of the data. We finally merged the results with the base predictions using a weighted mean (0.80 for the enhaced predictions in the subset of 256 genes and 60 drugs, 0.20 for the base prediction)","metadata":{}},{"cell_type":"code","source":"sub_drugs = df_sub_ix.index.get_level_values(\"sm_name\").unique().tolist()\nlen(sub_drugs)","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:37.758900Z","iopub.status.busy":"2024-03-30T21:35:37.758518Z","iopub.status.idle":"2024-03-30T21:35:37.766349Z","shell.execute_reply":"2024-03-30T21:35:37.765130Z","shell.execute_reply.started":"2024-03-30T21:35:37.758870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_n_top_drugs = 50\nn_genes = 256\n\n# This time, exclude control drugs for the calculation of the top genes, in order to\n# introduce more variability in the model\ntop_genes = scape.util.select_top_variable([df_de], k=n_genes, exclude_controls=True)","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:37.768546Z","iopub.status.busy":"2024-03-30T21:35:37.767987Z","iopub.status.idle":"2024-03-30T21:35:37.945432Z","shell.execute_reply":"2024-03-30T21:35:37.944224Z","shell.execute_reply.started":"2024-03-30T21:35:37.768507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_drug_effects = pd.DataFrame(df_de.T.pow(2).mean().pow(0.5).groupby(\"sm_name\").mean().sort_values(ascending=False), columns=[\"effect\"])\ndf_drug_effects[\"effect_norm\"] = (df_drug_effects[\"effect\"] / df_drug_effects[\"effect\"].sum())*100\ndf_drug_effects","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:37.947322Z","iopub.status.busy":"2024-03-30T21:35:37.946967Z","iopub.status.idle":"2024-03-30T21:35:38.121520Z","shell.execute_reply":"2024-03-30T21:35:38.120528Z","shell.execute_reply.started":"2024-03-30T21:35:37.947291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_sub_drugs = df_drug_effects.loc[sub_drugs].sort_values(\"effect\", ascending=False).head(min_n_top_drugs).index.tolist()\nlen(top_sub_drugs)","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:38.123639Z","iopub.status.busy":"2024-03-30T21:35:38.123337Z","iopub.status.idle":"2024-03-30T21:35:38.131993Z","shell.execute_reply":"2024-03-30T21:35:38.130779Z","shell.execute_reply.started":"2024-03-30T21:35:38.123613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_all_drugs = df_drug_effects.head(min_n_top_drugs).index.tolist()\ntop_drugs = set(top_all_drugs) | set(top_sub_drugs)\nlen(top_drugs)","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:38.133607Z","iopub.status.busy":"2024-03-30T21:35:38.133237Z","iopub.status.idle":"2024-03-30T21:35:38.144880Z","shell.execute_reply":"2024-03-30T21:35:38.143884Z","shell.execute_reply.started":"2024-03-30T21:35:38.133578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_de_c = df_de[df_de.index.get_level_values(\"sm_name\").isin(top_drugs)]\ndf_de_c = df_de_c.loc[:, top_genes]\ndf_de_c","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:38.146597Z","iopub.status.busy":"2024-03-30T21:35:38.146044Z","iopub.status.idle":"2024-03-30T21:35:38.204303Z","shell.execute_reply":"2024-03-30T21:35:38.203387Z","shell.execute_reply.started":"2024-03-30T21:35:38.146570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_lfc_c = df_lfc.loc[df_de_c.index, df_de_c.columns]\ndf_lfc_c.shape","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:38.206352Z","iopub.status.busy":"2024-03-30T21:35:38.205675Z","iopub.status.idle":"2024-03-30T21:35:38.219309Z","shell.execute_reply":"2024-03-30T21:35:38.218382Z","shell.execute_reply.started":"2024-03-30T21:35:38.206318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enhanced_predictions = []\nfor i, d in enumerate(top_drugs):\n    print(i, d)\n    scm = scape.model.create_default_model(n_genes, df_de_c, df_lfc_c)\n    result = scm.train(\n        val_cells=[cell], \n        val_drugs=[d],\n        input_columns=top_genes,\n        epochs=800,\n        output_folder=\"_models\",\n        config_file_name=\"enhanced_config.pkl\",\n        model_file_name=f\"enhanced_drug{i}.keras\",\n        baselines=[\"zero\", \"slogpval_drug\"],\n    )\n    # Collect prediction in the OOF data\n    df_pred = scm.predict(df_sub_ix)\n    enhanced_predictions.append(df_pred)\n\ndf_sub_enhanced = pd.DataFrame(np.median(enhanced_predictions, axis=0), index=df_sub_ix.index, columns=df_de_c.columns)\ndf_sub_enhanced.to_csv(\"enhanced_predictions.csv\")\ndf_sub_enhanced","metadata":{"execution":{"iopub.execute_input":"2024-03-30T21:35:38.223232Z","iopub.status.busy":"2024-03-30T21:35:38.222939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub_enhanced = pd.read_csv(\"enhanced_predictions.csv\")\ndisplay(df_sub_enhanced)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_focus = df_sub.copy()\ndisplay(df_focus)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_focus.update(df_sub_enhanced)\ndisplay(df_focus)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = 0.80 * df_focus + 0.20 * df_sub\ndf_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv(\"submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}