{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\n\nimport tensorflow as tf\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OrdinalEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import RandomForestRegressor\n\nfrom sklearn.model_selection import GridSearchCV","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":14.706293,"end_time":"2023-10-26T12:41:58.325892","exception":false,"start_time":"2023-10-26T12:41:43.619599","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:05.927651Z","iopub.execute_input":"2023-10-26T19:39:05.928622Z","iopub.status.idle":"2023-10-26T19:39:05.937391Z","shell.execute_reply.started":"2023-10-26T19:39:05.928576Z","shell.execute_reply":"2023-10-26T19:39:05.936240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nde_train = pl.scan_parquet('/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet')\nde_train_df = de_train.collect().to_pandas()\nadata_train = pl.scan_parquet('/kaggle/input/open-problems-single-cell-perturbations/adata_train.parquet')\n# adata_train_df = adata_train.collect().to_pandas()\nmultiome_train = pl.scan_parquet('/kaggle/input/open-problems-single-cell-perturbations/multiome_train.parquet')","metadata":{"papermill":{"duration":2.023386,"end_time":"2023-10-26T12:42:00.366147","exception":false,"start_time":"2023-10-26T12:41:58.342761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:05.939462Z","iopub.execute_input":"2023-10-26T19:39:05.939802Z","iopub.status.idle":"2023-10-26T19:39:07.692664Z","shell.execute_reply.started":"2023-10-26T19:39:05.939769Z","shell.execute_reply":"2023-10-26T19:39:07.691523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(de_train_df.columns[:5])\nprint(adata_train.columns)\nprint(multiome_train.columns)","metadata":{"papermill":{"duration":0.028608,"end_time":"2023-10-26T12:42:00.411160","exception":false,"start_time":"2023-10-26T12:42:00.382552","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:07.696639Z","iopub.execute_input":"2023-10-26T19:39:07.697357Z","iopub.status.idle":"2023-10-26T19:39:07.703729Z","shell.execute_reply.started":"2023-10-26T19:39:07.697320Z","shell.execute_reply":"2023-10-26T19:39:07.702513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adata_obs_meta = pl.scan_csv('/kaggle/input/open-problems-single-cell-perturbations/adata_obs_meta.csv')\nadata_obs_meta_df = adata_obs_meta.collect().to_pandas()\n\nid_map = pl.scan_csv('/kaggle/input/open-problems-single-cell-perturbations/id_map.csv')\nid_map_df = id_map.collect().to_pandas()\n\nmultiome_obs_meta = pl.scan_csv('/kaggle/input/open-problems-single-cell-perturbations/multiome_obs_meta.csv')\nmultiome_obs_meta_df = multiome_obs_meta.collect().to_pandas()\n\nmultiome_var_meta = pl.scan_csv('/kaggle/input/open-problems-single-cell-perturbations/multiome_var_meta.csv')\nmultiome_var_meta_df = multiome_var_meta.collect().to_pandas()","metadata":{"papermill":{"duration":0.747928,"end_time":"2023-10-26T12:42:01.177738","exception":false,"start_time":"2023-10-26T12:42:00.429810","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:07.706608Z","iopub.execute_input":"2023-10-26T19:39:07.706997Z","iopub.status.idle":"2023-10-26T19:39:08.207696Z","shell.execute_reply.started":"2023-10-26T19:39:07.706946Z","shell.execute_reply":"2023-10-26T19:39:08.206649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(adata_obs_meta_df.columns)\nprint(id_map_df.columns)\nprint(multiome_obs_meta_df.columns)\nprint(multiome_var_meta_df.columns)","metadata":{"papermill":{"duration":0.027487,"end_time":"2023-10-26T12:42:01.221712","exception":false,"start_time":"2023-10-26T12:42:01.194225","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.209597Z","iopub.execute_input":"2023-10-26T19:39:08.210388Z","iopub.status.idle":"2023-10-26T19:39:08.217500Z","shell.execute_reply.started":"2023-10-26T19:39:08.210348Z","shell.execute_reply":"2023-10-26T19:39:08.216426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map_df.shape","metadata":{"papermill":{"duration":0.028437,"end_time":"2023-10-26T12:42:01.266607","exception":false,"start_time":"2023-10-26T12:42:01.238170","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.219007Z","iopub.execute_input":"2023-10-26T19:39:08.219780Z","iopub.status.idle":"2023-10-26T19:39:08.239658Z","shell.execute_reply.started":"2023-10-26T19:39:08.219745Z","shell.execute_reply":"2023-10-26T19:39:08.238703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multiome_obs_meta_df.shape","metadata":{"papermill":{"duration":0.03266,"end_time":"2023-10-26T12:42:01.316654","exception":false,"start_time":"2023-10-26T12:42:01.283994","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.240951Z","iopub.execute_input":"2023-10-26T19:39:08.241299Z","iopub.status.idle":"2023-10-26T19:39:08.250520Z","shell.execute_reply.started":"2023-10-26T19:39:08.241266Z","shell.execute_reply":"2023-10-26T19:39:08.249343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore datasets: de_train","metadata":{"execution":{"iopub.execute_input":"2023-10-09T15:26:54.062427Z","iopub.status.busy":"2023-10-09T15:26:54.061906Z","iopub.status.idle":"2023-10-09T15:26:55.941709Z","shell.execute_reply":"2023-10-09T15:26:55.940772Z","shell.execute_reply.started":"2023-10-09T15:26:54.062389Z"},"papermill":{"duration":0.018166,"end_time":"2023-10-26T12:42:01.353972","exception":false,"start_time":"2023-10-26T12:42:01.335806","status":"completed"},"tags":[]}},{"cell_type":"code","source":"de_train_df.shape","metadata":{"papermill":{"duration":0.100518,"end_time":"2023-10-26T12:42:01.474453","exception":false,"start_time":"2023-10-26T12:42:01.373935","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.251934Z","iopub.execute_input":"2023-10-26T19:39:08.252633Z","iopub.status.idle":"2023-10-26T19:39:08.265417Z","shell.execute_reply.started":"2023-10-26T19:39:08.252591Z","shell.execute_reply":"2023-10-26T19:39:08.264219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_df.head()","metadata":{"papermill":{"duration":0.0602,"end_time":"2023-10-26T12:42:01.552886","exception":false,"start_time":"2023-10-26T12:42:01.492686","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.269923Z","iopub.execute_input":"2023-10-26T19:39:08.270290Z","iopub.status.idle":"2023-10-26T19:39:08.303696Z","shell.execute_reply.started":"2023-10-26T19:39:08.270257Z","shell.execute_reply":"2023-10-26T19:39:08.302650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['cell_type', 'sm_name', 'sm_lincs_id', 'SMILES', 'control']\nde_train_df_sub = de_train_df[features]","metadata":{"papermill":{"duration":0.032779,"end_time":"2023-10-26T12:42:01.607063","exception":false,"start_time":"2023-10-26T12:42:01.574284","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.305324Z","iopub.execute_input":"2023-10-26T19:39:08.305676Z","iopub.status.idle":"2023-10-26T19:39:08.313678Z","shell.execute_reply.started":"2023-10-26T19:39:08.305642Z","shell.execute_reply":"2023-10-26T19:39:08.312436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_df_sub.info()","metadata":{"papermill":{"duration":0.044072,"end_time":"2023-10-26T12:42:01.668766","exception":false,"start_time":"2023-10-26T12:42:01.624694","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.314885Z","iopub.execute_input":"2023-10-26T19:39:08.315246Z","iopub.status.idle":"2023-10-26T19:39:08.338202Z","shell.execute_reply.started":"2023-10-26T19:39:08.315213Z","shell.execute_reply":"2023-10-26T19:39:08.337195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_df_sub.apply(lambda x: len(x.unique()))","metadata":{"papermill":{"duration":0.036277,"end_time":"2023-10-26T12:42:01.722680","exception":false,"start_time":"2023-10-26T12:42:01.686403","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.339746Z","iopub.execute_input":"2023-10-26T19:39:08.340231Z","iopub.status.idle":"2023-10-26T19:39:08.353231Z","shell.execute_reply.started":"2023-10-26T19:39:08.340196Z","shell.execute_reply":"2023-10-26T19:39:08.352170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_types = de_train_df_sub.cell_type.value_counts()\nsm_namea = de_train_df_sub.sm_name.value_counts()\nsm_lincs_ida = de_train_df_sub.sm_lincs_id.value_counts()\nSMILESa = de_train_df_sub.SMILES.value_counts()\ncontrols = de_train_df_sub.control.value_counts()","metadata":{"papermill":{"duration":0.032251,"end_time":"2023-10-26T12:42:01.772713","exception":false,"start_time":"2023-10-26T12:42:01.740462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.354610Z","iopub.execute_input":"2023-10-26T19:39:08.354970Z","iopub.status.idle":"2023-10-26T19:39:08.364831Z","shell.execute_reply.started":"2023-10-26T19:39:08.354936Z","shell.execute_reply":"2023-10-26T19:39:08.363820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(x=cell_types.values, y=cell_types.index, color=\"b\")","metadata":{"papermill":{"duration":0.325021,"end_time":"2023-10-26T12:42:02.114929","exception":false,"start_time":"2023-10-26T12:42:01.789908","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.365956Z","iopub.execute_input":"2023-10-26T19:39:08.366304Z","iopub.status.idle":"2023-10-26T19:39:08.672041Z","shell.execute_reply.started":"2023-10-26T19:39:08.366272Z","shell.execute_reply":"2023-10-26T19:39:08.670944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore datasets: adata_obs_meta","metadata":{"papermill":{"duration":0.017374,"end_time":"2023-10-26T12:42:02.150714","exception":false,"start_time":"2023-10-26T12:42:02.133340","status":"completed"},"tags":[]}},{"cell_type":"code","source":"adata_obs_meta_df.shape","metadata":{"papermill":{"duration":0.029612,"end_time":"2023-10-26T12:42:02.198834","exception":false,"start_time":"2023-10-26T12:42:02.169222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.673429Z","iopub.execute_input":"2023-10-26T19:39:08.673754Z","iopub.status.idle":"2023-10-26T19:39:08.680481Z","shell.execute_reply.started":"2023-10-26T19:39:08.673726Z","shell.execute_reply":"2023-10-26T19:39:08.679417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adata_obs_meta_df.head()","metadata":{"papermill":{"duration":0.041301,"end_time":"2023-10-26T12:42:02.258486","exception":false,"start_time":"2023-10-26T12:42:02.217185","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.681832Z","iopub.execute_input":"2023-10-26T19:39:08.682197Z","iopub.status.idle":"2023-10-26T19:39:08.705784Z","shell.execute_reply.started":"2023-10-26T19:39:08.682168Z","shell.execute_reply":"2023-10-26T19:39:08.704481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adata_obs_meta_df.apply(lambda x: len(x.unique()))","metadata":{"papermill":{"duration":0.24699,"end_time":"2023-10-26T12:42:02.523559","exception":false,"start_time":"2023-10-26T12:42:02.276569","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.707509Z","iopub.execute_input":"2023-10-26T19:39:08.708163Z","iopub.status.idle":"2023-10-26T19:39:08.972950Z","shell.execute_reply.started":"2023-10-26T19:39:08.708105Z","shell.execute_reply":"2023-10-26T19:39:08.971772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(x=adata_obs_meta_df.plate_name.value_counts().values, y=adata_obs_meta_df.plate_name.value_counts().index, color=\"b\")","metadata":{"papermill":{"duration":0.285301,"end_time":"2023-10-26T12:42:02.827372","exception":false,"start_time":"2023-10-26T12:42:02.542071","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:08.974270Z","iopub.execute_input":"2023-10-26T19:39:08.974639Z","iopub.status.idle":"2023-10-26T19:39:09.303341Z","shell.execute_reply.started":"2023-10-26T19:39:08.974611Z","shell.execute_reply":"2023-10-26T19:39:09.302277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(x=adata_obs_meta_df.donor_id.value_counts().values, y=adata_obs_meta_df.donor_id.value_counts().index, color=\"b\")","metadata":{"papermill":{"duration":0.287478,"end_time":"2023-10-26T12:42:03.133378","exception":false,"start_time":"2023-10-26T12:42:02.845900","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:09.306115Z","iopub.execute_input":"2023-10-26T19:39:09.306482Z","iopub.status.idle":"2023-10-26T19:39:09.601629Z","shell.execute_reply.started":"2023-10-26T19:39:09.306454Z","shell.execute_reply":"2023-10-26T19:39:09.600517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(x=adata_obs_meta_df.row.value_counts().values, y=adata_obs_meta_df.row.value_counts().index, color=\"b\")","metadata":{"papermill":{"duration":0.320882,"end_time":"2023-10-26T12:42:03.473654","exception":false,"start_time":"2023-10-26T12:42:03.152772","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:09.603831Z","iopub.execute_input":"2023-10-26T19:39:09.604260Z","iopub.status.idle":"2023-10-26T19:39:09.970251Z","shell.execute_reply.started":"2023-10-26T19:39:09.604223Z","shell.execute_reply":"2023-10-26T19:39:09.969152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(y=adata_obs_meta_df.col.value_counts().values, x=adata_obs_meta_df.col.value_counts().index, color=\"b\")","metadata":{"papermill":{"duration":0.29703,"end_time":"2023-10-26T12:42:03.791716","exception":false,"start_time":"2023-10-26T12:42:03.494686","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:09.973322Z","iopub.execute_input":"2023-10-26T19:39:09.973658Z","iopub.status.idle":"2023-10-26T19:39:10.324778Z","shell.execute_reply.started":"2023-10-26T19:39:09.973631Z","shell.execute_reply":"2023-10-26T19:39:10.323703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore datasets: multiome_obs_meta","metadata":{"papermill":{"duration":0.01915,"end_time":"2023-10-26T12:42:03.830546","exception":false,"start_time":"2023-10-26T12:42:03.811396","status":"completed"},"tags":[]}},{"cell_type":"code","source":"multiome_obs_meta_df.head()","metadata":{"papermill":{"duration":0.034832,"end_time":"2023-10-26T12:42:03.885203","exception":false,"start_time":"2023-10-26T12:42:03.850371","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.326326Z","iopub.execute_input":"2023-10-26T19:39:10.326687Z","iopub.status.idle":"2023-10-26T19:39:10.338114Z","shell.execute_reply.started":"2023-10-26T19:39:10.326653Z","shell.execute_reply":"2023-10-26T19:39:10.337130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multiome_obs_meta_df.shape","metadata":{"papermill":{"duration":0.031585,"end_time":"2023-10-26T12:42:03.937151","exception":false,"start_time":"2023-10-26T12:42:03.905566","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.339538Z","iopub.execute_input":"2023-10-26T19:39:10.339866Z","iopub.status.idle":"2023-10-26T19:39:10.351348Z","shell.execute_reply.started":"2023-10-26T19:39:10.339835Z","shell.execute_reply":"2023-10-26T19:39:10.350291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multiome_obs_meta_df.info()","metadata":{"papermill":{"duration":0.039035,"end_time":"2023-10-26T12:42:03.997048","exception":false,"start_time":"2023-10-26T12:42:03.958013","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.359311Z","iopub.execute_input":"2023-10-26T19:39:10.359583Z","iopub.status.idle":"2023-10-26T19:39:10.377524Z","shell.execute_reply.started":"2023-10-26T19:39:10.359560Z","shell.execute_reply":"2023-10-26T19:39:10.376374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multiome_obs_meta_df.apply(lambda x: len(x.unique()))","metadata":{"papermill":{"duration":0.042534,"end_time":"2023-10-26T12:42:04.059433","exception":false,"start_time":"2023-10-26T12:42:04.016899","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.378862Z","iopub.execute_input":"2023-10-26T19:39:10.379249Z","iopub.status.idle":"2023-10-26T19:39:10.403097Z","shell.execute_reply.started":"2023-10-26T19:39:10.379216Z","shell.execute_reply":"2023-10-26T19:39:10.402055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(x=multiome_obs_meta_df.cell_type.value_counts().values, y=multiome_obs_meta_df.cell_type.value_counts().index, color=\"b\")","metadata":{"papermill":{"duration":0.264471,"end_time":"2023-10-26T12:42:04.343986","exception":false,"start_time":"2023-10-26T12:42:04.079515","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.404393Z","iopub.execute_input":"2023-10-26T19:39:10.406096Z","iopub.status.idle":"2023-10-26T19:39:10.680507Z","shell.execute_reply.started":"2023-10-26T19:39:10.406069Z","shell.execute_reply":"2023-10-26T19:39:10.679478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(x=multiome_obs_meta_df.donor_id.value_counts().values, y=multiome_obs_meta_df.donor_id.value_counts().index, color=\"b\")","metadata":{"papermill":{"duration":0.24703,"end_time":"2023-10-26T12:42:04.612667","exception":false,"start_time":"2023-10-26T12:42:04.365637","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.681805Z","iopub.execute_input":"2023-10-26T19:39:10.682139Z","iopub.status.idle":"2023-10-26T19:39:10.920732Z","shell.execute_reply.started":"2023-10-26T19:39:10.682111Z","shell.execute_reply":"2023-10-26T19:39:10.919502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore datasets: multiome_var_meta","metadata":{"papermill":{"duration":0.020726,"end_time":"2023-10-26T12:42:04.654402","exception":false,"start_time":"2023-10-26T12:42:04.633676","status":"completed"},"tags":[]}},{"cell_type":"code","source":"multiome_var_meta_df.head()","metadata":{"papermill":{"duration":0.038833,"end_time":"2023-10-26T12:42:04.715350","exception":false,"start_time":"2023-10-26T12:42:04.676517","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.922247Z","iopub.execute_input":"2023-10-26T19:39:10.922606Z","iopub.status.idle":"2023-10-26T19:39:10.936359Z","shell.execute_reply.started":"2023-10-26T19:39:10.922574Z","shell.execute_reply":"2023-10-26T19:39:10.935362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multiome_var_meta_df.shape","metadata":{"papermill":{"duration":0.034535,"end_time":"2023-10-26T12:42:04.771706","exception":false,"start_time":"2023-10-26T12:42:04.737171","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.937648Z","iopub.execute_input":"2023-10-26T19:39:10.938212Z","iopub.status.idle":"2023-10-26T19:39:10.950023Z","shell.execute_reply.started":"2023-10-26T19:39:10.938175Z","shell.execute_reply":"2023-10-26T19:39:10.949042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multiome_var_meta_df.apply(lambda x: len(x.unique()))","metadata":{"papermill":{"duration":0.160621,"end_time":"2023-10-26T12:42:04.953600","exception":false,"start_time":"2023-10-26T12:42:04.792979","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:10.951567Z","iopub.execute_input":"2023-10-26T19:39:10.952207Z","iopub.status.idle":"2023-10-26T19:39:11.087338Z","shell.execute_reply.started":"2023-10-26T19:39:10.952174Z","shell.execute_reply":"2023-10-26T19:39:11.086231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Explore datasets: id_map","metadata":{"papermill":{"duration":0.02149,"end_time":"2023-10-26T12:42:04.996466","exception":false,"start_time":"2023-10-26T12:42:04.974976","status":"completed"},"tags":[]}},{"cell_type":"code","source":"id_map_df.head()","metadata":{"papermill":{"duration":0.037586,"end_time":"2023-10-26T12:42:05.057873","exception":false,"start_time":"2023-10-26T12:42:05.020287","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:11.088598Z","iopub.execute_input":"2023-10-26T19:39:11.089002Z","iopub.status.idle":"2023-10-26T19:39:11.099505Z","shell.execute_reply.started":"2023-10-26T19:39:11.088950Z","shell.execute_reply":"2023-10-26T19:39:11.098325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map_df.info()","metadata":{"papermill":{"duration":0.038327,"end_time":"2023-10-26T12:42:05.121392","exception":false,"start_time":"2023-10-26T12:42:05.083065","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:11.101677Z","iopub.execute_input":"2023-10-26T19:39:11.102140Z","iopub.status.idle":"2023-10-26T19:39:11.115690Z","shell.execute_reply.started":"2023-10-26T19:39:11.102105Z","shell.execute_reply":"2023-10-26T19:39:11.114551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map_df.apply(lambda x: len(x.unique()))","metadata":{"papermill":{"duration":0.035417,"end_time":"2023-10-26T12:42:05.178622","exception":false,"start_time":"2023-10-26T12:42:05.143205","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:11.117065Z","iopub.execute_input":"2023-10-26T19:39:11.117398Z","iopub.status.idle":"2023-10-26T19:39:11.125920Z","shell.execute_reply.started":"2023-10-26T19:39:11.117351Z","shell.execute_reply":"2023-10-26T19:39:11.124742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(8,2))\nsns.barplot(x=id_map_df.cell_type.value_counts().values, y=id_map_df.cell_type.value_counts().index, color=\"b\")","metadata":{"papermill":{"duration":0.233267,"end_time":"2023-10-26T12:42:05.433677","exception":false,"start_time":"2023-10-26T12:42:05.200410","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:11.127893Z","iopub.execute_input":"2023-10-26T19:39:11.128383Z","iopub.status.idle":"2023-10-26T19:39:11.367074Z","shell.execute_reply.started":"2023-10-26T19:39:11.128350Z","shell.execute_reply":"2023-10-26T19:39:11.366023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,2))\nsns.barplot(y=id_map_df.sm_name.value_counts().values[10:50], x=id_map_df.sm_name.value_counts().index[10:50], color=\"b\")\na=plt.xticks(rotation=60)","metadata":{"papermill":{"duration":0.653278,"end_time":"2023-10-26T12:42:06.109251","exception":false,"start_time":"2023-10-26T12:42:05.455973","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:11.371261Z","iopub.execute_input":"2023-10-26T19:39:11.372034Z","iopub.status.idle":"2023-10-26T19:39:12.062180Z","shell.execute_reply.started":"2023-10-26T19:39:11.371975Z","shell.execute_reply":"2023-10-26T19:39:12.061070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare datasets","metadata":{"papermill":{"duration":0.023436,"end_time":"2023-10-26T12:42:06.157384","exception":false,"start_time":"2023-10-26T12:42:06.133948","status":"completed"},"tags":[]}},{"cell_type":"code","source":"Xfeatures = ['cell_type', 'sm_name']\nyfeatures = ['cell_type', 'sm_name', 'sm_lincs_id', 'SMILES', 'control']","metadata":{"papermill":{"duration":0.033006,"end_time":"2023-10-26T12:42:06.214416","exception":false,"start_time":"2023-10-26T12:42:06.181410","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.063575Z","iopub.execute_input":"2023-10-26T19:39:12.064010Z","iopub.status.idle":"2023-10-26T19:39:12.069066Z","shell.execute_reply.started":"2023-10-26T19:39:12.063952Z","shell.execute_reply":"2023-10-26T19:39:12.068054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged = id_map_df.merge(pd.DataFrame(adata_obs_meta_df[yfeatures].groupby('sm_name').sm_lincs_id, columns=['sm_name', 'sm_lincs_id']), on='sm_name')\ndf_merged['sm_lincs_id'] = df_merged.sm_lincs_id.apply(lambda x: x.unique()[0])\ndf_merged = df_merged.sort_values('id').reset_index(drop=True)\ndf_merged.head()","metadata":{"papermill":{"duration":0.149784,"end_time":"2023-10-26T12:42:06.388685","exception":false,"start_time":"2023-10-26T12:42:06.238901","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.070310Z","iopub.execute_input":"2023-10-26T19:39:12.071314Z","iopub.status.idle":"2023-10-26T19:39:12.193020Z","shell.execute_reply.started":"2023-10-26T19:39:12.071275Z","shell.execute_reply":"2023-10-26T19:39:12.192023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged = df_merged.merge(pd.DataFrame(adata_obs_meta_df[yfeatures].groupby('sm_name').SMILES, columns=['sm_name', 'SMILES']), on='sm_name')\ndf_merged['SMILES'] = df_merged.SMILES.apply(lambda x: x.unique()[0])\ndf_merged = df_merged.sort_values('id').reset_index(drop=True)\ndf_merged.head()","metadata":{"papermill":{"duration":0.156027,"end_time":"2023-10-26T12:42:06.569136","exception":false,"start_time":"2023-10-26T12:42:06.413109","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.194333Z","iopub.execute_input":"2023-10-26T19:39:12.194655Z","iopub.status.idle":"2023-10-26T19:39:12.318208Z","shell.execute_reply.started":"2023-10-26T19:39:12.194627Z","shell.execute_reply":"2023-10-26T19:39:12.317039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_merged = df_merged.merge(pd.DataFrame(adata_obs_meta_df[yfeatures].groupby('sm_name').control, columns=['sm_name', 'control']), on='sm_name')\ndf_merged['control'] = df_merged.control.apply(lambda x: x.unique()[0]).astype(int)\ndf_merged = df_merged.sort_values('id').reset_index(drop=True)\ndf_merged.head()","metadata":{"papermill":{"duration":0.118821,"end_time":"2023-10-26T12:42:06.712879","exception":false,"start_time":"2023-10-26T12:42:06.594058","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.319687Z","iopub.execute_input":"2023-10-26T19:39:12.320100Z","iopub.status.idle":"2023-10-26T19:39:12.407203Z","shell.execute_reply.started":"2023-10-26T19:39:12.320066Z","shell.execute_reply":"2023-10-26T19:39:12.406140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create new datasets","metadata":{"papermill":{"duration":0.024681,"end_time":"2023-10-26T12:42:06.762624","exception":false,"start_time":"2023-10-26T12:42:06.737943","status":"completed"},"tags":[]}},{"cell_type":"code","source":"Xfeatures = ['cell_type', 'sm_name', 'sm_lincs_id', 'SMILES', 'control']\nyfeatures = ['cell_type', 'sm_name', 'sm_lincs_id', 'SMILES', 'control']","metadata":{"papermill":{"duration":0.032213,"end_time":"2023-10-26T12:42:06.820195","exception":false,"start_time":"2023-10-26T12:42:06.787982","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.408590Z","iopub.execute_input":"2023-10-26T19:39:12.408953Z","iopub.status.idle":"2023-10-26T19:39:12.417241Z","shell.execute_reply.started":"2023-10-26T19:39:12.408921Z","shell.execute_reply":"2023-10-26T19:39:12.416308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_cols = de_train_df.drop(columns=yfeatures).columns","metadata":{"papermill":{"duration":0.080991,"end_time":"2023-10-26T12:42:06.925165","exception":false,"start_time":"2023-10-26T12:42:06.844174","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.418574Z","iopub.execute_input":"2023-10-26T19:39:12.418920Z","iopub.status.idle":"2023-10-26T19:39:12.460085Z","shell.execute_reply.started":"2023-10-26T19:39:12.418888Z","shell.execute_reply":"2023-10-26T19:39:12.459162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = de_train_df[Xfeatures].copy()\nX['control'] = X.control.astype(int)\ny = de_train_df[output_cols]","metadata":{"papermill":{"duration":0.068111,"end_time":"2023-10-26T12:42:07.018096","exception":false,"start_time":"2023-10-26T12:42:06.949985","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.461515Z","iopub.execute_input":"2023-10-26T19:39:12.461871Z","iopub.status.idle":"2023-10-26T19:39:12.504740Z","shell.execute_reply.started":"2023-10-26T19:39:12.461838Z","shell.execute_reply":"2023-10-26T19:39:12.503883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"papermill":{"duration":0.043153,"end_time":"2023-10-26T12:42:07.087122","exception":false,"start_time":"2023-10-26T12:42:07.043969","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.505832Z","iopub.execute_input":"2023-10-26T19:39:12.506114Z","iopub.status.idle":"2023-10-26T19:39:12.518005Z","shell.execute_reply.started":"2023-10-26T19:39:12.506090Z","shell.execute_reply":"2023-10-26T19:39:12.516811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocessing for numerical data\n# numerical_transformer = SimpleImputer(strategy='most_frequent')\n\n# Preprocessing for categorical data\ncategorical_transformer = Pipeline(steps=[\n#     ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))\n    ('onehot', OrdinalEncoder(handle_unknown='error'))\n])\n\n# Bundle preprocessing for numerical and categorical data\npreprocessor = ColumnTransformer(\n    transformers=[\n#         ('num', numerical_transformer, ['control']),\n        ('cat', categorical_transformer, Xfeatures)\n    ])","metadata":{"papermill":{"duration":0.035122,"end_time":"2023-10-26T12:42:07.147491","exception":false,"start_time":"2023-10-26T12:42:07.112369","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.519715Z","iopub.execute_input":"2023-10-26T19:39:12.520135Z","iopub.status.idle":"2023-10-26T19:39:12.529816Z","shell.execute_reply.started":"2023-10-26T19:39:12.520097Z","shell.execute_reply":"2023-10-26T19:39:12.529037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_transformer.fit(X)","metadata":{"papermill":{"duration":0.042978,"end_time":"2023-10-26T12:42:07.215045","exception":false,"start_time":"2023-10-26T12:42:07.172067","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.530869Z","iopub.execute_input":"2023-10-26T19:39:12.531269Z","iopub.status.idle":"2023-10-26T19:39:12.547267Z","shell.execute_reply.started":"2023-10-26T19:39:12.531240Z","shell.execute_reply":"2023-10-26T19:39:12.546001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_pipeline = Pipeline(steps=[('preprocessor', preprocessor)])\nmy_pipeline.fit(X)\nencoded = my_pipeline.transform(X)\n# df_merged_new['smiles_len'] = df_merged_new.SMILES.str.len()\ntest_encoded = pd.DataFrame(my_pipeline.transform(df_merged[Xfeatures]), columns=Xfeatures)","metadata":{"papermill":{"duration":0.051745,"end_time":"2023-10-26T12:42:07.294175","exception":false,"start_time":"2023-10-26T12:42:07.242430","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.548554Z","iopub.execute_input":"2023-10-26T19:39:12.548820Z","iopub.status.idle":"2023-10-26T19:39:12.577356Z","shell.execute_reply.started":"2023-10-26T19:39:12.548796Z","shell.execute_reply":"2023-10-26T19:39:12.576503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_encoded.head()","metadata":{"papermill":{"duration":0.044982,"end_time":"2023-10-26T12:42:07.364062","exception":false,"start_time":"2023-10-26T12:42:07.319080","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.578717Z","iopub.execute_input":"2023-10-26T19:39:12.579223Z","iopub.status.idle":"2023-10-26T19:39:12.594963Z","shell.execute_reply.started":"2023-10-26T19:39:12.579184Z","shell.execute_reply":"2023-10-26T19:39:12.593904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_encoded = pd.DataFrame(encoded, columns=Xfeatures)","metadata":{"papermill":{"duration":0.034064,"end_time":"2023-10-26T12:42:07.424260","exception":false,"start_time":"2023-10-26T12:42:07.390196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.596393Z","iopub.execute_input":"2023-10-26T19:39:12.597096Z","iopub.status.idle":"2023-10-26T19:39:12.604811Z","shell.execute_reply.started":"2023-10-26T19:39:12.597059Z","shell.execute_reply":"2023-10-26T19:39:12.603854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_encoded.head()","metadata":{"papermill":{"duration":0.043509,"end_time":"2023-10-26T12:42:07.492852","exception":false,"start_time":"2023-10-26T12:42:07.449343","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.605831Z","iopub.execute_input":"2023-10-26T19:39:12.606170Z","iopub.status.idle":"2023-10-26T19:39:12.624871Z","shell.execute_reply.started":"2023-10-26T19:39:12.606137Z","shell.execute_reply":"2023-10-26T19:39:12.623997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create model","metadata":{"papermill":{"duration":0.025032,"end_time":"2023-10-26T12:42:07.543283","exception":false,"start_time":"2023-10-26T12:42:07.518251","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def mean_rowwise_rmse_loss(y_true, y_pred):\n    \"\"\"\n    Custom loss function to calculate the Mean Rowwise Root Mean Squared Error (RMSE) loss.\n\n    Parameters:\n    - y_true: The true target values.\n    - y_pred: The predicted values.\n\n    Returns:\n    - Mean Rowwise RMSE loss as a scalar tensor.\n    \"\"\"\n    # Calculate RMSE for each row\n    rmse_per_row = tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred), axis=1))\n    # Calculate the mean of RMSE values across all rows\n    mean_rmse = tf.reduce_mean(rmse_per_row)\n    \n    return mean_rmse","metadata":{"papermill":{"duration":0.036309,"end_time":"2023-10-26T12:42:07.604776","exception":false,"start_time":"2023-10-26T12:42:07.568467","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.626322Z","iopub.execute_input":"2023-10-26T19:39:12.627019Z","iopub.status.idle":"2023-10-26T19:39:12.636822Z","shell.execute_reply.started":"2023-10-26T19:39:12.626960Z","shell.execute_reply":"2023-10-26T19:39:12.635879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_mean_rowwise_rmse(y_true, y_pred):\n    \"\"\"\n    Custom metric to calculate the Mean Rowwise Root Mean Squared Error (RMSE).\n\n    Parameters:\n    - y_true: The true target values.\n    - y_pred: The predicted values.\n\n    Returns:\n    - Mean Rowwise RMSE as a scalar tensor.\n    \"\"\"\n    # Calculate RMSE for each row\n    rmse_per_row = tf.sqrt(tf.reduce_mean(tf.square(y_true - y_pred), axis=1))\n    # Calculate the mean of RMSE values across all rows\n    mean_rmse = tf.reduce_mean(rmse_per_row)\n    \n    return mean_rmse","metadata":{"papermill":{"duration":0.034497,"end_time":"2023-10-26T12:42:07.664915","exception":false,"start_time":"2023-10-26T12:42:07.630418","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.638025Z","iopub.execute_input":"2023-10-26T19:39:12.638351Z","iopub.status.idle":"2023-10-26T19:39:12.651906Z","shell.execute_reply.started":"2023-10-26T19:39:12.638327Z","shell.execute_reply":"2023-10-26T19:39:12.651006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.03, random_state=32)\n# X_train, X_valid, y_train, y_valid = train_test_split(train_encoded, y, test_size=0.10, random_state=42)","metadata":{"papermill":{"duration":0.095948,"end_time":"2023-10-26T12:42:07.786861","exception":false,"start_time":"2023-10-26T12:42:07.690913","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.653058Z","iopub.execute_input":"2023-10-26T19:39:12.653780Z","iopub.status.idle":"2023-10-26T19:39:12.710915Z","shell.execute_reply.started":"2023-10-26T19:39:12.653736Z","shell.execute_reply":"2023-10-26T19:39:12.709819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"papermill":{"duration":0.04307,"end_time":"2023-10-26T12:42:07.855646","exception":false,"start_time":"2023-10-26T12:42:07.812576","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.712279Z","iopub.execute_input":"2023-10-26T19:39:12.712697Z","iopub.status.idle":"2023-10-26T19:39:12.728513Z","shell.execute_reply.started":"2023-10-26T19:39:12.712667Z","shell.execute_reply":"2023-10-26T19:39:12.727543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks_list = [tf.keras.callbacks.EarlyStopping(\n                  monitor='val_loss',\n                  restore_best_weights=True,\n                  patience=5)] ","metadata":{"papermill":{"duration":0.035101,"end_time":"2023-10-26T12:42:07.917404","exception":false,"start_time":"2023-10-26T12:42:07.882303","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.730175Z","iopub.execute_input":"2023-10-26T19:39:12.730440Z","iopub.status.idle":"2023-10-26T19:39:12.737654Z","shell.execute_reply.started":"2023-10-26T19:39:12.730417Z","shell.execute_reply":"2023-10-26T19:39:12.736723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EMBEDDING_DIM = 100\n# MAX_LENGTH = 100\n# def embedding_text(df, batch_size=64, AUTOTUNE=tf.data.AUTOTUNE, embedding_dim=EMBEDDING_DIM, sequence_length=MAX_LENGTH):\n#     dff = df.sum(axis=1)\n#     ds = tf.data.Dataset.from_tensor_slices((dff))\n#     ds = ds.batch(batch_size).cache().prefetch(buffer_size=AUTOTUNE)\n    \n# #     embedding_dim = EMBEDDING_DIM\n#     vocab_size = df.shape[0]\n# #     sequence_length = MAX_LENGTH\n\n#     vectorize_layer = tf.keras.layers.TextVectorization(\n#         max_tokens=vocab_size,\n#         output_sequence_length=sequence_length)\n\n#     vectorize_layer.adapt(ds)\n\n#     model = tf.keras.models.Sequential()\n#     model.add(vectorize_layer)\n#     model.add(tf.keras.layers.Embedding(vocab_size, embedding_dim, name=\"embedding\"))\n    \n#     weights = model.get_layer('embedding').get_weights()[0]\n#     w_cols = [f'w{i}' for i in range(embedding_dim)]\n#     new_df = pd.DataFrame()\n#     new_df[w_cols] = weights\n#     return new_df","metadata":{"papermill":{"duration":0.035382,"end_time":"2023-10-26T12:42:07.978107","exception":false,"start_time":"2023-10-26T12:42:07.942725","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.738873Z","iopub.execute_input":"2023-10-26T19:39:12.739232Z","iopub.status.idle":"2023-10-26T19:39:12.748216Z","shell.execute_reply.started":"2023-10-26T19:39:12.739198Z","shell.execute_reply":"2023-10-26T19:39:12.747250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_X_train = embedding_text(X_train, embedding_dim=64)\n# new_X_valid = embedding_text(X_valid, embedding_dim=64)","metadata":{"papermill":{"duration":0.033848,"end_time":"2023-10-26T12:42:08.037490","exception":false,"start_time":"2023-10-26T12:42:08.003642","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.749274Z","iopub.execute_input":"2023-10-26T19:39:12.749566Z","iopub.status.idle":"2023-10-26T19:39:12.762436Z","shell.execute_reply.started":"2023-10-26T19:39:12.749541Z","shell.execute_reply":"2023-10-26T19:39:12.761612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"papermill":{"duration":0.045044,"end_time":"2023-10-26T12:42:08.108488","exception":false,"start_time":"2023-10-26T12:42:08.063444","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.763550Z","iopub.execute_input":"2023-10-26T19:39:12.763899Z","iopub.status.idle":"2023-10-26T19:39:12.782401Z","shell.execute_reply.started":"2023-10-26T19:39:12.763866Z","shell.execute_reply":"2023-10-26T19:39:12.781275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    \n    ds_celltype = tf.data.Dataset.from_tensor_slices(X_train.cell_type).batch(64).cache().prefetch(buffer_size=tf.data.AUTOTUNE)\n    ds_smname = tf.data.Dataset.from_tensor_slices(X_train.sm_name).batch(64).cache().prefetch(buffer_size=tf.data.AUTOTUNE)\n    ds_smlincsid = tf.data.Dataset.from_tensor_slices(X_train.sm_lincs_id).batch(64).cache().prefetch(buffer_size=tf.data.AUTOTUNE)\n#     ds_smiles = tf.data.Dataset.from_tensor_slices(X_train.SMILES).batch(64).cache().prefetch(buffer_size=tf.data.AUTOTUNE)\n    \n    input_cell_type = tf.keras.Input(shape=(1, ), dtype=tf.string)\n    input_sm_name = tf.keras.Input(shape=(1, ), dtype=tf.string)\n    input_sm_lincs_id = tf.keras.Input(shape=(1, ), dtype=tf.string)\n#     input_smiles = tf.keras.Input(shape=(1, ), dtype=tf.string)\n    \n    vectorize_cell_type = tf.keras.layers.TextVectorization()\n    vectorize_sm_name = tf.keras.layers.TextVectorization()\n    vectorize_sm_lincs_id = tf.keras.layers.TextVectorization()\n#     vectorize_smiles = tf.keras.layers.TextVectorization()\n\n    vectorize_cell_type.adapt(X_train.cell_type)\n    vectorize_sm_name.adapt(X_train.sm_name)\n    vectorize_sm_lincs_id.adapt(X_train.sm_lincs_id)\n#     vectorize_smiles.adapt(X_train.SMILES)\n\n    x_cell_type = vectorize_cell_type(input_cell_type)\n    x_sm_name = vectorize_sm_name(input_sm_name)\n    x_sm_lincs_id = vectorize_sm_lincs_id(input_sm_lincs_id)\n#     x_smiles = vectorize_smiles(input_smiles)\n    \n    x_cell_type = tf.keras.layers.Embedding(1000, 16)(x_cell_type)\n    x_sm_name = tf.keras.layers.Embedding(1000, 16)(x_sm_name)\n    x_sm_lincs_id = tf.keras.layers.Embedding(1000, 16)(x_sm_lincs_id)\n#     x_smiles = tf.keras.layers.Embedding(1000, 16)(x_smiles)\n    \n    x_cell_type = tf.keras.layers.LSTM(32)(x_cell_type)\n    x_sm_name = tf.keras.layers.LSTM(32)(x_sm_name)\n    x_sm_lincs_id = tf.keras.layers.LSTM(32)(x_sm_lincs_id)\n#     x_smiles = tf.keras.layers.LSTM(32)(x_smiles)\n    \n    x = tf.keras.layers.concatenate([x_cell_type, x_sm_name, x_sm_lincs_id])\n    \n    x = tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.00001))(x)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    x = tf.keras.layers.Dense(32, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.000001))(x)\n    x = tf.keras.layers.Dropout(0.2)(x)\n    \n    output_layer = tf.keras.layers.Dense(y_train.shape[1])(x)\n    \n    model = tf.keras.Model(\n        inputs=[input_cell_type, input_sm_name, input_sm_lincs_id],\n        outputs=output_layer\n    )\n    \n    model.compile(loss=mean_rowwise_rmse_loss, \n                  optimizer=tf.keras.optimizers.Adam(),\n                  metrics=[custom_mean_rowwise_rmse])\n    \n    return model\n\nmodel = create_model()\n\nmodel.fit([X_train.cell_type, X_train.sm_name, X_train.sm_lincs_id], y_train,\n      epochs=100,\n      callbacks=callbacks_list,\n#       steps_per_epoch=8,\n      validation_data=([X_valid.cell_type, X_valid.sm_name, X_valid.sm_lincs_id], y_valid)\n         )","metadata":{"papermill":{"duration":16.797318,"end_time":"2023-10-26T12:42:24.931428","exception":false,"start_time":"2023-10-26T12:42:08.134110","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:12.783974Z","iopub.execute_input":"2023-10-26T19:39:12.784488Z","iopub.status.idle":"2023-10-26T19:39:34.440831Z","shell.execute_reply.started":"2023-10-26T19:39:12.784458Z","shell.execute_reply":"2023-10-26T19:39:34.439891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def create_model():\n#     input_cell_type = tf.keras.Input(shape=(1, ))\n#     input_sm_name = tf.keras.Input(shape=(1, ))\n#     input_sm_lincs_id = tf.keras.Input(shape=(1, ))\n#     input_smiles = tf.keras.Input(shape=(1, ))\n#     input_control = tf.keras.Input(shape=(1, ))\n    \n#     x_cell_type = tf.keras.layers.Embedding(1000, 16)(input_cell_type)\n#     x_sm_name = tf.keras.layers.Embedding(1000, 16)(input_sm_name)\n#     x_sm_lincs_id = tf.keras.layers.Embedding(1000, 16)(input_sm_lincs_id)\n#     x_smiles = tf.keras.layers.Embedding(1000, 16)(input_smiles)\n#     x_control = tf.keras.layers.Embedding(1000, 16)(input_control)\n    \n#     x_cell_type = tf.keras.layers.LSTM(32)(x_cell_type)\n#     x_sm_name = tf.keras.layers.LSTM(32)(x_sm_name)\n#     x_sm_lincs_id = tf.keras.layers.LSTM(32)(x_sm_lincs_id)\n#     x_smiles = tf.keras.layers.LSTM(32)(x_smiles)\n#     x_control = tf.keras.layers.LSTM(32)(x_control)\n    \n#     x = tf.keras.layers.concatenate([x_cell_type, x_sm_name, x_sm_lincs_id, x_smiles, x_control])\n    \n#     x = tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.00001))(x)\n#     x = tf.keras.layers.Dropout(0.2)(x)\n#     x = tf.keras.layers.Dense(32, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.000001))(x)\n#     x = tf.keras.layers.Dropout(0.2)(x)\n    \n#     output_layer = tf.keras.layers.Dense(y_train.shape[1])(x)\n    \n#     model = tf.keras.Model(\n#         inputs=[input_cell_type, input_sm_name, input_sm_lincs_id, input_smiles, input_control],\n#         outputs=output_layer\n#     )\n    \n#     model.compile(loss=mean_rowwise_rmse_loss, \n#                   optimizer=tf.keras.optimizers.Adam(),\n#                   metrics=[custom_mean_rowwise_rmse])\n    \n#     return model\n\n# model = create_model()\n\n# model.fit([X_train.cell_type, X_train.sm_name, X_train.sm_lincs_id, X_train.SMILES, X_train.control], y_train,\n#       epochs=100,\n#       callbacks=callbacks_list,\n# #       steps_per_epoch=8,\n#       validation_data=([X_valid.cell_type, X_valid.sm_name, X_valid.sm_lincs_id, X_valid.SMILES, X_valid.control], y_valid)\n#          )","metadata":{"papermill":{"duration":0.04498,"end_time":"2023-10-26T12:42:25.012726","exception":false,"start_time":"2023-10-26T12:42:24.967746","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:34.442255Z","iopub.execute_input":"2023-10-26T19:39:34.442909Z","iopub.status.idle":"2023-10-26T19:39:34.448730Z","shell.execute_reply.started":"2023-10-26T19:39:34.442870Z","shell.execute_reply":"2023-10-26T19:39:34.447596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = RandomForestRegressor()\n# model.fit(X_train, y_train)\n# print(mean_rowwise_rmse_loss(y_valid, model.predict(X_valid)))","metadata":{"papermill":{"duration":0.045922,"end_time":"2023-10-26T12:42:25.093047","exception":false,"start_time":"2023-10-26T12:42:25.047125","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:34.449972Z","iopub.execute_input":"2023-10-26T19:39:34.450289Z","iopub.status.idle":"2023-10-26T19:39:34.471348Z","shell.execute_reply.started":"2023-10-26T19:39:34.450264Z","shell.execute_reply":"2023-10-26T19:39:34.470294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.035815,"end_time":"2023-10-26T12:42:25.163318","exception":false,"start_time":"2023-10-26T12:42:25.127503","status":"completed"},"tags":[]}},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv')","metadata":{"papermill":{"duration":3.383604,"end_time":"2023-10-26T12:42:28.579985","exception":false,"start_time":"2023-10-26T12:42:25.196381","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:34.472618Z","iopub.execute_input":"2023-10-26T19:39:34.472883Z","iopub.status.idle":"2023-10-26T19:39:38.768846Z","shell.execute_reply.started":"2023-10-26T19:39:34.472860Z","shell.execute_reply":"2023-10-26T19:39:38.767799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"papermill":{"duration":0.06959,"end_time":"2023-10-26T12:42:28.686771","exception":false,"start_time":"2023-10-26T12:42:28.617181","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:38.771238Z","iopub.execute_input":"2023-10-26T19:39:38.771539Z","iopub.status.idle":"2023-10-26T19:39:38.807105Z","shell.execute_reply.started":"2023-10-26T19:39:38.771512Z","shell.execute_reply":"2023-10-26T19:39:38.806081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sample_submission.columns[1:])","metadata":{"papermill":{"duration":0.044668,"end_time":"2023-10-26T12:42:28.766456","exception":false,"start_time":"2023-10-26T12:42:28.721788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:38.810264Z","iopub.execute_input":"2023-10-26T19:39:38.810568Z","iopub.status.idle":"2023-10-26T19:39:38.818171Z","shell.execute_reply.started":"2023-10-26T19:39:38.810544Z","shell.execute_reply":"2023-10-26T19:39:38.817111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_test_list = []\n# cols_list = []\n# output_size = len(sample_submission.columns[1:])//SPLIT_SIZE + 1\n# for i in tqdm(range(SPLIT_SIZE)):\n# #     y_test = models[i].predict(embedding_text(id_map_df[Xfeatures]), verbose=False)    \n#     y_test = models[i].predict(test_encoded, verbose=False)\n#     y_test_list.append(y_test)\n#     cols_list.append(sample_submission.drop(columns='id').columns[i*output_size:(i+1)*output_size])\n","metadata":{"papermill":{"duration":0.044817,"end_time":"2023-10-26T12:42:28.846753","exception":false,"start_time":"2023-10-26T12:42:28.801936","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:38.819454Z","iopub.execute_input":"2023-10-26T19:39:38.820115Z","iopub.status.idle":"2023-10-26T19:39:38.826137Z","shell.execute_reply.started":"2023-10-26T19:39:38.820079Z","shell.execute_reply":"2023-10-26T19:39:38.825214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.sum([len(x) for x in cols_list])","metadata":{"papermill":{"duration":0.043703,"end_time":"2023-10-26T12:42:28.924942","exception":false,"start_time":"2023-10-26T12:42:28.881239","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:38.827275Z","iopub.execute_input":"2023-10-26T19:39:38.827538Z","iopub.status.idle":"2023-10-26T19:39:38.837747Z","shell.execute_reply.started":"2023-10-26T19:39:38.827516Z","shell.execute_reply":"2023-10-26T19:39:38.836905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.DataFrame()\n# for i in range(len(y_test_list)):\n#     df = pd.concat([df, pd.DataFrame(y_test_list[i], columns=cols_list[i])], axis=1)","metadata":{"papermill":{"duration":0.047912,"end_time":"2023-10-26T12:42:29.009225","exception":false,"start_time":"2023-10-26T12:42:28.961313","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:38.838927Z","iopub.execute_input":"2023-10-26T19:39:38.839228Z","iopub.status.idle":"2023-10-26T19:39:38.848058Z","shell.execute_reply.started":"2023-10-26T19:39:38.839204Z","shell.execute_reply":"2023-10-26T19:39:38.847203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission[sample_submission.columns[1:]] = model.predict(embedding_text(df_merged[Xfeatures], embedding_dim=64))\n# sample_submission[sample_submission.columns[1:]] = model.predict([test_encoded[col] for col in test_encoded.columns])\nsample_submission[sample_submission.columns[1:]] = model.predict([df_merged.cell_type, df_merged.sm_name, df_merged.sm_lincs_id])\n# submission = pd.concat([sample_submission[['id']], df], axis=1)","metadata":{"papermill":{"duration":9.037893,"end_time":"2023-10-26T12:42:38.081844","exception":false,"start_time":"2023-10-26T12:42:29.043951","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:38.849410Z","iopub.execute_input":"2023-10-26T19:39:38.849722Z","iopub.status.idle":"2023-10-26T19:39:46.875055Z","shell.execute_reply.started":"2023-10-26T19:39:38.849691Z","shell.execute_reply":"2023-10-26T19:39:46.874160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"papermill":{"duration":0.148311,"end_time":"2023-10-26T12:42:38.263760","exception":false,"start_time":"2023-10-26T12:42:38.115449","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:46.876512Z","iopub.execute_input":"2023-10-26T19:39:46.876876Z","iopub.status.idle":"2023-10-26T19:39:47.005167Z","shell.execute_reply.started":"2023-10-26T19:39:46.876842Z","shell.execute_reply":"2023-10-26T19:39:47.004162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.shape","metadata":{"papermill":{"duration":0.044777,"end_time":"2023-10-26T12:42:38.342733","exception":false,"start_time":"2023-10-26T12:42:38.297956","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:47.013148Z","iopub.execute_input":"2023-10-26T19:39:47.013422Z","iopub.status.idle":"2023-10-26T19:39:47.019343Z","shell.execute_reply.started":"2023-10-26T19:39:47.013400Z","shell.execute_reply":"2023-10-26T19:39:47.018388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":32.649682,"end_time":"2023-10-26T12:43:11.027619","exception":false,"start_time":"2023-10-26T12:42:38.377937","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-10-26T19:39:47.022356Z","iopub.execute_input":"2023-10-26T19:39:47.022662Z","iopub.status.idle":"2023-10-26T19:40:33.590146Z","shell.execute_reply.started":"2023-10-26T19:39:47.022636Z","shell.execute_reply":"2023-10-26T19:40:33.589012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.033576,"end_time":"2023-10-26T12:43:11.095815","exception":false,"start_time":"2023-10-26T12:43:11.062239","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}