{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":1767.896818,"end_time":"2023-10-03T19:16:48.560142","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-10-03T18:47:20.663324","version":"2.4.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7123483,"sourceType":"datasetVersion","datasetId":4109071}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\nt0start = time.time()\nfrom fastai.collab import *\nfrom fastai.tabular.all import *","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":6.509942,"end_time":"2023-10-03T18:47:30.760703","exception":false,"start_time":"2023-10-03T18:47:24.250761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:31.078300Z","iopub.execute_input":"2023-12-08T16:46:31.078729Z","iopub.status.idle":"2023-12-08T16:46:37.112678Z","shell.execute_reply.started":"2023-12-08T16:46:31.078696Z","shell.execute_reply":"2023-12-08T16:46:37.111508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_seed = 209","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:37.115206Z","iopub.execute_input":"2023-12-08T16:46:37.115992Z","iopub.status.idle":"2023-12-08T16:46:37.120952Z","shell.execute_reply.started":"2023-12-08T16:46:37.115961Z","shell.execute_reply":"2023-12-08T16:46:37.119656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading and Melting Train and Test Data","metadata":{"tags":[]}},{"cell_type":"markdown","source":"Here I read the training and test data and melt it to yield a ```DataFrame``` with three categorical features (```cell_type```, ```sm_name```, and ```gene```) and one target (```value```).","metadata":{}},{"cell_type":"code","source":"%%time\nfn = '/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet'\ndf_de_train = pd.read_parquet(fn)\ntrain_df = df_de_train.melt(id_vars=['cell_type', 'sm_name'], value_vars=df_de_train.iloc[:,5:].columns, var_name='gene', value_name='value')","metadata":{"papermill":{"duration":163.861162,"end_time":"2023-10-03T18:50:18.214746","exception":false,"start_time":"2023-10-03T18:47:34.353584","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:37.122738Z","iopub.execute_input":"2023-12-08T16:46:37.123085Z","iopub.status.idle":"2023-12-08T16:46:41.467542Z","shell.execute_reply.started":"2023-12-08T16:46:37.123059Z","shell.execute_reply":"2023-12-08T16:46:41.466019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = '/kaggle/input/open-problems-single-cell-perturbations/id_map.csv'\ndf_id_map = pd.read_csv(fn)\nfn = '/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv'\ndf = pd.read_csv(fn, index_col = 0)\n\ncols_to_add = df_de_train.iloc[:,5:].columns\ncols_to_add\n\ndf_zeros = pd.DataFrame(0.0, columns=cols_to_add, index=df_id_map.index)\ndf_zeros\n\ndf_id_map_preds = pd.concat([df_id_map, df_zeros], axis=1)\ntest_df = df_id_map_preds.melt(id_vars=['cell_type', 'sm_name'], value_vars=df_id_map_preds.iloc[:,3:].columns, var_name='gene', value_name='value')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:41.469906Z","iopub.execute_input":"2023-12-08T16:46:41.470251Z","iopub.status.idle":"2023-12-08T16:46:45.950628Z","shell.execute_reply.started":"2023-12-08T16:46:41.470223Z","shell.execute_reply":"2023-12-08T16:46:45.949520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Embeddings","metadata":{}},{"cell_type":"markdown","source":"The function ```reduce_emb_dim``` is used to reduce the dimensionality of molecular (```dim = 26```) and gene embeddings (```dim = 1000```) down to 10 components.","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n\ndef reduce_emb_dim(data, n_comp=35, random_state=42):\n    embname = data.columns[1][:-1]\n    Y = data.iloc[:,1:]\n    scaler = StandardScaler()\n    Y_std = scaler.fit_transform(Y)\n    reducer = PCA(n_components=n_comp, random_state=random_state)\n\n    Yr = reducer.fit_transform(Y_std)\n    column_names = [f'{embname}pca{n_comp}_{i}' for i in range(n_comp)]\n    reduced_data = pd.DataFrame(Yr, columns = column_names)\n    reduced_data = pd.concat([data.iloc[:, 0], reduced_data], axis=1)\n    return reduced_data","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:45.952679Z","iopub.execute_input":"2023-12-08T16:46:45.953376Z","iopub.status.idle":"2023-12-08T16:46:46.090439Z","shell.execute_reply.started":"2023-12-08T16:46:45.953335Z","shell.execute_reply":"2023-12-08T16:46:46.089455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_embs = pd.read_csv('/kaggle/input/op2-single-cell-perturbations-tabmodnn-embeddings/cell_embeddings_no_pca.csv', index_col = 0)\nmol_embs = pd.read_csv('/kaggle/input/op2-single-cell-perturbations-tabmodnn-embeddings/molecular_embeddings_no_pca.csv', index_col = 0)\ngene_embs = pd.read_parquet('/kaggle/input/op2-single-cell-perturbations-tabmodnn-embeddings/gene_embeddings_no_pca.parquet')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:46.094254Z","iopub.execute_input":"2023-12-08T16:46:46.094562Z","iopub.status.idle":"2023-12-08T16:46:47.340437Z","shell.execute_reply.started":"2023-12-08T16:46:46.094537Z","shell.execute_reply":"2023-12-08T16:46:47.338971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gene_embs = reduce_emb_dim(gene_embs, n_comp=10, random_state=random_seed)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:47.343007Z","iopub.execute_input":"2023-12-08T16:46:47.345634Z","iopub.status.idle":"2023-12-08T16:46:48.195503Z","shell.execute_reply.started":"2023-12-08T16:46:47.345600Z","shell.execute_reply":"2023-12-08T16:46:48.194695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mol_embs = reduce_emb_dim(mol_embs, n_comp=10, random_state=random_seed)","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:48.197309Z","iopub.execute_input":"2023-12-08T16:46:48.198490Z","iopub.status.idle":"2023-12-08T16:46:48.217951Z","shell.execute_reply.started":"2023-12-08T16:46:48.198449Z","shell.execute_reply":"2023-12-08T16:46:48.217158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Functions","metadata":{"tags":[]}},{"cell_type":"code","source":"def splitter(df):\n    train = df.index[~df['is_valid']].tolist()\n    valid = df.index[df['is_valid']].tolist()\n    return L(train), L(valid)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T16:46:48.219238Z","iopub.execute_input":"2023-12-08T16:46:48.219732Z","iopub.status.idle":"2023-12-08T16:46:48.224369Z","shell.execute_reply.started":"2023-12-08T16:46:48.219701Z","shell.execute_reply":"2023-12-08T16:46:48.223512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The function ```get_train_valid_data``` takes the train and test data as inputs and returns a ```TabularPandas``` object using the ```valid``` scheme and ```split_func``` provided.","metadata":{}},{"cell_type":"code","source":"def get_train_valid_data(train_data, test_data, valid=None, random_state=None, split_func=splitter):\n    train_data['is_valid'] = False\n    cont, cat = cont_cat_split(train_data, 1, dep_var='value')\n    if valid in ['NK cells', 'T cells CD4+', 'T cells CD8+', 'T regulatory cells']:\n        test_compounds = test_data['sm_name'].unique().tolist()\n        valid_indices = train_data.loc[(train_data['cell_type']==valid) & train_data['sm_name'].isin(test_compounds)].index.sort_values().tolist()\n        train_data.loc[valid_indices, 'is_valid'] = True\n        splits = splitter(train_data)\n        cont, cat = cont_cat_split(train_data, 1, dep_var='value')\n        cat.remove('is_valid')\n        to = TabularPandas(train_data, procs=[Categorify, FillMissing], cont_names=cont, cat_names=cat, y_names='value', splits=splits)\n    elif type(valid)==float:\n        valid_indices = train_data.sample(frac=valid, random_state=random_state).index.sort_values().tolist()\n        train_data.loc[valid_indices, 'is_valid'] = True\n        splits = splitter(train_data)\n        cat.remove('is_valid')\n        to = TabularPandas(train_data, procs=[Categorify, FillMissing], cont_names=cont, cat_names=cat, y_names='value', splits=splits)\n    else:\n        cat.remove('is_valid')\n        to = TabularPandas(train_data, procs=[Categorify, FillMissing], cont_names=cont, cat_names=cat, y_names='value', splits=None)\n    return to","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:48.230551Z","iopub.execute_input":"2023-12-08T16:46:48.231487Z","iopub.status.idle":"2023-12-08T16:46:48.250718Z","shell.execute_reply.started":"2023-12-08T16:46:48.231457Z","shell.execute_reply":"2023-12-08T16:46:48.249825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmse(preds, targs): return round(math.sqrt(((targs-preds)**2).mean()), 6)\ndef m_rmse(m, xs, y): return rmse(m.predict(xs), y)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T16:46:48.251787Z","iopub.execute_input":"2023-12-08T16:46:48.252537Z","iopub.status.idle":"2023-12-08T16:46:48.261500Z","shell.execute_reply.started":"2023-12-08T16:46:48.252510Z","shell.execute_reply":"2023-12-08T16:46:48.260559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The ```rf``` function trains a random forest and facilitates quick experimenting.","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\ndef rf(xs, y, n_estimators=100, max_samples=0.66, max_features='sqrt', min_samples_leaf=5, **kwargs):\n    return RandomForestRegressor(n_jobs=-1, n_estimators=n_estimators, max_samples=max_samples, max_features=max_features, min_samples_leaf=min_samples_leaf, oob_score=True).fit(xs, y)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T16:46:48.262475Z","iopub.execute_input":"2023-12-08T16:46:48.263053Z","iopub.status.idle":"2023-12-08T16:46:48.426356Z","shell.execute_reply.started":"2023-12-08T16:46:48.263027Z","shell.execute_reply":"2023-12-08T16:46:48.425061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add Embeddings","metadata":{}},{"cell_type":"markdown","source":"The following lines merge the embeddings with the train and test data.","metadata":{}},{"cell_type":"code","source":"train_df_c = pd.merge(train_df, cell_embs, on='cell_type', how='left')\ntrain_df_cm = pd.merge(train_df_c, mol_embs, on='sm_name', how='left')\ntrain_df_cmg = pd.merge(train_df_cm, gene_embs, on='gene', how='left')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:48.428291Z","iopub.execute_input":"2023-12-08T16:46:48.428691Z","iopub.status.idle":"2023-12-08T16:46:53.256899Z","shell.execute_reply.started":"2023-12-08T16:46:48.428658Z","shell.execute_reply":"2023-12-08T16:46:53.255518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df_c = pd.merge(test_df, cell_embs, on='cell_type', how='left')\ntest_df_cm = pd.merge(test_df_c, mol_embs, on='sm_name', how='left')\ntest_df_cmg = pd.merge(test_df_cm, gene_embs, on='gene', how='left')","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:53.258230Z","iopub.execute_input":"2023-12-08T16:46:53.258780Z","iopub.status.idle":"2023-12-08T16:46:55.266927Z","shell.execute_reply.started":"2023-12-08T16:46:53.258751Z","shell.execute_reply":"2023-12-08T16:46:55.266230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Delete Unused Data","metadata":{}},{"cell_type":"markdown","source":"And collect garbage to save some memory.","metadata":{}},{"cell_type":"code","source":"import gc\ndel train_df_c\ndel train_df_cm\ndel test_df_c\ndel test_df_cm\ndel cell_embs\ndel mol_embs\ndel gene_embs\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T16:46:55.268016Z","iopub.execute_input":"2023-12-08T16:46:55.268479Z","iopub.status.idle":"2023-12-08T16:46:55.725761Z","shell.execute_reply.started":"2023-12-08T16:46:55.268451Z","shell.execute_reply":"2023-12-08T16:46:55.724267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Hierarchie","metadata":{}},{"cell_type":"markdown","source":"Convert data and check features for redundancy.","metadata":{}},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_cmg, test_df_cmg, valid=None, random_state=random_seed)\nxs, y = to.train.xs, to.train.y","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:46:55.727087Z","iopub.execute_input":"2023-12-08T16:46:55.727785Z","iopub.status.idle":"2023-12-08T16:47:06.450182Z","shell.execute_reply.started":"2023-12-08T16:46:55.727749Z","shell.execute_reply":"2023-12-08T16:47:06.447912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom scipy.cluster import hierarchy\nfrom scipy.spatial.distance import squareform\ncorrelation_matrix = xs.corr()\ndistances = 1 - np.abs(correlation_matrix)\nnp.fill_diagonal(distances.values, 0)\ncondensed_distances = squareform(distances)\nlinkage_matrix = hierarchy.linkage(condensed_distances, method='complete')\nplt.figure(figsize=(10,10))\nhierarchy.dendrogram(linkage_matrix, labels=xs.columns.tolist(), orientation='left')\nplt.show()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T16:47:06.454407Z","iopub.execute_input":"2023-12-08T16:47:06.455263Z","iopub.status.idle":"2023-12-08T16:47:23.501378Z","shell.execute_reply.started":"2023-12-08T16:47:06.455196Z","shell.execute_reply":"2023-12-08T16:47:23.500605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Forest All Data","metadata":{}},{"cell_type":"markdown","source":"Finally I train a random forest using all training data, predict test scores and submit.","metadata":{}},{"cell_type":"code","source":"%%time\nm = rf(xs, y, random_state=random_seed)\nm_rmse(m, xs, y), rmse(m.oob_prediction_, y), m.oob_score_","metadata":{"execution":{"iopub.status.busy":"2023-12-08T16:47:23.502432Z","iopub.execute_input":"2023-12-08T16:47:23.502702Z","iopub.status.idle":"2023-12-08T17:28:46.837224Z","shell.execute_reply.started":"2023-12-08T16:47:23.502674Z","shell.execute_reply":"2023-12-08T17:28:46.836580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference and Submission","metadata":{}},{"cell_type":"code","source":"test_to = to.new(test_df_cmg)\ntest_to.process()\ntest_xs = test_to.xs\ntest_xs","metadata":{"papermill":{"duration":7.510266,"end_time":"2023-10-03T19:15:32.667807","exception":false,"start_time":"2023-10-03T19:15:25.157541","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-08T17:28:46.840840Z","iopub.execute_input":"2023-12-08T17:28:46.841445Z","iopub.status.idle":"2023-12-08T17:28:48.852925Z","shell.execute_reply.started":"2023-12-08T17:28:46.841414Z","shell.execute_reply":"2023-12-08T17:28:48.851992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = tensor(m.predict(test_xs))\npreds","metadata":{"papermill":{"duration":65.204612,"end_time":"2023-10-03T19:16:37.884386","exception":false,"start_time":"2023-10-03T19:15:32.679774","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-08T17:28:48.854592Z","iopub.execute_input":"2023-12-08T17:28:48.855337Z","iopub.status.idle":"2023-12-08T17:29:51.807761Z","shell.execute_reply.started":"2023-12-08T17:28:48.855304Z","shell.execute_reply":"2023-12-08T17:29:51.805866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.min(), preds.max()","metadata":{"tags":[],"execution":{"iopub.status.busy":"2023-12-08T17:29:51.809061Z","iopub.execute_input":"2023-12-08T17:29:51.809437Z","iopub.status.idle":"2023-12-08T17:29:51.818819Z","shell.execute_reply.started":"2023-12-08T17:29:51.809405Z","shell.execute_reply":"2023-12-08T17:29:51.816975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the following step I reshape the predictions back into a 255 x 18211 tensor for submission:","metadata":{}},{"cell_type":"code","source":"to_submit = preds.view(18211, -1).t().numpy()\nsubmit = pd.DataFrame(to_submit, columns=df_de_train.iloc[:,5:].columns)\nsubmit.index.name = 'id'\nsubmit","metadata":{"papermill":{"duration":0.152745,"end_time":"2023-10-03T19:16:38.048033","exception":false,"start_time":"2023-10-03T19:16:37.895288","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-08T17:29:51.820705Z","iopub.execute_input":"2023-12-08T17:29:51.821149Z","iopub.status.idle":"2023-12-08T17:29:51.880172Z","shell.execute_reply.started":"2023-12-08T17:29:51.821077Z","shell.execute_reply":"2023-12-08T17:29:51.878896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit.to_csv('submission.csv')","metadata":{"papermill":{"duration":7.117791,"end_time":"2023-10-03T19:16:45.177690","exception":false,"start_time":"2023-10-03T19:16:38.059899","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-12-08T17:29:51.882084Z","iopub.execute_input":"2023-12-08T17:29:51.882413Z","iopub.status.idle":"2023-12-08T17:29:56.448339Z","shell.execute_reply.started":"2023-12-08T17:29:51.882384Z","shell.execute_reply":"2023-12-08T17:29:56.446769Z"},"trusted":true},"execution_count":null,"outputs":[]}]}