{"metadata":{"kernelspec":{"display_name":"saturn (Python 3)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.15"},"papermill":{"default_parameters":{},"duration":1767.896818,"end_time":"2023-10-03T19:16:48.560142","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-10-03T18:47:20.663324","version":"2.4.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7178058,"sourceType":"datasetVersion","datasetId":4148415}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\nt0start = time.time()\nfrom fastai.collab import *\nfrom fastai.tabular.all import *","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2023-11-14T15:24:45.320965Z","iopub.status.busy":"2023-11-14T15:24:45.320568Z","iopub.status.idle":"2023-11-14T15:24:50.452076Z","shell.execute_reply":"2023-11-14T15:24:50.451397Z","shell.execute_reply.started":"2023-11-14T15:24:45.320933Z"},"papermill":{"duration":6.509942,"end_time":"2023-10-03T18:47:30.760703","exception":false,"start_time":"2023-10-03T18:47:24.250761","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_seed = 42 #272 #582","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:24:50.453709Z","iopub.status.busy":"2023-11-14T15:24:50.453325Z","iopub.status.idle":"2023-11-14T15:24:50.456662Z","shell.execute_reply":"2023-11-14T15:24:50.456153Z","shell.execute_reply.started":"2023-11-14T15:24:50.453686Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading and Melting Train and Test Data","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nfn = '/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet'\ndf_de_train = pd.read_parquet(fn)# , index_col = 0)\ntrain_df = df_de_train.melt(id_vars=['cell_type', 'sm_name'], value_vars=df_de_train.iloc[:,5:].columns, var_name='gene', value_name='value')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:24:50.457549Z","iopub.status.busy":"2023-11-14T15:24:50.457354Z","iopub.status.idle":"2023-11-14T15:24:52.231928Z","shell.execute_reply":"2023-11-14T15:24:52.231313Z","shell.execute_reply.started":"2023-11-14T15:24:50.457531Z"},"papermill":{"duration":163.861162,"end_time":"2023-10-03T18:50:18.214746","exception":false,"start_time":"2023-10-03T18:47:34.353584","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = '/kaggle/input/open-problems-single-cell-perturbations/id_map.csv'\ndf_id_map = pd.read_csv(fn)\nfn = '/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv'\ndf = pd.read_csv(fn, index_col = 0)\n\ncols_to_add = df_de_train.iloc[:,5:].columns\ncols_to_add\n\ndf_zeros = pd.DataFrame(0.0, columns=cols_to_add, index=df_id_map.index)\ndf_zeros\n\ndf_id_map_preds = pd.concat([df_id_map, df_zeros], axis=1)\ntest_df = df_id_map_preds.melt(id_vars=['cell_type', 'sm_name'], value_vars=df_id_map_preds.iloc[:,3:].columns, var_name='gene', value_name='value')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:24:52.233493Z","iopub.status.busy":"2023-11-14T15:24:52.233196Z","iopub.status.idle":"2023-11-14T15:24:54.471084Z","shell.execute_reply":"2023-11-14T15:24:54.470439Z","shell.execute_reply.started":"2023-11-14T15:24:52.233471Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baseline","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"def splitter(df):\n    train = df.index[~df['is_valid']].tolist()\n    valid = df.index[df['is_valid']].tolist()\n    return L(train), L(valid)\n\ndef get_train_valid_data(train_data, test_data, valid=None, random_state=None, split_func=splitter):\n    train_data['is_valid'] = False\n    cont, cat = cont_cat_split(train_data, 1, dep_var='value')\n    if valid in ['NK cells', 'T cells CD4+', 'T cells CD8+', 'T regulatory cells']:\n        test_compounds = test_data['sm_name'].unique().tolist()\n        valid_indices = train_data.loc[(train_data['cell_type']==valid) & train_data['sm_name'].isin(test_compounds)].index.sort_values().tolist()\n        train_data.loc[valid_indices, 'is_valid'] = True\n        splits = splitter(train_data)\n        cont, cat = cont_cat_split(train_data, 1, dep_var='value')\n        cat.remove('is_valid')\n        to = TabularPandas(train_data, procs=[Categorify, FillMissing], cont_names=cont, cat_names=cat, y_names='value', splits=splits)\n    elif type(valid)==float:\n        valid_indices = train_data.sample(frac=valid, random_state=random_state).index.sort_values().tolist()\n        train_data.loc[valid_indices, 'is_valid'] = True\n        splits = splitter(train_data)\n        cat.remove('is_valid')\n        to = TabularPandas(train_data, procs=[Categorify, FillMissing], cont_names=cont, cat_names=cat, y_names='value', splits=splits)\n    else:\n        cat.remove('is_valid')\n        to = TabularPandas(train_data, procs=[Categorify, FillMissing], cont_names=cont, cat_names=cat, y_names='value', splits=None)\n    return to\n\ndef rmse(preds, targs): return round(math.sqrt(((targs-preds)**2).mean()), 6)\ndef m_rmse(m, xs, y): return rmse(m.predict(xs), y)\n\nfrom sklearn.ensemble import RandomForestRegressor\n\ndef rf(xs, y, n_estimators=100, max_samples=0.66, max_features='sqrt', min_samples_leaf=5, **kwargs):\n    return RandomForestRegressor(n_jobs=-1, n_estimators=n_estimators, max_samples=max_samples, max_features=max_features, min_samples_leaf=min_samples_leaf, oob_score=True).fit(xs, y)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:24:54.472379Z","iopub.status.busy":"2023-11-14T15:24:54.472070Z","iopub.status.idle":"2023-11-14T15:24:54.760525Z","shell.execute_reply":"2023-11-14T15:24:54.759921Z","shell.execute_reply.started":"2023-11-14T15:24:54.472357Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df, test_df, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y","metadata":{"execution":{"iopub.execute_input":"2023-11-13T14:29:14.272837Z","iopub.status.busy":"2023-11-13T14:29:14.272453Z","iopub.status.idle":"2023-11-13T14:29:20.775556Z","shell.execute_reply":"2023-11-13T14:29:20.774954Z","shell.execute_reply.started":"2023-11-13T14:29:14.272813Z"},"papermill":{"duration":29.185442,"end_time":"2023-10-03T18:52:39.937817","exception":false,"start_time":"2023-10-03T18:52:10.752375","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Baseline Check Feature Redundance","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nfrom scipy.cluster import hierarchy\nfrom scipy.spatial.distance import squareform\ncorrelation_matrix = xs.corr()\ndistances = 1 - np.abs(correlation_matrix)\nnp.fill_diagonal(distances.values, 0)\ncondensed_distances = squareform(distances)\nlinkage_matrix = hierarchy.linkage(condensed_distances, method='complete')\nplt.figure(figsize=(4,2))\nhierarchy.dendrogram(linkage_matrix, labels=xs.columns.tolist(), orientation='left')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T14:29:20.776969Z","iopub.status.busy":"2023-11-13T14:29:20.776684Z","iopub.status.idle":"2023-11-13T14:29:21.264735Z","shell.execute_reply":"2023-11-13T14:29:21.264163Z","shell.execute_reply.started":"2023-11-13T14:29:20.776948Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Baseline Random Forest (Random Data)","metadata":{"jp-MarkdownHeadingCollapsed":true,"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nm = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:30:26.458964Z","iopub.status.busy":"2023-11-13T03:30:26.458683Z","iopub.status.idle":"2023-11-13T03:34:56.448421Z","shell.execute_reply":"2023-11-13T03:34:56.447751Z","shell.execute_reply.started":"2023-11-13T03:30:26.458944Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_plain = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nresults_plain","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:34:56.455838Z","iopub.status.busy":"2023-11-13T03:34:56.455532Z","iopub.status.idle":"2023-11-13T03:35:15.790077Z","shell.execute_reply":"2023-11-13T03:35:15.789461Z","shell.execute_reply.started":"2023-11-13T03:34:56.455819Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Baseline Unseen Cell Type Cross Validation","metadata":{}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(train_df, test_df, valid='NK cells')\ncd4_to = get_train_valid_data(train_df, test_df, valid='T cells CD4+')\ncd8_to = get_train_valid_data(train_df, test_df, valid='T cells CD8+')\nreg_to = get_train_valid_data(train_df, test_df, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:55:11.556016Z","iopub.status.busy":"2023-11-13T03:55:11.555627Z","iopub.status.idle":"2023-11-13T03:55:35.899773Z","shell.execute_reply":"2023-11-13T03:55:35.899156Z","shell.execute_reply.started":"2023-11-13T03:55:11.555993Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_plain = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_plain[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}: {cv_results_plain[key]}')\n    \nav_cv_result_plain = tuple(round(i.item(), 6) for i in tensor(list(cv_results_plain.values())).mean(0))\nprint(f'Average Result: {av_cv_result_plain}')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T04:11:26.373223Z","iopub.status.busy":"2023-11-13T04:11:26.372832Z","iopub.status.idle":"2023-11-13T04:30:35.499993Z","shell.execute_reply":"2023-11-13T04:30:35.499391Z","shell.execute_reply.started":"2023-11-13T04:11:26.373199Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Baseline Feature Importance","metadata":{"papermill":{"duration":0.009248,"end_time":"2023-10-03T19:15:24.281571","exception":false,"start_time":"2023-10-03T19:15:24.272323","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rf_feat_importance(m, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}).sort_values('imp', ascending=False)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:25:58.973504Z","iopub.status.busy":"2023-11-14T15:25:58.973099Z","iopub.status.idle":"2023-11-14T15:25:58.977087Z","shell.execute_reply":"2023-11-14T15:25:58.976519Z","shell.execute_reply.started":"2023-11-14T15:25:58.973478Z"},"papermill":{"duration":0.020571,"end_time":"2023-10-03T19:15:24.312495","exception":false,"start_time":"2023-10-03T19:15:24.291924","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = rf_feat_importance(m, xs)\nfi","metadata":{"execution":{"iopub.execute_input":"2023-11-13T14:29:35.133327Z","iopub.status.busy":"2023-11-13T14:29:35.132928Z","iopub.status.idle":"2023-11-13T14:29:35.447075Z","shell.execute_reply":"2023-11-13T14:29:35.446320Z","shell.execute_reply.started":"2023-11-13T14:29:35.133301Z"},"papermill":{"duration":0.214009,"end_time":"2023-10-03T19:15:24.537400","exception":false,"start_time":"2023-10-03T19:15:24.323391","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_fi(fi, figsize=(10,7)):\n    return fi.plot('cols', 'imp', 'barh', figsize=figsize, legend=False)\nplot_fi(fi, figsize=(4,2));","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:01.354922Z","iopub.status.busy":"2023-11-14T15:26:01.354520Z","iopub.status.idle":"2023-11-14T15:26:01.375581Z","shell.execute_reply":"2023-11-14T15:26:01.374850Z","shell.execute_reply.started":"2023-11-14T15:26:01.354897Z"},"papermill":{"duration":0.555166,"end_time":"2023-10-03T19:15:25.103016","exception":false,"start_time":"2023-10-03T19:15:24.547850","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Molecular Descriptors","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"compounds = df_de_train.iloc[:, 1:4].drop(columns='sm_lincs_id')\ncompounds = compounds.drop_duplicates(keep='first').reset_index(drop=True)\ncompounds","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:25:15.985884Z","iopub.status.busy":"2023-11-14T15:25:15.985472Z","iopub.status.idle":"2023-11-14T15:25:16.004745Z","shell.execute_reply":"2023-11-14T15:25:16.004216Z","shell.execute_reply.started":"2023-11-14T15:25:15.985858Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Calculate Molecular Descriptors","metadata":{"execution":{"iopub.execute_input":"2023-11-13T02:08:58.680456Z","iopub.status.busy":"2023-11-13T02:08:58.680077Z","iopub.status.idle":"2023-11-13T02:08:58.683390Z","shell.execute_reply":"2023-11-13T02:08:58.682845Z","shell.execute_reply.started":"2023-11-13T02:08:58.680434Z"},"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"import rdkit, rdkit.Chem, rdkit.Chem.Draw\nimport mordred, mordred.descriptors\n\n# make object that can compute descriptors\ncalc = mordred.Calculator(mordred.descriptors, ignore_3D=True)\n# make subsample from pandas df\nmolecules = [rdkit.Chem.MolFromSmiles(smi) for smi in compounds['SMILES']]\n# view one molecule to make sure things look good.\nmolecules[0]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:25:17.041703Z","iopub.status.busy":"2023-11-14T15:25:17.041314Z","iopub.status.idle":"2023-11-14T15:25:17.300746Z","shell.execute_reply":"2023-11-14T15:25:17.300174Z","shell.execute_reply.started":"2023-11-14T15:25:17.041677Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Uncomment to calculate descriptors\n# features = calc.pandas(molecules)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:25:18.097613Z","iopub.status.busy":"2023-11-14T15:25:18.097243Z","iopub.status.idle":"2023-11-14T15:25:18.100463Z","shell.execute_reply":"2023-11-14T15:25:18.099891Z","shell.execute_reply.started":"2023-11-14T15:25:18.097589Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Uncomment to concatenate calculated features with drug names\n# mol_descriptors = pd.concat([compounds.drop(columns=['SMILES']), features.astype(float)], axis=1)\nmol_descriptors = pd.read_parquet('/kaggle/input/mol_descriptors.parquet') ## This is the saved file comment out if descriptors were calculated from scratch\nmol_descriptors","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:25:20.232191Z","iopub.status.busy":"2023-11-14T15:25:20.231788Z","iopub.status.idle":"2023-11-14T15:25:20.358940Z","shell.execute_reply":"2023-11-14T15:25:20.358392Z","shell.execute_reply.started":"2023-11-14T15:25:20.232165Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Uncomment to save file:\n# mol_descriptors.to_csv('/kaggle/input/mol_descriptors.csv')\n# mol_descriptors.to_parquet('/kaggle/input/mol_descriptors.parquet')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:25:20.766886Z","iopub.status.busy":"2023-11-14T15:25:20.766552Z","iopub.status.idle":"2023-11-14T15:25:20.769685Z","shell.execute_reply":"2023-11-14T15:25:20.769112Z","shell.execute_reply.started":"2023-11-14T15:25:20.766864Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following ```dataframe``` contains feature importance values obtained training a random forrest with all 1614 molecular descriptors:","metadata":{}},{"cell_type":"code","source":"mol_desc_feature_importance = pd.read_csv('/kaggle/input/mol_desc_feat_imp.csv', index_col = 0)\nprint(mol_desc_feature_importance[:10])\nplot_fi(mol_desc_feature_importance[2:30]);","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:15.858515Z","iopub.status.busy":"2023-11-14T15:26:15.858099Z","iopub.status.idle":"2023-11-14T15:26:16.230451Z","shell.execute_reply":"2023-11-14T15:26:16.229903Z","shell.execute_reply.started":"2023-11-14T15:26:15.858490Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The next file contains feature importance values obtained training a random forest using only the molecular descriptors that achieved feature importance values greater or equal to 0.001, which yielded 33 descriptors (including ```sm_name```) from which only those that achieved feature importance values greater or equal to 0.005 were chosen.","metadata":{}},{"cell_type":"code","source":"important_moldesc = pd.read_csv('/kaggle/input/important_mol_descriptors.csv', index_col = 0)\nimportant_moldesc.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:18.753790Z","iopub.status.busy":"2023-11-14T15:26:18.753405Z","iopub.status.idle":"2023-11-14T15:26:18.763363Z","shell.execute_reply":"2023-11-14T15:26:18.762768Z","shell.execute_reply.started":"2023-11-14T15:26:18.753765Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add Most Important Molecular Descriptors","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"to_keep = important_moldesc.cols.to_list()\nimportant_mol_descriptors = mol_descriptors[to_keep]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:28:07.635495Z","iopub.status.busy":"2023-11-14T15:28:07.635095Z","iopub.status.idle":"2023-11-14T15:28:07.639741Z","shell.execute_reply":"2023-11-14T15:28:07.639160Z","shell.execute_reply.started":"2023-11-14T15:28:07.635470Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_moldesc = pd.merge(train_df, important_mol_descriptors, on='sm_name', how='left')\ncols = train_df_moldesc.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntrain_df_moldesc = train_df_moldesc[cols]\n\ntest_df_moldesc = pd.merge(test_df, important_mol_descriptors, on='sm_name', how='left')\ncols = test_df_moldesc.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntest_df_moldesc = test_df_moldesc[cols]","metadata":{"execution":{"iopub.execute_input":"2023-11-13T14:30:00.740129Z","iopub.status.busy":"2023-11-13T14:30:00.739742Z","iopub.status.idle":"2023-11-13T14:30:07.206006Z","shell.execute_reply":"2023-11-13T14:30:07.205366Z","shell.execute_reply.started":"2023-11-13T14:30:00.740103Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Improvement After Adding Important Molecular Descriptors","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_moldesc, test_df, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:35:24.510832Z","iopub.status.busy":"2023-11-13T03:35:24.510530Z","iopub.status.idle":"2023-11-13T03:35:38.826885Z","shell.execute_reply":"2023-11-13T03:35:38.826246Z","shell.execute_reply.started":"2023-11-13T03:35:24.510812Z"},"papermill":{"duration":0.15509,"end_time":"2023-10-03T18:52:10.743315","exception":false,"start_time":"2023-10-03T18:52:10.588225","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Molecular Descriptors Check Feature Redundance","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nfrom scipy.cluster import hierarchy\nfrom scipy.spatial.distance import squareform\ncorrelation_matrix = xs.corr()\ndistances = 1 - np.abs(correlation_matrix)\nnp.fill_diagonal(distances.values, 0)\ncondensed_distances = squareform(distances)\nlinkage_matrix = hierarchy.linkage(condensed_distances, method='complete')\nplt.figure(figsize=(10,5))\nhierarchy.dendrogram(linkage_matrix, labels=xs.columns.tolist(), orientation='left')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:35:38.828026Z","iopub.status.busy":"2023-11-13T03:35:38.827728Z","iopub.status.idle":"2023-11-13T03:35:54.393347Z","shell.execute_reply":"2023-11-13T03:35:54.392757Z","shell.execute_reply.started":"2023-11-13T03:35:38.828006Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Molecular Descriptors Random Forest","metadata":{"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Let's fit a model after adding the molecular descriptors and compare it to the results of the model without the descriptors:","metadata":{"execution":{"iopub.execute_input":"2023-11-13T02:26:23.283687Z","iopub.status.busy":"2023-11-13T02:26:23.283308Z","iopub.status.idle":"2023-11-13T02:26:23.287693Z","shell.execute_reply":"2023-11-13T02:26:23.287010Z","shell.execute_reply.started":"2023-11-13T02:26:23.283665Z"},"tags":[]}},{"cell_type":"code","source":"m = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:35:54.394517Z","iopub.status.busy":"2023-11-13T03:35:54.394220Z","iopub.status.idle":"2023-11-13T03:43:58.798970Z","shell.execute_reply":"2023-11-13T03:43:58.798329Z","shell.execute_reply.started":"2023-11-13T03:35:54.394497Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_mdesc = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with molecular descriptors:\\n{results_mdesc}')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:43:58.800385Z","iopub.status.busy":"2023-11-13T03:43:58.799822Z","iopub.status.idle":"2023-11-13T03:44:40.749005Z","shell.execute_reply":"2023-11-13T03:44:40.748361Z","shell.execute_reply.started":"2023-11-13T03:43:58.800365Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Molecular Descriptors Unseen Cell Type Cross Validation","metadata":{}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(train_df_moldesc, test_df_moldesc, valid='NK cells')\ncd4_to = get_train_valid_data(train_df_moldesc, test_df_moldesc, valid='T cells CD4+')\ncd8_to = get_train_valid_data(train_df_moldesc, test_df_moldesc, valid='T cells CD8+')\nreg_to = get_train_valid_data(train_df_moldesc, test_df_moldesc, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T04:51:19.164868Z","iopub.status.busy":"2023-11-13T04:51:19.164484Z","iopub.status.idle":"2023-11-13T04:52:06.182251Z","shell.execute_reply":"2023-11-13T04:52:06.181613Z","shell.execute_reply.started":"2023-11-13T04:51:19.164843Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_moldesc = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_moldesc[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}: {cv_results_moldesc[key]}')\n    print(f'Results plain: {cv_results_plain[key]}')\n    \nav_cv_result_moldesc = tuple(round(i.item(), 6) for i in tensor(list(cv_results_moldesc.values())).mean(0))\nprint(f'Average Result: {av_cv_result_moldesc}')\nprint(f'Average Plain : {av_cv_result_plain}')\n\n# Training model for NK cells...\n# Results for NK cells: (1.741441, 1.813849, 2.025264, 0.318259)\n# Results plain: (1.739662, 4.170631, 2.018356, 0.32290143296121143)\n# Training model for T cells CD4+...\n# Results for T cells CD4+: (1.789305, 1.60049, 2.083509, 0.335666)\n# Results plain: (1.788045, 2.050092, 2.077544, 0.3394645135326362)\n# Training model for T cells CD8+...\n# Results for T cells CD8+: (1.847783, 1.071253, 2.149825, 0.31715)\n# Results plain: (1.846364, 1.159838, 2.141735, 0.3222795700496216)\n# Training model for T regulatory cells...\n# Results for T regulatory cells: (1.838012, 1.637333, 2.136911, 0.303395)\n# Results plain: (1.836061, 4.709332, 2.128992, 0.3085484285086162)\n# Average Result: (1.804135, 1.530731, 2.098877, 0.318617)\n# Average Plain : (1.8025, 3.0225, 2.0917, 0.3233)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T04:52:06.183617Z","iopub.status.busy":"2023-11-13T04:52:06.183316Z","iopub.status.idle":"2023-11-13T05:26:03.787719Z","shell.execute_reply":"2023-11-13T05:26:03.787094Z","shell.execute_reply.started":"2023-11-13T04:52:06.183596Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Molecular Descriptors Feature Importance","metadata":{"papermill":{"duration":0.009248,"end_time":"2023-10-03T19:15:24.281571","exception":false,"start_time":"2023-10-03T19:15:24.272323","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def rf_feat_importance(m, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}).sort_values('imp', ascending=False)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:44:40.752169Z","iopub.status.busy":"2023-11-13T03:44:40.751859Z","iopub.status.idle":"2023-11-13T03:44:40.755588Z","shell.execute_reply":"2023-11-13T03:44:40.755001Z","shell.execute_reply.started":"2023-11-13T03:44:40.752148Z"},"papermill":{"duration":0.020571,"end_time":"2023-10-03T19:15:24.312495","exception":false,"start_time":"2023-10-03T19:15:24.291924","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = rf_feat_importance(m, xs)\nfi.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:44:40.756509Z","iopub.status.busy":"2023-11-13T03:44:40.756302Z","iopub.status.idle":"2023-11-13T03:44:40.914738Z","shell.execute_reply":"2023-11-13T03:44:40.914111Z","shell.execute_reply.started":"2023-11-13T03:44:40.756491Z"},"papermill":{"duration":0.214009,"end_time":"2023-10-03T19:15:24.537400","exception":false,"start_time":"2023-10-03T19:15:24.323391","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_fi(fi, figsize=(10,7)):\n    return fi.plot('cols', 'imp', 'barh', figsize=figsize, legend=False)\nplot_fi(fi);","metadata":{"execution":{"iopub.execute_input":"2023-11-13T03:44:40.915853Z","iopub.status.busy":"2023-11-13T03:44:40.915569Z","iopub.status.idle":"2023-11-13T03:44:41.195469Z","shell.execute_reply":"2023-11-13T03:44:41.194901Z","shell.execute_reply.started":"2023-11-13T03:44:40.915835Z"},"papermill":{"duration":0.555166,"end_time":"2023-10-03T19:15:25.103016","exception":false,"start_time":"2023-10-03T19:15:24.547850","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Gene Information","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"markdown","source":"### Load Geneinfo Data Set (data obtained from the ensembl data base using pybiomart)","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nfn = '/kaggle/input/geneinfo.parquet'\ngeneinfo = pd.read_parquet(fn)# , index_col = 0)\ngeneinfo","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:49.136694Z","iopub.status.busy":"2023-11-14T15:26:49.136287Z","iopub.status.idle":"2023-11-14T15:26:49.508325Z","shell.execute_reply":"2023-11-14T15:26:49.507751Z","shell.execute_reply.started":"2023-11-14T15:26:49.136670Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add Gene Information to Dataset","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"def get_peptide_length(x):\n    if x: return len(x)-1\n    else: return None","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:51.329545Z","iopub.status.busy":"2023-11-14T15:26:51.329144Z","iopub.status.idle":"2023-11-14T15:26:51.333058Z","shell.execute_reply":"2023-11-14T15:26:51.332462Z","shell.execute_reply.started":"2023-11-14T15:26:51.329520Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_number_of_terms(x):\n    if x: return len(x.split(';'))","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:51.538338Z","iopub.status.busy":"2023-11-14T15:26:51.538053Z","iopub.status.idle":"2023-11-14T15:26:51.541234Z","shell.execute_reply":"2023-11-14T15:26:51.540688Z","shell.execute_reply.started":"2023-11-14T15:26:51.538318Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_number_of_terms(geneinfo['go_evidences'][0])","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:51.733017Z","iopub.status.busy":"2023-11-14T15:26:51.732747Z","iopub.status.idle":"2023-11-14T15:26:51.737004Z","shell.execute_reply":"2023-11-14T15:26:51.736505Z","shell.execute_reply.started":"2023-11-14T15:26:51.732998Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def term_isin_string(x, term=None):\n    if x: return term in x\n    else: return x","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:51.884066Z","iopub.status.busy":"2023-11-14T15:26:51.883580Z","iopub.status.idle":"2023-11-14T15:26:51.886734Z","shell.execute_reply":"2023-11-14T15:26:51.886231Z","shell.execute_reply.started":"2023-11-14T15:26:51.884047Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"term_isin_string(geneinfo['go_names'][0], 'molecular_function')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:52.070470Z","iopub.status.busy":"2023-11-14T15:26:52.069924Z","iopub.status.idle":"2023-11-14T15:26:52.074262Z","shell.execute_reply":"2023-11-14T15:26:52.073766Z","shell.execute_reply.started":"2023-11-14T15:26:52.070449Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def truncate_tsl(x):\n    if x: return x[:4]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:52.226511Z","iopub.status.busy":"2023-11-14T15:26:52.226141Z","iopub.status.idle":"2023-11-14T15:26:52.229132Z","shell.execute_reply":"2023-11-14T15:26:52.228618Z","shell.execute_reply.started":"2023-11-14T15:26:52.226492Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"more_geneinfo = geneinfo\nmore_geneinfo['bio_proc'] = list(map(partial(term_isin_string, term='biological_process'), more_geneinfo['go_names']))\nmore_geneinfo['mol_func'] = list(map(partial(term_isin_string, term='molecular_function'), more_geneinfo['go_names']))\nmore_geneinfo['cell_comp'] = list(map(partial(term_isin_string, term='cellular_component'), more_geneinfo['go_names']))\n# more_geneinfo['chem_resp'] = list(map(partial(term_isin_string, term='response to chemical'), more_geneinfo['go_names']))\n# more_geneinfo['xbio_resp'] = list(map(partial(term_isin_string, term='response to xenobiotic stimulus'), more_geneinfo['go_names']))\n\nmore_geneinfo['peptide_length'] = list(map(get_peptide_length, more_geneinfo['peptide_sequence'].replace(['Sequence unavailable', np.nan], None)))\nmore_geneinfo['num_go_ids'] = list(map(get_number_of_terms, more_geneinfo['go_ids'].replace([np.nan], None)))\n# train_df_expanded['num_go_names'] = list(map(get_number_of_terms, train_df_expanded['go_names'].replace([np.nan], None))) ## same as num_go_ids\nmore_geneinfo['num_go_evidences'] = list(map(get_number_of_terms, more_geneinfo['go_evidences'].replace([np.nan], None)))\nmore_geneinfo['tsl'] = list(map(truncate_tsl, more_geneinfo['tsl']))","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:52.792781Z","iopub.status.busy":"2023-11-14T15:26:52.792239Z","iopub.status.idle":"2023-11-14T15:26:52.862843Z","shell.execute_reply":"2023-11-14T15:26:52.862280Z","shell.execute_reply.started":"2023-11-14T15:26:52.792761Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# all_go_terms_agg = more_geneinfo['go_ids'].dropna().unique().tolist()\n# all_go_terms = []\n# for string in all_go_terms_agg:\n#     all_go_terms.extend(string.split(';'))\n\n# unique_go_terms = pd.Series(all_go_terms).unique()\n# print('Total number of GO terms:', len(unique_go_terms))\n\n# new_columns = {}\n\n# ## Adding first 500 go terms\n# for i, term in enumerate(unique_go_terms[:10]):\n#     if not i%50: print(i)\n#     new_columns[term] = more_geneinfo['go_ids'].str.contains(term)\n\n# # Create a new DataFrame from the dictionary\n# go_terms_df = pd.DataFrame(new_columns)\n\n# # Concatenate the new DataFrame with the original DataFrame\n# more_geneinfo = pd.concat([more_geneinfo, go_terms_df], axis=1)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:53.317359Z","iopub.status.busy":"2023-11-14T15:26:53.316703Z","iopub.status.idle":"2023-11-14T15:26:53.320207Z","shell.execute_reply":"2023-11-14T15:26:53.319674Z","shell.execute_reply.started":"2023-11-14T15:26:53.317334Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_geneinfo = more_geneinfo # pd.concat([more_geneinfo.iloc[:, 0], more_geneinfo.iloc[:, -10:]], axis=1)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:26:53.798179Z","iopub.status.busy":"2023-11-14T15:26:53.797854Z","iopub.status.idle":"2023-11-14T15:26:53.801183Z","shell.execute_reply":"2023-11-14T15:26:53.800654Z","shell.execute_reply.started":"2023-11-14T15:26:53.798156Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_ginf = pd.merge(train_df, selected_geneinfo, on='gene', how='left')\ntrain_df_ginf = train_df_ginf.drop(columns=['is_canonical', 'transcript_source', 'gencode', 'strand', 'end', 'appris', 'bio_proc', 'mol_func', 'cell_comp'])\n# train_df_ginf['strand'] = train_df_ginf['strand'].fillna(0).astype('int')\ntrain_df_ginf['transcript_count'] = train_df_ginf['transcript_count'].fillna(0).astype('int')\ntrain_df_ginf['start'] = train_df_ginf['start'].fillna(0).astype('int')\ntrain_df_ginf = train_df_ginf.rename(columns={'start': 'position'})\n# train_df_ginf['end'] = train_df_ginf['end'].fillna(0).astype('int')\ntrain_df_ginf['length'] = train_df_ginf['length'].fillna(0).astype('int')\ntrain_df_ginf['peptide_length'] = train_df_ginf['peptide_length'].fillna(0).astype('int')\ntrain_df_ginf['num_go_ids'] = train_df_ginf['num_go_ids'].fillna(0).astype('int')\n# train_df_expanded['num_go_names'] = train_df_expanded['num_go_names'].fillna(0).astype('int')\ntrain_df_ginf['num_go_evidences'] = train_df_ginf['num_go_evidences'].fillna(0).astype('int')\ntrain_df_ginf['nG'] = train_df_ginf['nG'].fillna(0).astype('int')\ntrain_df_ginf['nC'] = train_df_ginf['nC'].fillna(0).astype('int')\ntrain_df_ginf['nA'] = train_df_ginf['nA'].fillna(0).astype('int')\ntrain_df_ginf['nT'] = train_df_ginf['nT'].fillna(0).astype('int')\ncols = train_df_ginf.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntrain_df_ginf = train_df_ginf[cols]","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:26:17.276632Z","iopub.status.busy":"2023-11-13T17:26:17.276233Z","iopub.status.idle":"2023-11-13T17:26:35.241759Z","shell.execute_reply":"2023-11-13T17:26:35.241080Z","shell.execute_reply.started":"2023-11-13T17:26:17.276607Z"},"papermill":{"duration":12.646394,"end_time":"2023-10-03T18:50:58.470110","exception":false,"start_time":"2023-10-03T18:50:45.823716","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df_ginf = pd.merge(test_df, selected_geneinfo, on='gene', how='left')\ntest_df_ginf = test_df_ginf.drop(columns=['is_canonical', 'transcript_source', 'gencode', 'strand', 'end', 'appris', 'bio_proc', 'mol_func', 'cell_comp'])\n# test_df_ginf['strand'] = test_df_ginf['strand'].fillna(0).astype('int')\ntest_df_ginf['transcript_count'] = test_df_ginf['transcript_count'].fillna(0).astype('int')\ntest_df_ginf['start'] = test_df_ginf['start'].fillna(0).astype('int')\ntest_df_ginf = test_df_ginf.rename(columns={'start': 'position'})\n# test_df_ginf['end'] = test_df_ginf['end'].fillna(0).astype('int')\ntest_df_ginf['length'] = test_df_ginf['length'].fillna(0).astype('int')\ntest_df_ginf['peptide_length'] = test_df_ginf['peptide_length'].fillna(0).astype('int')\ntest_df_ginf['num_go_ids'] = test_df_ginf['num_go_ids'].fillna(0).astype('int')\ntest_df_ginf['num_go_evidences'] = test_df_ginf['num_go_evidences'].fillna(0).astype('int')\ntest_df_ginf['nG'] = test_df_ginf['nG'].fillna(0).astype('int')\ntest_df_ginf['nC'] = test_df_ginf['nC'].fillna(0).astype('int')\ntest_df_ginf['nA'] = test_df_ginf['nA'].fillna(0).astype('int')\ntest_df_ginf['nT'] = test_df_ginf['nT'].fillna(0).astype('int')\ncols = test_df_ginf.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntest_df_ginf = test_df_ginf[cols]","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:26:35.243354Z","iopub.status.busy":"2023-11-13T17:26:35.243054Z","iopub.status.idle":"2023-11-13T17:26:42.460920Z","shell.execute_reply":"2023-11-13T17:26:42.460279Z","shell.execute_reply.started":"2023-11-13T17:26:35.243332Z"},"papermill":{"duration":2.928413,"end_time":"2023-10-03T18:52:10.122057","exception":false,"start_time":"2023-10-03T18:52:07.193644","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Improvement After Adding Gene Information","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_ginf, test_df_ginf, valid=None, random_state=random_seed)\nxs, y = to.train.xs, to.train.y","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:22:52.695881Z","iopub.status.busy":"2023-11-13T16:22:52.695577Z","iopub.status.idle":"2023-11-13T16:23:08.358919Z","shell.execute_reply":"2023-11-13T16:23:08.358316Z","shell.execute_reply.started":"2023-11-13T16:22:52.695859Z"},"papermill":{"duration":0.15509,"end_time":"2023-10-03T18:52:10.743315","exception":false,"start_time":"2023-10-03T18:52:10.588225","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Info Check Feature Redundance","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nfrom scipy.cluster import hierarchy\nfrom scipy.spatial.distance import squareform\ncorrelation_matrix = xs.corr()\ndistances = 1 - np.abs(correlation_matrix)\nnp.fill_diagonal(distances.values, 0)\ncondensed_distances = squareform(distances)\nlinkage_matrix = hierarchy.linkage(condensed_distances, method='complete')\nplt.figure(figsize=(7,5))\nhierarchy.dendrogram(linkage_matrix, labels=xs.columns.tolist(), orientation='left')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:23:36.580876Z","iopub.status.busy":"2023-11-13T16:23:36.580496Z","iopub.status.idle":"2023-11-13T16:23:54.442591Z","shell.execute_reply":"2023-11-13T16:23:54.442010Z","shell.execute_reply.started":"2023-11-13T16:23:36.580852Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Info Feature Importance","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:21:36.206479Z","iopub.status.busy":"2023-11-13T16:21:36.206079Z","iopub.status.idle":"2023-11-13T16:21:36.209419Z","shell.execute_reply":"2023-11-13T16:21:36.208838Z","shell.execute_reply.started":"2023-11-13T16:21:36.206455Z"},"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"m = rf(xs, y, n_estimators=10, random_state=random_seed)\nm_rmse(m, xs, y), rmse(m.oob_prediction_, y), m.oob_score_","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:43:07.897541Z","iopub.status.busy":"2023-11-13T16:43:07.897113Z","iopub.status.idle":"2023-11-13T16:43:56.849342Z","shell.execute_reply":"2023-11-13T16:43:56.848623Z","shell.execute_reply.started":"2023-11-13T16:43:07.897515Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = rf_feat_importance(m, xs)\nfi.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:43:56.850831Z","iopub.status.busy":"2023-11-13T16:43:56.850520Z","iopub.status.idle":"2023-11-13T16:43:56.895435Z","shell.execute_reply":"2023-11-13T16:43:56.894884Z","shell.execute_reply.started":"2023-11-13T16:43:56.850807Z"},"papermill":{"duration":0.214009,"end_time":"2023-10-03T19:15:24.537400","exception":false,"start_time":"2023-10-03T19:15:24.323391","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fi(fi[2:], figsize=(7,4));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:43:56.896496Z","iopub.status.busy":"2023-11-13T16:43:56.896231Z","iopub.status.idle":"2023-11-13T16:43:57.160953Z","shell.execute_reply":"2023-11-13T16:43:57.160398Z","shell.execute_reply.started":"2023-11-13T16:43:56.896477Z"},"papermill":{"duration":0.555166,"end_time":"2023-10-03T19:15:25.103016","exception":false,"start_time":"2023-10-03T19:15:24.547850","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Info Permutation Importance","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"def rf_perm_importance(pi, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':pi['importances_mean'], 'imp_std':pi['importances_std']}).sort_values('imp', ascending=False)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:58:01.307038Z","iopub.status.busy":"2023-11-13T16:58:01.306630Z","iopub.status.idle":"2023-11-13T16:58:01.311532Z","shell.execute_reply":"2023-11-13T16:58:01.310968Z","shell.execute_reply.started":"2023-11-13T16:58:01.307013Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.inspection import permutation_importance\npi = permutation_importance(m, xs, y, n_repeats=3)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:51:05.244943Z","iopub.status.busy":"2023-11-13T16:51:05.244541Z","iopub.status.idle":"2023-11-13T16:51:05.247892Z","shell.execute_reply":"2023-11-13T16:51:05.247305Z","shell.execute_reply.started":"2023-11-13T16:51:05.244916Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perm_imp = rf_perm_importance(pi, xs)\nperm_imp.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:24:41.158836Z","iopub.status.busy":"2023-11-13T17:24:41.158444Z","iopub.status.idle":"2023-11-13T17:24:41.167386Z","shell.execute_reply":"2023-11-13T17:24:41.166876Z","shell.execute_reply.started":"2023-11-13T17:24:41.158810Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fi(perm_imp[2:], figsize=(7,4));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:59:28.021427Z","iopub.status.busy":"2023-11-13T16:59:28.021015Z","iopub.status.idle":"2023-11-13T16:59:28.273767Z","shell.execute_reply":"2023-11-13T16:59:28.273223Z","shell.execute_reply.started":"2023-11-13T16:59:28.021401Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Information Random Forest","metadata":{"jp-MarkdownHeadingCollapsed":true,"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Let's fit a model after adding the gene info and compare it to the results of the baseline model:","metadata":{"execution":{"iopub.execute_input":"2023-11-13T02:26:23.283687Z","iopub.status.busy":"2023-11-13T02:26:23.283308Z","iopub.status.idle":"2023-11-13T02:26:23.287693Z","shell.execute_reply":"2023-11-13T02:26:23.287010Z","shell.execute_reply.started":"2023-11-13T02:26:23.283665Z"},"tags":[]}},{"cell_type":"code","source":"try: results_plain\nexcept NameError: results_plain = 'NA'\n\ntry: results_moldesc\nexcept NameError: results_moldesc = 'NA'","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:16:55.475184Z","iopub.status.busy":"2023-11-13T16:16:55.474872Z","iopub.status.idle":"2023-11-13T16:16:55.478419Z","shell.execute_reply":"2023-11-13T16:16:55.477892Z","shell.execute_reply.started":"2023-11-13T16:16:55.475162Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_ginf, test_df_ginf, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y\nm = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:24:36.696064Z","iopub.status.busy":"2023-11-13T16:24:36.695647Z","iopub.status.idle":"2023-11-13T16:24:36.699058Z","shell.execute_reply":"2023-11-13T16:24:36.698498Z","shell.execute_reply.started":"2023-11-13T16:24:36.696029Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_ginf = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with molecular descriptors:\\n{results_mdesc}\\nResults with gene info:\\n{results_ginf}')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T14:59:34.522632Z","iopub.status.busy":"2023-11-13T14:59:34.522330Z","iopub.status.idle":"2023-11-13T14:59:51.561183Z","shell.execute_reply":"2023-11-13T14:59:51.560546Z","shell.execute_reply.started":"2023-11-13T14:59:34.522610Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Information Unseen Cell Type Cross Validation","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='NK cells')\ncd4_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='T cells CD4+')\ncd8_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='T cells CD8+')\nreg_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T14:59:51.562533Z","iopub.status.busy":"2023-11-13T14:59:51.562061Z","iopub.status.idle":"2023-11-13T15:01:14.345419Z","shell.execute_reply":"2023-11-13T15:01:14.344794Z","shell.execute_reply.started":"2023-11-13T14:59:51.562509Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_ginf = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_ginf[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}:\\t{cv_results_ginf[key]}')\n    #print(f'Results moldesc:\\t{cv_results_moldesc[key]}')\n    #print(f'Results plain:\\t{cv_results_plain[key]}')\n    \nav_cv_result_ginf = tuple(round(i.item(), 6) for i in tensor(list(cv_results_ginf.values())).mean(0))\nprint(f'Average Result: {av_cv_result_ginf}')\n#print(f'Average Moldesc: {av_cv_result_moldesc}')\n#print(f'Average Plain : {av_cv_result_plain}')\n","metadata":{"execution":{"iopub.execute_input":"2023-11-13T15:01:14.346923Z","iopub.status.busy":"2023-11-13T15:01:14.346628Z","iopub.status.idle":"2023-11-13T15:22:00.245993Z","shell.execute_reply":"2023-11-13T15:22:00.245376Z","shell.execute_reply.started":"2023-11-13T15:01:14.346901Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Remove Unreasonable Features","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"markdown","source":"After removing features that don't seem reasonable (e.g. because they should be equivalent to another like ```go_names``` to ```go_ids``` or beacuse they have a low feature importance score):","metadata":{}},{"cell_type":"code","source":"train_df_ginf = train_df_ginf.drop(columns=['protein_id', 'phenotype_description', 'go_names', 'gene_id', 'canonical_cdna_sequence', 'protein_id', 'peptide_sequence', 'tsl', 'num_go_evidences', 'go_ids', 'go_evidences'])\ntest_df_ginf = test_df_ginf.drop(columns=['protein_id', 'phenotype_description', 'go_names', 'gene_id', 'canonical_cdna_sequence', 'protein_id', 'peptide_sequence', 'tsl', 'num_go_evidences', 'go_ids', 'go_evidences'])","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:26:42.462098Z","iopub.status.busy":"2023-11-13T17:26:42.461764Z","iopub.status.idle":"2023-11-13T17:26:43.978217Z","shell.execute_reply":"2023-11-13T17:26:43.977573Z","shell.execute_reply.started":"2023-11-13T17:26:42.462076Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_ginf, test_df_ginf, valid=None, random_state=random_seed)\nxs, y = to.train.xs, to.train.y","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:26:43.979905Z","iopub.status.busy":"2023-11-13T17:26:43.979601Z","iopub.status.idle":"2023-11-13T17:26:51.235942Z","shell.execute_reply":"2023-11-13T17:26:51.235335Z","shell.execute_reply.started":"2023-11-13T17:26:43.979884Z"},"papermill":{"duration":0.15509,"end_time":"2023-10-03T18:52:10.743315","exception":false,"start_time":"2023-10-03T18:52:10.588225","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Info Check Feature Redundance","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nfrom scipy.cluster import hierarchy\nfrom scipy.spatial.distance import squareform\ncorrelation_matrix = xs.corr()\ndistances = 1 - np.abs(correlation_matrix)\nnp.fill_diagonal(distances.values, 0)\ncondensed_distances = squareform(distances)\nlinkage_matrix = hierarchy.linkage(condensed_distances, method='complete')\nplt.figure(figsize=(7,5))\nhierarchy.dendrogram(linkage_matrix, labels=xs.columns.tolist(), orientation='left')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:26:51.237079Z","iopub.status.busy":"2023-11-13T17:26:51.236780Z","iopub.status.idle":"2023-11-13T17:26:58.273613Z","shell.execute_reply":"2023-11-13T17:26:58.273019Z","shell.execute_reply.started":"2023-11-13T17:26:51.237057Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Info Feature Importance","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:21:36.206479Z","iopub.status.busy":"2023-11-13T16:21:36.206079Z","iopub.status.idle":"2023-11-13T16:21:36.209419Z","shell.execute_reply":"2023-11-13T16:21:36.208838Z","shell.execute_reply.started":"2023-11-13T16:21:36.206455Z"},"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"m = rf(xs, y, n_estimators=10, random_state=random_seed)\nm_rmse(m, xs, y), rmse(m.oob_prediction_, y), m.oob_score_","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:28:51.332329Z","iopub.status.busy":"2023-11-13T17:28:51.331932Z","iopub.status.idle":"2023-11-13T17:29:38.641479Z","shell.execute_reply":"2023-11-13T17:29:38.640627Z","shell.execute_reply.started":"2023-11-13T17:28:51.332304Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = rf_feat_importance(m, xs)\nfi.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:29:45.727511Z","iopub.status.busy":"2023-11-13T17:29:45.727108Z","iopub.status.idle":"2023-11-13T17:29:45.774831Z","shell.execute_reply":"2023-11-13T17:29:45.774240Z","shell.execute_reply.started":"2023-11-13T17:29:45.727485Z"},"papermill":{"duration":0.214009,"end_time":"2023-10-03T19:15:24.537400","exception":false,"start_time":"2023-10-03T19:15:24.323391","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fi(fi, figsize=(7,4));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:30:12.551519Z","iopub.status.busy":"2023-11-13T17:30:12.551127Z","iopub.status.idle":"2023-11-13T17:30:12.758580Z","shell.execute_reply":"2023-11-13T17:30:12.758026Z","shell.execute_reply.started":"2023-11-13T17:30:12.551494Z"},"papermill":{"duration":0.555166,"end_time":"2023-10-03T19:15:25.103016","exception":false,"start_time":"2023-10-03T19:15:24.547850","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Info Permutation Importance","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"pi = permutation_importance(m, xs, y, n_repeats=3)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:31:16.074762Z","iopub.status.busy":"2023-11-13T17:31:16.074352Z","iopub.status.idle":"2023-11-13T17:35:01.104894Z","shell.execute_reply":"2023-11-13T17:35:01.104233Z","shell.execute_reply.started":"2023-11-13T17:31:16.074735Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perm_imp = rf_perm_importance(pi, xs)\nperm_imp","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:35:44.516038Z","iopub.status.busy":"2023-11-13T17:35:44.515629Z","iopub.status.idle":"2023-11-13T17:35:44.525320Z","shell.execute_reply":"2023-11-13T17:35:44.524747Z","shell.execute_reply.started":"2023-11-13T17:35:44.516012Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fi(perm_imp[:], figsize=(7,4));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:35:56.737729Z","iopub.status.busy":"2023-11-13T17:35:56.737330Z","iopub.status.idle":"2023-11-13T17:35:56.952049Z","shell.execute_reply":"2023-11-13T17:35:56.951486Z","shell.execute_reply.started":"2023-11-13T17:35:56.737705Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Validation Random","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_ginf, test_df_ginf, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:36:36.685322Z","iopub.status.busy":"2023-11-13T17:36:36.684912Z","iopub.status.idle":"2023-11-13T17:36:49.594249Z","shell.execute_reply":"2023-11-13T17:36:49.593595Z","shell.execute_reply.started":"2023-11-13T17:36:36.685295Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:36:49.595694Z","iopub.status.busy":"2023-11-13T17:36:49.595372Z","iopub.status.idle":"2023-11-13T17:41:05.541922Z","shell.execute_reply":"2023-11-13T17:41:05.541271Z","shell.execute_reply.started":"2023-11-13T17:36:49.595670Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_ginf = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with gene info:\\n{results_ginf}')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:41:05.543107Z","iopub.status.busy":"2023-11-13T17:41:05.542812Z","iopub.status.idle":"2023-11-13T17:41:22.108984Z","shell.execute_reply":"2023-11-13T17:41:22.108371Z","shell.execute_reply.started":"2023-11-13T17:41:05.543085Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gene Information Unseen Cell Type Cross Validation","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='NK cells')\ncd4_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='T cells CD4+')\ncd8_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='T cells CD8+')\nreg_to = get_train_valid_data(train_df_ginf, test_df_ginf, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:41:22.110564Z","iopub.status.busy":"2023-11-13T17:41:22.110280Z","iopub.status.idle":"2023-11-13T17:42:10.609407Z","shell.execute_reply":"2023-11-13T17:42:10.608773Z","shell.execute_reply.started":"2023-11-13T17:41:22.110543Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_ginf = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_ginf[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}:\\t{cv_results_ginf[key]}')\n    #print(f'Results moldesc:\\t{cv_results_moldesc[key]}')\n    #print(f'Results plain:\\t{cv_results_plain[key]}')\n    \nav_cv_result_ginf = tuple(round(i.item(), 6) for i in tensor(list(cv_results_ginf.values())).mean(0))\nprint(f'Average Result: {av_cv_result_ginf}')\n#print(f'Average Moldesc: {av_cv_result_moldesc}')\n#print(f'Average Plain : {av_cv_result_plain}')\n","metadata":{"execution":{"iopub.execute_input":"2023-11-13T17:42:10.610559Z","iopub.status.busy":"2023-11-13T17:42:10.610262Z","iopub.status.idle":"2023-11-13T18:00:18.080625Z","shell.execute_reply":"2023-11-13T18:00:18.079856Z","shell.execute_reply.started":"2023-11-13T17:42:10.610538Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding Cell Information","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"markdown","source":"### Create Cell Information","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"cellinfo = pd.DataFrame(train_df['cell_type'].drop_duplicates(keep='first').reset_index(drop=True))\n\ncellinfo['organ_of_origin'] = ''\ncellinfo.loc[cellinfo['cell_type'].isin(['T cells CD4+', 'T cells CD8+', 'T regulatory cells']), 'organ_of_origin'] = 'thymus'\ncellinfo.loc[cellinfo['cell_type'].isin(['NK cells']), 'organ_of_origin'] = 'bone marrow;lymph nodes;spleen;tonsils;thymus'\ncellinfo.loc[cellinfo['cell_type'].isin(['B cells']), 'organ_of_origin'] = 'bone marrow'\ncellinfo.loc[cellinfo['cell_type'].isin(['Myeloid cells']), 'organ_of_origin'] = 'bone marrow'\n\ncellinfo['num_origins'] = list(map(get_number_of_terms, cellinfo['organ_of_origin']))\n\n\ncellinfo['is_lymphoid'] = True\ncellinfo.loc[cellinfo['cell_type'].isin(['Myeloid cells']), 'is_lymphoid'] = False\n\ncellinfo['immune_type'] = 'adaptive'\ncellinfo.loc[cellinfo['cell_type'].isin(['Myeloid cells', 'NK cells']), 'immune_type'] = 'innate'\n\n# Autoimmunity: From Bench to Bedside; R Luz Elena Cano and H. Damaris E. Lopera, Chapter 5 Introduction to T and B lymphocytes.\n# From the morphological point of view, T and B lymphocytes are indistinguishable since they are both small cells (8–10 microns in diameter) and each possesses a large nucleus...\n# Hematopoietic and Lymphoid Tissues, Michael Linden, ... Sindhu Cherian, in Comparative Anatomy and Histology, 2012, Myeloid Cells.\n# Myeloid blasts are infrequent (<2% of white blood cells) and vary in diameter from 15 to 20 µm, with a high nuclear:cytoplasmic ratio and scant pale blue agranular cytoplasm.\n# Analysis of sphingosine kinase activity in single natural killer cells from peripheral blood\n# Alexandra J. Dickinson,a Megan Meyer,b Erica A. Pawlak,c Shawn Gomez,d Ilona Jaspers,b,c,e and Nancy L. Allbritton*,a,d\n# Integr Biol (Camb). 2015 Apr; 7(4): 392–401. DOI: 10.1039/c5ib00007f\n# In order to efficiently trap NK cells, which are small (6–7 μm diameter) relative to the size of most tissue-cultured cells (12–15 μm diameter), the cell-trap size was optimized for capture of single NK cells.\ncellinfo['av_size_um'] = 9.0\ncellinfo.loc[cellinfo['cell_type'].isin(['Myeloid cells']), 'av_size_um'] = 17.5\ncellinfo.loc[cellinfo['cell_type'].isin(['NK cells']), 'av_size_um'] = 6.5\n\ncellinfo['cell_function'] = 'cytotoxicity'\ncellinfo.loc[cellinfo['cell_type'].isin(['B cells']), 'cell_function'] = 'producing antibodies'\ncellinfo.loc[cellinfo['cell_type'].isin(['T cells CD4+', 'T regulatory cells', 'Myeloid cells']), 'cell_function'] = 'regulate'\ncellinfo.loc[cellinfo['cell_type'].isin(['Myeloid cells']), 'cell_function'] = 'complex'\n\ncellinfo","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:27:11.208315Z","iopub.status.busy":"2023-11-14T15:27:11.207898Z","iopub.status.idle":"2023-11-14T15:27:11.412878Z","shell.execute_reply":"2023-11-14T15:27:11.412313Z","shell.execute_reply.started":"2023-11-14T15:27:11.208289Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cellinfo.to_csv('/kaggle/input/cellinfo.csv')\n# cellinfo.to_parquet('/kaggle/input/cellinfo.parquet')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:27:12.930566Z","iopub.status.busy":"2023-11-14T15:27:12.930190Z","iopub.status.idle":"2023-11-14T15:27:12.933361Z","shell.execute_reply":"2023-11-14T15:27:12.932825Z","shell.execute_reply.started":"2023-11-14T15:27:12.930541Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add Cell Information","metadata":{"tags":[]}},{"cell_type":"code","source":"train_df_cinf = pd.merge(train_df, cellinfo, on='cell_type', how='left')\ncols = train_df_cinf.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntrain_df_cinf = train_df_cinf[cols]\n\ntest_df_cinf = pd.merge(test_df, cellinfo, on='cell_type', how='left')\ncols = test_df_cinf.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntest_df_cinf = test_df_cinf[cols]","metadata":{"execution":{"iopub.execute_input":"2023-11-13T18:53:54.982888Z","iopub.status.busy":"2023-11-13T18:53:54.982494Z","iopub.status.idle":"2023-11-13T18:54:00.429313Z","shell.execute_reply":"2023-11-13T18:54:00.428650Z","shell.execute_reply.started":"2023-11-13T18:53:54.982863Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Improvement After Adding Cell Information","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_cinf, test_df_cinf, valid=None, random_state=random_seed)\nxs, y = to.train.xs, to.train.y","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:08:53.620296Z","iopub.status.busy":"2023-11-13T19:08:53.619895Z","iopub.status.idle":"2023-11-13T19:09:08.270536Z","shell.execute_reply":"2023-11-13T19:09:08.269736Z","shell.execute_reply.started":"2023-11-13T19:08:53.620269Z"},"papermill":{"duration":0.15509,"end_time":"2023-10-03T18:52:10.743315","exception":false,"start_time":"2023-10-03T18:52:10.588225","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cell Info Check Feature Redundance","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nfrom scipy.cluster import hierarchy\nfrom scipy.spatial.distance import squareform\ncorrelation_matrix = xs.corr()\ndistances = 1 - np.abs(correlation_matrix)\nnp.fill_diagonal(distances.values, 0)\ncondensed_distances = squareform(distances)\nlinkage_matrix = hierarchy.linkage(condensed_distances, method='complete')\nplt.figure(figsize=(7,5))\nhierarchy.dendrogram(linkage_matrix, labels=xs.columns.tolist(), orientation='left')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:09:08.272766Z","iopub.status.busy":"2023-11-13T19:09:08.272101Z","iopub.status.idle":"2023-11-13T19:09:11.034875Z","shell.execute_reply":"2023-11-13T19:09:11.034124Z","shell.execute_reply.started":"2023-11-13T19:09:08.272730Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cell Info Feature Importance","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:21:36.206479Z","iopub.status.busy":"2023-11-13T16:21:36.206079Z","iopub.status.idle":"2023-11-13T16:21:36.209419Z","shell.execute_reply":"2023-11-13T16:21:36.208838Z","shell.execute_reply.started":"2023-11-13T16:21:36.206455Z"},"tags":[]}},{"cell_type":"code","source":"m = rf(xs, y, n_estimators=10, random_state=random_seed)\nm_rmse(m, xs, y), rmse(m.oob_prediction_, y), m.oob_score_","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:09:49.349736Z","iopub.status.busy":"2023-11-13T19:09:49.349347Z","iopub.status.idle":"2023-11-13T19:10:46.347060Z","shell.execute_reply":"2023-11-13T19:10:46.346452Z","shell.execute_reply.started":"2023-11-13T19:09:49.349712Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = rf_feat_importance(m, xs)\nfi.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:10:46.348483Z","iopub.status.busy":"2023-11-13T19:10:46.348199Z","iopub.status.idle":"2023-11-13T19:10:46.392875Z","shell.execute_reply":"2023-11-13T19:10:46.392337Z","shell.execute_reply.started":"2023-11-13T19:10:46.348461Z"},"papermill":{"duration":0.214009,"end_time":"2023-10-03T19:15:24.537400","exception":false,"start_time":"2023-10-03T19:15:24.323391","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fi(fi[2:], figsize=(7,4));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:14:40.814042Z","iopub.status.busy":"2023-11-13T19:14:40.813652Z","iopub.status.idle":"2023-11-13T19:14:40.969991Z","shell.execute_reply":"2023-11-13T19:14:40.969473Z","shell.execute_reply.started":"2023-11-13T19:14:40.814022Z"},"papermill":{"duration":0.555166,"end_time":"2023-10-03T19:15:25.103016","exception":false,"start_time":"2023-10-03T19:15:24.547850","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cell Info Permutation Importance","metadata":{"tags":[]}},{"cell_type":"code","source":"def rf_perm_importance(pi, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':pi['importances_mean'], 'imp_std':pi['importances_std']}).sort_values('imp', ascending=False)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:11:30.289273Z","iopub.status.busy":"2023-11-13T19:11:30.288865Z","iopub.status.idle":"2023-11-13T19:11:30.292958Z","shell.execute_reply":"2023-11-13T19:11:30.292416Z","shell.execute_reply.started":"2023-11-13T19:11:30.289247Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.inspection import permutation_importance\npi = permutation_importance(m, xs, y, n_repeats=3)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:11:35.281256Z","iopub.status.busy":"2023-11-13T19:11:35.280859Z","iopub.status.idle":"2023-11-13T19:14:40.651502Z","shell.execute_reply":"2023-11-13T19:14:40.650872Z","shell.execute_reply.started":"2023-11-13T19:11:35.281230Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"perm_imp = rf_perm_importance(pi, xs)\nperm_imp.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:14:40.653024Z","iopub.status.busy":"2023-11-13T19:14:40.652736Z","iopub.status.idle":"2023-11-13T19:14:40.661025Z","shell.execute_reply":"2023-11-13T19:14:40.660537Z","shell.execute_reply.started":"2023-11-13T19:14:40.653002Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fi(perm_imp[2:], figsize=(7,4));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:14:40.662220Z","iopub.status.busy":"2023-11-13T19:14:40.661922Z","iopub.status.idle":"2023-11-13T19:14:40.812456Z","shell.execute_reply":"2023-11-13T19:14:40.811920Z","shell.execute_reply.started":"2023-11-13T19:14:40.662200Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cell Information Random Forest","metadata":{"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Let's fit a model after adding the cell info and compare it to the results of the baseline model:","metadata":{"execution":{"iopub.execute_input":"2023-11-13T02:26:23.283687Z","iopub.status.busy":"2023-11-13T02:26:23.283308Z","iopub.status.idle":"2023-11-13T02:26:23.287693Z","shell.execute_reply":"2023-11-13T02:26:23.287010Z","shell.execute_reply.started":"2023-11-13T02:26:23.283665Z"},"tags":[]}},{"cell_type":"code","source":"try: results_plain\nexcept NameError: results_plain = 'NA'\n\ntry: results_moldesc\nexcept NameError: results_moldesc = 'NA'\n\ntry: results_ginf\nexcept NameError: results_ginf = 'NA'","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:33:41.899186Z","iopub.status.busy":"2023-11-13T19:33:41.898620Z","iopub.status.idle":"2023-11-13T19:33:41.905123Z","shell.execute_reply":"2023-11-13T19:33:41.904095Z","shell.execute_reply.started":"2023-11-13T19:33:41.899144Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_cinf, test_df_cinf, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y\nm = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:33:43.400820Z","iopub.status.busy":"2023-11-13T19:33:43.400247Z","iopub.status.idle":"2023-11-13T19:39:05.000076Z","shell.execute_reply":"2023-11-13T19:39:04.999387Z","shell.execute_reply.started":"2023-11-13T19:33:43.400774Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_cinf = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with molecular descriptors:\\n{results_moldesc}\\nResults with gene info:\\n{results_ginf}\\nResults with cell info:\\n{results_cinf}')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:39:05.001561Z","iopub.status.busy":"2023-11-13T19:39:05.001259Z","iopub.status.idle":"2023-11-13T19:39:28.557508Z","shell.execute_reply":"2023-11-13T19:39:28.556862Z","shell.execute_reply.started":"2023-11-13T19:39:05.001539Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cell Information Unseen Cell Type Cross Validation","metadata":{}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(train_df_cinf, test_df_cinf, valid='NK cells')\ncd4_to = get_train_valid_data(train_df_cinf, test_df_cinf, valid='T cells CD4+')\ncd8_to = get_train_valid_data(train_df_cinf, test_df_cinf, valid='T cells CD8+')\nreg_to = get_train_valid_data(train_df_cinf, test_df_cinf, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:39:28.558725Z","iopub.status.busy":"2023-11-13T19:39:28.558421Z","iopub.status.idle":"2023-11-13T19:40:42.270184Z","shell.execute_reply":"2023-11-13T19:40:42.269563Z","shell.execute_reply.started":"2023-11-13T19:39:28.558702Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_cinf = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_cinf[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}:\\t{cv_results_cinf[key]}')\n    print(f'Results ginf:\\t{cv_results_ginf[key]}')\n    #print(f'Results moldesc:\\t{cv_results_moldesc[key]}')\n    #print(f'Results plain:\\t{cv_results_plain[key]}')\n    \nav_cv_result_cinf = tuple(round(i.item(), 6) for i in tensor(list(cv_results_cinf.values())).mean(0))\nprint(f'Average Result: {av_cv_result_cinf}')\nprint(f'Average Ginf: {av_cv_result_ginf}')\n#print(f'Average Moldesc: {av_cv_result_moldesc}')\n#print(f'Average Plain : {av_cv_result_plain}')\n","metadata":{"execution":{"iopub.execute_input":"2023-11-13T19:40:42.271719Z","iopub.status.busy":"2023-11-13T19:40:42.271428Z","iopub.status.idle":"2023-11-13T20:02:07.059694Z","shell.execute_reply":"2023-11-13T20:02:07.059079Z","shell.execute_reply.started":"2023-11-13T19:40:42.271697Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adding All Mined Features","metadata":{}},{"cell_type":"markdown","source":"### Merge Data","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"train_df_cinf = pd.merge(train_df, cellinfo, on='cell_type', how='left')\n\ntrain_df_cinf_ginf = pd.merge(train_df_cinf, selected_geneinfo, on='gene', how='left')\ntrain_df_cinf_ginf = train_df_cinf_ginf.drop(columns=['is_canonical', 'transcript_source', 'gencode', 'strand', 'end', 'appris', 'bio_proc', 'mol_func', 'cell_comp'])\n# train_df_cinf_ginf['strand'] = train_df_cinf_ginf['strand'].fillna(0).astype('int')\ntrain_df_cinf_ginf['transcript_count'] = train_df_cinf_ginf['transcript_count'].fillna(0).astype('int')\ntrain_df_cinf_ginf['start'] = train_df_cinf_ginf['start'].fillna(0).astype('int')\ntrain_df_cinf_ginf = train_df_cinf_ginf.rename(columns={'start': 'position'})\n# train_df_cinf_ginf['end'] = train_df_cinf_ginf['end'].fillna(0).astype('int')\ntrain_df_cinf_ginf['length'] = train_df_cinf_ginf['length'].fillna(0).astype('int')\ntrain_df_cinf_ginf['peptide_length'] = train_df_cinf_ginf['peptide_length'].fillna(0).astype('int')\ntrain_df_cinf_ginf['num_go_ids'] = train_df_cinf_ginf['num_go_ids'].fillna(0).astype('int')\n# train_df_cinf_ginf['num_go_names'] = train_df_cinf_ginf['num_go_names'].fillna(0).astype('int')\ntrain_df_cinf_ginf['num_go_evidences'] = train_df_cinf_ginf['num_go_evidences'].fillna(0).astype('int')\ntrain_df_cinf_ginf['nG'] = train_df_cinf_ginf['nG'].fillna(0).astype('int')\ntrain_df_cinf_ginf['nC'] = train_df_cinf_ginf['nC'].fillna(0).astype('int')\ntrain_df_cinf_ginf['nA'] = train_df_cinf_ginf['nA'].fillna(0).astype('int')\ntrain_df_cinf_ginf['nT'] = train_df_cinf_ginf['nT'].fillna(0).astype('int')\n\ntrain_df_cinf_ginf_mdesc = pd.merge(train_df_cinf_ginf, important_mol_descriptors, on='sm_name', how='left')\ncols = train_df_cinf_ginf_mdesc.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntrain_df_cinf_ginf_mdesc = train_df_cinf_ginf_mdesc[cols]\n\ntest_df_cinf = pd.merge(test_df, cellinfo, on='cell_type', how='left')\n\ntest_df_cinf_ginf = pd.merge(test_df_cinf, selected_geneinfo, on='gene', how='left')\ntest_df_cinf_ginf = test_df_cinf_ginf.drop(columns=['is_canonical', 'transcript_source', 'gencode', 'strand', 'end', 'appris', 'bio_proc', 'mol_func', 'cell_comp'])\n# test_df_cinf_ginf['strand'] = test_df_cinf_ginf['strand'].fillna(0).astype('int')\ntest_df_cinf_ginf['transcript_count'] = test_df_cinf_ginf['transcript_count'].fillna(0).astype('int')\ntest_df_cinf_ginf['start'] = test_df_cinf_ginf['start'].fillna(0).astype('int')\ntest_df_cinf_ginf = test_df_cinf_ginf.rename(columns={'start': 'position'})\n# test_df_cinf_ginf['end'] = test_df_cinf_ginf['end'].fillna(0).astype('int')\ntest_df_cinf_ginf['length'] = test_df_cinf_ginf['length'].fillna(0).astype('int')\ntest_df_cinf_ginf['peptide_length'] = test_df_cinf_ginf['peptide_length'].fillna(0).astype('int')\ntest_df_cinf_ginf['num_go_ids'] = test_df_cinf_ginf['num_go_ids'].fillna(0).astype('int')\n# test_df_cinf_ginf['num_go_names'] = test_df_cinf_ginf['num_go_names'].fillna(0).astype('int')\ntest_df_cinf_ginf['num_go_evidences'] = test_df_cinf_ginf['num_go_evidences'].fillna(0).astype('int')\ntest_df_cinf_ginf['nG'] = test_df_cinf_ginf['nG'].fillna(0).astype('int')\ntest_df_cinf_ginf['nC'] = test_df_cinf_ginf['nC'].fillna(0).astype('int')\ntest_df_cinf_ginf['nA'] = test_df_cinf_ginf['nA'].fillna(0).astype('int')\ntest_df_cinf_ginf['nT'] = test_df_cinf_ginf['nT'].fillna(0).astype('int')\n\ntest_df_cinf_ginf_mdesc = pd.merge(test_df_cinf_ginf, important_mol_descriptors, on='sm_name', how='left')\ncols = test_df_cinf_ginf_mdesc.columns.tolist()\ncols.remove('value')\ncols.append('value')\ntest_df_cinf_ginf_mdesc = test_df_cinf_ginf_mdesc[cols]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:28:16.241201Z","iopub.status.busy":"2023-11-14T15:28:16.240598Z","iopub.status.idle":"2023-11-14T15:28:59.661181Z","shell.execute_reply":"2023-11-14T15:28:59.660525Z","shell.execute_reply.started":"2023-11-14T15:28:16.241175Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Improvement After Adding All Information","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(train_df_cinf_ginf_mdesc, test_df_cinf_ginf_mdesc, valid=None, random_state=random_seed)\nxs, y = to.train.xs, to.train.y","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:28:59.662681Z","iopub.status.busy":"2023-11-14T15:28:59.662373Z","iopub.status.idle":"2023-11-14T15:29:34.334093Z","shell.execute_reply":"2023-11-14T15:29:34.333473Z","shell.execute_reply.started":"2023-11-14T15:28:59.662660Z"},"papermill":{"duration":0.15509,"end_time":"2023-10-03T18:52:10.743315","exception":false,"start_time":"2023-10-03T18:52:10.588225","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cell Info Check Feature Redundance","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nfrom scipy.cluster import hierarchy\nfrom scipy.spatial.distance import squareform\ncorrelation_matrix = xs.corr()\ndistances = 1 - np.abs(correlation_matrix)\nnp.fill_diagonal(distances.values, 0)\ncondensed_distances = squareform(distances)\nlinkage_matrix = hierarchy.linkage(condensed_distances, method='complete')\nplt.figure(figsize=(7,5))\nhierarchy.dendrogram(linkage_matrix, labels=xs.columns.tolist(), orientation='left')\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:29:34.335295Z","iopub.status.busy":"2023-11-14T15:29:34.334982Z","iopub.status.idle":"2023-11-14T15:30:52.787030Z","shell.execute_reply":"2023-11-14T15:30:52.786435Z","shell.execute_reply.started":"2023-11-14T15:29:34.335274Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### All Info Feature Importance","metadata":{"execution":{"iopub.execute_input":"2023-11-13T16:21:36.206479Z","iopub.status.busy":"2023-11-13T16:21:36.206079Z","iopub.status.idle":"2023-11-13T16:21:36.209419Z","shell.execute_reply":"2023-11-13T16:21:36.208838Z","shell.execute_reply.started":"2023-11-13T16:21:36.206455Z"},"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"m = rf(xs, y, n_estimators=10, random_state=random_seed)\nm_rmse(m, xs, y), rmse(m.oob_prediction_, y), m.oob_score_","metadata":{"execution":{"iopub.execute_input":"2023-11-13T20:37:56.883058Z","iopub.status.busy":"2023-11-13T20:37:56.882758Z","iopub.status.idle":"2023-11-13T20:39:55.057762Z","shell.execute_reply":"2023-11-13T20:39:55.057156Z","shell.execute_reply.started":"2023-11-13T20:37:56.883036Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rf_feat_importance(m, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}).sort_values('imp', ascending=False)\nfi = rf_feat_importance(m, xs)\nfi.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-13T20:39:55.059294Z","iopub.status.busy":"2023-11-13T20:39:55.059001Z","iopub.status.idle":"2023-11-13T20:39:55.094237Z","shell.execute_reply":"2023-11-13T20:39:55.093675Z","shell.execute_reply.started":"2023-11-13T20:39:55.059273Z"},"papermill":{"duration":0.214009,"end_time":"2023-10-03T19:15:24.537400","exception":false,"start_time":"2023-10-03T19:15:24.323391","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_fi(fi, figsize=(10,7)):\n    return fi.plot('cols', 'imp', 'barh', figsize=figsize, legend=False)\nplot_fi(fi, figsize=(10,7));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T20:41:44.084372Z","iopub.status.busy":"2023-11-13T20:41:44.083950Z","iopub.status.idle":"2023-11-13T20:41:44.560070Z","shell.execute_reply":"2023-11-13T20:41:44.559499Z","shell.execute_reply.started":"2023-11-13T20:41:44.084346Z"},"papermill":{"duration":0.555166,"end_time":"2023-10-03T19:15:25.103016","exception":false,"start_time":"2023-10-03T19:15:24.547850","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### All Info Permutation Importance","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"def rf_perm_importance(pi, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':pi['importances_mean'], 'imp_std':pi['importances_std']}).sort_values('imp', ascending=False)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T20:42:03.960198Z","iopub.status.busy":"2023-11-13T20:42:03.959792Z","iopub.status.idle":"2023-11-13T20:42:03.963863Z","shell.execute_reply":"2023-11-13T20:42:03.963279Z","shell.execute_reply.started":"2023-11-13T20:42:03.960172Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.inspection import permutation_importance\n# pi = permutation_importance(m, xs, y, n_repeats=3)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T20:42:06.662087Z","iopub.status.busy":"2023-11-13T20:42:06.661689Z","iopub.status.idle":"2023-11-13T21:15:00.537224Z","shell.execute_reply":"2023-11-13T21:15:00.536534Z","shell.execute_reply.started":"2023-11-13T20:42:06.662062Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# perm_imp = rf_perm_importance(pi, xs).sort_values('imp', ascending=False)\n# perm_imp.head()","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:22:06.078792Z","iopub.status.busy":"2023-11-14T16:22:06.078384Z","iopub.status.idle":"2023-11-14T16:22:06.086611Z","shell.execute_reply":"2023-11-14T16:22:06.086049Z","shell.execute_reply.started":"2023-11-14T16:22:06.078766Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# perm_imp.to_csv('/kaggle/input/permutation_importance_scores_10trees_3repeats.csv')\nperm_imp = pd.read_csv('/kaggle/input/permutation_importance_scores_10trees_3repeats.csv')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:32:00.619022Z","iopub.status.busy":"2023-11-14T15:32:00.618627Z","iopub.status.idle":"2023-11-14T15:32:00.624464Z","shell.execute_reply":"2023-11-14T15:32:00.623845Z","shell.execute_reply.started":"2023-11-14T15:32:00.618997Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_fi(perm_imp, figsize=(10,7));","metadata":{"execution":{"iopub.execute_input":"2023-11-13T21:20:54.914791Z","iopub.status.busy":"2023-11-13T21:20:54.914395Z","iopub.status.idle":"2023-11-13T21:20:55.385283Z","shell.execute_reply":"2023-11-13T21:20:55.384683Z","shell.execute_reply.started":"2023-11-13T21:20:54.914765Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 20 Selected Additional Features Random Forest","metadata":{"jp-MarkdownHeadingCollapsed":true,"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Let's fit a model after adding the top ten features and compare it to the results of the baseline model:","metadata":{"execution":{"iopub.execute_input":"2023-11-13T02:26:23.283687Z","iopub.status.busy":"2023-11-13T02:26:23.283308Z","iopub.status.idle":"2023-11-13T02:26:23.287693Z","shell.execute_reply":"2023-11-13T02:26:23.287010Z","shell.execute_reply.started":"2023-11-13T02:26:23.283665Z"},"tags":[]}},{"cell_type":"code","source":"all_features = perm_imp['cols'].tolist()\n\nselected_features = ['cell_type', 'cell_function', 'organ_of_origin', 'immune_type', 'sm_name', 'BCUTpe-1l', 'ATSC8s', 'BIC4', 'AATSC1dv', 'BCUTi-1h', 'MATS1dv', 'SaaO', 'ATSC3c', 'BCUTse-1l', 'BIC5', 'gene', 'length', 'transcript_count', 'nC', 'nA', 'nT', 'nG', 'peptide_length', 'value']\nlen(selected_features)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:32:07.451496Z","iopub.status.busy":"2023-11-14T15:32:07.451082Z","iopub.status.idle":"2023-11-14T15:32:07.456889Z","shell.execute_reply":"2023-11-14T15:32:07.456340Z","shell.execute_reply.started":"2023-11-14T15:32:07.451470Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = train_df_cinf_ginf_mdesc.loc[:, selected_features]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:32:11.300317Z","iopub.status.busy":"2023-11-14T15:32:11.299900Z","iopub.status.idle":"2023-11-14T15:32:12.253329Z","shell.execute_reply":"2023-11-14T15:32:12.252685Z","shell.execute_reply.started":"2023-11-14T15:32:11.300291Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: results_plain\nexcept NameError: results_plain = 'NA'\n\ntry: results_moldesc\nexcept NameError: results_moldesc = 'NA'\n\ntry: results_ginf\nexcept NameError: results_ginf = 'NA'\n\ntry: results_cinf\nexcept NameError: results_cinf = 'NA'","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:32:12.572426Z","iopub.status.busy":"2023-11-14T15:32:12.572058Z","iopub.status.idle":"2023-11-14T15:32:12.576030Z","shell.execute_reply":"2023-11-14T15:32:12.575489Z","shell.execute_reply.started":"2023-11-14T15:32:12.572402Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(data_train, test_df, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y\nm = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-13T21:44:42.538485Z","iopub.status.busy":"2023-11-13T21:44:42.538096Z","iopub.status.idle":"2023-11-13T21:52:19.870783Z","shell.execute_reply":"2023-11-13T21:52:19.870142Z","shell.execute_reply.started":"2023-11-13T21:44:42.538460Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_20 = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with molecular descriptors:\\n{results_moldesc}\\nResults with gene info:\\n{results_ginf}\\nResults with cell info:\\n{results_cinf}\\nResults with 20 selected:\\n{results_20}')","metadata":{"execution":{"iopub.execute_input":"2023-11-13T21:52:19.872229Z","iopub.status.busy":"2023-11-13T21:52:19.871917Z","iopub.status.idle":"2023-11-13T21:53:03.769916Z","shell.execute_reply":"2023-11-13T21:53:03.769281Z","shell.execute_reply.started":"2023-11-13T21:52:19.872206Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 20 Selected Unseen Cell Type Cross Validation","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(data_train, test_df, valid='NK cells')\ncd4_to = get_train_valid_data(data_train, test_df, valid='T cells CD4+')\ncd8_to = get_train_valid_data(data_train, test_df, valid='T cells CD8+')\nreg_to = get_train_valid_data(data_train, test_df, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:32:16.330624Z","iopub.status.busy":"2023-11-14T15:32:16.330215Z","iopub.status.idle":"2023-11-14T15:33:10.391964Z","shell.execute_reply":"2023-11-14T15:33:10.391340Z","shell.execute_reply.started":"2023-11-14T15:32:16.330599Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_20 = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_20[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}:\\t{cv_results_20[key]}')\n    #print(f'Results ginf:\\t{cv_results_ginf[key]}')\n    #print(f'Results moldesc:\\t{cv_results_moldesc[key]}')\n    #print(f'Results plain:\\t{cv_results_plain[key]}')\n    \nav_cv_result_20 = tuple(round(i.item(), 6) for i in tensor(list(cv_results_20.values())).mean(0))\nprint(f'Average Result: {av_cv_result_20}')\n# print(f'Average Ginf: {av_cv_result_ginf}')\n#print(f'Average Moldesc: {av_cv_result_moldesc}')\n#print(f'Average Plain : {av_cv_result_plain}')\n","metadata":{"execution":{"iopub.execute_input":"2023-11-14T15:45:15.055972Z","iopub.status.busy":"2023-11-14T15:45:15.055578Z","iopub.status.idle":"2023-11-14T16:16:59.663324Z","shell.execute_reply":"2023-11-14T16:16:59.662696Z","shell.execute_reply.started":"2023-11-14T15:45:15.055943Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Add remaining molecular descriptors","metadata":{"jp-MarkdownHeadingCollapsed":true,"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"code","source":"#all_features","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:27:03.221784Z","iopub.status.busy":"2023-11-14T16:27:03.221378Z","iopub.status.idle":"2023-11-14T16:27:03.224746Z","shell.execute_reply":"2023-11-14T16:27:03.224152Z","shell.execute_reply.started":"2023-11-14T16:27:03.221759Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = ['cell_type', 'cell_function', 'organ_of_origin', 'immune_type', 'sm_name', 'BCUTpe-1l', 'ATSC8s', 'BIC4', 'AATSC1dv', 'BCUTi-1h', 'MATS1dv', 'SaaO', 'ATSC3c', 'BCUTse-1l', 'BIC5', 'AATSC1are', 'AATS1i', 'BCUTm-1l', 'ATSC7s', 'ATSC6c', 'AATS5s', 'MATS3v', 'VSA_EState2', 'ATSC5s', 'AATS3i', 'JGI2', 'AATSC4pe', 'ATSC4Z', 'gene', 'length', 'transcript_count', 'nC', 'nA', 'nT', 'nG', 'peptide_length', 'value']\nlen(selected_features)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:27:05.063542Z","iopub.status.busy":"2023-11-14T16:27:05.063153Z","iopub.status.idle":"2023-11-14T16:27:05.068810Z","shell.execute_reply":"2023-11-14T16:27:05.068234Z","shell.execute_reply.started":"2023-11-14T16:27:05.063517Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = train_df_cinf_ginf_mdesc.loc[:, selected_features]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:27:10.126804Z","iopub.status.busy":"2023-11-14T16:27:10.126404Z","iopub.status.idle":"2023-11-14T16:27:11.540432Z","shell.execute_reply":"2023-11-14T16:27:11.539733Z","shell.execute_reply.started":"2023-11-14T16:27:10.126779Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: results_plain\nexcept NameError: results_plain = 'NA'\n\ntry: results_moldesc\nexcept NameError: results_moldesc = 'NA'\n\ntry: results_ginf\nexcept NameError: results_ginf = 'NA'\n\ntry: results_cinf\nexcept NameError: results_cinf = 'NA'","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:27:13.375468Z","iopub.status.busy":"2023-11-14T16:27:13.375037Z","iopub.status.idle":"2023-11-14T16:27:13.379168Z","shell.execute_reply":"2023-11-14T16:27:13.378599Z","shell.execute_reply.started":"2023-11-14T16:27:13.375444Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(data_train, test_df, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y\nm = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:27:22.504525Z","iopub.status.busy":"2023-11-14T16:27:22.504116Z","iopub.status.idle":"2023-11-14T16:36:51.653829Z","shell.execute_reply":"2023-11-14T16:36:51.653206Z","shell.execute_reply.started":"2023-11-14T16:27:22.504500Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_20 = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with molecular descriptors:\\n{results_moldesc}\\nResults with gene info:\\n{results_ginf}\\nResults with cell info:\\n{results_cinf}\\nResults with 20 selected:\\n{results_20}')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:36:51.655260Z","iopub.status.busy":"2023-11-14T16:36:51.654960Z","iopub.status.idle":"2023-11-14T16:37:38.260925Z","shell.execute_reply":"2023-11-14T16:37:38.260316Z","shell.execute_reply.started":"2023-11-14T16:36:51.655238Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Add remaining molecular descriptors Unseen Cell Type Cross Validation","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(data_train, test_df, valid='NK cells')\ncd4_to = get_train_valid_data(data_train, test_df, valid='T cells CD4+')\ncd8_to = get_train_valid_data(data_train, test_df, valid='T cells CD8+')\nreg_to = get_train_valid_data(data_train, test_df, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:37:38.262274Z","iopub.status.busy":"2023-11-14T16:37:38.261888Z","iopub.status.idle":"2023-11-14T16:38:42.664328Z","shell.execute_reply":"2023-11-14T16:38:42.663706Z","shell.execute_reply.started":"2023-11-14T16:37:38.262253Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_20 = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_20[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}:\\t{cv_results_20[key]}')\n    #print(f'Results ginf:\\t{cv_results_ginf[key]}')\n    #print(f'Results moldesc:\\t{cv_results_moldesc[key]}')\n    #print(f'Results plain:\\t{cv_results_plain[key]}')\n    \nav_cv_result_20 = tuple(round(i.item(), 6) for i in tensor(list(cv_results_20.values())).mean(0))\nprint(f'Average Result: {av_cv_result_20}')\n# print(f'Average Ginf: {av_cv_result_ginf}')\n#print(f'Average Moldesc: {av_cv_result_moldesc}')\n#print(f'Average Plain : {av_cv_result_plain}')\n","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:38:42.665870Z","iopub.status.busy":"2023-11-14T16:38:42.665566Z","iopub.status.idle":"2023-11-14T17:20:40.118428Z","shell.execute_reply":"2023-11-14T17:20:40.117800Z","shell.execute_reply.started":"2023-11-14T16:38:42.665849Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Use All Features","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"# all_features\n# all_features.append('value')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T17:42:09.554250Z","iopub.status.busy":"2023-11-14T17:42:09.553850Z","iopub.status.idle":"2023-11-14T17:42:09.557193Z","shell.execute_reply":"2023-11-14T17:42:09.556567Z","shell.execute_reply.started":"2023-11-14T17:42:09.554224Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = all_features\nlen(selected_features)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T17:42:10.332875Z","iopub.status.busy":"2023-11-14T17:42:10.332489Z","iopub.status.idle":"2023-11-14T17:42:10.337237Z","shell.execute_reply":"2023-11-14T17:42:10.336660Z","shell.execute_reply.started":"2023-11-14T17:42:10.332850Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = train_df_cinf_ginf_mdesc.loc[:, selected_features]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T17:42:12.983270Z","iopub.status.busy":"2023-11-14T17:42:12.982861Z","iopub.status.idle":"2023-11-14T17:42:15.484029Z","shell.execute_reply":"2023-11-14T17:42:15.483364Z","shell.execute_reply.started":"2023-11-14T17:42:12.983245Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: results_plain\nexcept NameError: results_plain = 'NA'\n\ntry: results_moldesc\nexcept NameError: results_moldesc = 'NA'\n\ntry: results_ginf\nexcept NameError: results_ginf = 'NA'\n\ntry: results_cinf\nexcept NameError: results_cinf = 'NA'","metadata":{"execution":{"iopub.execute_input":"2023-11-14T17:42:15.954020Z","iopub.status.busy":"2023-11-14T17:42:15.953622Z","iopub.status.idle":"2023-11-14T17:42:15.957812Z","shell.execute_reply":"2023-11-14T17:42:15.957195Z","shell.execute_reply.started":"2023-11-14T17:42:15.953994Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(data_train, test_df, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y\nm = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T17:42:46.126574Z","iopub.status.busy":"2023-11-14T17:42:46.126168Z","iopub.status.idle":"2023-11-14T17:55:23.125852Z","shell.execute_reply":"2023-11-14T17:55:23.125169Z","shell.execute_reply.started":"2023-11-14T17:42:46.126549Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_all = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with molecular descriptors:\\n{results_moldesc}\\nResults with gene info:\\n{results_ginf}\\nResults with cell info:\\n{results_cinf}\\nResults with all:\\n{results_all}')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T17:55:23.127498Z","iopub.status.busy":"2023-11-14T17:55:23.127169Z","iopub.status.idle":"2023-11-14T17:56:11.183744Z","shell.execute_reply":"2023-11-14T17:56:11.183094Z","shell.execute_reply.started":"2023-11-14T17:55:23.127476Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### All Features Unseen Cell Type Cross Validation","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(data_train, test_df, valid='NK cells')\ncd4_to = get_train_valid_data(data_train, test_df, valid='T cells CD4+')\ncd8_to = get_train_valid_data(data_train, test_df, valid='T cells CD8+')\nreg_to = get_train_valid_data(data_train, test_df, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T17:56:11.184960Z","iopub.status.busy":"2023-11-14T17:56:11.184646Z","iopub.status.idle":"2023-11-14T17:58:46.528161Z","shell.execute_reply":"2023-11-14T17:58:46.527474Z","shell.execute_reply.started":"2023-11-14T17:56:11.184937Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_all = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_all[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}:\\t{cv_results_all[key]}')\n    #print(f'Results ginf:\\t{cv_results_ginf[key]}')\n    #print(f'Results moldesc:\\t{cv_results_moldesc[key]}')\n    #print(f'Results plain:\\t{cv_results_plain[key]}')\n    \nav_cv_result_all = tuple(round(i.item(), 6) for i in tensor(list(cv_results_all.values())).mean(0))\nprint(f'Average Result: {av_cv_result_all}')\n# print(f'Average Ginf: {av_cv_result_ginf}')\n#print(f'Average Moldesc: {av_cv_result_moldesc}')\n#print(f'Average Plain : {av_cv_result_plain}')\n","metadata":{"execution":{"iopub.execute_input":"2023-11-14T19:17:10.519821Z","iopub.status.busy":"2023-11-14T19:17:10.519385Z","iopub.status.idle":"2023-11-14T19:17:10.523597Z","shell.execute_reply":"2023-11-14T19:17:10.522866Z","shell.execute_reply.started":"2023-11-14T19:17:10.519795Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_results_all","metadata":{"execution":{"iopub.execute_input":"2023-11-14T19:16:17.793766Z","iopub.status.busy":"2023-11-14T19:16:17.793337Z","iopub.status.idle":"2023-11-14T19:16:17.798657Z","shell.execute_reply":"2023-11-14T19:16:17.797975Z","shell.execute_reply.started":"2023-11-14T19:16:17.793739Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Add sensible features to the selection of 37","metadata":{"jp-MarkdownHeadingCollapsed":true,"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# all_features","metadata":{"execution":{"iopub.execute_input":"2023-11-14T20:25:58.700791Z","iopub.status.busy":"2023-11-14T20:25:58.700424Z","iopub.status.idle":"2023-11-14T20:25:58.703894Z","shell.execute_reply":"2023-11-14T20:25:58.703214Z","shell.execute_reply.started":"2023-11-14T20:25:58.700766Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = ['cell_type', 'cell_function', 'organ_of_origin', 'immune_type', 'sm_name', 'BCUTpe-1l', 'ATSC8s', 'BIC4', 'AATSC1dv', 'BCUTi-1h', 'MATS1dv', 'SaaO', 'ATSC3c', 'BCUTse-1l', 'BIC5', 'AATSC1are', 'AATS1i', 'BCUTm-1l', 'ATSC7s', 'ATSC6c', 'AATS5s', 'MATS3v', 'VSA_EState2', 'ATSC5s', 'AATS3i', 'JGI2', 'AATSC4pe', 'ATSC4Z', 'gene', 'mane', 'length', 'transcript_count', 'nC', 'nA', 'nT', 'nG', 'peptide_length', 'value']\nlen(selected_features)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T19:27:03.207684Z","iopub.status.busy":"2023-11-14T19:27:03.207251Z","iopub.status.idle":"2023-11-14T19:27:03.213662Z","shell.execute_reply":"2023-11-14T19:27:03.212953Z","shell.execute_reply.started":"2023-11-14T19:27:03.207658Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = train_df_cinf_ginf_mdesc.loc[:, selected_features]","metadata":{"execution":{"iopub.execute_input":"2023-11-14T19:27:36.865381Z","iopub.status.busy":"2023-11-14T19:27:36.864957Z","iopub.status.idle":"2023-11-14T19:27:38.321844Z","shell.execute_reply":"2023-11-14T19:27:38.321107Z","shell.execute_reply.started":"2023-11-14T19:27:36.865353Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try: results_plain\nexcept NameError: results_plain = 'NA'\n\ntry: results_moldesc\nexcept NameError: results_moldesc = 'NA'\n\ntry: results_ginf\nexcept NameError: results_ginf = 'NA'\n\ntry: results_cinf\nexcept NameError: results_cinf = 'NA'","metadata":{"execution":{"iopub.execute_input":"2023-11-14T19:27:38.323500Z","iopub.status.busy":"2023-11-14T19:27:38.323142Z","iopub.status.idle":"2023-11-14T19:27:38.327560Z","shell.execute_reply":"2023-11-14T19:27:38.326831Z","shell.execute_reply.started":"2023-11-14T19:27:38.323476Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto = get_train_valid_data(data_train, test_df, valid=0.2, random_state=random_seed)\nxs, y, valid_xs, valid_y = to.train.xs, to.train.y, to.valid.xs, to.valid.y\nm = rf(xs, y, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:27:22.504525Z","iopub.status.busy":"2023-11-14T16:27:22.504116Z","iopub.status.idle":"2023-11-14T16:36:51.653829Z","shell.execute_reply":"2023-11-14T16:36:51.653206Z","shell.execute_reply.started":"2023-11-14T16:27:22.504500Z"},"papermill":{"duration":1045.906017,"end_time":"2023-10-03T19:10:11.083784","exception":false,"start_time":"2023-10-03T18:52:45.177767","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nresults_35 = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), m.oob_score_\nprint(f'Results plain model:\\n{results_plain}\\nResults with molecular descriptors:\\n{results_moldesc}\\nResults with gene info:\\n{results_ginf}\\nResults with cell info:\\n{results_cinf}\\nResults with 20 selected:\\n{results_20}')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T16:36:51.655260Z","iopub.status.busy":"2023-11-14T16:36:51.654960Z","iopub.status.idle":"2023-11-14T16:37:38.260925Z","shell.execute_reply":"2023-11-14T16:37:38.260316Z","shell.execute_reply.started":"2023-11-14T16:36:51.655238Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Add sensible features to the selection of 37 Unseen Cell Type Cross Validation","metadata":{"jp-MarkdownHeadingCollapsed":true,"tags":[]}},{"cell_type":"code","source":"%%time\nnk_to = get_train_valid_data(data_train, test_df, valid='NK cells')\ncd4_to = get_train_valid_data(data_train, test_df, valid='T cells CD4+')\ncd8_to = get_train_valid_data(data_train, test_df, valid='T cells CD8+')\nreg_to = get_train_valid_data(data_train, test_df, valid='T regulatory cells')","metadata":{"execution":{"iopub.execute_input":"2023-11-14T19:28:19.114795Z","iopub.status.busy":"2023-11-14T19:28:19.114366Z","iopub.status.idle":"2023-11-14T19:29:30.627797Z","shell.execute_reply":"2023-11-14T19:29:30.627006Z","shell.execute_reply.started":"2023-11-14T19:28:19.114771Z"},"papermill":{"duration":313.113778,"end_time":"2023-10-03T19:15:24.262733","exception":false,"start_time":"2023-10-03T19:10:11.148955","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nto_dict = {'NK cells': nk_to, 'T cells CD4+': cd4_to, 'T cells CD8+': cd8_to, 'T regulatory cells': reg_to}\ncv_results_35 = {}\n\nfor key, value in to_dict.items():\n    print(f'Training model for {key}...')\n    xs, y, valid_xs, valid_y = value.train.xs, value.train.y, value.valid.xs, value.valid.y\n    m = rf(xs, y, random_state=random_seed)\n    cv_results_35[key] = m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y), rmse(m.oob_prediction_, y), round(m.oob_score_, 6)\n    print(f'Results for {key}:\\t{cv_results_35[key]}')\n    #print(f'Results ginf:\\t{cv_results_ginf[key]}')\n    #print(f'Results moldesc:\\t{cv_results_moldesc[key]}')\n    #print(f'Results plain:\\t{cv_results_plain[key]}')\n    \nav_cv_result_35 = tuple(round(i.item(), 6) for i in tensor(list(cv_results_35.values())).mean(0))\nprint(f'Average Result: {av_cv_result_35}')\n# print(f'Average Ginf: {av_cv_result_ginf}')\n#print(f'Average Moldesc: {av_cv_result_moldesc}')\n#print(f'Average Plain : {av_cv_result_plain}')\n","metadata":{"execution":{"iopub.execute_input":"2023-11-14T19:29:30.629573Z","iopub.status.busy":"2023-11-14T19:29:30.629195Z","iopub.status.idle":"2023-11-14T20:25:58.699087Z","shell.execute_reply":"2023-11-14T20:25:58.698315Z","shell.execute_reply.started":"2023-11-14T19:29:30.629549Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Final Random Forests","metadata":{"papermill":{"duration":0.009024,"end_time":"2023-10-03T18:52:41.244635","exception":false,"start_time":"2023-10-03T18:52:41.235611","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2023-11-14T20:38:20.024532Z","iopub.status.busy":"2023-11-14T20:38:20.024098Z","iopub.status.idle":"2023-11-14T20:38:22.505133Z","shell.execute_reply":"2023-11-14T20:38:22.504433Z","shell.execute_reply.started":"2023-11-14T20:38:20.024504Z"},"papermill":{"duration":0.551699,"end_time":"2023-10-03T18:52:41.806231","exception":false,"start_time":"2023-10-03T18:52:41.254532","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = ['cell_type', 'cell_function', 'organ_of_origin', 'immune_type', 'sm_name', 'BCUTpe-1l', 'ATSC8s', 'BIC4', 'AATSC1dv', 'BCUTi-1h', 'MATS1dv', 'SaaO', 'ATSC3c', 'BCUTse-1l', 'BIC5', 'AATSC1are', 'AATS1i', 'BCUTm-1l', 'ATSC7s', 'ATSC6c', 'AATS5s', 'MATS3v', 'VSA_EState2', 'ATSC5s', 'AATS3i', 'JGI2', 'AATSC4pe', 'ATSC4Z', 'gene', 'length', 'transcript_count', 'nC', 'nA', 'nT', 'nG', 'peptide_length', 'value']\nlen(selected_features)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T21:03:13.473083Z","iopub.status.busy":"2023-11-14T21:03:13.472658Z","iopub.status.idle":"2023-11-14T21:03:13.479655Z","shell.execute_reply":"2023-11-14T21:03:13.478739Z","shell.execute_reply.started":"2023-11-14T21:03:13.473056Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = train_df_cinf_ginf_mdesc.loc[:, selected_features]\ndata_test = test_df_cinf_ginf_mdesc.loc[:, selected_features]\ntrain_cells = ['NK cells', 'T cells CD4+', 'T cells CD8+', 'T regulatory cells']","metadata":{"execution":{"iopub.execute_input":"2023-11-14T21:03:14.501177Z","iopub.status.busy":"2023-11-14T21:03:14.500765Z","iopub.status.idle":"2023-11-14T21:03:16.591699Z","shell.execute_reply":"2023-11-14T21:03:16.590970Z","shell.execute_reply.started":"2023-11-14T21:03:14.501151Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ct in train_cells:\n    print(f'preparing {ct} data...')\n    trd = data_train.loc[data_train['cell_type'].isin([ct, 'B cells', 'Myeloid cells'])].reset_index(drop=True)\n    tsd = data_test\n    to = get_train_valid_data(trd, tsd, valid=None, random_state=random_seed)\n    xs, y = to.train.xs, to.train.y\n    print(f'fitting {ct} model...')\n    m = rf(xs, y, random_state=random_seed)\n    print(f'{ct}:\\t{m_rmse(m, xs, y), rmse(m.oob_prediction_, y), m.oob_score_}')\n    test_to = to.new(tsd)\n    test_to.process()\n    test_xs = test_to.xs\n    preds = tensor(m.predict(test_xs))\n    print(preds.min(), preds.max())\n    to_submit = preds.view(18211, -1).t().numpy()\n    submit = pd.DataFrame(to_submit, columns=df_de_train.iloc[:,5:].columns)\n    submit.index.name = 'id'\n    filename = f\"submission_forest100r67csq_sl35_100pct_v{''.join(ct.split(' '))}_seed42.csv\"\n    submit.to_csv(filename)","metadata":{"execution":{"iopub.execute_input":"2023-11-14T21:03:16.593280Z","iopub.status.busy":"2023-11-14T21:03:16.592932Z","iopub.status.idle":"2023-11-14T21:19:02.181848Z","shell.execute_reply":"2023-11-14T21:19:02.181131Z","shell.execute_reply.started":"2023-11-14T21:03:16.593257Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}