{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install tables","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:33:23.352073Z","iopub.execute_input":"2023-04-12T18:33:23.352881Z","iopub.status.idle":"2023-04-12T18:33:37.722867Z","shell.execute_reply.started":"2023-04-12T18:33:23.352772Z","shell.execute_reply":"2023-04-12T18:33:37.721396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os, gc, pickle\nimport time\nfrom sklearn.decomposition import PCA\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.preprocessing import MinMaxScaler, minmax_scale\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import Ridge, Lasso\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error, log_loss, make_scorer\nfrom scipy.stats import pearsonr\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport random\nimport warnings\nfrom scipy.linalg import LinAlgWarning\nwarnings.filterwarnings(action='ignore', category=LinAlgWarning)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-12T18:33:37.726254Z","iopub.execute_input":"2023-04-12T18:33:37.727428Z","iopub.status.idle":"2023-04-12T18:33:39.601550Z","shell.execute_reply.started":"2023-04-12T18:33:37.727378Z","shell.execute_reply":"2023-04-12T18:33:39.600232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef pearson(X, y):\n    n=X.shape[1]\n    assert n==y.shape[1]\n    xmean=np.mean(X,axis=1)\n    ymean=np.mean(y,axis=1)\n    A=np.sum(X**2, axis=1) - n*xmean**2\n    B=np.sum(y**2, axis=1) - n*ymean**2\n    C=np.sum(X*y, axis=1)\n    corr=(C-n*xmean*ymean)/ np.sqrt(A*B)\n    return corr","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us compare denoised data with different parameters. ","metadata":{}},{"cell_type":"markdown","source":"The data are denoised as following and stored here https://www.kaggle.com/datasets/geraseva/citeseq-denoised, https://www.kaggle.com/datasets/geraseva/citeseq-denoised-dca","metadata":{}},{"cell_type":"markdown","source":"```\nimport magic\n\ntrain_cite_inputs=pd.read_hdf(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\")\ntest_cite_inputs=pd.read_hdf(\"/kaggle/input/open-problems-multimodal/test_cite_inputs.h5\")\nextra_cite_inputs=pd.read_hdf(\"/kaggle/input/open-problems-multimodal/test_cite_inputs_day_2_donor_27678.h5\")\n\ncols1=(train_cite_inputs > 0).sum(axis=0)\ncols2=(test_cite_inputs > 0).sum(axis=0)\n\nnonzero_columns=list(cols1[cols1>0].index)\nnonzero_columns.extend(list(cols2[cols2>0].index))\nnonzero_columns=list(set(nonzero_columns))\n\ntrain_cite_inputs=train_cite_inputs[nonzero_columns]\ntest_cite_inputs=test_cite_inputs[nonzero_columns]\nextra_cite_inputs=extra_cite_inputs[nonzero_columns]\n\nknn_states=[5,7,11]\nt_states=[3,5,7]\nfor knn in knn_states:\n    for t in t_states:\n        magic_operator = magic.MAGIC(random_state=32, n_jobs=16, knn=knn, t=t, verbose=0)\n        denoised=magic_operator.fit_transform(pd.concat([train_cite_inputs,test_cite_inputs,extra_cite_inputs], copy=False))\n\n        denoised=denoised.astype('float32', copy=False)\n\n        denoised.loc[train_cite_inputs.index].to_hdf('train_cite_inputs_denoised_knn_'+str(knn)+'_t_'+str(t)+'.h5', key='0', mode='w')\n        denoised.loc[test_cite_inputs.index].to_hdf('test_cite_inputs_denoised_knn_'+str(knn)+'_t_'+str(t)+'.h5', key='0', mode='w')\n        denoised.loc[extra_cite_inputs.index].to_hdf('test_cite_inputs_day_2_donor_27678_knn_'+str(knn)+'_t_'+str(t)+'.h5', key='0', mode='w')\n        del denoised, magic_operator\n```","metadata":{"execution":{"iopub.status.busy":"2022-10-03T12:36:03.928447Z","iopub.execute_input":"2022-10-03T12:36:03.928989Z","iopub.status.idle":"2022-10-03T12:36:03.935412Z","shell.execute_reply.started":"2022-10-03T12:36:03.928919Z","shell.execute_reply":"2022-10-03T12:36:03.934157Z"}}},{"cell_type":"markdown","source":"Load metadata","metadata":{}},{"cell_type":"code","source":"meta = pd.read_csv(\"/kaggle/input/open-problems-multimodal/metadata.csv\")\n# set index as cell_id\nmeta.set_index('cell_id', inplace=True)\nmeta=meta[meta.technology=='citeseq'].drop('technology', axis=1)\nprint(meta.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:33:39.612302Z","iopub.execute_input":"2023-04-12T18:33:39.613305Z","iopub.status.idle":"2023-04-12T18:33:40.134644Z","shell.execute_reply.started":"2023-04-12T18:33:39.613270Z","shell.execute_reply":"2023-04-12T18:33:40.133286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use the 4th day as a validation set","metadata":{}},{"cell_type":"code","source":"meta=meta[meta.day!=7]\nmeta=meta[meta.donor!=27678]\nmetatrain=meta[meta.day!=4]\nmetaval=meta[meta.day==4]\ndel meta","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:33:40.136012Z","iopub.execute_input":"2023-04-12T18:33:40.136517Z","iopub.status.idle":"2023-04-12T18:33:40.158623Z","shell.execute_reply.started":"2023-04-12T18:33:40.136481Z","shell.execute_reply":"2023-04-12T18:33:40.157688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cite_targets=pd.read_hdf(\n        \"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\")\ntarget_cols=train_cite_targets.columns\nval_cite_targets=train_cite_targets.loc[metaval.index].values\ntrain_cite_targets=train_cite_targets.loc[metatrain.index].values","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:33:40.160027Z","iopub.execute_input":"2023-04-12T18:33:40.160371Z","iopub.status.idle":"2023-04-12T18:33:40.998124Z","shell.execute_reply.started":"2023-04-12T18:33:40.160339Z","shell.execute_reply":"2023-04-12T18:33:40.997193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models trained on denoised data","metadata":{}},{"cell_type":"markdown","source":"Compare performance of a simple model trained on original and denoised data","metadata":{}},{"cell_type":"code","source":"scores={}\nprint('original')\ntrain_cite_inputs = pd.read_hdf(\n\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\")\nval_cite_inputs=train_cite_inputs.loc[metaval.index].values\ntrain_cite_inputs=train_cite_inputs.loc[metatrain.index].values\nprint('loaded')\nmodel = Pipeline([('decomp',PCA(n_components=512, copy=False)),\n                  ('model',MultiOutputRegressor(Ridge()))])\ntime0=time.time()\nrandom.seed(62)\nmodel.fit(train_cite_inputs, train_cite_targets)\nprint('time:',time.time()-time0)\nscore = pearson(model.predict(val_cite_inputs), val_cite_targets)\nprint(np.mean(score))\nscores['original']=score\ndel train_cite_inputs, val_cite_inputs, model\ngc.collect()\n\nprint('dca')\ntrain_cite_inputs = pd.read_hdf(\n\"/kaggle/input/citeseq-denoised-dca/train_cite_inputs_denoised_dca.h5\")\nval_cite_inputs=train_cite_inputs.loc[metaval.index].values\ntrain_cite_inputs=train_cite_inputs.loc[metatrain.index].values\nprint('loaded')\nmodel = Pipeline([('decomp',PCA(n_components=512, copy=False)),\n                  ('model',MultiOutputRegressor(Ridge()))])\ntime0=time.time()\nrandom.seed(62)\nmodel.fit(train_cite_inputs, train_cite_targets)\nprint('time:',time.time()-time0)\nscore = pearson(model.predict(val_cite_inputs), val_cite_targets)\nprint(np.mean(score))\nscores['dca']=score\ndel train_cite_inputs, val_cite_inputs, model\ngc.collect()\n\nfor knn in [5,7,11]:\n    for t in [3,5,7]:\n        s='_knn_'+str(knn)+'_t_'+str(t)\n        k=s if (knn!=5 or t!=3) else ''\n        print(s)\n        train_cite_inputs = pd.read_hdf(\n        \"/kaggle/input/citeseq-denoised/train_cite_inputs_denoised\"+k+\".h5\")\n        val_cite_inputs=train_cite_inputs.loc[metaval.index].values\n        train_cite_inputs=train_cite_inputs.loc[metatrain.index].values\n        print('loaded')\n        model = Pipeline([('decomp',PCA(n_components=512, copy=False)),\n                  ('model',MultiOutputRegressor(Ridge()))])\n        time0=time.time()\n        random.seed(62)\n        model.fit(train_cite_inputs, train_cite_targets)\n        print('time:',time.time()-time0)\n        score = pearson(model.predict(val_cite_inputs), val_cite_targets)\n        print(np.mean(score))\n        scores[s]=score\n        del train_cite_inputs, val_cite_inputs, model\n        gc.collect()\n        \n        ","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:33:40.999416Z","iopub.execute_input":"2023-04-12T18:33:40.999767Z","iopub.status.idle":"2023-04-12T18:34:30.137820Z","shell.execute_reply.started":"2023-04-12T18:33:40.999734Z","shell.execute_reply":"2023-04-12T18:34:30.134924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x=[a for a in scores.keys()]\ny=[scores[a] for a in x]\nx=[a if a!='' else '_knn_5_t_3' for a in x]\nplt.figure(figsize=(15,10))\nplt.grid()\nplt.boxplot(y,\n            labels=x,\n           showfliers=False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:34:30.139450Z","iopub.status.idle":"2023-04-12T18:34:30.140344Z","shell.execute_reply.started":"2023-04-12T18:34:30.140017Z","shell.execute_reply":"2023-04-12T18:34:30.140050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that models trained on denoised data show a subtle improvement. MAGIC with default parameters performs the best","metadata":{}},{"cell_type":"markdown","source":"## Correlation matrices","metadata":{}},{"cell_type":"code","source":"def corr_matrix(X_data, y_data):\n    batch_size=128 \n    X=np.array(X_data)\n    y=np.array(y_data)\n    n=X.shape[0]\n    assert n==y.shape[0]\n    xmean=np.mean(X,axis=0)\n    ymean=np.mean(y,axis=0)\n    A=np.sum(X**2, axis=0) - n*xmean**2\n    B=np.sum(y**2, axis=0) - n*ymean**2\n    C=np.zeros((X.shape[1],y.shape[1]))\n    for batch in range(0,n,batch_size):\n        C+=np.sum(X[batch:batch+batch_size,:,None]*y[batch:batch+batch_size,None,:], axis=0)\n    corr=(C-n*xmean[:,None]*ymean[None,:])/ np.sqrt(A[:,None]*B[None,:])\n    return corr","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:34:35.815757Z","iopub.execute_input":"2023-04-12T18:34:35.816723Z","iopub.status.idle":"2023-04-12T18:34:35.825853Z","shell.execute_reply.started":"2023-04-12T18:34:35.816680Z","shell.execute_reply":"2023-04-12T18:34:35.824456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Compute correlation matrices for original and denoised data","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(20,20))\nscores={}\ntrain_cite_inputs = pd.read_hdf(\n\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\")\ntrain_cite_targets=pd.read_hdf(\n        \"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\")\nm=corr_matrix(train_cite_inputs.values,train_cite_targets)\nm=pd.DataFrame(columns=target_cols,\n            index=train_cite_inputs.columns, data=m)\nm=m.loc[m.abs().median(axis=1).sort_values(ascending=False).index]\nm.to_csv('correlations_original.csv')\nplt.subplot(2,3,1)\nplt.title('Correlations distribution original')\nplt.hist(m.to_numpy().flatten(), bins=40, log=True)\n\nplt.subplot(2,3,4)\nplt.title('Correlations heatmap original')\nax = sns.heatmap(m.values, cmap='hot')\n\ntrain_cite_inputs = pd.read_hdf(\n\"/kaggle/input/citeseq-denoised-dca/train_cite_inputs_denoised_dca.h5\")\n\nm=corr_matrix(train_cite_inputs.values,train_cite_targets)\nm=pd.DataFrame(columns=target_cols,\n            index=train_cite_inputs.columns, data=m)\nm=m.loc[m.abs().median(axis=1).sort_values(ascending=False).index]\nm.to_csv('correlations_dca.csv')\nplt.subplot(2,3,2)\nplt.title('Correlations distribution dca')\nplt.hist(m.to_numpy().flatten(), bins=40, log=True)\n\nplt.subplot(2,3,5)\nplt.title('Correlations heatmap dca')\nax = sns.heatmap(m.values, cmap='hot')\n\ntrain_cite_inputs = pd.read_hdf(\n\"/kaggle/input/citeseq-denoised/train_cite_inputs_denoised.h5\")\n\nm=corr_matrix(train_cite_inputs.values,train_cite_targets)\nm=pd.DataFrame(columns=target_cols,\n            index=train_cite_inputs.columns, data=m)\nm=m.loc[m.abs().median(axis=1).sort_values(ascending=False).index]\nm.to_csv('correlations_magic.csv')\nplt.subplot(2,3,3)\nplt.title('Correlations distribution magic')\nplt.hist(m.to_numpy().flatten(), bins=40, log=True)\n\nplt.subplot(2,3,6)\nplt.title('Correlations heatmap magic')\nax = sns.heatmap(m.values, cmap='hot')\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T18:34:41.640588Z","iopub.execute_input":"2023-04-12T18:34:41.641145Z","iopub.status.idle":"2023-04-12T19:09:07.368336Z","shell.execute_reply.started":"2023-04-12T18:34:41.641084Z","shell.execute_reply":"2023-04-12T19:09:07.364099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CD RNAs","metadata":{}},{"cell_type":"markdown","source":"Let us look on dependencies between target proteins and corresponding RNAs","metadata":{}},{"cell_type":"code","source":"train_cite_inputs = pd.read_hdf(\n    \"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\", stop=1)\ncite_cols=train_cite_inputs.columns","metadata":{"execution":{"iopub.status.busy":"2023-04-12T19:09:07.380370Z","iopub.execute_input":"2023-04-12T19:09:07.381239Z","iopub.status.idle":"2023-04-12T19:09:07.611152Z","shell.execute_reply.started":"2023-04-12T19:09:07.381132Z","shell.execute_reply":"2023-04-12T19:09:07.603528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_cols_important={'CD86': ['ENSG00000114013_CD86'],\n             'CD274': ['ENSG00000120217_CD274'],\n             'CD270': ['ENSG00000157873_TNFRSF14'],\n             'CD155': ['ENSG00000073008_PVR'],\n             'CD112': ['ENSG00000130202_NECTIN2'],\n             'CD47': ['ENSG00000196776_CD47'],\n             'CD48': ['ENSG00000117091_CD48'],\n             'CD40': ['ENSG00000101017_CD40'],\n             'CD154': ['ENSG00000102245_CD40LG'],\n             'CD52': ['ENSG00000169442_CD52'],\n             'CD3': ['ENSG00000167286_CD3D'],\n             'CD8': [],\n             'CD56': ['ENSG00000149294_NCAM1'],\n             'CD19': ['ENSG00000177455_CD19'],\n             'CD33': ['ENSG00000105383_CD33'],\n             'CD11c': ['ENSG00000140678_ITGAX'],\n             'HLA-A-B-C': ['ENSG00000204525_HLA-C',\n              'ENSG00000206503_HLA-A',\n              'ENSG00000234745_HLA-B'],\n             'CD45RA': ['ENSG00000081237_PTPRC'],\n             'CD123': ['ENSG00000185291_IL3RA'],\n             'CD7': ['ENSG00000173762_CD7'],\n             'CD105': ['ENSG00000106991_ENG'],\n             'CD49f': ['ENSG00000091409_ITGA6'],\n             'CD194': ['ENSG00000183813_CCR4'],\n             'CD4': ['ENSG00000010610_CD4'],\n             'CD44': ['ENSG00000026508_CD44'],\n             'CD14': ['ENSG00000170458_CD14'],\n             'CD16': [],\n             'CD25': ['ENSG00000134460_IL2RA'],\n             'CD45RO': ['ENSG00000081237_PTPRC'],\n             'CD279': [],\n             'TIGIT': [],\n             'Mouse-IgG1': [],\n             'Mouse-IgG2a': [],\n             'Mouse-IgG2b': [],\n             'Rat-IgG2b': [],\n             'CD20': ['ENSG00000156738_MS4A1'],\n             'CD335': ['ENSG00000189430_NCR1'],\n             'CD31': ['ENSG00000261371_PECAM1'],\n             'Podoplanin': [],\n             'CD146': ['ENSG00000076706_MCAM'],\n             'IgM': ['ENSG00000211899_IGHM'],\n             'CD5': [],\n             'CD195': ['ENSG00000160791_CCR5'],\n             'CD32': ['ENSG00000143226_FCGR2A'],\n             'CD196': [],\n             'CD185': ['ENSG00000160683_CXCR5'],\n             'CD103': ['ENSG00000083457_ITGAE'],\n             'CD69': ['ENSG00000110848_CD69'],\n             'CD62L': ['ENSG00000188404_SELL'],\n             'CD161': ['ENSG00000111796_KLRB1'],\n             'CD152': [],\n             'CD223': ['ENSG00000089692_LAG3'],\n             'KLRG1': ['ENSG00000139187_KLRG1'],\n             'CD27': ['ENSG00000139193_CD27'],\n             'CD107a': ['ENSG00000185896_LAMP1'],\n             'CD95': ['ENSG00000026103_FAS'],\n             'CD134': ['ENSG00000186827_TNFRSF4'],\n             'HLA-DR': ['ENSG00000204287_HLA-DRA'],\n             'CD1c': ['ENSG00000158481_CD1C'],\n             'CD11b': ['ENSG00000169896_ITGAM'],\n             'CD64': ['ENSG00000150337_FCGR1A'],\n             'CD141': ['ENSG00000178726_THBD'],\n             'CD1d': ['ENSG00000158473_CD1D'],\n             'CD314': [],\n             'CD35': ['ENSG00000203710_CR1'],\n             'CD57': [],\n             'CD272': [],\n             'CD278': ['ENSG00000163600_ICOS'],\n             'CD58': ['ENSG00000116815_CD58'],\n             'CD39': ['ENSG00000138185_ENTPD1'],\n             'CX3CR1': ['ENSG00000168329_CX3CR1'],\n             'CD24': ['ENSG00000272398_CD24'],\n             'CD21': ['ENSG00000117322_CR2'],\n             'CD11a': ['ENSG00000005844_ITGAL'],\n             'CD79b': ['ENSG00000007312_CD79B'],\n             'CD244': ['ENSG00000122223_CD244'],\n             'CD169': [],\n             'integrinB7': ['ENSG00000139626_ITGB7'],\n             'CD268': ['ENSG00000159958_TNFRSF13C'],\n             'CD42b': ['ENSG00000185245_GP1BA'],\n             'CD54': ['ENSG00000090339_ICAM1'],\n             'CD62P': ['ENSG00000174175_SELP'],\n             'CD119': ['ENSG00000027697_IFNGR1'],\n             'TCR': [],\n             'Rat-IgG1': [],\n             'Rat-IgG2a': [],\n             'CD192': ['ENSG00000121807_CCR2'],\n             'CD122': ['ENSG00000100385_IL2RB'],\n             'FceRIa': ['ENSG00000179639_FCER1A'],\n             'CD41': ['ENSG00000005961_ITGA2B'],\n             'CD137': ['ENSG00000049249_TNFRSF9'],\n             'CD163': ['ENSG00000177575_CD163'],\n             'CD83': ['ENSG00000112149_CD83'],\n             'CD124': ['ENSG00000077238_IL4R'],\n             'CD13': ['ENSG00000166825_ANPEP'],\n             'CD2': ['ENSG00000116824_CD2'],\n             'CD226': ['ENSG00000150637_CD226'],\n             'CD29': ['ENSG00000150093_ITGB1'],\n             'CD303': ['ENSG00000198178_CLEC4C'],\n             'CD49b': ['ENSG00000164171_ITGA2'],\n             'CD81': ['ENSG00000110651_CD81'],\n             'IgD': ['ENSG00000211898_IGHD'],\n             'CD18': ['ENSG00000160255_ITGB2'],\n             'CD28': [],\n             'CD38': ['ENSG00000004468_CD38'],\n             'CD127': ['ENSG00000168685_IL7R'],\n             'CD45': ['ENSG00000081237_PTPRC'],\n             'CD22': ['ENSG00000012124_CD22'],\n             'CD71': ['ENSG00000072274_TFRC'],\n             'CD26': ['ENSG00000197635_DPP4'],\n             'CD115': ['ENSG00000182578_CSF1R'],\n             'CD63': ['ENSG00000135404_CD63'],\n             'CD304': ['ENSG00000099250_NRP1'],\n             'CD36': ['ENSG00000135218_CD36'],\n             'CD172a': ['ENSG00000198053_SIRPA'],\n             'CD72': ['ENSG00000137101_CD72'],\n             'CD158': [],\n             'CD93': ['ENSG00000125810_CD93'],\n             'CD49a': ['ENSG00000213949_ITGA1'],\n             'CD49d': ['ENSG00000115232_ITGA4'],\n             'CD73': [],\n             'CD9': ['ENSG00000010278_CD9'],\n             'TCRVa7.2': [],\n             'TCRVd2': [],\n             'LOX-1': ['ENSG00000173391_OLR1'],\n             'CD158b': [],\n             'CD158e1': [],\n             'CD142': ['ENSG00000117525_F3'],\n             'CD319': ['ENSG00000026751_SLAMF7'],\n             'CD352': ['ENSG00000162739_SLAMF6'],\n             'CD94': ['ENSG00000134539_KLRD1'],\n             'CD162': ['ENSG00000110876_SELPLG'],\n             'CD85j': ['ENSG00000104972_LILRB1'],\n             'CD23': ['ENSG00000104921_FCER2'],\n             'CD328': ['ENSG00000168995_SIGLEC7'],\n             'HLA-E': ['ENSG00000204592_HLA-E'],\n             'CD82': ['ENSG00000085117_CD82'],\n             'CD101': ['ENSG00000134256_CD101'],\n             'CD88': ['ENSG00000197405_C5AR1'],\n             'CD224': ['ENSG00000100031_GGT1']}","metadata":{"execution":{"iopub.status.busy":"2023-04-12T19:09:07.614632Z","iopub.execute_input":"2023-04-12T19:09:07.615126Z","iopub.status.idle":"2023-04-12T19:09:07.682198Z","shell.execute_reply.started":"2023-04-12T19:09:07.615086Z","shell.execute_reply":"2023-04-12T19:09:07.678081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m=pd.read_csv('correlations_original.csv', index_col=0)\norig_dict={}\nfor k in cite_cols_important:\n    orig_dict[k]=list(m[k][cite_cols_important[k]])\nm=pd.read_csv('correlations_dca.csv', index_col=0)\ndca_dict={}\nfor k in cite_cols_important:\n    dca_dict[k]=list(m[k][cite_cols_important[k]])\nm=pd.read_csv('correlations_magic.csv', index_col=0)\nmagic_dict={}\nfor k in cite_cols_important:\n    magic_dict[k]=list(m[k][cite_cols_important[k]])\n    \n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T19:21:36.408853Z","iopub.execute_input":"2023-04-12T19:21:36.409758Z","iopub.status.idle":"2023-04-12T19:21:38.846713Z","shell.execute_reply.started":"2023-04-12T19:21:36.409713Z","shell.execute_reply":"2023-04-12T19:21:38.845689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_df=pd.DataFrame(columns=['protein','gene','orig','magic','dca'],\n                     data=[[k, x, orig_dict[k][i],magic_dict[k][i],dca_dict[k][i]] \n                            for k in cite_cols_important \n                            for i, x in enumerate(cite_cols_important[k])])\ncorr_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-04-12T19:21:42.074991Z","iopub.execute_input":"2023-04-12T19:21:42.075469Z","iopub.status.idle":"2023-04-12T19:21:42.103979Z","shell.execute_reply.started":"2023-04-12T19:21:42.075433Z","shell.execute_reply":"2023-04-12T19:21:42.102519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_df.sort_values('orig', ascending=False)[:20]","metadata":{"execution":{"iopub.status.busy":"2023-04-12T20:03:13.077988Z","iopub.execute_input":"2023-04-12T20:03:13.078608Z","iopub.status.idle":"2023-04-12T20:03:13.107293Z","shell.execute_reply.started":"2023-04-12T20:03:13.078560Z","shell.execute_reply":"2023-04-12T20:03:13.104831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,10))\nplt.title('Distribution of correlations between target proteins and corresponding RNAs')\nsns.histplot([corr_df.orig,corr_df.magic,corr_df.dca], bins=20, multiple='dodge', kde=True, legend=True)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-12T19:54:39.602586Z","iopub.execute_input":"2023-04-12T19:54:39.603161Z","iopub.status.idle":"2023-04-12T19:54:40.107442Z","shell.execute_reply.started":"2023-04-12T19:54:39.603117Z","shell.execute_reply":"2023-04-12T19:54:40.106072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(21,7))\nplt.subplot(131)\nplt.title('Correlations of original data against magic')\nplt.xlabel('orig')\nplt.ylabel('magic')\nplt.plot([-0.1,1],[-0.1,1], color='black', linestyle='--')\nplt.scatter(corr_df.orig,corr_df.magic)\nplt.subplot(132)\nplt.title('Correlations of original data against DCA')\nplt.xlabel('orig')\nplt.ylabel('DCA')\nplt.plot([-0.1,1],[-0.1,1], color='black', linestyle='--')\nplt.scatter(corr_df.orig,corr_df.dca)\nplt.subplot(133)\nplt.title('Correlations of MAGIC against DCA')\nplt.xlabel('MAGIC')\nplt.ylabel('DCA')\nplt.plot([-0.1,1],[-0.1,1], color='black', linestyle='--')\nplt.scatter(corr_df.magic,corr_df.dca)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-12T20:04:30.856909Z","iopub.execute_input":"2023-04-12T20:04:30.857546Z","iopub.status.idle":"2023-04-12T20:04:31.877552Z","shell.execute_reply.started":"2023-04-12T20:04:30.857490Z","shell.execute_reply":"2023-04-12T20:04:31.875753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cite_inputs = pd.read_hdf(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\")[corr_df.gene.unique()]\ntrain_denoised_magic=pd.read_hdf(\"/kaggle/input/citeseq-denoised/train_cite_inputs_denoised_knn_11_t_3.h5\")[corr_df.gene.unique()]\ntrain_denoised_dca=pd.read_hdf(\"/kaggle/input/citeseq-denoised-dca/train_cite_inputs_denoised_dca.h5\")[corr_df.gene.unique()]\ntrain_cite_targets=pd.read_hdf(\"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\")","metadata":{"execution":{"iopub.status.busy":"2023-04-12T20:07:09.515423Z","iopub.execute_input":"2023-04-12T20:07:09.515907Z","iopub.status.idle":"2023-04-12T20:10:02.927057Z","shell.execute_reply.started":"2023-04-12T20:07:09.515867Z","shell.execute_reply":"2023-04-12T20:10:02.925633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(corr_df.shape[0]):\n        plt.figure(figsize=(20,5))\n        plt.subplot(131)\n        plt.title(f'No denoise \\n Pearson corr: {corr_df.iloc[i].orig:.3f}')\n        plt.xlabel(corr_df.iloc[i].protein)\n        plt.ylabel(corr_df.iloc[i].gene)\n        plt.scatter(train_cite_targets[corr_df.iloc[i].protein],train_cite_inputs[corr_df.iloc[i].gene], color='blue', s=2)\n        plt.subplot(132)\n        plt.title(f'Magic denoise \\n Pearson corr: {corr_df.iloc[i].magic:.3f}')\n        plt.xlabel(corr_df.iloc[i].protein)\n        plt.ylabel(corr_df.iloc[i].gene)\n        plt.scatter(train_cite_targets[corr_df.iloc[i].protein],train_denoised_magic[corr_df.iloc[i].gene], color='blue', s=2)\n        plt.subplot(133)\n        plt.title(f'DCA denoise \\n Pearson corr: {corr_df.iloc[i].dca:.3f}')\n        plt.xlabel(corr_df.iloc[i].protein)\n        plt.ylabel(corr_df.iloc[i].gene)\n        plt.scatter(train_cite_targets[corr_df.iloc[i].protein],train_denoised_dca[corr_df.iloc[i].gene], color='blue', s=2)\n\n        plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-12T19:33:38.262579Z","iopub.execute_input":"2023-04-12T19:33:38.263124Z","iopub.status.idle":"2023-04-12T19:34:53.464991Z","shell.execute_reply.started":"2023-04-12T19:33:38.263068Z","shell.execute_reply":"2023-04-12T19:34:53.463639Z"},"trusted":true},"execution_count":null,"outputs":[]}]}