{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os, gc, pickle\nimport time\nfrom sklearn.decomposition import PCA\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.preprocessing import MinMaxScaler, minmax_scale\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import Ridge, Lasso\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error, log_loss, make_scorer\nfrom scipy.stats import pearsonr\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport random","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-10-13T15:25:38.689405Z","iopub.execute_input":"2022-10-13T15:25:38.690028Z","iopub.status.idle":"2022-10-13T15:25:40.835928Z","shell.execute_reply.started":"2022-10-13T15:25:38.689944Z","shell.execute_reply":"2022-10-13T15:25:40.834968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('/opt/conda/lib/python3.7/site-packages/tables'):\n    !pip install --quiet tables","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:26:41.559485Z","iopub.execute_input":"2022-10-13T15:26:41.559869Z","iopub.status.idle":"2022-10-13T15:26:53.679957Z","shell.execute_reply.started":"2022-10-13T15:26:41.559839Z","shell.execute_reply":"2022-10-13T15:26:53.679251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us compare MAGIC denoised data (https://github.com/KrishnaswamyLab/MAGIC) with different parameters. ","metadata":{}},{"cell_type":"markdown","source":"The data are denoised as following and stored here https://www.kaggle.com/datasets/geraseva/citeseq-denoised","metadata":{}},{"cell_type":"markdown","source":"```\nimport magic\n\ntrain_cite_inputs=pd.read_hdf(\"/kaggle/input/open-problems-multimodal/train_cite_inputs.h5\")\ntest_cite_inputs=pd.read_hdf(\"/kaggle/input/open-problems-multimodal/test_cite_inputs.h5\")\nextra_cite_inputs=pd.read_hdf(\"/kaggle/input/open-problems-multimodal/test_cite_inputs_day_2_donor_27678.h5\")\n\ncols1=(train_cite_inputs > 0).sum(axis=0)\ncols2=(test_cite_inputs > 0).sum(axis=0)\n\nnonzero_columns=list(cols1[cols1>0].index)\nnonzero_columns.extend(list(cols2[cols2>0].index))\nnonzero_columns=list(set(nonzero_columns))\n\ntrain_cite_inputs=train_cite_inputs[nonzero_columns]\ntest_cite_inputs=test_cite_inputs[nonzero_columns]\nextra_cite_inputs=extra_cite_inputs[nonzero_columns]\n\nknn_states=[5,7,11]\nt_states=[3,5,7]\nfor knn in knn_states:\n    for t in t_states:\n        magic_operator = magic.MAGIC(random_state=32, n_jobs=16, knn=knn, t=t, verbose=0)\n        denoised=magic_operator.fit_transform(pd.concat([train_cite_inputs,test_cite_inputs,extra_cite_inputs], copy=False))\n\n        denoised=denoised.astype('float32', copy=False)\n\n        denoised.loc[train_cite_inputs.index].to_hdf('train_cite_inputs_denoised_knn_'+str(knn)+'_t_'+str(t)+'.h5', key='0', mode='w')\n        denoised.loc[test_cite_inputs.index].to_hdf('test_cite_inputs_denoised_knn_'+str(knn)+'_t_'+str(t)+'.h5', key='0', mode='w')\n        denoised.loc[extra_cite_inputs.index].to_hdf('test_cite_inputs_day_2_donor_27678_knn_'+str(knn)+'_t_'+str(t)+'.h5', key='0', mode='w')\n        del denoised, magic_operator\n```","metadata":{"execution":{"iopub.status.busy":"2022-10-03T12:36:03.928447Z","iopub.execute_input":"2022-10-03T12:36:03.928989Z","iopub.status.idle":"2022-10-03T12:36:03.935412Z","shell.execute_reply.started":"2022-10-03T12:36:03.928919Z","shell.execute_reply":"2022-10-03T12:36:03.934157Z"}}},{"cell_type":"code","source":"meta = pd.read_csv(\"/kaggle/input/open-problems-multimodal/metadata.csv\")\n# set index as cell_id\nmeta.set_index('cell_id', inplace=True)\nmeta=meta[meta.technology=='citeseq'].drop('technology', axis=1)\nmeta.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:51:52.776663Z","iopub.execute_input":"2022-10-13T15:51:52.777522Z","iopub.status.idle":"2022-10-13T15:51:53.108307Z","shell.execute_reply.started":"2022-10-13T15:51:52.777466Z","shell.execute_reply":"2022-10-13T15:51:53.107319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_cite_targets=pd.read_hdf(\n        \"/kaggle/input/open-problems-multimodal/train_cite_targets.h5\")\ntarget_cols=train_cite_targets.columns","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:49:24.308654Z","iopub.execute_input":"2022-10-13T15:49:24.309022Z","iopub.status.idle":"2022-10-13T15:49:24.995230Z","shell.execute_reply.started":"2022-10-13T15:49:24.308995Z","shell.execute_reply":"2022-10-13T15:49:24.994351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr_matrix(X_data, y_data):\n    batch_size=128 \n    X=np.array(X_data)\n    y=np.array(y_data)\n    n=X.shape[0]\n    assert n==y.shape[0]\n    xmean=np.mean(X,axis=0)\n    ymean=np.mean(y,axis=0)\n    A=np.sum(X**2, axis=0) - n*xmean**2\n    B=np.sum(y**2, axis=0) - n*ymean**2\n    C=np.zeros((X.shape[1],y.shape[1]))\n    for batch in range(0,n,batch_size):\n        C+=np.sum(X[batch:batch+batch_size,:,None]*y[batch:batch+batch_size,None,:], axis=0)\n    corr=(C-n*xmean[:,None]*ymean[None,:])/ np.sqrt(A[:,None]*B[None,:])\n    return corr","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:27:54.323986Z","iopub.execute_input":"2022-10-13T15:27:54.325065Z","iopub.status.idle":"2022-10-13T15:27:54.333806Z","shell.execute_reply.started":"2022-10-13T15:27:54.325033Z","shell.execute_reply":"2022-10-13T15:27:54.332710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cite_cols_important={'CD86': ['ENSG00000114013_CD86'],\n             'CD274': ['ENSG00000120217_CD274'],\n             'CD270': ['ENSG00000157873_TNFRSF14'],\n             'CD155': ['ENSG00000073008_PVR'],\n             'CD112': ['ENSG00000130202_NECTIN2'],\n             'CD47': ['ENSG00000196776_CD47'],\n             'CD48': ['ENSG00000117091_CD48'],\n             'CD40': ['ENSG00000101017_CD40'],\n             'CD154': ['ENSG00000102245_CD40LG'],\n             'CD52': ['ENSG00000169442_CD52'],\n             'CD3': ['ENSG00000167286_CD3D'],\n             'CD8': [],\n             'CD56': ['ENSG00000149294_NCAM1'],\n             'CD19': ['ENSG00000177455_CD19'],\n             'CD33': ['ENSG00000105383_CD33'],\n             'CD11c': ['ENSG00000140678_ITGAX'],\n             'HLA-A-B-C': ['ENSG00000204525_HLA-C',\n              'ENSG00000206503_HLA-A',\n              'ENSG00000234745_HLA-B'],\n             'CD45RA': ['ENSG00000081237_PTPRC'],\n             'CD123': ['ENSG00000185291_IL3RA'],\n             'CD7': ['ENSG00000173762_CD7'],\n             'CD105': ['ENSG00000106991_ENG'],\n             'CD49f': ['ENSG00000091409_ITGA6'],\n             'CD194': ['ENSG00000183813_CCR4'],\n             'CD4': ['ENSG00000010610_CD4'],\n             'CD44': ['ENSG00000026508_CD44'],\n             'CD14': ['ENSG00000170458_CD14'],\n             'CD16': [],\n             'CD25': ['ENSG00000134460_IL2RA'],\n             'CD45RO': ['ENSG00000081237_PTPRC'],\n             'CD279': [],\n             'TIGIT': [],\n             'Mouse-IgG1': [],\n             'Mouse-IgG2a': [],\n             'Mouse-IgG2b': [],\n             'Rat-IgG2b': [],\n             'CD20': ['ENSG00000156738_MS4A1'],\n             'CD335': ['ENSG00000189430_NCR1'],\n             'CD31': ['ENSG00000261371_PECAM1'],\n             'Podoplanin': [],\n             'CD146': ['ENSG00000076706_MCAM'],\n             'IgM': ['ENSG00000211899_IGHM'],\n             'CD5': [],\n             'CD195': ['ENSG00000160791_CCR5'],\n             'CD32': ['ENSG00000143226_FCGR2A'],\n             'CD196': [],\n             'CD185': ['ENSG00000160683_CXCR5'],\n             'CD103': ['ENSG00000083457_ITGAE'],\n             'CD69': ['ENSG00000110848_CD69'],\n             'CD62L': ['ENSG00000188404_SELL'],\n             'CD161': ['ENSG00000111796_KLRB1'],\n             'CD152': [],\n             'CD223': ['ENSG00000089692_LAG3'],\n             'KLRG1': ['ENSG00000139187_KLRG1'],\n             'CD27': ['ENSG00000139193_CD27'],\n             'CD107a': ['ENSG00000185896_LAMP1'],\n             'CD95': ['ENSG00000026103_FAS'],\n             'CD134': ['ENSG00000186827_TNFRSF4'],\n             'HLA-DR': ['ENSG00000204287_HLA-DRA'],\n             'CD1c': ['ENSG00000158481_CD1C'],\n             'CD11b': ['ENSG00000169896_ITGAM'],\n             'CD64': ['ENSG00000150337_FCGR1A'],\n             'CD141': ['ENSG00000178726_THBD'],\n             'CD1d': ['ENSG00000158473_CD1D'],\n             'CD314': [],\n             'CD35': ['ENSG00000203710_CR1'],\n             'CD57': [],\n             'CD272': [],\n             'CD278': ['ENSG00000163600_ICOS'],\n             'CD58': ['ENSG00000116815_CD58'],\n             'CD39': ['ENSG00000138185_ENTPD1'],\n             'CX3CR1': ['ENSG00000168329_CX3CR1'],\n             'CD24': ['ENSG00000272398_CD24'],\n             'CD21': ['ENSG00000117322_CR2'],\n             'CD11a': ['ENSG00000005844_ITGAL'],\n             'CD79b': ['ENSG00000007312_CD79B'],\n             'CD244': ['ENSG00000122223_CD244'],\n             'CD169': [],\n             'integrinB7': ['ENSG00000139626_ITGB7'],\n             'CD268': ['ENSG00000159958_TNFRSF13C'],\n             'CD42b': ['ENSG00000185245_GP1BA'],\n             'CD54': ['ENSG00000090339_ICAM1'],\n             'CD62P': ['ENSG00000174175_SELP'],\n             'CD119': ['ENSG00000027697_IFNGR1'],\n             'TCR': [],\n             'Rat-IgG1': [],\n             'Rat-IgG2a': [],\n             'CD192': ['ENSG00000121807_CCR2'],\n             'CD122': ['ENSG00000100385_IL2RB'],\n             'FceRIa': ['ENSG00000179639_FCER1A'],\n             'CD41': ['ENSG00000005961_ITGA2B'],\n             'CD137': ['ENSG00000049249_TNFRSF9'],\n             'CD163': ['ENSG00000177575_CD163'],\n             'CD83': ['ENSG00000112149_CD83'],\n             'CD124': ['ENSG00000077238_IL4R'],\n             'CD13': ['ENSG00000166825_ANPEP'],\n             'CD2': ['ENSG00000116824_CD2'],\n             'CD226': ['ENSG00000150637_CD226'],\n             'CD29': ['ENSG00000150093_ITGB1'],\n             'CD303': ['ENSG00000198178_CLEC4C'],\n             'CD49b': ['ENSG00000164171_ITGA2'],\n             'CD81': ['ENSG00000110651_CD81'],\n             'IgD': ['ENSG00000211898_IGHD'],\n             'CD18': ['ENSG00000160255_ITGB2'],\n             'CD28': [],\n             'CD38': ['ENSG00000004468_CD38'],\n             'CD127': ['ENSG00000168685_IL7R'],\n             'CD45': ['ENSG00000081237_PTPRC'],\n             'CD22': ['ENSG00000012124_CD22'],\n             'CD71': ['ENSG00000072274_TFRC'],\n             'CD26': ['ENSG00000197635_DPP4'],\n             'CD115': ['ENSG00000182578_CSF1R'],\n             'CD63': ['ENSG00000135404_CD63'],\n             'CD304': ['ENSG00000099250_NRP1'],\n             'CD36': ['ENSG00000135218_CD36'],\n             'CD172a': ['ENSG00000198053_SIRPA'],\n             'CD72': ['ENSG00000137101_CD72'],\n             'CD158': [],\n             'CD93': ['ENSG00000125810_CD93'],\n             'CD49a': ['ENSG00000213949_ITGA1'],\n             'CD49d': ['ENSG00000115232_ITGA4'],\n             'CD73': [],\n             'CD9': ['ENSG00000010278_CD9'],\n             'TCRVa7.2': [],\n             'TCRVd2': [],\n             'LOX-1': ['ENSG00000173391_OLR1'],\n             'CD158b': [],\n             'CD158e1': [],\n             'CD142': ['ENSG00000117525_F3'],\n             'CD319': ['ENSG00000026751_SLAMF7'],\n             'CD352': ['ENSG00000162739_SLAMF6'],\n             'CD94': ['ENSG00000134539_KLRD1'],\n             'CD162': ['ENSG00000110876_SELPLG'],\n             'CD85j': ['ENSG00000104972_LILRB1'],\n             'CD23': ['ENSG00000104921_FCER2'],\n             'CD328': ['ENSG00000168995_SIGLEC7'],\n             'HLA-E': ['ENSG00000204592_HLA-E'],\n             'CD82': ['ENSG00000085117_CD82'],\n             'CD101': ['ENSG00000134256_CD101'],\n             'CD88': ['ENSG00000197405_C5AR1'],\n             'CD224': ['ENSG00000100031_GGT1']}","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:52:01.229499Z","iopub.execute_input":"2022-10-13T15:52:01.229885Z","iopub.status.idle":"2022-10-13T15:52:01.250672Z","shell.execute_reply.started":"2022-10-13T15:52:01.229853Z","shell.execute_reply":"2022-10-13T15:52:01.249767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncite_cols_imp=list(set(x for s in cite_cols_important.values() for x in s))\nlen(cite_cols_imp)","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:52:02.663117Z","iopub.execute_input":"2022-10-13T15:52:02.663554Z","iopub.status.idle":"2022-10-13T15:52:02.671908Z","shell.execute_reply.started":"2022-10-13T15:52:02.663520Z","shell.execute_reply":"2022-10-13T15:52:02.670710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_denoised_t3=pd.read_hdf(\"/kaggle/input/citeseq-denoised/train_cite_inputs_denoised.h5\")[cite_cols_imp]\nmeta=meta.join(train_denoised_t3, how='right')\nmeta=meta.join(train_cite_targets)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:52:30.767952Z","iopub.execute_input":"2022-10-13T15:52:30.768301Z","iopub.status.idle":"2022-10-13T15:53:32.592748Z","shell.execute_reply.started":"2022-10-13T15:52:30.768275Z","shell.execute_reply":"2022-10-13T15:53:32.591602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctl = list(meta.cell_type.value_counts().index)\n   \nfor key in cite_cols_important.keys():\n    for value in cite_cols_important[key]:\n        fig, axs = plt.subplots(ncols =  7, figsize=(30, 4))\n        for j in range(7):\n            ct = ctl[j]\n            temp_df = meta[meta.cell_type == ct]\n            plt.subplot(1,7,j+1)\n            plt.title(f'Cell type: {ct} \\n Pearson corr: {np.corrcoef(temp_df[key], temp_df[value])[0,1]:.3f}')\n            plt.xlabel(key)\n            plt.ylabel(value)\n            plt.scatter(temp_df[temp_df.day==2][key],temp_df[temp_df.day==2][value], color='red', s=2, alpha=0.3)\n            plt.scatter(temp_df[temp_df.day==3][key],temp_df[temp_df.day==3][value], color='green', s=2, alpha=0.3)\n            plt.scatter(temp_df[temp_df.day==4][key],temp_df[temp_df.day==4][value], color='blue', s=2, alpha=0.3)\n\n\n            ","metadata":{"execution":{"iopub.status.busy":"2022-10-13T15:58:09.396364Z","iopub.execute_input":"2022-10-13T15:58:09.396751Z","iopub.status.idle":"2022-10-13T16:00:58.592778Z","shell.execute_reply.started":"2022-10-13T15:58:09.396722Z","shell.execute_reply":"2022-10-13T16:00:58.591927Z"},"trusted":true},"execution_count":null,"outputs":[]}]}