{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport time\nt0start = time.time()\n\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:57:45.108030Z","iopub.execute_input":"2023-01-16T06:57:45.109038Z","iopub.status.idle":"2023-01-16T06:57:45.117596Z","shell.execute_reply.started":"2023-01-16T06:57:45.108983Z","shell.execute_reply":"2023-01-16T06:57:45.116489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_top = 100\n\nslow_calculations_on = 1 # calculations which take hours e.g. LGB for each feauture and corrlation  based on it - it might take 6 hours ","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:57:45.118998Z","iopub.execute_input":"2023-01-16T06:57:45.120303Z","iopub.status.idle":"2023-01-16T06:57:45.135251Z","shell.execute_reply.started":"2023-01-16T06:57:45.120233Z","shell.execute_reply":"2023-01-16T06:57:45.133108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', 500)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:57:47.082527Z","iopub.execute_input":"2023-01-16T06:57:47.083243Z","iopub.status.idle":"2023-01-16T06:57:47.090651Z","shell.execute_reply.started":"2023-01-16T06:57:47.083187Z","shell.execute_reply":"2023-01-16T06:57:47.089363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_y = pd.read_hdf('/kaggle/input/open-problems-multimodal/train_cite_targets.h5')\ndf_y","metadata":{"execution":{"iopub.status.busy":"2023-01-10T10:55:47.987200Z","iopub.execute_input":"2023-01-10T10:55:47.988188Z","iopub.status.idle":"2023-01-10T10:55:48.999067Z","shell.execute_reply.started":"2023-01-10T10:55:47.988133Z","shell.execute_reply":"2023-01-10T10:55:48.997371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf_rna = pd.read_hdf('/kaggle/input/open-problems-multimodal/train_cite_inputs.h5')\ndisplay(df_rna) ","metadata":{"execution":{"iopub.status.busy":"2023-01-16T06:57:52.523120Z","iopub.execute_input":"2023-01-16T06:57:52.523536Z","iopub.status.idle":"2023-01-16T06:58:56.885147Z","shell.execute_reply.started":"2023-01-16T06:57:52.523501Z","shell.execute_reply":"2023-01-16T06:58:56.883853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"part1=df_y.columns[0:20]\npart2=df_y.columns[20:40]\npart3=df_y.columns[40:60]\npart4=df_y.columns[60:80]\npart5=df_y.columns[80:100]\npart6=df_y.columns[100:120]\npart7=df_y.columns[120:140]","metadata":{"execution":{"iopub.status.busy":"2023-01-10T10:56:48.449237Z","iopub.execute_input":"2023-01-10T10:56:48.449627Z","iopub.status.idle":"2023-01-10T10:56:48.457174Z","shell.execute_reply.started":"2023-01-10T10:56:48.449591Z","shell.execute_reply":"2023-01-10T10:56:48.455796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"current_part = part1","metadata":{"execution":{"iopub.status.busy":"2023-01-10T10:56:48.458486Z","iopub.execute_input":"2023-01-10T10:56:48.458818Z","iopub.status.idle":"2023-01-10T10:56:48.467900Z","shell.execute_reply.started":"2023-01-10T10:56:48.458789Z","shell.execute_reply":"2023-01-10T10:56:48.466359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%time\n#target_name = 'CD36'\nfrom scipy import stats\n#through all prots\nfor target_name in current_part:\n    pearson = []#list of corrs for current CD\n    spearman = []#list of corrs for current CD\n    y = df_y[target_name]\n    #throught all features\n    for col in df_rna.columns:\n        pears_c = np.corrcoef(y,df_rna[col])[0,1]\n        spear_c = stats.spearmanr(y,df_rna[col])[0]\n        pearson.append(pears_c)\n        spearman.append(spear_c)\n\n        #if (len(l)%5000) == 1:\n        #    print(len(l))\n    #get full DS pearson\n    pearsonDS = pd.DataFrame(index=df_rna.columns,columns=[target_name,'ABS_'+target_name])\n    pearsonDS[target_name]=pearson\n    pearsonDS['ABS_'+target_name]=pearsonDS[target_name].abs()\n    pearsonDS.to_csv(target_name+'_pearson_rna_all.csv')\n    #get full DS spearman\n    spearmanDS = pd.DataFrame(index=df_rna.columns,columns=[target_name,'ABS_'+target_name])\n    spearmanDS[target_name]=spearman\n    spearmanDS['ABS_'+target_name]=spearmanDS[target_name].abs()\n    spearmanDS.to_csv(target_name+'_spearman_rna_all.csv')\n    #take top100 corrs\n    topcorrpearson = pearsonDS.sort_values('ABS_'+target_name, ascending = False).head(n_top).index\n    topcorrspearman = spearmanDS.sort_values('ABS_'+target_name, ascending = False).head(n_top).index\n    #read cd info\n    cdinfo = pd.read_csv('/kaggle/input/silogram-fi-by-cd/'+target_name+'_protein_feature_importances.csv')\n    cdinfo['Corr Pearson NIPS22'] =topcorrpearson\n    cdinfo['Corr Spearman NIPS22'] =topcorrspearman\n    cdinfo.to_csv(target_name+'_fi_corr.csv')\n    \n        \n        \n#col = 'Corr Pearson NIPS22'\n#df_stat = pd.DataFrame(index = df_rna.columns, data = l , columns = [col] )\n#df_stat['Abs '+col] = df_stat[col].abs()\n#df_stat.sort_values('Abs '+col, ascending = False).head(n_top)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T10:56:48.469417Z","iopub.execute_input":"2023-01-10T10:56:48.470089Z","iopub.status.idle":"2023-01-10T13:01:58.423850Z","shell.execute_reply.started":"2023-01-10T10:56:48.470050Z","shell.execute_reply":"2023-01-10T13:01:58.422264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"RNA-RNA correlations","metadata":{}},{"cell_type":"code","source":"cd44 = df_rna.columns[df_rna.columns.str.contains('CD44')]","metadata":{"execution":{"iopub.status.busy":"2023-01-16T08:06:28.084278Z","iopub.execute_input":"2023-01-16T08:06:28.084697Z","iopub.status.idle":"2023-01-16T08:06:28.101364Z","shell.execute_reply.started":"2023-01-16T08:06:28.084646Z","shell.execute_reply":"2023-01-16T08:06:28.099848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd44df = df_rna[cd44]","metadata":{"execution":{"iopub.status.busy":"2023-01-16T08:26:48.662081Z","iopub.execute_input":"2023-01-16T08:26:48.663140Z","iopub.status.idle":"2023-01-16T08:26:48.669371Z","shell.execute_reply.started":"2023-01-16T08:26:48.663100Z","shell.execute_reply":"2023-01-16T08:26:48.668172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd44corrs = pd.DataFrame(index=df_rna.columns)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T08:48:44.995155Z","iopub.execute_input":"2023-01-16T08:48:44.995614Z","iopub.status.idle":"2023-01-16T08:48:45.001760Z","shell.execute_reply.started":"2023-01-16T08:48:44.995578Z","shell.execute_reply":"2023-01-16T08:48:45.000293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor col1 in cd44df.columns:\n    pearsCD44=[]\n    y=cd44df[col1]\n    for col in df_rna.columns:\n            pears_c = np.corrcoef(y,df_rna[col])[0,1]\n            pearsCD44.append(pears_c)\n    cd44corrs[col1]=pearsCD44\n    cd44corrs[col1+'_ABS']=cd44corrs[col1].abs()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T08:58:27.019359Z","iopub.execute_input":"2023-01-16T08:58:27.020415Z","iopub.status.idle":"2023-01-16T08:59:02.757624Z","shell.execute_reply.started":"2023-01-16T08:58:27.020374Z","shell.execute_reply":"2023-01-16T08:59:02.756330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd44corrs.sort_values('ENSG00000255443_CD44-AS1_ABS',ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-16T09:00:40.255974Z","iopub.execute_input":"2023-01-16T09:00:40.256446Z","iopub.status.idle":"2023-01-16T09:00:40.284945Z","shell.execute_reply.started":"2023-01-16T09:00:40.256406Z","shell.execute_reply":"2023-01-16T09:00:40.283673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cd44corrs.to_csv('CD44 RNA-RNA Pearson.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-16T09:09:55.323547Z","iopub.execute_input":"2023-01-16T09:09:55.324557Z","iopub.status.idle":"2023-01-16T09:09:55.513081Z","shell.execute_reply.started":"2023-01-16T09:09:55.324515Z","shell.execute_reply":"2023-01-16T09:09:55.511842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_rna['ENSG00000178175_ZNF366'].sort_values()","metadata":{"execution":{"iopub.status.busy":"2023-01-16T09:09:13.808390Z","iopub.execute_input":"2023-01-16T09:09:13.808761Z","iopub.status.idle":"2023-01-16T09:09:13.834672Z","shell.execute_reply.started":"2023-01-16T09:09:13.808729Z","shell.execute_reply":"2023-01-16T09:09:13.833518Z"},"trusted":true},"execution_count":null,"outputs":[]}]}