{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7277663,"sourceType":"datasetVersion","datasetId":4219336},{"sourceId":7279447,"sourceType":"datasetVersion","datasetId":1608748}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What is about \n\nHere we look on housekeeping genes (list from: https://www.cell.com/trends/genetics/fulltext/S0168-9525(13)00089-9, Human housekeeping genes, Eli Eisenberg Erez Y. Levanon Published:June 28, 2013 )\n\nMotivated by findings:\nhttps://www.kaggle.com/competitions/open-problems-single-cell-perturbations/discussion/461159\n20th Place Solution Writeup For Open Problems - Single-cell\nPerturbations Competition\nAntoine Passemiers and Jalil Nourisa\n\n\nWho observed (and found ways to quantify) that housekeeping are less changed than generic genes - which is quite natural.\n\n\n\nWe also observe some sub-clusters in housekeeping genes \n","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"version 1 - sirtuin family\nversion 2 - sirtuin family + related genes\nversion 3 - autophagy genes","metadata":{}},{"cell_type":"markdown","source":"# Load list of housekeeping genes ( Eisenberg,  Levanon , 2013)\n\nHuman housekeeping genes, revisited\nEli Eisenberg\nErez Y. Levanon\nPublished:June 28, 2013\n\nKaggle dataset: 'Genes information' https://www.kaggle.com/datasets/alexandervc/genes-information/\ncollects various genes groups, lists, info, etc. ","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n#fn = '/kaggle/input/genes-information/Human_housekeeping_genes_Eisenberg_Levanon_2013_TrendsGenetics.txt'\n# df_hk_genes = pd.read_csv(fn, header = None, sep = '\\t')\n# df_hk_genes[0] = [t.replace(' ','') for t in df_hk_genes[0]]\n# df_hk_genes.columns = ['Symbol', 'NM_id(RefSeq_mRNA_curated)']\nfn = '/kaggle/input/genes-information/Human_housekeeping_genes_Eisenberg_Levanon_2013_TrendsGenetics.csv'\ndf_hk_genes = pd.read_csv(fn,index_col = 0)\n# df_hk_genes.to_csv('df_hk_genes.csv')\ndf_hk_genes","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:47:54.141831Z","iopub.execute_input":"2023-12-29T12:47:54.142513Z","iopub.status.idle":"2023-12-29T12:47:54.618828Z","shell.execute_reply.started":"2023-12-29T12:47:54.142474Z","shell.execute_reply":"2023-12-29T12:47:54.617767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preliminaries","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport time\nt0start = time.time() \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport pandas as pd\nimport tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:47:54.620575Z","iopub.execute_input":"2023-12-29T12:47:54.620969Z","iopub.status.idle":"2023-12-29T12:48:10.162804Z","shell.execute_reply.started":"2023-12-29T12:47:54.620937Z","shell.execute_reply":"2023-12-29T12:48:10.161474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"%%time\nfn = '/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet'\ndf_de_train = pd.read_parquet(fn)# , index_col = 0)\nprint(df_de_train.shape)\ndisplay(df_de_train )\n\nplt.figure(figsize = (20,4) )\nv = df_de_train.iloc[:,5:].max(axis = 0 ).sort_values(ascending = False, key = abs )\nplt.plot(v.values,'*-')\nplt.title('Max-abs DE for genes',fontsize = 20 )\nplt.grid()\nplt.show()\ndisplay(v.head(15))\n\n# %%time\nfn = '/kaggle/input/open-problems-single-cell-perturbations/id_map.csv'\ndf_id_map = pd.read_csv(fn,index_col = 0)\nprint(df_id_map.shape)\ndisplay(df_id_map)\nfn = '/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv'\ndf_sample_submit = pd.read_csv(fn, index_col = 0)\nprint(df_sample_submit.shape)\ndisplay( df_sample_submit )\n\n\nfeatures_columns = ['cell_type','sm_name'] # \n# selected_features_columns = ['cell_type','sm_name']  # [ 'sm_name'] # What features to consider - others not used \n#     # For OP2 task we have two features - cell-type and compound - both categorical\n#     # Some simple models (like Ridge) better work when one uses only one feature - 'sm_name' (=\"compound\") - so you can try that option. \n#     # But seems pyboost is powerful enough to proper work with the both features.\n#     # Table with results for Ridge can be found here: https://www.kaggle.com/code/alexandervc/op2-target-encoders?scriptVersionId=148576642&cellId=1\n#     # Or here: https://docs.google.com/spreadsheets/d/1APN63PMaWZygVjYimK9Ivt0RvifdAU5JRYkxiDn4szw/edit?usp=sharing (sheet \"Encoders\") and discussed here:\n    \nX_submit_categorical = df_id_map[features_columns] # selected_features_columns] # pd.DataFrame(df_id_map, columns= features_columns )\nX_full_categorical = df_de_train[features_columns]# selected_features_columns ]\n\nY_full = df_de_train.iloc[:,5:].values\nprint('X_submit_categorical.shape, X_full_categorical.shape ,  Y_full.shape', X_submit_categorical.shape, X_full_categorical.shape ,  Y_full.shape)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:48:10.164753Z","iopub.execute_input":"2023-12-29T12:48:10.165921Z","iopub.status.idle":"2023-12-29T12:48:16.890016Z","shell.execute_reply.started":"2023-12-29T12:48:10.165869Z","shell.execute_reply":"2023-12-29T12:48:16.887681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Look on intersection - 3518 out of 3804 - quite good ","metadata":{}},{"cell_type":"code","source":"l = df_hk_genes.iloc[:,0]\ns = set(l) & set(df_de_train.columns)\nlist_not_hk = list( set(df_de_train.columns[5:]) - set(l) )\nlist_hk = list(s)\nprint(list_hk[:10])\nlen(s), len(set(l) ), len(list_not_hk)\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:48:16.897515Z","iopub.execute_input":"2023-12-29T12:48:16.899606Z","iopub.status.idle":"2023-12-29T12:48:16.925811Z","shell.execute_reply.started":"2023-12-29T12:48:16.899318Z","shell.execute_reply":"2023-12-29T12:48:16.924120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Housekeeping genes are less affected than others\n\nAnother look on results of: \n\nhttps://www.kaggle.com/competitions/open-problems-single-cell-perturbations/discussion/461159\n20th Place Solution Writeup For Open Problems - Single-cell\nPerturbations Competition\nAntoine Passemiers and Jalil Nourisa\n","metadata":{}},{"cell_type":"code","source":"d = pd.DataFrame(); IX = -1\nlist_thresholds = [0.1,1e-2,1e-3,1e-4,1e-5,1e-6,1e-7,1e-8,1e-9,1e-10,1e-11,1e-12,1e-13,1e-14,1e-15,1e-16]\nfor t in list_thresholds:\n    v  = df_de_train[list_not_hk].values.ravel()\n    v = 10**(-np.abs(v))\n    m = v < t\n    #print(m.sum()/len(v) )\n    IX+=1\n    d.loc[IX, 'Threshold'] = t\n    d.loc[IX, 'Not housekeeping percent'] = 100*m.sum()/len(v)\n    \n    \n    v  = df_de_train[list_hk].values.ravel()\n    v = 10**(-np.abs(v))\n    m = v < t\n    #print(m.sum()/len(v) )\n    d.loc[IX, 'Housekeeping percent'] = 100*m.sum()/len(v)\nplt.figure(figsize = (15,4))    \nfor col in d.columns[1:]:\n    plt.plot(d[col].values , label = col )\nplt.grid()\nplt.legend()\nplt.xticks( )\nplt.xticks(range(len(list_thresholds)), list_thresholds)\nplt.xlabel('Threshold', fontsize = 20)\nplt.title('Percent DE expressed stronger than theshold on p-value', fontsize = 20)\nplt.show()\nd['Difference'] = d[d.columns[1]] - d[d.columns[2]] \nd","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:48:16.929540Z","iopub.execute_input":"2023-12-29T12:48:16.930081Z","iopub.status.idle":"2023-12-29T12:48:24.535324Z","shell.execute_reply.started":"2023-12-29T12:48:16.930033Z","shell.execute_reply":"2023-12-29T12:48:24.534423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# From direct visualization effect is not so clear seen","metadata":{}},{"cell_type":"code","source":"v  = df_de_train[list_not_hk].values.ravel()\nv = 10**(-np.abs(v))\nm = v < 0.01\nprint(m.sum()/len(v) )\nplt.hist(v, bins = 100)\nplt.show()\n\nv  = df_de_train[list_hk].values.ravel()\nv = 10**(-np.abs(v))\nm = v < 0.01\nprint(m.sum()/len(v) )\nplt.hist(v, bins = 100)\nplt.show()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:48:24.536582Z","iopub.execute_input":"2023-12-29T12:48:24.537723Z","iopub.status.idle":"2023-12-29T12:48:25.878997Z","shell.execute_reply.started":"2023-12-29T12:48:24.537679Z","shell.execute_reply":"2023-12-29T12:48:25.877977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clusters in housekeeping genes","metadata":{}},{"cell_type":"code","source":"%%time\nd2 = df_de_train[list_hk]\ncm = d2.corr()\nsns.clustermap(cm)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:48:25.880519Z","iopub.execute_input":"2023-12-29T12:48:25.880994Z","iopub.status.idle":"2023-12-29T12:49:31.789991Z","shell.execute_reply.started":"2023-12-29T12:48:25.880954Z","shell.execute_reply":"2023-12-29T12:49:31.788892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spearman","metadata":{}},{"cell_type":"code","source":"%%time\nd2 = df_de_train[list_hk]\ncm = d2.corr(method = 'spearman')\nsns.clustermap(cm)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:49:31.791440Z","iopub.execute_input":"2023-12-29T12:49:31.792482Z","iopub.status.idle":"2023-12-29T12:50:39.171605Z","shell.execute_reply.started":"2023-12-29T12:49:31.792441Z","shell.execute_reply":"2023-12-29T12:50:39.170384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Clustermap genes/samples  - no so good","metadata":{}},{"cell_type":"code","source":"%%time\nd2 = df_de_train[list_hk[:200]]\n# cm = d2.corr()\nsns.clustermap(d2)\nplt.show()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:39.173260Z","iopub.execute_input":"2023-12-29T12:50:39.173633Z","iopub.status.idle":"2023-12-29T12:50:40.343027Z","shell.execute_reply.started":"2023-12-29T12:50:39.173601Z","shell.execute_reply":"2023-12-29T12:50:40.341915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Not about housekeeping genes , but some other eda\n","metadata":{}},{"cell_type":"markdown","source":"# Uniform distribution under null hypothesis\n\nBased on   https://www.kaggle.com/competitions/open-problems-single-cell-perturbations/discussion/461159\n\nJALIL NOURISA · 20TH IN THIS COMPETITION · POSTED 5 DAYS AGO\n","metadata":{}},{"cell_type":"code","source":"X = df_de_train.iloc[:,5:].values\nv = X.ravel()\nv = 10**(-np.abs(v))\nplt.hist(v,bins = 300)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:50:40.344254Z","iopub.execute_input":"2023-12-29T12:50:40.344585Z","iopub.status.idle":"2023-12-29T12:50:41.558460Z","shell.execute_reply.started":"2023-12-29T12:50:40.344556Z","shell.execute_reply":"2023-12-29T12:50:41.557385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for uv in df_de_train['cell_type'].unique():\n    m = df_de_train['cell_type'] == uv \n    X = df_de_train[m].iloc[:,5:].values\n    v = X.ravel()\n    v = 10**(-np.abs(v))\n    print(uv, m.sum(),  np.min(v))\n    plt.hist(v,bins = 100)\n    plt.title(uv, fontsize = 20 )\n    plt.show()\n    plt.hist(v[v<1e-5],bins = 100)\n    plt.title(uv, fontsize = 20 )\n    plt.show()    ","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:41.562334Z","iopub.execute_input":"2023-12-29T12:50:41.562668Z","iopub.status.idle":"2023-12-29T12:50:46.980930Z","shell.execute_reply.started":"2023-12-29T12:50:41.562641Z","shell.execute_reply":"2023-12-29T12:50:46.979855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_de_train['cell_type'].unique()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:46.982594Z","iopub.execute_input":"2023-12-29T12:50:46.983159Z","iopub.status.idle":"2023-12-29T12:50:46.990190Z","shell.execute_reply.started":"2023-12-29T12:50:46.983128Z","shell.execute_reply":"2023-12-29T12:50:46.989253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df_de_train.iloc[:,5:]\nX.shape","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:46.991707Z","iopub.execute_input":"2023-12-29T12:50:46.992196Z","iopub.status.idle":"2023-12-29T12:50:47.038768Z","shell.execute_reply.started":"2023-12-29T12:50:46.992157Z","shell.execute_reply":"2023-12-29T12:50:47.037595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df_de_train['cell_type'] ==  'Myeloid cells'#  'B cells'\nv = X[m].values.ravel()\ns1 = pd.Series(v).describe()\ns1","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:47.040495Z","iopub.execute_input":"2023-12-29T12:50:47.040932Z","iopub.status.idle":"2023-12-29T12:50:47.071533Z","shell.execute_reply.started":"2023-12-29T12:50:47.040894Z","shell.execute_reply":"2023-12-29T12:50:47.070513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_df = []\nfor col in ['B cells' , 'Myeloid cells' ]:\n    m = df_de_train['cell_type'] ==  col\n    v = X[m].values.ravel()\n    s2 = pd.Series(v).describe()\n    s2.name = col\n    list_df.append( s2.to_frame() ) \nd = pd.concat( list_df, axis = 1)\nd","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:47.072923Z","iopub.execute_input":"2023-12-29T12:50:47.073252Z","iopub.status.idle":"2023-12-29T12:50:47.125757Z","shell.execute_reply.started":"2023-12-29T12:50:47.073224Z","shell.execute_reply":"2023-12-29T12:50:47.124659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ['B cells' , 'Myeloid cells' ]:\n    m = df_de_train['cell_type'] ==  col\n    v = X[m].values.ravel()\n    plt.plot(v)\n    plt.title(col,fontsize = 20)\n    plt.show()\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:47.127005Z","iopub.execute_input":"2023-12-29T12:50:47.127335Z","iopub.status.idle":"2023-12-29T12:50:47.807866Z","shell.execute_reply.started":"2023-12-29T12:50:47.127307Z","shell.execute_reply":"2023-12-29T12:50:47.806624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df_de_train['cell_type'] == 'Myeloid cells' \ndf_de_train[m]['sm_name'].unique()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:47.809003Z","iopub.execute_input":"2023-12-29T12:50:47.809308Z","iopub.status.idle":"2023-12-29T12:50:47.822772Z","shell.execute_reply.started":"2023-12-29T12:50:47.809283Z","shell.execute_reply":"2023-12-29T12:50:47.821681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Got from that:\n# Drugs higher effect on myeloids:\nl6 = ['Crizotinib',\n'Dabrafenib',\n'R428',\n'Porcn Inhibitor III',\n'Foretinib',\n'Dactolisib']\n\n\nprint( col  )\n\nm = df_de_train['cell_type'] == 'Myeloid cells' \ni = 0;\nfor drug in df_de_train['sm_name'][m].unique(): \n    plt.figure(figsize = (20,3) )\n    plt.subplot(1,2,1)\n    i+=1\n    print(i, drug)\n    col = 'Myeloid cells'\n    m = df_de_train['cell_type'] == col\n    #plt.subplot(1,2,1)\n    m2 = df_de_train['sm_name'] == drug\n    m2 = m2 & m\n    plt.plot( X[m2].values.ravel() , label = col )\n    plt.title(drug + ' ' + col  ,fontsize = 20)\n    \n    #plt.subplot(1,2,2)\n    col = 'B cells'\n    m = df_de_train['cell_type'] == col\n    m2 = df_de_train['sm_name'] == drug\n    m2 = m2 & m\n    plt.plot( X[m2].values.ravel(), label = col )\n    plt.title(drug   ,fontsize = 20)\n    plt.legend(fontsize = 20 )\n    \n    \n    plt.subplot(1,2,2)\n    #print(drug)\n    \n    #plt.subplot(1,2,2)\n    col = 'B cells'\n    m = df_de_train['cell_type'] == col\n    m2 = df_de_train['sm_name'] == drug\n    m2 = m2 & m\n    plt.plot( X[m2].values.ravel(), label = col )\n    plt.title(drug   ,fontsize = 20)\n    plt.legend(fontsize = 20 )\n\n    col = 'Myeloid cells'\n    m = df_de_train['cell_type'] == col\n    #plt.subplot(1,2,1)\n    m2 = df_de_train['sm_name'] == drug\n    m2 = m2 & m\n    plt.plot( X[m2].values.ravel() , label = col )\n    plt.title(drug + ' ' + col  ,fontsize = 20)\n    plt.legend(fontsize = 20 )\n    \n    plt.show()\n    ","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:50:47.824126Z","iopub.execute_input":"2023-12-29T12:50:47.824425Z","iopub.status.idle":"2023-12-29T12:51:04.095010Z","shell.execute_reply.started":"2023-12-29T12:50:47.824400Z","shell.execute_reply":"2023-12-29T12:51:04.093916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df_de_train['cell_type'] == 'NK cells'\nm.sum()\ndrug = 'Crizotinib'\nm2 = df_de_train['sm_name'] == drug\n(m & m2).sum()\nX[m&m2]","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:04.096389Z","iopub.execute_input":"2023-12-29T12:51:04.096753Z","iopub.status.idle":"2023-12-29T12:51:04.124962Z","shell.execute_reply.started":"2023-12-29T12:51:04.096720Z","shell.execute_reply":"2023-12-29T12:51:04.123916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drugs higher effect on myeloids:\nl6 = ['Crizotinib',\n'Dabrafenib',\n'R428',\n'Porcn Inhibitor III',\n'Foretinib',\n'Dactolisib']\n\nfor ct in [ 'B cells', 'Myeloid cells']:\n    m = df_de_train['cell_type'] == ct\n    print( m.sum() )\n    cm = X[m].T.corr() \n    ll = df_de_train[m]['sm_name']\n    dd = {True : '   Higher on Myeloids  ', False: ''}\n    ll = [t +' '+ str( dd[t in l6] ) for t in ll]\n    cm.index = cm.columns = ll\n\n    sns.clustermap( cm.round(2), annot= True)\n    plt.title(ct, fontsize = 20 )\n    plt.show()\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:04.126509Z","iopub.execute_input":"2023-12-29T12:51:04.126874Z","iopub.status.idle":"2023-12-29T12:51:06.872962Z","shell.execute_reply.started":"2023-12-29T12:51:04.126832Z","shell.execute_reply":"2023-12-29T12:51:06.872162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor ct in [ 'NK cells']:#, 'Myeloid cells']:\n    m = df_de_train['cell_type'] == ct\n    print( m.sum() )\n    cm = X[m].T.corr() \n    cm.index = cm.columns = df_de_train[m]['sm_name']\n\n    clustergrid = sns.clustermap( cm.round(2), annot= False)\n    \n    reordered_columns = clustergrid.dendrogram_col.reordered_ind\n    reordered_rows = clustergrid.dendrogram_row.reordered_ind\n    print(len(reordered_rows), len(reordered_columns))\n    print(list(cm.index[reordered_rows]))\n\n    plt.title(ct, fontsize = 20 )\n    plt.show()\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:06.874074Z","iopub.execute_input":"2023-12-29T12:51:06.875008Z","iopub.status.idle":"2023-12-29T12:51:09.399371Z","shell.execute_reply.started":"2023-12-29T12:51:06.874974Z","shell.execute_reply":"2023-12-29T12:51:09.398227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor ct in [ 'NK cells']:#, 'Myeloid cells']:\n    m = df_de_train['cell_type'] == ct\n    print( m.sum() )\n    cm = X[m].T.corr() \n    cm.index = cm.columns = df_de_train[m]['sm_name']\n\n    sns.clustermap( cm.round(2), annot= True)\n    plt.title(ct, fontsize = 20 )\n    plt.show()\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:09.401156Z","iopub.execute_input":"2023-12-29T12:51:09.401526Z","iopub.status.idle":"2023-12-29T12:51:53.333533Z","shell.execute_reply.started":"2023-12-29T12:51:09.401496Z","shell.execute_reply":"2023-12-29T12:51:53.332343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"public_ids = [\n    'LSM-43216', 'LSM-1050', 'LSM-45849', 'LSM-42800', 'LSM-1131', 'LSM-6335', 'LSM-1211',\n    'LSM-45239', 'LSM-1130', 'LSM-45786', 'LSM-5199', 'LSM-45281',\n    'LSM-6324', # 'ACY-1215' -> 'Ricolinostat'\n    'LSM-3309', 'LSM-1056', 'LSM-45591', 'LSM-46203', 'LSM-5662',\n    'LSM-47134',  # 'SB-2342' -> '5-(9-Isopropyl-8-methyl-2-morpholino-9H-purin-6-yl)pyrimidin-2-amine  '\n    'LSM-45637', 'LSM-1127', 'LSM-46971', 'LSM-1172', 'LSM-46042', 'LSM-1101', 'LSM-45758',\n    'LSM-5218', 'LSM-2287', 'LSM-1014',\n    'LSM-1040', #  'fostamatinib' -> 'Tamatinib'\n    'LSM-1476;LSM-5290',\n    'LSM-45680',  # 'basimglurant' -> 'RG7090'\n    'LSM-4349',  # '5-iodotubercidin' -> 'IN1451'\n    'LSM-3425', 'LSM-45806',\n    'LSM-45616',  # 'SB-683698' -> 'TR-14035'\n    'LSM-1055',\n    'LSM-43281',  # 'C-646' -> 'STK219801'\n    'LSM-5690', 'LSM-1155', 'LSM-2499',\n    'LSM-2382',  # 'JTC-801' -> 'UNII-BXU45ZH6LI'\n    'LSM-45220', 'LSM-1037', 'LSM-1005', 'LSM-1180', 'LSM-36812',\n    'LSM-45924',  # 'filgotinib' -> 'GLPG0634'\n    'LSM-2013',  # 'TL-HRAS-61' -> TL_HRAS26'\n    'LSM-4738'\n]\nlist17 = ['Idelalisib', 'Crizotinib', 'Linagliptin', 'Palbociclib',\n       'Dabrafenib', 'Alvocidib', 'LDN 193189', 'R428',\n       'Porcn Inhibitor III', 'Belinostat', 'Foretinib', 'MLN 2238',\n       'Penfluridol', 'Dactolisib', 'O-Demethylated Adapalene',\n       'Oprozomib (ONX 0912)', 'CHIR-99021']\nprint(len(list17))\nprint(len(public_ids))\nm = df_de_train['sm_lincs_id'].isin(public_ids )\npublic_names = list( df_de_train[m]['sm_name'].unique() )\nlist_publ = public_names\nprint('public_names', public_names)\nm =( ~df_de_train['sm_lincs_id'].isin(public_ids )) & ( ~df_de_train['sm_name'].isin(list17 ) )\nprint(m.sum() )\nlist_priv = list( df_de_train['sm_name'][m].unique() )\nprint(len(list_priv))\nprint('list_priv', list_priv )","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:53.334966Z","iopub.execute_input":"2023-12-29T12:51:53.336005Z","iopub.status.idle":"2023-12-29T12:51:53.372864Z","shell.execute_reply.started":"2023-12-29T12:51:53.335963Z","shell.execute_reply":"2023-12-29T12:51:53.371890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df_de_train['sm_name'].isin( list_priv)\nm = m & (df_de_train['cell_type'] == 'NK cells' )\ncm = X[m].T.corr()\ncm.index = cm.columns = df_de_train[m]['sm_name']\nclustergrid = sns.clustermap( cm.round(2), annot= False)\n\nreordered_columns = clustergrid.dendrogram_col.reordered_ind\nreordered_rows = clustergrid.dendrogram_row.reordered_ind\nprint(len(reordered_rows), len(reordered_columns))\nprint(list(cm.index[reordered_rows]))\n\nplt.show()\nprint(list(cm.index[reordered_rows]))\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:53.375225Z","iopub.execute_input":"2023-12-29T12:51:53.376240Z","iopub.status.idle":"2023-12-29T12:51:54.808601Z","shell.execute_reply.started":"2023-12-29T12:51:53.376190Z","shell.execute_reply":"2023-12-29T12:51:54.807718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df_de_train['sm_name'].isin( list_priv)\nm = m | df_de_train['sm_name'].isin( list17)\nm = m & (df_de_train['cell_type'] == 'NK cells' )\ncm = X[m].T.corr()\n\nll = df_de_train[m]['sm_name']\ndd = {True : '   Higher on Myeloids  ', False: ''}\nll = [t +' '+ str( dd[t in l6] ) for t in ll]\n\ncm.index = cm.columns = ll\nclustergrid = sns.clustermap( cm.round(2), annot= False, cmap = 'coolwarm')\n\nreordered_columns = clustergrid.dendrogram_col.reordered_ind\nreordered_rows = clustergrid.dendrogram_row.reordered_ind\nprint(len(reordered_rows), len(reordered_columns))\nprint(list(cm.index[reordered_rows]))\n\nplt.show()\nprint(list(cm.index[reordered_rows]))\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:54.809936Z","iopub.execute_input":"2023-12-29T12:51:54.810488Z","iopub.status.idle":"2023-12-29T12:51:56.297101Z","shell.execute_reply.started":"2023-12-29T12:51:54.810453Z","shell.execute_reply":"2023-12-29T12:51:56.296019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm2 = cm.copy()\ncm2.columns =  df_de_train[m]['sm_name']\ncm2 = cm2[l6]\ncm2 = cm2[~cm2.index.isin(list17)]\ncm2['Mean'] = cm2.mean(axis = 1)\ncm2['Mean'].sort_values(ascending = False ).head(30)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:56.298717Z","iopub.execute_input":"2023-12-29T12:51:56.299167Z","iopub.status.idle":"2023-12-29T12:51:56.324174Z","shell.execute_reply.started":"2023-12-29T12:51:56.299110Z","shell.execute_reply":"2023-12-29T12:51:56.322886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = df_de_train['sm_name'].isin( list_publ)\nm = m | df_de_train['sm_name'].isin( list17)\nm = m & (df_de_train['cell_type'] == 'NK cells' )\ncm = X[m].T.corr()\n\nll = df_de_train[m]['sm_name']\ndd = {True : '   Higher on Myeloids  ', False: ''}\nll = [t +' '+ str( dd[t in l6] ) for t in ll]\n\ncm.index = cm.columns = ll\nclustergrid = sns.clustermap( cm.round(2), annot= False, cmap = 'coolwarm')\n\nreordered_columns = clustergrid.dendrogram_col.reordered_ind\nreordered_rows = clustergrid.dendrogram_row.reordered_ind\nprint(len(reordered_rows), len(reordered_columns))\nprint(list(cm.index[reordered_rows]))\n\nplt.show()\nprint(list(cm.index[reordered_rows]))\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:56.325945Z","iopub.execute_input":"2023-12-29T12:51:56.326661Z","iopub.status.idle":"2023-12-29T12:51:57.587502Z","shell.execute_reply.started":"2023-12-29T12:51:56.326617Z","shell.execute_reply":"2023-12-29T12:51:57.586401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm2 = cm.copy()\ncm2.columns =  df_de_train[m]['sm_name']\ncm2 = cm2[l6]\ncm2 = cm2[~cm2.index.isin(list17)]\ncm2['Mean'] = cm2.mean(axis = 1)\ncm2['Mean'].sort_values(ascending = False ).head(30)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:57.589146Z","iopub.execute_input":"2023-12-29T12:51:57.589493Z","iopub.status.idle":"2023-12-29T12:51:57.612331Z","shell.execute_reply.started":"2023-12-29T12:51:57.589463Z","shell.execute_reply":"2023-12-29T12:51:57.611262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm2.index","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:57.613685Z","iopub.execute_input":"2023-12-29T12:51:57.614036Z","iopub.status.idle":"2023-12-29T12:51:57.621185Z","shell.execute_reply.started":"2023-12-29T12:51:57.614006Z","shell.execute_reply":"2023-12-29T12:51:57.620124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Housekeeper cluster ","metadata":{}},{"cell_type":"code","source":"transcription_factors = [\n    \"ATF1\", \"ATF2\", \"ATF4\", \"ATF6\", \"ATF7\", \"ATF7IP\", \"BTF3\", \"E2F4\", \"ERH\", \"HMGB1\", \n    \"ILF2\", \"IER2\", \"JUND\", \"TCEB2\"\n]\n\nrna_splicing_factors = [\n    \"BAT1\", \"HNRPD\", \"HNRPK\", \"PABPN1\", \"SRSF3\",\n    \"SNRPB\", \"SRSF1\", \"U2AF1\", \"PRPF8\", \"SF3B1\",\n    \"SNRPA\", \"SRSF7\", \"SNRPC\", \"U2AF2\", \"SRSF2\"\n]\n\ntranslation_factors = [\n    \"EIF1\", \"EIF1AD\", \"EIF1B\", \"EIF2A\", \"EIF2AK1\", \"EIF2AK3\", \"EIF2AK4\", \"EIF2B2\", \"EIF2B3\", \n    \"EIF2B4\", \"EIF2S2\", \"EIF3A\", \"EIF3B\", \"EIF3D\", \"EIF3G\", \"EIF3I\", \"EIF3H\", \"EIF3J\", \"EIF3K\", \n    \"EIF3L\", \"EIF3M\", \"EIF3S5\", \"EIF3S8\", \"EIF4A1\", \"EIF4A2\", \"EIF4A3\", \"EIF4E2\", \"EIF4G1\", \n    \"EIF4G2\", \"EIF4G3\", \"EIF4H\", \"EIF5\", \"EIF5A\", \"EIF5AL1\", \"EIF5B\", \"EIF6\", \"TUFM\"\n]\n\ntrna_synthesis_genes = [\n    \"AARS\", \"AARS2\", \"AARSD1\", \"CARS\", \"CARS2\", \"DARS\", \"DARS2\", \"EARS2\", \"FARS2\", \"FARSA\", \n    \"FARSB\", \"GARS\", \"HARS\", \"HARS2\", \"IARS\", \"IARS2\", \"KARS\", \"LARS2\", \"MARS\", \"MARS2\", \n    \"NARS\", \"NARS2\", \"QARS\", \"RARS\", \"RARS2\", \"SARS\", \"TARS\", \"VARS2\", \"WARS2\", \"YARS\", \"YARS2\"\n]\n\nribosomal_proteins = [\n    \"RPL5\", \"RPL8\", \"RPL9\", \"RPL10A\", \"RPL11\", \"RPL14\", \"RPL25\", \"RPL26L1\", \"RPL27\", \"RPL30\", \n    \"RPL32\", \"RPL34\", \"RPL35\", \"RPL35A\", \"RPL36AL\", \"RPS5\", \"RPS6\", \"RPS6KA3\", \"RPS6KB1\", \"RPS6KB2\", \n    \"RPS13\", \"RPS19BP1\", \"RPS20\", \"RPS23\", \"RPS24\", \"RPS27\", \"RPN1\"\n]\n\nmitochondrial_ribosomal_proteins = [\n    \"MRPL9\", \"MRPL1\", \"MRPL10\", \"MRPL11\", \"MRPL12\", \"MRPL13\", \"MRPL14\", \"MRPL15\", \"MRPL16\", \"MRPL17\", \n    \"MRPL18\", \"MRPL19\", \"MRPL2\", \"MRPL20\", \"MRPL21\", \"MRPL22\", \"MRPL23\", \"MRPL24\", \"MRPL27\", \"MRPL28\", \n    \"MRPL3\", \"MRPL30\", \"MRPL32\", \"MRPL33\", \"MRPL35\", \"MRPL36\", \"MRPL37\", \"MRPL38\", \"MRPL4\", \"MRPL40\", \n    \"MRPL41\", \"MRPL42\", \"MRPL43\", \"MRPL44\", \"MRPL45\", \"MRPL46\", \"MRPL47\", \"MRPL48\", \"MRPL49\", \"MRPL50\", \n    \"MRPL51\", \"MRPL52\", \"MRPL53\", \"MRPL54\", \"MRPL55\", \"MRPS10\", \"MRPS11\", \"MRPS12\", \"MRPS14\", \"MRPS15\", \n    \"MRPS16\", \"MRPS17\", \"MRPS18A\", \"MRPS18B\", \"MRPS18C\", \"MRPS2\", \"MRPS21\", \"MRPS22\", \"MRPS23\", \"MRPS24\", \n    \"MRPS25\", \"MRPS26\", \"MRPS27\", \"MRPS28\", \"MRPS30\", \"MRPS31\", \"MRPS33\", \"MRPS34\", \"MRPS35\", \"MRPS5\", \n    \"MRPS6\", \"MRPS7\", \"MRPS9\"\n]\n\nnadh_dehydrogenase_enzymes = [\n    \"NDUFA2\", \"NDUFA3\", \"NDUFA4\", \"NDUFA5\", \"NDUFA6\", \"NDUFA7\", \"NDUFA8\", \"NDUFA9\", \"NDUFA10\", \"NDUFA11\", \n    \"NDUFA12\", \"NDUFA13\", \"NDUFAF2\", \"NDUFAF3\", \"NDUFAF4\", \"NDUFB2\", \"NDUFB3\", \"NDUFB4\", \"NDUFB5\", \"NDUFB6\", \n    \"NDUFB7\", \"NDUFB10\", \"NDUFB11\", \"NDUFB8\", \"NDUFB9\", \"NDUFC1\", \"NDUFC2\", \"NDUFC2-KCTD14\", \"NDUFS2\", \n    \"NDUFS3\", \"NDUFS4\", \"NDUFS5\", \"NDUFS6\", \"NDUFS7\", \"NDUFS8\", \"NDUFV1\", \"NDUFV2\"\n]\n\ncytochrome_c_oxidase_enzymes = [\n    \"COX4I1\", \"COX5B\", \"COX6B1\", \"COX6C\", \"COX7A2\", \"COX7A2L\", \"COX7C\", \"COX8\", \"COX8A\", \"COX11\", \n    \"COX14\", \"COX15\", \"COX16\", \"COX19\", \"COX20\", \"CYC1\", \"UQCC\", \"UQCR10\", \"UQCR11\", \"UQCRB\", \"UQCRC1\", \n    \"UQCRC2\", \"UQCRHL\", \"UQCRQ\"\n]","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:51:57.622920Z","iopub.execute_input":"2023-12-29T12:51:57.623764Z","iopub.status.idle":"2023-12-29T12:51:57.638109Z","shell.execute_reply.started":"2023-12-29T12:51:57.623730Z","shell.execute_reply":"2023-12-29T12:51:57.637058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"G1S_genes_Tirosh = ['MCM5', 'PCNA', 'TYMS', 'FEN1', 'MCM2', 'MCM4', 'RRM1', 'UNG', 'GINS2', 'MCM6', 'CDCA7', 'DTL', 'PRIM1', 'UHRF1', 'MLF1IP', 'HELLS', 'RFC2', 'RPA2', 'NASP', 'RAD51AP1', 'GMNN', 'WDR76', 'SLBP', 'CCNE2', 'UBR7', 'POLD3', 'MSH2', 'ATAD2', 'RAD51', 'RRM2', 'CDC45', 'CDC6', 'EXO1', 'TIPIN', 'DSCC1', 'BLM', 'CASP8AP2', 'USP1', 'CLSPN', 'POLA1', 'CHAF1B', 'BRIP1', 'E2F8']\nG2M_genes_Tirosh = ['HMGB2', 'CDK1', 'NUSAP1', 'UBE2C', 'BIRC5', 'TPX2', 'TOP2A', 'NDC80', 'CKS2', 'NUF2', 'CKS1B', 'MKI67', 'TMPO', 'CENPF', 'TACC3', 'FAM64A', 'SMC4', 'CCNB2', 'CKAP2L', 'CKAP2', 'AURKB', 'BUB1', 'KIF11', 'ANP32E', 'TUBB4B', 'GTSE1', 'KIF20B', 'HJURP', 'CDCA3', 'HN1', 'CDC20', 'TTK', 'CDC25C', 'KIF2C', 'RANGAP1', 'NCAPD2', 'DLGAP5', 'CDCA2', 'CDCA8', 'ECT2', 'KIF23', 'HMMR', 'AURKA', 'PSRC1', 'ANLN', 'LBR', 'CKAP5', 'CENPE', 'CTCF', 'NEK2', 'G2E3', 'GAS2L3', 'CBX5', 'CENPA']\ngenes_Tirosh = G1S_genes_Tirosh + G2M_genes_Tirosh\n\n# Subset of Tirosh genes to capture \"fast\" cell cycle pattern = see https://arxiv.org/abs/2208.05229\nlist_genes_fastCCsign = ['CDK1', 'UBE2C', 'TOP2A', 'TMPO', 'HJURP', 'RRM1', 'RAD51AP1', 'RRM2', 'CDC45', 'BLM', 'BRIP1', 'E2F8', 'HIST2H2AC']\n\nG1S_genes_Freeman = ['ADAMTS1', 'ASF1B', 'ATAD2', 'BARD1', 'BLM', 'BRCA1', 'BRIP1', 'C17orf75', 'C9orf40', 'CACYBP', 'CASP8AP2', 'CCDC15', 'CCNE1', 'CCNE2', 'CCP110', 'CDC25A', 'CDC45', 'CDC6', 'CDC7', 'CDK2', 'CDT1', 'CENPJ', 'CENPQ', 'CENPU', 'CEP57', 'CHAF1A', 'CHAF1B', 'CHEK1', 'CLSPN', 'CREBZF', 'CRYL1', 'CSE1L', 'DCLRE1B', 'DCTPP1', 'DEK', 'DERA', 'DHFR', 'DNA2', 'DNAJC9', 'DNMT1', 'DONSON', 'DSCC1', 'DSN1', 'DTL', 'E2F8', 'EED', 'EFCAB11', 'ENDOD1', 'ETAA1', 'EXO1', 'EYA2', 'EZH2', 'FAM111A', 'FANCE', 'FANCG', 'FANCI', 'FANCL', 'FBXO5', 'FEN1', 'GGH', 'GINS1', 'GINS2', 'GINS3', 'GLMN', 'GMNN', 'GMPS', 'GPD2', 'HADH', 'HELLS', 'HSF2', 'ITGB3BP', 'KIAA0101', 'KNTC1', 'LIG1', 'MCM10', 'MCM2', 'MCM3', 'MCM4', 'MCM5', 'MCM6', 'MCM7', 'MCMBP', 'METTL9', 'MMD', 'MNS1', 'MPP1', 'MRE11A', 'MSH2', 'MSH6', 'MYO19', 'NASP', 'NPAT', 'NSMCE4A', 'ORC1', 'OSGEPL1', 'PAK1', 'PAQR4', 'PARP2', 'PASK', 'PAXIP1', 'PBX3', 'PCNA', 'PKMYT1', 'PMS1', 'POLA1', 'POLA2', 'POLD3', 'POLE2', 'PRIM1', 'PRPS2', 'PSMC3IP', 'RAB23', 'RAD51', 'RAD51AP1', 'RAD54L', 'RBBP8', 'RBL1', 'RDX', 'RFC2', 'RFC3', 'RFC4', 'RMI1', 'RNASEH2A', 'RPA1', 'RRM1', 'RRM2', 'SLBP', 'SLC25A40', 'SMC2', 'SMC3', 'SSX2IP', 'SUPT16H', 'TEX30', 'TFDP1', 'THAP10', 'THEM6', 'TIMELESS', 'TIPIN', 'TMEM106C', 'TMEM38B', 'TRIM45', 'TRIP13', 'TSPYL4', 'TTI1', 'TUBGCP5', 'TYMS', 'UBR7', 'UNG', 'USP1', 'WDHD1', 'WDR76', 'WRB', 'YEATS4', 'ZBTB14', 'ZWINT']\nG2M_genes_Freeman = ['ADGRE5', 'ARHGAP11A', 'ARHGDIB', 'ARL6IP1', 'ASPM', 'AURKA', 'AURKB', 'BIRC5', 'BORA', 'BRD8', 'BUB1', 'BUB1B', 'BUB3', 'CCNA2', 'CCNB1', 'CCNB2', 'CCNF', 'CDC20', 'CDC25B', 'CDC25C', 'CDC27', 'CDCA3', 'CDCA8', 'CDK1', 'CDKN1B', 'CDKN3', 'CENPE', 'CENPF', 'CENPI', 'CENPN', 'CEP55', 'CEP70', 'CEP85', 'CKAP2', 'CKAP5', 'CKS1B', 'CKS2', 'CTCF', 'DBF4', 'DBF4B', 'DCAF7', 'DEPDC1', 'DLGAP5', 'ECT2', 'ERCC6L', 'ESPL1', 'FAM64A', 'FOXM1', 'FZD2', 'FZD7', 'FZR1', 'GPSM2', 'GTF2E1', 'GTSE1', 'H2AFX', 'HJURP', 'HMGB2', 'HMGB3', 'HMMR', 'HN1', 'INCENP', 'JADE2', 'KIF11', 'KIF14', 'KIF15', 'KIF18A', 'KIF18B', 'KIF20A', 'KIF20B', 'KIF22', 'KIF23', 'KIF2C', 'KIF4A', 'KIF5B', 'KIFC1', 'KPNA2', 'LBR', 'LMNB2', 'MAD2L1', 'MELK', 'MET', 'METTL4', 'MIS18BP1', 'MKI67', 'MPHOSPH9', 'MTMR6', 'NCAPD2', 'NCAPG', 'NCAPG2', 'NCAPH', 'NDC1', 'NDC80', 'NDE1', 'NEIL3', 'NEK2', 'NRF1', 'NUSAP1', 'OIP5', 'PAFAH2', 'PARPBP', 'PBK', 'PLEKHG3', 'PLK1', 'PLK4', 'PRC1', 'PRR11', 'PSRC1', 'PTTG1', 'PTTG3P', 'RACGAP1', 'RAD21', 'RASSF1', 'REEP4', 'SAP30', 'SHCBP1', 'SKA1', 'SLCO1B3', 'SOGA1', 'SPA17', 'SPAG5', 'SPC25', 'SPDL1', 'STIL', 'STK17B', 'TACC3', 'TAF5', 'TBC1D2', 'TBC1D31', 'TMPO', 'TOP2A', 'TPX2', 'TROAP', 'TTF2', 'TTK', 'TUBB4B', 'TUBD1', 'UBE2C', 'UBE2S', 'VANGL1', 'WEE1', 'WHSC1', 'XPO1', 'ZMYM1']\ngenes_Freeman = G1S_genes_Freeman + G2M_genes_Freeman","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:51:57.639890Z","iopub.execute_input":"2023-12-29T12:51:57.640333Z","iopub.status.idle":"2023-12-29T12:51:57.663199Z","shell.execute_reply.started":"2023-12-29T12:51:57.640299Z","shell.execute_reply":"2023-12-29T12:51:57.662292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"transcription_factors\": transcription_factors,\n    \"rna_splicing_factors\": rna_splicing_factors,\n    \"translation_factors\": translation_factors,\n    \"trna_synthesis_genes\": trna_synthesis_genes,\n    \"ribosomal_proteins\": ribosomal_proteins,\n    \"mitochondrial_ribosomal_proteins\": mitochondrial_ribosomal_proteins,\n    \"nadh_dehydrogenase_enzymes\": nadh_dehydrogenase_enzymes,\n    \"cytochrome_c_oxidase_enzymes\": cytochrome_c_oxidase_enzymes,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\n\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list: \n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nplt.figure(figsize=(15, 4))\n\nfor col in d.columns[1:]:\n    plt.plot(d['Threshold'], d[col], label=col)\n\nplt.grid()\nplt.legend()\nplt.xscale('log')\nplt.xlabel('Threshold', fontsize=20)\nplt.ylabel('Percent DE stronger than threshold', fontsize=20)\nplt.title('Percent DE Expressed Stronger Than Threshold by Gene Group', fontsize=20)\n\nx_min, x_max = plt.xlim()\nplt.xlim(x_max, x_min)\n\nplt.show()\n","metadata":{"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:51:57.669453Z","iopub.execute_input":"2023-12-29T12:51:57.669863Z","iopub.status.idle":"2023-12-29T12:52:05.303991Z","shell.execute_reply.started":"2023-12-29T12:51:57.669816Z","shell.execute_reply":"2023-12-29T12:52:05.302906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"genes_Tirosh\": genes_Tirosh,\n    \"genes_Freeman\": genes_Freeman,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\n\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\n\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nplt.figure(figsize=(20, 10))\nfor col in d.columns[1:]:\n    plt.plot(d['Threshold'], d[col], label=col)\n\nplt.grid()\nplt.legend()\nplt.xscale('log')\nplt.xlabel('Threshold', fontsize=20)\nplt.ylabel('Percent DE stronger than threshold', fontsize=20)\nplt.title('Percent DE Expressed Stronger Than Threshold by Gene Group', fontsize=20)\nx_min, x_max = plt.xlim()\nplt.xlim(x_max, x_min)\nplt.show()","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:52:05.305393Z","iopub.execute_input":"2023-12-29T12:52:05.305815Z","iopub.status.idle":"2023-12-29T12:52:12.742090Z","shell.execute_reply.started":"2023-12-29T12:52:05.305778Z","shell.execute_reply":"2023-12-29T12:52:12.740903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HALLMARK_APICAL_JUNCTION = ['ACTA1', 'ACTB', 'ACTC1', 'ACTG1', 'ACTG2', 'ACTN1', 'ACTN2', 'ACTN3', 'ACTN4', 'ADAM15', 'ADAM23', 'ADAM9', 'ADAMTS5', 'ADRA1B', 'AKT2', 'AKT3', 'ALOX15B', 'AMH', 'AMIGO1', 'AMIGO2', 'ARHGEF6', 'ARPC2', 'ATP1A3', 'B4GALT1', 'BAIAP2', 'BMP1', 'CADM2', 'CADM3', 'CALB2', 'CAP1', 'CD209', 'CD274', 'CD276', 'CD34', 'CD86', 'CD99', 'CDH1', 'CDH11', 'CDH15', 'CDH3', 'CDH4', 'CDH6', 'CDH8', 'CDK8', 'CDSN', 'CERCAM', 'CLDN11', 'CLDN14', 'CLDN15', 'CLDN18', 'CLDN19', 'CLDN4', 'CLDN5', 'CLDN6', 'CLDN7', 'CLDN8', 'CLDN9', 'CNN2', 'CNTN1', 'COL16A1', 'COL17A1', 'COL9A1', 'CRAT', 'CRB3', 'CTNNA1', 'CTNND1', 'CX3CL1', 'DHX16', 'DLG1', 'DMP1', 'DSC1', 'DSC3', 'EGFR', 'EPB41L2', 'EVL', 'EXOC4', 'FBN1', 'FLNC', 'FSCN1', 'FYB1', 'GAMT', 'GNAI1', 'GNAI2', 'GRB7', 'GTF2F1', 'HADH', 'HRAS', 'ICAM1', 'ICAM2', 'ICAM4', 'ICAM5', 'IKBKG', 'INPPL1', 'INSIG1', 'IRS1', 'ITGA10', 'ITGA2', 'ITGA3', 'ITGA9', 'ITGB1', 'ITGB4', 'JAM3', 'JUP', 'KCNH2', 'KRT31', 'LAMA3', 'LAMB3', 'LAMC2', 'LAYN', 'LDLRAP1', 'LIMA1', 'MADCAM1', 'MAP3K20', 'MAP4K2', 'MAPK11', 'MAPK13', 'MAPK14', 'MDK', 'MMP2', 'MMP9', 'MPZL1', 'MPZL2', 'MSN', 'MVD', 'MYH10', 'MYH9', 'MYL12B', 'MYL9', 'NECTIN1', 'NECTIN2', 'NECTIN3', 'NECTIN4', 'NEGR1', 'NEXN', 'NF1', 'NF2', 'NFASC', 'NHERF4', 'NLGN2', 'NLGN3', 'NRAP', 'NRTN', 'NRXN2', 'PALS1', 'PARD6G', 'PARVA', 'PBX2', 'PCDH1', 'PECAM1', 'PFN1', 'PIK3CB', 'PIK3R3', 'PKD1', 'PLCG1', 'PPP2R2C', 'PTEN', 'PTK2', 'PTPRC', 'RAC2', 'RASA1', 'RHOF', 'RRAS', 'RSU1', 'SDC3', 'SGCE', 'SHC1', 'SHROOM2', 'SIRPA', 'SKAP2', 'SLC30A3', 'SLIT2', 'SORBS3', 'SPEG', 'SRC', 'STX4', 'SYK', 'SYMPK', 'TAOK2', 'TGFBI', 'THBS3', 'THY1', 'TIAL1', 'TJP1', 'TMEM8B', 'TNFRSF11B', 'TRAF1', 'TRO', 'TSC1', 'TSPAN4', 'TUBG1', 'VASP', 'VAV2', 'VCAM1', 'VCAN', 'VCL', 'VWF', 'WASL', 'WNK4', 'YWHAH', 'ZYX']\nHALLMARK_ESTROGEN_RESPONSE_LATE = ['ABCA3', 'ABHD2', 'ACOX2', 'ADD3', 'AFF1', 'AGR2', 'ALDH3A2', 'ALDH3B1', 'AMFR', 'ANXA9', 'AREG', 'ARL3', 'ASCL1', 'ASS1', 'ATP2B4', 'BAG1', 'BATF', 'BCL2', 'BLVRB', 'BTG3', 'CA12', 'CA2', 'CACNA2D2', 'CALCR', 'CAV1', 'CCN5', 'CCNA1', 'CCND1', 'CD44', 'CD9', 'CDC20', 'CDC6', 'CDH1', 'CELSR2', 'CHPT1', 'CHST8', 'CISH', 'CKB', 'CLIC3', 'COX6C', 'CPE', 'CXCL12', 'CXCL14', 'CYP26B1', 'CYP4F11', 'DCXR', 'DHCR7', 'DHRS2', 'DLG5', 'DNAJC1', 'DNAJC12', 'DUSP2', 'DYNLT3', 'EEIG1', 'EGR3', 'ELOVL5', 'EMP2', 'ETFB', 'FABP5', 'FARP1', 'FDFT1', 'FGFR3', 'FKBP4', 'FKBP5', 'FLNB', 'FOS', 'FOXC1', 'FRK', 'GAL', 'GALE', 'GFUS', 'GINS2', 'GJB3', 'GLA', 'GPER1', 'HMGCS2', 'HOMER2', 'HPRT1', 'HR', 'HSPA4L', 'HSPB8', 'ID2', 'IDH2', 'IGFBP4', 'IGSF1', 'IL17RB', 'IL6ST', 'IMPA2', 'ISG20', 'ITPK1', 'JAK1', 'JAK2', 'KCNK5', 'KIF20A', 'KLF4', 'KLK10', 'KLK11', 'KRT13', 'KRT19', 'LAMC2', 'LARGE1', 'LLGL2', 'LSR', 'LTF', 'MAPK13', 'MAPT', 'MDK', 'MEST', 'METTL3', 'MICB', 'MOCS2', 'MYB', 'MYOF', 'NAB2', 'NBL1', 'NCOR2', 'NHERF1', 'NMU', 'NPY1R', 'NRIP1', 'NXT1', 'OLFM1', 'OPN3', 'OVOL2', 'PAPSS2', 'PCP4', 'PDCD4', 'PDLIM3', 'PDZK1', 'PERP', 'PGR', 'PKP3', 'PLAAT3', 'PLAC1', 'PLK4', 'PLXNB1', 'PPIF', 'PRKAR2B', 'PRLR', 'PRSS23', 'PTGER3', 'PTGES', 'PTPN6', 'RAB31', 'RABEP1', 'RAPGEFL1', 'RBBP8', 'RET', 'RNASEH2A', 'RPS6KA2', 'S100A9', 'SCARB1', 'SCNN1A', 'SCUBE2', 'SEMA3B', 'SERPINA1', 'SERPINA3', 'SERPINA5', 'SFN', 'SGK1', 'SIAH2', 'SLC16A1', 'SLC1A4', 'SLC22A5', 'SLC24A3', 'SLC26A2', 'SLC27A2', 'SLC29A1', 'SLC2A8', 'SLC7A5', 'SNX10', 'SORD', 'SOX3', 'ST14', 'ST6GALNAC2', 'STIL', 'SULT2B1', 'TFAP2C', 'TFF1', 'TFF3', 'TFPI2', 'TH', 'TIAM1', 'TJP3', 'TMPRSS3', 'TNNC1', 'TOB1', 'TOP2A', 'TPBG', 'TPD52L1', 'TPSAB1', 'TRIM29', 'TSPAN13', 'TST', 'UGDH', 'UNC13B', 'WFS1', 'XBP1', 'XRCC3', 'ZFP36']\nHALLMARK_TNFA_SIGNALING_VIA_NFKB = ['ABCA1', 'ACKR3', 'AREG', 'ATF3', 'ATP2B1', 'B4GALT1', 'B4GALT5', 'BCL2A1', 'BCL3', 'BCL6', 'BHLHE40', 'BIRC2', 'BIRC3', 'BMP2', 'BTG1', 'BTG2', 'BTG3', 'CCL2', 'CCL20', 'CCL4', 'CCL5', 'CCN1', 'CCND1', 'CCNL1', 'CCRL2', 'CD44', 'CD69', 'CD80', 'CD83', 'CDKN1A', 'CEBPB', 'CEBPD', 'CFLAR', 'CLCF1', 'CSF1', 'CSF2', 'CXCL1', 'CXCL10', 'CXCL11', 'CXCL2', 'CXCL3', 'CXCL6', 'DENND5A', 'DNAJB4', 'DRAM1', 'DUSP1', 'DUSP2', 'DUSP4', 'DUSP5', 'EDN1', 'EFNA1', 'EGR1', 'EGR2', 'EGR3', 'EHD1', 'EIF1', 'ETS2', 'F2RL1', 'F3', 'FJX1', 'FOS', 'FOSB', 'FOSL1', 'FOSL2', 'FUT4', 'G0S2', 'GADD45A', 'GADD45B', 'GCH1', 'GEM', 'GFPT2', 'GPR183', 'HBEGF', 'HES1', 'ICAM1', 'ICOSLG', 'ID2', 'IER2', 'IER3', 'IER5', 'IFIH1', 'IFIT2', 'IFNGR2', 'IL12B', 'IL15RA', 'IL18', 'IL1A', 'IL1B', 'IL23A', 'IL6', 'IL6ST', 'IL7R', 'INHBA', 'IRF1', 'IRS2', 'JAG1', 'JUN', 'JUNB', 'KDM6B', 'KLF10', 'KLF2', 'KLF4', 'KLF6', 'KLF9', 'KYNU', 'LAMB3', 'LDLR', 'LIF', 'LITAF', 'MAFF', 'MAP2K3', 'MAP3K8', 'MARCKS', 'MCL1', 'MSC', 'MXD1', 'MYC', 'NAMPT', 'NFAT5', 'NFE2L2', 'NFIL3', 'NFKB1', 'NFKB2', 'NFKBIA', 'NFKBIE', 'NINJ1', 'NR4A1', 'NR4A2', 'NR4A3', 'OLR1', 'PANX1', 'PDE4B', 'PDLIM5', 'PER1', 'PFKFB3', 'PHLDA1', 'PHLDA2', 'PLAU', 'PLAUR', 'PLEK', 'PLK2', 'PLPP3', 'PMEPA1', 'PNRC1', 'PPP1R15A', 'PTGER4', 'PTGS2', 'PTPRE', 'PTX3', 'RCAN1', 'REL', 'RELA', 'RELB', 'RHOB', 'RIGI', 'RIPK2', 'RNF19B', 'SAT1', 'SDC4', 'SERPINB2', 'SERPINB8', 'SERPINE1', 'SGK1', 'SIK1', 'SLC16A6', 'SLC2A3', 'SLC2A6', 'SMAD3', 'SNN', 'SOCS3', 'SOD2', 'SPHK1', 'SPSB1', 'SQSTM1', 'STAT5A', 'TANK', 'TAP1', 'TGIF1', 'TIPARP', 'TLR2', 'TNC', 'TNF', 'TNFAIP2', 'TNFAIP3', 'TNFAIP6', 'TNFAIP8', 'TNFRSF9', 'TNFSF9', 'TNIP1', 'TNIP2', 'TRAF1', 'TRIB1', 'TRIP10', 'TSC22D1', 'TUBB2A', 'VEGFA', 'YRDC', 'ZBTB10', 'ZC3H12A', 'ZFP36']\nHALLMARK_KRAS_SIGNALING_UP = ['ABCB1', 'ACE', 'ADAM17', 'ADAM8', 'ADAMDEC1', 'ADGRA2', 'ADGRL4', 'AKAP12', 'AKT2', 'ALDH1A2', 'ALDH1A3', 'AMMECR1', 'ANGPTL4', 'ANKH', 'ANO1', 'ANXA10', 'APOD', 'ARG1', 'ATG10', 'AVL9', 'BIRC3', 'BMP2', 'BPGM', 'BTBD3', 'BTC', 'C3AR1', 'CA2', 'CAB39L', 'CBL', 'CBR4', 'CBX8', 'CCL20', 'CCND2', 'CCSER2', 'CD37', 'CDADC1', 'CFB', 'CFH', 'CFHR2', 'CIDEA', 'CLEC4A', 'CMKLR1', 'CPE', 'CROT', 'CSF2', 'CSF2RA', 'CTSS', 'CXCL10', 'CXCR4', 'DCBLD2', 'DNMBP', 'DOCK2', 'DUSP6', 'EMP1', 'ENG', 'EPB41L3', 'EPHB2', 'EREG', 'ERO1A', 'ETS1', 'ETV1', 'ETV4', 'ETV5', 'EVI5', 'F13A1', 'F2RL1', 'FBXO4', 'FCER1G', 'FGF9', 'FLT4', 'FUCA1', 'G0S2', 'GABRA3', 'GADD45G', 'GALNT3', 'GFPT2', 'GLRX', 'GNG11', 'GPNMB', 'GPRC5B', 'GUCY1A1', 'GYPC', 'H2BC3', 'HBEGF', 'HDAC9', 'HKDC1', 'HOXD11', 'HSD11B1', 'ID2', 'IGF2', 'IGFBP3', 'IKZF1', 'IL10RA', 'IL1B', 'IL1RL2', 'IL2RG', 'IL33', 'IL7R', 'INHBA', 'IRF8', 'ITGA2', 'ITGB2', 'ITGBL1', 'JUP', 'KCNN4', 'KIF5C', 'KLF4', 'LAPTM5', 'LAT2', 'LCP1', 'LIF', 'LY96', 'MAFB', 'MALL', 'MAP3K1', 'MAP4K1', 'MAP7', 'MMD', 'MMP10', 'MMP11', 'MMP9', 'MPZL2', 'MTMR10', 'MYCN', 'NAP1L2', 'NGF', 'NIN', 'NR0B2', 'NR1H4', 'NRP1', 'PCP4', 'PCSK1N', 'PDCD1LG2', 'PECAM1', 'PEG3', 'PIGR', 'PLAT', 'PLAU', 'PLAUR', 'PLEK2', 'PLVAP', 'PPBP', 'PPP1R15A', 'PRDM1', 'PRELID3B', 'PRKG2', 'PRRX1', 'PSMB8', 'PTBP2', 'PTCD2', 'PTGS2', 'PTPRR', 'RABGAP1L', 'RBM4', 'RBP4', 'RELN', 'RETN', 'RGS16', 'SATB1', 'SCG3', 'SCG5', 'SCN1B', 'SDCCAG8', 'SEMA3B', 'SERPINA3', 'SLPI', 'SNAP25', 'SNAP91', 'SOX9', 'SPARCL1', 'SPON1', 'SPP1', 'SPRY2', 'ST6GAL1', 'STRN', 'TFPI', 'TLR8', 'TMEM100', 'TMEM158', 'TMEM176A', 'TMEM176B', 'TNFAIP3', 'TNFRSF1B', 'TNNT2', 'TOR1AIP2', 'TPH1', 'TRAF1', 'TRIB1', 'TRIB2', 'TSPAN1', 'TSPAN13', 'TSPAN7', 'USH1C', 'USP12', 'VWA5A', 'WDR33', 'WNT7A', 'YRDC', 'ZNF277', 'ZNF639']\nHALLMARK_COAGULATION = ['A2M', 'ACOX2', 'ADAM9', 'ANG', 'ANXA1', 'APOA1', 'APOC1', 'APOC2', 'APOC3', 'ARF4', 'BMP1', 'C1QA', 'C1R', 'C1S', 'C2', 'C3', 'C8A', 'C8B', 'C8G', 'C9', 'CAPN2', 'CAPN5', 'CASP9', 'CD9', 'CFB', 'CFD', 'CFH', 'CFI', 'CLU', 'COMP', 'CPB2', 'CPN1', 'CPQ', 'CRIP2', 'CSRP1', 'CTSB', 'CTSE', 'CTSH', 'CTSK', 'CTSO', 'CTSV', 'DCT', 'DPP4', 'DUSP14', 'DUSP6', 'F10', 'F11', 'F12', 'F13B', 'F2', 'F2RL2', 'F3', 'F8', 'F9', 'FBN1', 'FGA', 'FGG', 'FN1', 'FURIN', 'FYN', 'GDA', 'GNB2', 'GNG12', 'GP1BA', 'GP9', 'GSN', 'HMGCS2', 'HNF4A', 'HPN', 'HRG', 'HTRA1', 'ISCU', 'ITGA2', 'ITGB3', 'ITIH1', 'KLF7', 'KLK8', 'KLKB1', 'LAMP2', 'LEFTY2', 'LGMN', 'LRP1', 'LTA4H', 'MAFF', 'MASP2', 'MBL2', 'MEP1A', 'MMP1', 'MMP10', 'MMP11', 'MMP14', 'MMP15', 'MMP2', 'MMP3', 'MMP7', 'MMP8', 'MMP9', 'MSRB2', 'MST1', 'OLR1', 'P2RY1', 'PDGFB', 'PECAM1', 'PEF1', 'PF4', 'PLAT', 'PLAU', 'PLEK', 'PLG', 'PREP', 'PROC', 'PROS1', 'PROZ', 'PRSS23', 'RABIF', 'RAC1', 'RAPGEF3', 'RGN', 'S100A1', 'S100A13', 'SERPINA1', 'SERPINB2', 'SERPINC1', 'SERPINE1', 'SERPING1', 'SH2B2', 'SIRT2', 'SPARC', 'TF', 'TFPI2', 'THBD', 'THBS1', 'TIMP1', 'TIMP3', 'TMPRSS6', 'USP11', 'VWF', 'WDR1']\nHALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION = ['ABI3BP', 'ACTA2', 'ADAM12', 'ANPEP', 'APLP1', 'AREG', 'BASP1', 'BDNF', 'BGN', 'BMP1', 'CADM1', 'CALD1', 'CALU', 'CAP2', 'CAPG', 'CCN1', 'CCN2', 'CD44', 'CD59', 'CDH11', 'CDH2', 'CDH6', 'COL11A1', 'COL12A1', 'COL16A1', 'COL1A1', 'COL1A2', 'COL3A1', 'COL4A1', 'COL4A2', 'COL5A1', 'COL5A2', 'COL5A3', 'COL6A2', 'COL6A3', 'COL7A1', 'COL8A2', 'COLGALT1', 'COMP', 'COPA', 'CRLF1', 'CTHRC1', 'CXCL1', 'CXCL12', 'CXCL6', 'CXCL8', 'DAB2', 'DCN', 'DKK1', 'DPYSL3', 'DST', 'ECM1', 'ECM2', 'EDIL3', 'EFEMP2', 'ELN', 'EMP3', 'ENO2', 'FAP', 'FAS', 'FBLN1', 'FBLN2', 'FBLN5', 'FBN1', 'FBN2', 'FERMT2', 'FGF2', 'FLNA', 'FMOD', 'FN1', 'FOXC2', 'FSTL1', 'FSTL3', 'FUCA1', 'FZD8', 'GADD45A', 'GADD45B', 'GAS1', 'GEM', 'GJA1', 'GLIPR1', 'GPC1', 'GPX7', 'GREM1', 'HTRA1', 'ID2', 'IGFBP2', 'IGFBP3', 'IGFBP4', 'IL15', 'IL32', 'IL6', 'INHBA', 'ITGA2', 'ITGA5', 'ITGAV', 'ITGB1', 'ITGB3', 'ITGB5', 'JUN', 'LAMA1', 'LAMA2', 'LAMA3', 'LAMC1', 'LAMC2', 'LGALS1', 'LOX', 'LOXL1', 'LOXL2', 'LRP1', 'LRRC15', 'LUM', 'MAGEE1', 'MATN2', 'MATN3', 'MCM7', 'MEST', 'MFAP5', 'MGP', 'MMP1', 'MMP14', 'MMP2', 'MMP3', 'MSX1', 'MXRA5', 'MYL9', 'MYLK', 'NID2', 'NNMT', 'NOTCH2', 'NT5E', 'NTM', 'OXTR', 'P3H1', 'PCOLCE', 'PCOLCE2', 'PDGFRB', 'PDLIM4', 'PFN2', 'PLAUR', 'PLOD1', 'PLOD2', 'PLOD3', 'PMEPA1', 'PMP22', 'POSTN', 'PPIB', 'PRRX1', 'PRSS2', 'PTHLH', 'PTX3', 'PVR', 'QSOX1', 'RGS4', 'RHOB', 'SAT1', 'SCG2', 'SDC1', 'SDC4', 'SERPINE1', 'SERPINE2', 'SERPINH1', 'SFRP1', 'SFRP4', 'SGCB', 'SGCD', 'SGCG', 'SLC6A8', 'SLIT2', 'SLIT3', 'SNAI2', 'SNTB1', 'SPARC', 'SPOCK1', 'SPP1', 'TAGLN', 'TFPI2', 'TGFB1', 'TGFBI', 'TGFBR3', 'TGM2', 'THBS1', 'THBS2', 'THY1', 'TIMP1', 'TIMP3', 'TNC', 'TNFAIP3', 'TNFRSF11B', 'TNFRSF12A', 'TPM1', 'TPM2', 'TPM4', 'VCAM1', 'VCAN', 'VEGFA', 'VEGFC', 'VIM', 'WIPF1', 'WNT5A']\nHALLMARK_HYPOXIA = ['ACKR3', 'ADM', 'ADORA2B', 'AK4', 'AKAP12', 'ALDOA', 'ALDOB', 'ALDOC', 'AMPD3', 'ANGPTL4', 'ANKZF1', 'ANXA2', 'ATF3', 'ATP7A', 'B3GALT6', 'B4GALNT2', 'BCAN', 'BCL2', 'BGN', 'BHLHE40', 'BNIP3L', 'BRS3', 'BTG1', 'CA12', 'CASP6', 'CAV1', 'CAVIN1', 'CAVIN3', 'CCN1', 'CCN2', 'CCN5', 'CCNG2', 'CDKN1A', 'CDKN1B', 'CDKN1C', 'CHST2', 'CHST3', 'CITED2', 'COL5A1', 'CP', 'CSRP2', 'CXCR4', 'DCN', 'DDIT3', 'DDIT4', 'DPYSL4', 'DTNA', 'DUSP1', 'EDN2', 'EFNA1', 'EFNA3', 'EGFR', 'ENO1', 'ENO2', 'ENO3', 'ERO1A', 'ERRFI1', 'ETS1', 'EXT1', 'F3', 'FAM162A', 'FBP1', 'FOS', 'FOSL2', 'FOXO3', 'GAA', 'GALK1', 'GAPDH', 'GAPDHS', 'GBE1', 'GCK', 'GCNT2', 'GLRX', 'GPC1', 'GPC3', 'GPC4', 'GPI', 'GRHPR', 'GYS1', 'HAS1', 'HDLBP', 'HEXA', 'HK1', 'HK2', 'HMOX1', 'HOXB9', 'HS3ST1', 'HSPA5', 'IDS', 'IER3', 'IGFBP1', 'IGFBP3', 'IL6', 'ILVBL', 'INHA', 'IRS2', 'ISG20', 'JMJD6', 'JUN', 'KDELR3', 'KDM3A', 'KIF5A', 'KLF6', 'KLF7', 'KLHL24', 'LALBA', 'LARGE1', 'LDHA', 'LDHC', 'LOX', 'LXN', 'MAFF', 'MAP3K1', 'MIF', 'MT1E', 'MT2A', 'MXI1', 'MYH9', 'NAGK', 'NCAN', 'NDRG1', 'NDST1', 'NDST2', 'NEDD4L', 'NFIL3', 'NOCT', 'NR3C1', 'P4HA1', 'P4HA2', 'PAM', 'PCK1', 'PDGFB', 'PDK1', 'PDK3', 'PFKFB3', 'PFKL', 'PFKP', 'PGAM2', 'PGF', 'PGK1', 'PGM1', 'PGM2', 'PHKG1', 'PIM1', 'PKLR', 'PKP1', 'PLAC8', 'PLAUR', 'PLIN2', 'PNRC1', 'PPARGC1A', 'PPFIA4', 'PPP1R15A', 'PPP1R3C', 'PRDX5', 'PRKCA', 'PYGM', 'RBPJ', 'RORA', 'RRAGD', 'S100A4', 'SAP30', 'SCARB1', 'SDC2', 'SDC3', 'SDC4', 'SELENBP1', 'SERPINE1', 'SIAH2', 'SLC25A1', 'SLC2A1', 'SLC2A3', 'SLC2A5', 'SLC37A4', 'SLC6A6', 'SRPX', 'STBD1', 'STC1', 'STC2', 'SULT2B1', 'TES', 'TGFB3', 'TGFBI', 'TGM2', 'TIPARP', 'TKTL1', 'TMEM45A', 'TNFAIP3', 'TPBG', 'TPD52', 'TPI1', 'TPST2', 'UGP2', 'VEGFA', 'VHL', 'VLDLR', 'WSB1', 'XPNPEP1', 'ZFP36', 'ZNF292']\nHALLMARK_COMPLEMENT = ['ACTN2', 'ADAM9', 'ADRA2B', 'AKAP10', 'ANG', 'ANXA5', 'APOA4', 'APOBEC3F', 'APOBEC3G', 'APOC1', 'ATOX1', 'BRPF3', 'C1QA', 'C1QC', 'C1R', 'C1S', 'C2', 'C3', 'C4BPB', 'C9', 'CA2', 'CALM1', 'CALM3', 'CASP1', 'CASP10', 'CASP3', 'CASP4', 'CASP5', 'CASP7', 'CASP9', 'CBLB', 'CCL5', 'CD36', 'CD40LG', 'CD46', 'CD55', 'CD59', 'CDA', 'CDH13', 'CDK5R1', 'CEBPB', 'CFB', 'CFH', 'CLU', 'COL4A2', 'CP', 'CPM', 'CPQ', 'CR1', 'CR2', 'CSRP1', 'CTSB', 'CTSC', 'CTSD', 'CTSH', 'CTSL', 'CTSO', 'CTSS', 'CTSV', 'CXCL1', 'DGKG', 'DGKH', 'DOCK10', 'DOCK4', 'DOCK9', 'DPP4', 'DUSP5', 'DUSP6', 'DYRK2', 'EHD1', 'ERAP2', 'F10', 'F2', 'F3', 'F5', 'F7', 'F8', 'FCER1G', 'FCN1', 'FDX1', 'FN1', 'FYN', 'GATA3', 'GCA', 'GMFB', 'GNAI2', 'GNAI3', 'GNB2', 'GNB4', 'GNG2', 'GNGT2', 'GP1BA', 'GP9', 'GPD2', 'GRB2', 'GZMA', 'GZMB', 'GZMK', 'HNF4A', 'HPCAL4', 'HSPA1A', 'HSPA5', 'IL6', 'IRF1', 'IRF2', 'IRF7', 'ITGAM', 'ITIH1', 'JAK2', 'KCNIP2', 'KCNIP3', 'KIF2A', 'KLK1', 'KLKB1', 'KYNU', 'L3MBTL4', 'LAMP2', 'LAP3', 'LCK', 'LCP2', 'LGALS3', 'LGMN', 'LIPA', 'LRP1', 'LTA4H', 'LTF', 'LYN', 'MAFF', 'ME1', 'MMP12', 'MMP13', 'MMP14', 'MMP15', 'MMP8', 'MSRB1', 'MT3', 'NOTCH4', 'OLR1', 'PCLO', 'PCSK9', 'PDGFB', 'PDP1', 'PFN1', 'PHEX', 'PIK3CA', 'PIK3CG', 'PIK3R5', 'PIM1', 'PLA2G4A', 'PLA2G7', 'PLAT', 'PLAUR', 'PLEK', 'PLG', 'PLSCR1', 'PPP2CB', 'PPP4C', 'PRCP', 'PRDM4', 'PREP', 'PRKCD', 'PRSS3', 'PRSS36', 'PSEN1', 'PSMB9', 'RABIF', 'RAF1', 'RASGRP1', 'RBSN', 'RCE1', 'RHOG', 'RNF4', 'S100A12', 'S100A13', 'S100A9', 'SCG3', 'SERPINA1', 'SERPINB2', 'SERPINC1', 'SERPINE1', 'SERPING1', 'SH2B3', 'SIRT6', 'SPOCK2', 'SRC', 'STX4', 'TFPI2', 'TIMP1', 'TIMP2', 'TMPRSS6', 'TNFAIP3', 'USP14', 'USP15', 'USP16', 'USP8', 'VCPIP1', 'WAS', 'XPNPEP1', 'ZEB1', 'ZFPM2']\nHALLMARK_UV_RESPONSE_UP = ['ABCB1', 'ACAA1', 'AGO2', 'ALAS1', 'ALDOA', 'AMD1', 'AP2S1', 'APOM', 'AQP3', 'ARRB2', 'ASNS', 'ATF3', 'ATP6V1C1', 'ATP6V1F', 'BAK1', 'BCL2L11', 'BID', 'BMP2', 'BSG', 'BTG1', 'BTG2', 'BTG3', 'C4BPB', 'CA2', 'CASP3', 'CCK', 'CCND3', 'CCNE1', 'CDC34', 'CDC5L', 'CDK2', 'CDKN1C', 'CDKN2B', 'CDO1', 'CEBPG', 'CHKA', 'CHRNA5', 'CLCN2', 'CLTB', 'CNP', 'COL2A1', 'CREG1', 'CTSV', 'CXCL2', 'CYB5B', 'CYB5R1', 'CYP1A1', 'DDX21', 'DGAT1', 'DLG4', 'DNAJA1', 'DNAJB1', 'E2F5', 'EIF2S3', 'EIF5', 'ENO2', 'EPCAM', 'EPHX1', 'FEN1', 'FGF18', 'FKBP4', 'FMO1', 'FOS', 'FOSB', 'FURIN', 'GAL', 'GCH1', 'GGH', 'GLS', 'GPX3', 'GRINA', 'GRPEL1', 'H2AX', 'HLA-F', 'HMOX1', 'HNRNPU', 'HSPA13', 'HSPA2', 'HTR7', 'HYAL2', 'ICAM1', 'IGFBP2', 'IL6', 'IL6ST', 'IRF1', 'JUNB', 'KCNH2', 'KLHDC3', 'LHX2', 'LYN', 'MAOA', 'MAPK8IP2', 'MARK2', 'MGAT1', 'MMP14', 'MRPL23', 'MSX1', 'NAT1', 'NFKBIA', 'NKX2-5', 'NPTX2', 'NPTXR', 'NR4A1', 'NTRK3', 'NUP58', 'NXF1', 'OLFM1', 'ONECUT1', 'PARP2', 'PDAP1', 'PDLIM3', 'PLCL1', 'POLE3', 'POLG2', 'POLR2H', 'PPAT', 'PPIF', 'PPP1R2', 'PPT1', 'PRKACA', 'PRKCD', 'PRPF3', 'PSMC3', 'PTPRD', 'RAB27A', 'RASGRP1', 'RET', 'RFC4', 'RHOB', 'RPN1', 'RRAD', 'RXRB', 'SELENOW', 'SHOX2', 'SIGMAR1', 'SLC25A4', 'SLC6A12', 'SLC6A8', 'SOD2', 'SPR', 'SQSTM1', 'STARD3', 'STIP1', 'STK25', 'SULT1A1', 'TACR3', 'TAP1', 'TARS1', 'TCHH', 'TFRC', 'TGFBRAP1', 'TMBIM6', 'TST', 'TUBA4A', 'TYRO3', 'UROD', 'WIZ', 'YKT6']\nHALLMARK_APOPTOSIS = ['ADD1', 'AIFM3', 'ANKH', 'ANXA1', 'APP', 'ATF3', 'AVPR1A', 'BAX', 'BCAP31', 'BCL10', 'BCL2L1', 'BCL2L10', 'BCL2L11', 'BCL2L2', 'BGN', 'BID', 'BIK', 'BIRC3', 'BMF', 'BMP2', 'BNIP3L', 'BRCA1', 'BTG2', 'BTG3', 'CASP1', 'CASP2', 'CASP3', 'CASP4', 'CASP6', 'CASP7', 'CASP8', 'CASP9', 'CAV1', 'CCNA1', 'CCND1', 'CCND2', 'CD14', 'CD2', 'CD38', 'CD44', 'CD69', 'CDC25B', 'CDK2', 'CDKN1A', 'CDKN1B', 'CFLAR', 'CLU', 'CREBBP', 'CTH', 'CTNNB1', 'CYLD', 'DAP', 'DAP3', 'DCN', 'DDIT3', 'DFFA', 'DIABLO', 'DNAJA1', 'DNAJC3', 'DNM1L', 'DPYD', 'EBP', 'EGR3', 'EMP1', 'ENO2', 'ERBB2', 'ERBB3', 'EREG', 'ETF1', 'F2', 'F2R', 'FAS', 'FASLG', 'FDXR', 'FEZ1', 'GADD45A', 'GADD45B', 'GCH1', 'GNA15', 'GPX1', 'GPX3', 'GPX4', 'GSN', 'GSR', 'GSTM1', 'GUCY2D', 'H1-0', 'HGF', 'HMGB2', 'HMOX1', 'HSPB1', 'IER3', 'IFITM3', 'IFNB1', 'IFNGR1', 'IGF2R', 'IGFBP6', 'IL18', 'IL1A', 'IL1B', 'IL6', 'IRF1', 'ISG20', 'JUN', 'KRT18', 'LEF1', 'LGALS3', 'LMNA', 'LUM', 'MADD', 'MCL1', 'MGMT', 'MMP2', 'NEDD9', 'NEFH', 'PAK1', 'PDCD4', 'PDGFRB', 'PEA15', 'PLAT', 'PLCB2', 'PLPPR4', 'PMAIP1', 'PPP2R5B', 'PPP3R1', 'PPT1', 'PRF1', 'PSEN1', 'PSEN2', 'PTK2', 'RARA', 'RELA', 'RETSAT', 'RHOB', 'RHOT2', 'RNASEL', 'ROCK1', 'SAT1', 'SATB1', 'SC5D', 'SLC20A1', 'SMAD7', 'SOD1', 'SOD2', 'SPTAN1', 'SQSTM1', 'TAP1', 'TGFB2', 'TGFBR3', 'TIMP1', 'TIMP2', 'TIMP3', 'TNF', 'TNFRSF12A', 'TNFSF10', 'TOP2A', 'TSPO', 'TXNIP', 'VDAC2', 'WEE1', 'XIAP']","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:12.743548Z","iopub.execute_input":"2023-12-29T12:52:12.744427Z","iopub.status.idle":"2023-12-29T12:52:12.805165Z","shell.execute_reply.started":"2023-12-29T12:52:12.744379Z","shell.execute_reply":"2023-12-29T12:52:12.804155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"HALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION\": HALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION,\n    \"HALLMARK_HYPOXIA\": HALLMARK_HYPOXIA,\n    \"HALLMARK_COMPLEMENT\": HALLMARK_COMPLEMENT,\n    \"HALLMARK_UV_RESPONSE_UP\": HALLMARK_UV_RESPONSE_UP,\n    \"HALLMARK_APOPTOSIS\": HALLMARK_APOPTOSIS,\n    \"HALLMARK_APICAL_JUNCTION\": HALLMARK_APICAL_JUNCTION,\n    \"HALLMARK_ESTROGEN_RESPONSE_LATE\": HALLMARK_ESTROGEN_RESPONSE_LATE,\n    \"HALLMARK_TNFA_SIGNALING_VIA_NFKB\": HALLMARK_TNFA_SIGNALING_VIA_NFKB,\n    \"HALLMARK_KRAS_SIGNALING_UP\": HALLMARK_KRAS_SIGNALING_UP,\n    \"HALLMARK_COAGULATION\": HALLMARK_COAGULATION,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\n\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nplt.figure(figsize=(20, 10))\nfor col in d.columns[1:]:\n    if col in ['Not housekeeping percent', 'Housekeeping percent']:\n        plt.plot(d['Threshold'], d[col], label=col, linewidth=3) \n    else:\n        plt.plot(d['Threshold'], d[col], label=col)\n\n\nplt.grid()\nplt.legend()\nplt.xscale('log')\nplt.xlabel('Threshold', fontsize=20)\nplt.ylabel('Percent DE stronger than threshold', fontsize=20)\nplt.title('Percent DE Expressed Stronger Than Threshold by Gene Group', fontsize=20)\nx_min, x_max = plt.xlim()\nplt.xlim(x_max, x_min)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:12.806428Z","iopub.execute_input":"2023-12-29T12:52:12.807705Z","iopub.status.idle":"2023-12-29T12:52:20.845936Z","shell.execute_reply.started":"2023-12-29T12:52:12.807667Z","shell.execute_reply":"2023-12-29T12:52:20.844887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"HALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION\": HALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION,\n    \"HALLMARK_HYPOXIA\": HALLMARK_HYPOXIA,\n    \"HALLMARK_COMPLEMENT\": HALLMARK_COMPLEMENT,\n    \"HALLMARK_UV_RESPONSE_UP\": HALLMARK_UV_RESPONSE_UP,\n    \"HALLMARK_APOPTOSIS\": HALLMARK_APOPTOSIS,\n    \"HALLMARK_APICAL_JUNCTION\": HALLMARK_APICAL_JUNCTION,\n    \"HALLMARK_ESTROGEN_RESPONSE_LATE\": HALLMARK_ESTROGEN_RESPONSE_LATE,\n    \"HALLMARK_TNFA_SIGNALING_VIA_NFKB\": HALLMARK_TNFA_SIGNALING_VIA_NFKB,\n    \"HALLMARK_KRAS_SIGNALING_UP\": HALLMARK_KRAS_SIGNALING_UP,\n    \"HALLMARK_COAGULATION\": HALLMARK_COAGULATION,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nfor group_name in filtered_gene_groups:\n    if group_name in [\"Housekeeping\", \"Not housekeeping\"]:\n        continue\n    plt.figure(figsize=(15, 3))\n    plt.plot(d['Threshold'], d[f'{group_name} percent'], label=group_name, linewidth=2)\n    plt.plot(d['Threshold'], d['Housekeeping percent'], label='Housekeeping', linewidth=1)\n    plt.plot(d['Threshold'], d['Not housekeeping percent'], label='Not housekeeping', linewidth=1)\n\n    plt.grid()\n    plt.legend()\n    plt.xscale('log')\n    plt.xlabel('Threshold')\n    plt.ylabel('Percent DE stronger than threshold')\n    plt.title(f'Percent DE Expressed Stronger Than Threshold: {group_name}')\n    x_min, x_max = plt.xlim()\n    plt.xlim(x_max, x_min)\n    plt.show()\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-12-29T12:52:20.847317Z","iopub.execute_input":"2023-12-29T12:52:20.847654Z","iopub.status.idle":"2023-12-29T12:52:32.265967Z","shell.execute_reply.started":"2023-12-29T12:52:20.847625Z","shell.execute_reply":"2023-12-29T12:52:32.264894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIOCARTA_HSP27_PATHWAY = ['ACTA1', 'APAF1', 'BCL2', 'CASP3', 'CASP9', 'DAXX', 'FAS', 'FASLG', 'HSPB1', 'HSPB2', 'IL1A', 'MAPKAPK2', 'MAPKAPK3', 'TNF']\nBIOCARTA_HIF_PATHWAY = ['ARNT', 'ASPH', 'COPS5', 'CREB1', 'EDN1', 'EP300', 'EPO', 'HIF1A', 'HSP90AA1', 'JUN', 'LDHA', 'NOS3', 'P4HB', 'VEGFA', 'VHL']\nGOBP_CELLULAR_RESPONSE_TO_HEAT = ['ANO1', 'ATM', 'ATR', 'ATXN3', 'BAG3', 'CDKN1A', 'CETN1', 'CHORDC1', 'CLPB', 'CREBBP', 'CXCL10', 'DAXX', 'DHX36', 'DNAJB1', 'DNAJB6', 'DNAJC2', 'DNAJC7', 'EIF2S1', 'EP300', 'FGF1', 'GSK3B', 'HDAC2', 'HIKESHI', 'HMOX1', 'HSBP1', 'HSBP1L1', 'HSF1', 'HSP90AA1', 'HSP90AA2P', 'HSP90AA4P', 'HSP90AB1', 'HSP90AB2P', 'HSP90AB3P', 'HSP90AB4P', 'HSPA1A', 'HSPA1B', 'HSPA6', 'HTRA2', 'IER5', 'IL1A', 'IRAK1', 'LYN', 'MAPKAPK2', 'MAPT', 'MTOR', 'NF1', 'PDCD6', 'PRKACA', 'PTGS2', 'RBBP7', 'SCARA5', 'SIRT1', 'SLC52A3', 'SLU7', 'ST8SIA1', 'STAC', 'STUB1', 'SUMO1', 'TCIM', 'TFEC', 'THBS1', 'TPR', 'TRPV1', 'TRPV4', 'VCP', 'YWHAE', 'ZFAND1']\nHALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION = ['ABI3BP', 'ACTA2', 'ADAM12', 'ANPEP', 'APLP1', 'AREG', 'BASP1', 'BDNF', 'BGN', 'BMP1', 'CADM1', 'CALD1', 'CALU', 'CAP2', 'CAPG', 'CCN1', 'CCN2', 'CD44', 'CD59', 'CDH11', 'CDH2', 'CDH6', 'COL11A1', 'COL12A1', 'COL16A1', 'COL1A1', 'COL1A2', 'COL3A1', 'COL4A1', 'COL4A2', 'COL5A1', 'COL5A2', 'COL5A3', 'COL6A2', 'COL6A3', 'COL7A1', 'COL8A2', 'COLGALT1', 'COMP', 'COPA', 'CRLF1', 'CTHRC1', 'CXCL1', 'CXCL12', 'CXCL6', 'CXCL8', 'DAB2', 'DCN', 'DKK1', 'DPYSL3', 'DST', 'ECM1', 'ECM2', 'EDIL3', 'EFEMP2', 'ELN', 'EMP3', 'ENO2', 'FAP', 'FAS', 'FBLN1', 'FBLN2', 'FBLN5', 'FBN1', 'FBN2', 'FERMT2', 'FGF2', 'FLNA', 'FMOD', 'FN1', 'FOXC2', 'FSTL1', 'FSTL3', 'FUCA1', 'FZD8', 'GADD45A', 'GADD45B', 'GAS1', 'GEM', 'GJA1', 'GLIPR1', 'GPC1', 'GPX7', 'GREM1', 'HTRA1', 'ID2', 'IGFBP2', 'IGFBP3', 'IGFBP4', 'IL15', 'IL32', 'IL6', 'INHBA', 'ITGA2', 'ITGA5', 'ITGAV', 'ITGB1', 'ITGB3', 'ITGB5', 'JUN', 'LAMA1', 'LAMA2', 'LAMA3', 'LAMC1', 'LAMC2', 'LGALS1', 'LOX', 'LOXL1', 'LOXL2', 'LRP1', 'LRRC15', 'LUM', 'MAGEE1', 'MATN2', 'MATN3', 'MCM7', 'MEST', 'MFAP5', 'MGP', 'MMP1', 'MMP14', 'MMP2', 'MMP3', 'MSX1', 'MXRA5', 'MYL9', 'MYLK', 'NID2', 'NNMT', 'NOTCH2', 'NT5E', 'NTM', 'OXTR', 'P3H1', 'PCOLCE', 'PCOLCE2', 'PDGFRB', 'PDLIM4', 'PFN2', 'PLAUR', 'PLOD1', 'PLOD2', 'PLOD3', 'PMEPA1', 'PMP22', 'POSTN', 'PPIB', 'PRRX1', 'PRSS2', 'PTHLH', 'PTX3', 'PVR', 'QSOX1', 'RGS4', 'RHOB', 'SAT1', 'SCG2', 'SDC1', 'SDC4', 'SERPINE1', 'SERPINE2', 'SERPINH1', 'SFRP1', 'SFRP4', 'SGCB', 'SGCD', 'SGCG', 'SLC6A8', 'SLIT2', 'SLIT3', 'SNAI2', 'SNTB1', 'SPARC', 'SPOCK1', 'SPP1', 'TAGLN', 'TFPI2', 'TGFB1', 'TGFBI', 'TGFBR3', 'TGM2', 'THBS1', 'THBS2', 'THY1', 'TIMP1', 'TIMP3', 'TNC', 'TNFAIP3', 'TNFRSF11B', 'TNFRSF12A', 'TPM1', 'TPM2', 'TPM4', 'VCAM1', 'VCAN', 'VEGFA', 'VEGFC', 'VIM', 'WIPF1', 'WNT5A']\nPID_HIF1A_PATHWAY = ['ARNT', 'CDKN2A', 'COPS5', 'CUL2', 'EGLN1', 'EGLN2', 'EGLN3', 'ELOB', 'ELOC', 'HIF1A', 'HIF1AN', 'HIF3A', 'HSP90AA1', 'NAA10', 'OS9', 'RACK1', 'RBX1', 'TP53', 'VHL']\nGOMF_HEAT_SHOCK_PROTEIN_BINDING = ['ADORA1', 'AHR', 'AHSA1', 'APAF1', 'APOA1', 'APOA2', 'BAG2', 'BAG6', 'BAK1', 'BAX', 'BCOR', 'BMAL1', 'CAMKMT', 'CDC37', 'CDC37L1', 'CDK1', 'CDK5', 'CDKN1B', 'CHORDC1', 'CSNK2A1', 'CYP1A1', 'CYP2E1', 'DAXX', 'DNAJA1', 'DNAJA2', 'DNAJA3', 'DNAJA4', 'DNAJB1', 'DNAJB12', 'DNAJB14', 'DNAJB2', 'DNAJB3', 'DNAJB6', 'DNAJB7', 'DNAJB8', 'DNAJB9', 'DNAJC10', 'DNAJC18', 'DNAJC2', 'DNAJC7', 'DNAJC8', 'DNAJC9', 'EEF1AKMT3', 'EIF2AK3', 'ERN1', 'ETFBKMT', 'FAF1', 'FGF1', 'FICD', 'FKBP4', 'FKBP5', 'FKBP6', 'GBP1', 'GMEB1', 'GPR37', 'GUCY1B1', 'HDAC2', 'HDAC6', 'HDAC8', 'HIF1A', 'HIKESHI', 'HSF1', 'HSP90AB1', 'HSPA13', 'HSPA14', 'HSPA1A', 'HSPA1B', 'HSPA1L', 'HSPA2', 'HSPA5', 'HSPA6', 'HSPA7', 'HSPA8', 'HSPA9', 'HTT', 'IQCG', 'IRAK1', 'ITGAM', 'ITGB2', 'KDR', 'KPNB1', 'LIMK1', 'LMAN2', 'MAPT', 'METTL18', 'METTL21A', 'METTL21C', 'METTL22', 'METTL23', 'MVD', 'NOD2', 'NPAS2', 'NR3C1', 'NUP62', 'PACRG', 'PDXP', 'PGLYRP1', 'PPEF2', 'PPID', 'PPP5C', 'PRKD1', 'PRKN', 'PTGES3', 'PTGES3L', 'RNF207', 'RPS3', 'SACS', 'SGTB', 'SLC12A2', 'SNCA', 'SPN', 'SSUH2', 'ST13', 'STAU2', 'STIP1', 'STUB1', 'TELO2', 'TFAM', 'TFRC', 'TOMM34', 'TPR', 'TSC1', 'TSC2', 'TTC4', 'UNC45A', 'UNC45B', 'USP19', 'ZFP36']","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:32.267756Z","iopub.execute_input":"2023-12-29T12:52:32.268192Z","iopub.status.idle":"2023-12-29T12:52:32.287509Z","shell.execute_reply.started":"2023-12-29T12:52:32.268151Z","shell.execute_reply":"2023-12-29T12:52:32.286265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"BIOCARTA_HSP27_PATHWAY\": BIOCARTA_HSP27_PATHWAY,\n    \"BIOCARTA_HIF_PATHWAY\": BIOCARTA_HIF_PATHWAY,\n    \"GOBP_CELLULAR_RESPONSE_TO_HEAT\": GOBP_CELLULAR_RESPONSE_TO_HEAT,\n    \"HALLMARK_UV_RESPONSE_UP\": HALLMARK_UV_RESPONSE_UP,\n    \"HALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION\": HALLMARK_EPITHELIAL_MESENCHYMAL_TRANSITION,\n    \"PID_HIF1A_PATHWAY \": PID_HIF1A_PATHWAY,\n    \"GOMF_HEAT_SHOCK_PROTEIN_BINDING\": GOMF_HEAT_SHOCK_PROTEIN_BINDING,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nfor group_name in filtered_gene_groups:\n    if group_name in [\"Housekeeping\", \"Not housekeeping\"]:\n        continue\n    plt.figure(figsize=(15, 3))\n    plt.plot(d['Threshold'], d[f'{group_name} percent'], label=group_name, linewidth=2)\n    plt.plot(d['Threshold'], d['Housekeeping percent'], label='Housekeeping', linewidth=1)\n    plt.plot(d['Threshold'], d['Not housekeeping percent'], label='Not housekeeping', linewidth=1)\n\n    plt.grid()\n    plt.legend()\n    plt.xscale('log')\n    plt.xlabel('Threshold')\n    plt.ylabel('Percent DE stronger than threshold')\n    plt.title(f'Percent DE Expressed Stronger Than Threshold: {group_name}')\n    x_min, x_max = plt.xlim()\n    plt.xlim(x_max, x_min)\n    plt.show()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-12-29T12:52:32.289351Z","iopub.execute_input":"2023-12-29T12:52:32.289718Z","iopub.status.idle":"2023-12-29T12:52:42.008360Z","shell.execute_reply.started":"2023-12-29T12:52:32.289690Z","shell.execute_reply":"2023-12-29T12:52:42.007287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf = pd.read_csv('/kaggle/input/cellular-response-to-stress/results.csv')\ndf_gene_name = df.iloc[:, 1:2].copy()\ndf_gene_name.columns = ['Gene Name']\ndf_gene_name['Gene Name'] = df_gene_name['Gene Name'].str.upper()\ngene_name_list = df_gene_name['Gene Name'].tolist()\nsirt_family = ['SIRT1','SIRT2','SIRT3','SIRT4','SIRT5','SIRT6','SIRT7']\ngene_name_list.extend(sirt_family)\n\nprint(gene_name_list)","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:42.010163Z","iopub.execute_input":"2023-12-29T12:52:42.010924Z","iopub.status.idle":"2023-12-29T12:52:42.036540Z","shell.execute_reply.started":"2023-12-29T12:52:42.010880Z","shell.execute_reply":"2023-12-29T12:52:42.035568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"cellular-response-to-stress\":gene_name_list,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nfor group_name in filtered_gene_groups:\n    if group_name in [\"Housekeeping\", \"Not housekeeping\"]:\n        continue\n    plt.figure(figsize=(15, 3))\n    plt.plot(d['Threshold'], d[f'{group_name} percent'], label=group_name, linewidth=2)\n    plt.plot(d['Threshold'], d['Housekeeping percent'], label='Housekeeping', linewidth=1)\n    plt.plot(d['Threshold'], d['Not housekeeping percent'], label='Not housekeeping', linewidth=1)\n\n    plt.grid()\n    plt.legend()\n    plt.xscale('log')\n    plt.xlabel('Threshold')\n    plt.ylabel('Percent DE stronger than threshold')\n    plt.title(f'Percent DE Expressed Stronger Than Threshold: {group_name}')\n    x_min, x_max = plt.xlim()\n    plt.xlim(x_max, x_min)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:42.037769Z","iopub.execute_input":"2023-12-29T12:52:42.038189Z","iopub.status.idle":"2023-12-29T12:52:49.769951Z","shell.execute_reply.started":"2023-12-29T12:52:42.038160Z","shell.execute_reply":"2023-12-29T12:52:49.768747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BIOCARTA_EIF_PATHWAY = [\"EEF2\",\"EEF2K\",\"EIF1\",\"EIF1AX\",\"EIF2S1\",\"EIF2S2\",\"EIF2S3\",\"EIF3A\",\"EIF4A1\",\"EIF4A2\",\"EIF4E\",\"EIF4G1\",\"EIF4G2\",\"EIF4G3\",\"EIF5\",\"EIF6\"]\nBIOCARTA_EIF2_PATHWAY = [\"EIF2AK1\",\"EIF2AK2\",\"EIF2AK3\",\"EIF2AK4\",\"EIF2B5\",\"EIF2S1\",\"EIF2S2\",\"EIF2S3\",\"EIF5\",\"GSK3B\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:49.771522Z","iopub.execute_input":"2023-12-29T12:52:49.771894Z","iopub.status.idle":"2023-12-29T12:52:49.777640Z","shell.execute_reply.started":"2023-12-29T12:52:49.771862Z","shell.execute_reply":"2023-12-29T12:52:49.776620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sirt_family = ['SIRT1','SIRT2','SIRT3','SIRT4','SIRT5','SIRT6','SIRT7']\n","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:49.779007Z","iopub.execute_input":"2023-12-29T12:52:49.780014Z","iopub.status.idle":"2023-12-29T12:52:49.791257Z","shell.execute_reply.started":"2023-12-29T12:52:49.779970Z","shell.execute_reply":"2023-12-29T12:52:49.790154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"SIRT_GENES\": sirt_family,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nfor group_name in filtered_gene_groups:\n    if group_name in [\"Housekeeping\", \"Not housekeeping\"]:\n        continue\n    plt.figure(figsize=(15, 3))\n    plt.plot(d['Threshold'], d[f'{group_name} percent'], label=group_name, linewidth=2)\n    plt.plot(d['Threshold'], d['Housekeeping percent'], label='Housekeeping', linewidth=1)\n    plt.plot(d['Threshold'], d['Not housekeeping percent'], label='Not housekeeping', linewidth=1)\n\n    plt.grid()\n    plt.legend()\n    plt.xscale('log')\n    plt.xlabel('Threshold')\n    plt.ylabel('Percent DE stronger than threshold')\n    plt.title(f'Percent DE Expressed Stronger Than Threshold: {group_name}')\n    x_min, x_max = plt.xlim()\n    plt.xlim(x_max, x_min)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:54:59.512219Z","iopub.execute_input":"2023-12-29T12:54:59.512598Z","iopub.status.idle":"2023-12-29T12:55:06.720672Z","shell.execute_reply.started":"2023-12-29T12:54:59.512570Z","shell.execute_reply":"2023-12-29T12:55:06.719519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sirt_related_genes = [\n    'AMPK','PKA','CTRP1','HO1','CRIF1','CAMkkB','DEPTOR','S6K1','SREBP1',\n    'Raptor','MCP-1','IL-B','ICAM-1','MCPIP1','KLF4',\n    'FGF21','RORa','KLF2','ABCA1','ABCG1','CCR7','eNOS','TLR-2','TAK1','IRAK4',\n    'IRF9','DBC1','PGRN','miR-9','PARP-1','SIRT1','Arf','p53',\n    'MDM2','Nrf2','Fox01','IFN-Y','NT5C3A','HMGB1','PGC-1a/PPARa',\n    'FXR','TRPV1','SIRT6','K458',\n    'HNF4a','XBP1','PI3K','HIF-1a','STAT3','Akt','mTOR','MMP2, 9, 13','DMP1',\n    'PAI-1','IL-6','GFAP','JNK','p38','XPB1/NLRP3','RORYT','IL-17',\n    'IL-17f','IL-22','TLR-4','TNF-a','SIRT2','Fox03','NLRP3','Ikkß','TGF-B','Ebi3','MKP1',\n    'Foxp3','ICOS','CTLA-4','SIRT3','E-selectin','p38 MAPK','TREM TGR5',\n    'MIP2','p-eNOS','TNF-aH','IL-8','NF-KB',\n    'SIRT7','SIRT3, 6','NF-KB p65','HICAM-1','COX2','MCP1'\n]\n\n# Cleaned list to ensure consistency\ncleaned_genes = [gene.strip().upper() for gene in sirt_related_genes]\n\n# Check for duplicates\nduplicate_genes = set()\nunique_genes = set()\n\nfor gene in cleaned_genes:\n    if gene in unique_genes:\n        duplicate_genes.add(gene)\n    else:\n        unique_genes.add(gene)\n\nif duplicate_genes:\n    print(\"Duplicates found:\", duplicate_genes)\nelse:\n    print(\"No duplicates found.\")","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:49.792705Z","iopub.execute_input":"2023-12-29T12:52:49.793152Z","iopub.status.idle":"2023-12-29T12:52:49.804890Z","shell.execute_reply.started":"2023-12-29T12:52:49.793110Z","shell.execute_reply":"2023-12-29T12:52:49.803757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes_in_caps = [gene.upper() for gene in sirt_related_genes]\n\n# Print the list in uppercase\nprint(genes_in_caps)","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:49.806517Z","iopub.execute_input":"2023-12-29T12:52:49.807213Z","iopub.status.idle":"2023-12-29T12:52:49.816318Z","shell.execute_reply.started":"2023-12-29T12:52:49.807170Z","shell.execute_reply":"2023-12-29T12:52:49.814890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"SIRT_FAMILY_RELATEDs\": genes_in_caps,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nfor group_name in filtered_gene_groups:\n    if group_name in [\"Housekeeping\", \"Not housekeeping\"]:\n        continue\n    plt.figure(figsize=(15, 3))\n    plt.plot(d['Threshold'], d[f'{group_name} percent'], label=group_name, linewidth=2)\n    plt.plot(d['Threshold'], d['Housekeeping percent'], label='Housekeeping', linewidth=1)\n    plt.plot(d['Threshold'], d['Not housekeeping percent'], label='Not housekeeping', linewidth=1)\n\n    plt.grid()\n    plt.legend()\n    plt.xscale('log')\n    plt.xlabel('Threshold')\n    plt.ylabel('Percent DE stronger than threshold')\n    plt.title(f'Percent DE Expressed Stronger Than Threshold: {group_name}')\n    x_min, x_max = plt.xlim()\n    plt.xlim(x_max, x_min)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:49.817987Z","iopub.execute_input":"2023-12-29T12:52:49.818321Z","iopub.status.idle":"2023-12-29T12:52:57.024621Z","shell.execute_reply.started":"2023-12-29T12:52:49.818293Z","shell.execute_reply":"2023-12-29T12:52:57.023704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"autophagy_genes = [\n    'ATG10', 'ATG14', 'ATG16L1', 'ATG3', 'ATG4', 'ATG7', 'ATG9A', 'BCL2', 'GABARAP',\n    'GABARAPL1', 'MAP1LC3B', 'NBR1', 'OPTN', 'PINK1', 'SQSTM1', 'ULK1', 'ULK2',\n    'UVRAG', 'WDR45', 'WIPI1'\n]\nautophagy_genes","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:57.025765Z","iopub.execute_input":"2023-12-29T12:52:57.026153Z","iopub.status.idle":"2023-12-29T12:52:57.034213Z","shell.execute_reply.started":"2023-12-29T12:52:57.026122Z","shell.execute_reply":"2023-12-29T12:52:57.033070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"Autophagy\": autophagy_genes,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nfor group_name in filtered_gene_groups:\n    if group_name in [\"Housekeeping\", \"Not housekeeping\"]:\n        continue\n    plt.figure(figsize=(15, 3))\n    plt.plot(d['Threshold'], d[f'{group_name} percent'], label=group_name, linewidth=2)\n    plt.plot(d['Threshold'], d['Housekeeping percent'], label='Housekeeping', linewidth=1)\n    plt.plot(d['Threshold'], d['Not housekeeping percent'], label='Not housekeeping', linewidth=1)\n\n    plt.grid()\n    plt.legend()\n    plt.xscale('log')\n    plt.xlabel('Threshold')\n    plt.ylabel('Percent DE stronger than threshold')\n    plt.title(f'Percent DE Expressed Stronger Than Threshold: {group_name}')\n    x_min, x_max = plt.xlim()\n    plt.xlim(x_max, x_min)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:52:57.035909Z","iopub.execute_input":"2023-12-29T12:52:57.036525Z","iopub.status.idle":"2023-12-29T12:53:04.247767Z","shell.execute_reply.started":"2023-12-29T12:52:57.036424Z","shell.execute_reply":"2023-12-29T12:53:04.246640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"genes = [\n    'NMNAT1', 'Rab7', 'LOX-1', 'EGFR', 'SIRT2', 'Fox01a', 'PGC', 'P', 'NF-KB', 'MAPK',\n    'PTEN', 'HMGB1', 'NAT10', 'PAK1 -Ac', 'MFN2', 'TFEB', 'Fox01', 'ATGL', 'HO-1', 'Ub',\n    'PARP1', 'SIRT1', 'HIF1a', 'C', 'CAMP', 'Fox03a', 'P53', 'PRKA', 'IPMK', 'PPAR-a',\n    'ATG12', 'ATG5', 'ATG7', 'CUL4B', 'Beclin 1', 'AMPK', 'LKB1', 'mTOR', 'PPAR-a', 'ULK1',\n    'SIRT3', 'LDHB', 'FoxM1', 'HI', 'REGY', 'SMAD4', 'SIRT5', 'PI3K', 'SIRT7', 'SIRT1',\n    'Fox01', 'PINK', 'SIRT6', 'SQSTM1/p62', 'Akt', 'Parkin', 'LC3-11', 'LC3-1', 'LC3', 'IGF',\n    'FBXW7 CERKL', 'Fox03a', 'SIRT5', 'IncR-TUG1', 'miR-122-5p', 'Bnip3', 'SIRT4', 'OPA1', 'SIRT3'\n]\n\n# Convert to uppercase and remove duplicates\nsirt_autophagy_genes = list(set([gene.upper() for gene in genes]))\nsirt_autophagy_genes","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:53:04.249415Z","iopub.execute_input":"2023-12-29T12:53:04.250032Z","iopub.status.idle":"2023-12-29T12:53:04.259408Z","shell.execute_reply.started":"2023-12-29T12:53:04.250000Z","shell.execute_reply":"2023-12-29T12:53:04.258606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_gene_groups = {\n    \"Autophagy_sirt\": sirt_autophagy_genes,\n    \"Not housekeeping\": list_not_hk,\n    \"Housekeeping\": list_hk\n}\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\n\ndef filter_existing_genes(genes, df):\n    return [gene for gene in genes if gene in df.columns]\nfiltered_gene_groups = {group: filter_existing_genes(genes, df_de_train) for group, genes in list_gene_groups.items()}\n\nd = pd.DataFrame()\nIX = -1\n\nfor group_name, gene_list in filtered_gene_groups.items():\n    if not gene_list:\n        continue\n    for t in list_thresholds:\n        v = df_de_train[gene_list].values.ravel()\n        v = 10**(-np.abs(v))\n        m = v < t\n        IX += 1\n        d.loc[IX, 'Threshold'] = t\n        d.loc[IX, f'{group_name} percent'] = 100 * m.sum() / len(v)\n\nfor group_name in filtered_gene_groups:\n    if group_name in [\"Housekeeping\", \"Not housekeeping\"]:\n        continue\n    plt.figure(figsize=(15, 3))\n    plt.plot(d['Threshold'], d[f'{group_name} percent'], label=group_name, linewidth=2)\n    plt.plot(d['Threshold'], d['Housekeeping percent'], label='Housekeeping', linewidth=1)\n    plt.plot(d['Threshold'], d['Not housekeeping percent'], label='Not housekeeping', linewidth=1)\n\n    plt.grid()\n    plt.legend()\n    plt.xscale('log')\n    plt.xlabel('Threshold')\n    plt.ylabel('Percent DE stronger than threshold')\n    plt.title(f'Percent DE Expressed Stronger Than Threshold: {group_name}')\n    x_min, x_max = plt.xlim()\n    plt.xlim(x_max, x_min)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-29T12:53:04.260602Z","iopub.execute_input":"2023-12-29T12:53:04.261757Z","iopub.status.idle":"2023-12-29T12:53:11.464581Z","shell.execute_reply.started":"2023-12-29T12:53:04.261714Z","shell.execute_reply":"2023-12-29T12:53:11.463522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final timing","metadata":{}},{"cell_type":"code","source":"print('%.1f seconds passed total '%(time.time()-t0start) )\nprint('%.1f minutes passed total '%( (time.time()-t0start)/60)  )\nprint('%.2f hours passed total '%( (time.time()-t0start)/3600)  )","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-12-29T12:53:11.465945Z","iopub.execute_input":"2023-12-29T12:53:11.466242Z","iopub.status.idle":"2023-12-29T12:53:11.472124Z","shell.execute_reply.started":"2023-12-29T12:53:11.466216Z","shell.execute_reply":"2023-12-29T12:53:11.471299Z"},"trusted":true},"execution_count":null,"outputs":[]}]}