{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Using PyWGCNA to find clusters (modules) of highly correlated genes\n- PyWGCNA [https://github.com/mortazavilab/PyWGCNA](https://github.com/mortazavilab/PyWGCNA)\n- I have not been able to get this to run successfully on kaggle due to errors.\n- This [dataset](https://www.kaggle.com/datasets/aemulcahy/opscp001) contains the modules that were created locally using the code below.\n","metadata":{}},{"cell_type":"code","source":"import os\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nfrom mpl_toolkits.mplot3d import Axes3D, proj3d\nimport random\nimport seaborn as sns\nimport sys\nfrom typing import List\n\nimport plotly\nimport matplotlib.image as mpimg\nimport io\n\nimport warnings\nimport pickle\n\nfrom IPython.display import Image","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-11-12T03:10:59.389256974Z","start_time":"2023-11-12T03:10:59.361096902Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:34:09.311542Z","iopub.execute_input":"2023-11-12T03:34:09.312116Z","iopub.status.idle":"2023-11-12T03:34:10.332290Z","shell.execute_reply.started":"2023-11-12T03:34:09.312078Z","shell.execute_reply":"2023-11-12T03:34:10.331287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_path = '../data/'\nif os.environ.get('KAGGLE_KERNEL_RUN_TYPE', 'Localhost') != 'Localhost':\n    file_path = '/kaggle/input/open-problems-single-cell-perturbations/'\n\nid_map = pd.read_csv(file_path+'id_map.csv', index_col='id')\nde_train = pd.read_parquet(file_path+'de_train.parquet')\ndisplay(de_train)\n","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-11-12T03:11:05.528768221Z","start_time":"2023-11-12T03:10:59.392126983Z"},"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.environ.get('KAGGLE_KERNEL_RUN_TYPE', 'Localhost') == 'Localhost':\n    # black_lst = pickle.load(open('black_lst.pkl', 'rb'))\n    # darkgrey_lst = pickle.load(open('darkgrey_lst.pkl', 'rb'))\n    # silver_lst = pickle.load(open('silver_lst.pkl', 'rb'))\n    import PyWGCNA\n\n    geneExp = de_train.iloc[:, 5:]\n    pyWGCNA_CC = PyWGCNA.WGCNA(name=' Open Problems – Single-Cell Perturbations',\n                               geneExp=geneExp,\n                               outputPath='',\n                               save=False)\n    pyWGCNA_CC.geneExpr.to_df().head(5)\n    pyWGCNA_CC.preprocess()\n    pyWGCNA_CC.findModules()\n\n    modules = pyWGCNA_CC.getModuleName()\n    display(modules)\n\n    black_lst = list(pyWGCNA_CC.getGeneModule('black')['black'].index)\n    darkgrey_lst = list(pyWGCNA_CC.getGeneModule('darkgrey')['darkgrey'].index)\n    silver_lst = list(pyWGCNA_CC.getGeneModule('silver')['silver'].index)\n    assert(set(black_lst).isdisjoint(darkgrey_lst))\n    assert(set(darkgrey_lst).isdisjoint(silver_lst))\n    assert(set(silver_lst).isdisjoint(black_lst))\n    assert(not set(black_lst).isdisjoint(black_lst))\n    pickle.dump(black_lst,open('black_lst.pkl','wb'))\n    pickle.dump(darkgrey_lst,open('darkgrey_lst.pkl','wb'))\n    pickle.dump(silver_lst,open('silver_lst.pkl','wb'))\n\n\n","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-11-12T03:18:55.963832192Z","start_time":"2023-11-12T03:11:05.536411031Z"},"jupyter":{"outputs_hidden":false}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pre-processing...\n\tDetecting genes and samples with too many missing values...\n\tDone pre-processing..\n\nRun WGCNA...\npickSoftThreshold: calculating connectivity for given powers...\nwill use block size  2456\n    Power  SFT.R.sq     slope truncated R.sq      mean(k)    median(k)  \\\n0       1  0.001083 -0.027167       0.685088  5860.509123   7043.16878   \n1       2  0.100577 -0.243164       0.915438  3782.030759  3880.518577   \n2       3  0.186078 -0.313104       0.969346  2786.965671   2274.74913   \n3       4  0.286136 -0.361544       0.960505   2190.51075  1379.922437   \n4       5  0.393929 -0.400788       0.934255  1788.226479   855.506908   \n5       6  0.485254 -0.433883       0.907235  1497.098521   537.982846   \n6       7   0.56364 -0.457743       0.882286  1276.414474   342.793583   \n7       8  0.630266 -0.483151        0.85937  1103.548154   220.568312   \n8       9  0.677994 -0.502587       0.839004   964.775402   145.057807   \n9      10  0.719021 -0.524513       0.820845   851.221849    98.976758   \n10     11  0.751535 -0.543868       0.813583   756.863779    70.140813   \n11     13  0.797349  -0.57915       0.803497   609.905867    37.881287   \n12     15  0.833109 -0.608557       0.823123   501.739267    20.766932   \n13     17  0.867242 -0.632676       0.851229   419.671684    12.004468   \n14     19  0.895137 -0.654172       0.880473   355.904771     7.056125   \n\n         max(k)  \n0   9825.363361  \n1   7815.530238  \n2   6650.225116  \n3   5834.503437  \n4   5209.289591  \n5   4711.774899  \n6   4298.499604  \n7   3954.350741  \n8   3656.988771  \n9   3404.250852  \n10  3181.892786  \n11  2805.674301  \n12  2498.490582  \n13  2242.348646  \n14  2025.281818  \nNo power detected to have scale free network!\nFound the best given power which is 19.\ncalculating adjacency matrix ...\n\tDone..\n\ncalculating TOM similarity matrix ...\n\tDone..\n\nGoing through the merge tree...\n..cutHeight not given, setting it to 0.995  ===>  99% of the (truncated) height range in dendro.\n\tDone..\n\nCalculating 4 module eigengenes in given set...\n\tDone..\n\nmergeCloseModules: Merging modules whose distance is less than 0.2\nfixDataStructure: data is not a Dictionary: converting it into one.\nmultiSetMEs: Calculating module MEs.\n  Working on set 1 ...\nCalculating 4 module eigengenes in given set...\n\tDone..\n\nmultiSetMEs: Calculating module MEs.\n  Working on set 1 ...\nCalculating 3 module eigengenes in given set...\n\tDone..\n\n  Calculating new MEs...\nmultiSetMEs: Calculating module MEs.\n  Working on set 1 ...\nCalculating 3 module eigengenes in given set...\n\tDone..\n\nCalculating 3 module eigengenes in given set...\n\tDone..\n\nfixDataStructure: data is not a Dictionary: converting it into one.\norderMEs: order not given, calculating using given set 0\n\tDone running WGCNA..\n","metadata":{}},{"cell_type":"code","source":"Image(\"/kaggle/input/opscp001/output_1.png\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:34:31.362595Z","iopub.execute_input":"2023-11-12T03:34:31.362934Z","iopub.status.idle":"2023-11-12T03:34:31.376822Z","shell.execute_reply.started":"2023-11-12T03:34:31.362907Z","shell.execute_reply":"2023-11-12T03:34:31.375225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image(\"/kaggle/input/opscp001/output_2.png\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:34:36.060169Z","iopub.execute_input":"2023-11-12T03:34:36.060536Z","iopub.status.idle":"2023-11-12T03:34:36.073855Z","shell.execute_reply.started":"2023-11-12T03:34:36.060510Z","shell.execute_reply":"2023-11-12T03:34:36.072372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image(\"/kaggle/input/opscp001/output_3.png\")\n","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:34:40.323718Z","iopub.execute_input":"2023-11-12T03:34:40.324075Z","iopub.status.idle":"2023-11-12T03:34:40.333503Z","shell.execute_reply.started":"2023-11-12T03:34:40.324051Z","shell.execute_reply":"2023-11-12T03:34:40.332063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.environ.get('KAGGLE_KERNEL_RUN_TYPE', 'Localhost') != 'Localhost':\n    # !pip install gseapy\n    # !pip install pyvis\n    # !pip install reactome2py\n    # !pip install anndata\n    # !pip install biomart\n    # !pip install rsrc\n    # !pip install --no-deps PyWGCNA\n\n    black_lst = pickle.load(open('/kaggle/input/opscp001/black_lst.pkl', 'rb'))\n    darkgrey_lst = pickle.load(open('/kaggle/input/opscp001/darkgrey_lst.pkl', 'rb'))\n    silver_lst = pickle.load(open('/kaggle/input/opscp001/silver_lst.pkl', 'rb'))\n","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-11-12T03:18:55.964414886Z","start_time":"2023-11-12T03:18:55.464692383Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:34:52.742154Z","iopub.execute_input":"2023-11-12T03:34:52.742530Z","iopub.status.idle":"2023-11-12T03:34:52.763226Z","shell.execute_reply.started":"2023-11-12T03:34:52.742507Z","shell.execute_reply":"2023-11-12T03:34:52.762435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}