{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-20T05:15:59.344717Z","iopub.execute_input":"2023-02-20T05:15:59.346100Z","iopub.status.idle":"2023-02-20T05:15:59.372902Z","shell.execute_reply.started":"2023-02-20T05:15:59.345977Z","shell.execute_reply":"2023-02-20T05:15:59.371588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_cite_inputs = pd.read_hdf('/kaggle/input/open-problems-multimodal/train_cite_inputs.h5')","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:15:59.378521Z","iopub.execute_input":"2023-02-20T05:15:59.379883Z","iopub.status.idle":"2023-02-20T05:17:03.547274Z","shell.execute_reply.started":"2023-02-20T05:15:59.379820Z","shell.execute_reply":"2023-02-20T05:17:03.545532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_cite_inputs.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:03.549236Z","iopub.execute_input":"2023-02-20T05:17:03.550398Z","iopub.status.idle":"2023-02-20T05:17:03.561157Z","shell.execute_reply.started":"2023-02-20T05:17:03.550348Z","shell.execute_reply":"2023-02-20T05:17:03.560220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_training_set_rna = pd.read_csv('/kaggle/input/machine-learning-challenge-2-prediction/training_set_rna.csv')","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:03.563962Z","iopub.execute_input":"2023-02-20T05:17:03.564688Z","iopub.status.idle":"2023-02-20T05:17:04.636477Z","shell.execute_reply.started":"2023-02-20T05:17:03.564648Z","shell.execute_reply":"2023-02-20T05:17:04.634601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_training_set_rna.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:04.638108Z","iopub.execute_input":"2023-02-20T05:17:04.638571Z","iopub.status.idle":"2023-02-20T05:17:04.648503Z","shell.execute_reply.started":"2023-02-20T05:17:04.638533Z","shell.execute_reply":"2023-02-20T05:17:04.646778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_cite_inputs = df_train_cite_inputs.rename(columns=lambda x: x.split('_')[1])","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:04.650682Z","iopub.execute_input":"2023-02-20T05:17:04.652177Z","iopub.status.idle":"2023-02-20T05:17:08.683189Z","shell.execute_reply.started":"2023-02-20T05:17:04.652120Z","shell.execute_reply":"2023-02-20T05:17:08.681778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_cite_inputs.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.684784Z","iopub.execute_input":"2023-02-20T05:17:08.685195Z","iopub.status.idle":"2023-02-20T05:17:08.733179Z","shell.execute_reply.started":"2023-02-20T05:17:08.685159Z","shell.execute_reply":"2023-02-20T05:17:08.731809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_training_set_rna.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.735605Z","iopub.execute_input":"2023-02-20T05:17:08.736540Z","iopub.status.idle":"2023-02-20T05:17:08.770884Z","shell.execute_reply.started":"2023-02-20T05:17:08.736483Z","shell.execute_reply":"2023-02-20T05:17:08.769011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rna_list = df_training_set_rna['Unnamed: 0'].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.773494Z","iopub.execute_input":"2023-02-20T05:17:08.773998Z","iopub.status.idle":"2023-02-20T05:17:08.784020Z","shell.execute_reply.started":"2023-02-20T05:17:08.773960Z","shell.execute_reply":"2023-02-20T05:17:08.782058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rna_list[:3]","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.786379Z","iopub.execute_input":"2023-02-20T05:17:08.787045Z","iopub.status.idle":"2023-02-20T05:17:08.802845Z","shell.execute_reply.started":"2023-02-20T05:17:08.786955Z","shell.execute_reply":"2023-02-20T05:17:08.801166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(rna_list)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.805916Z","iopub.execute_input":"2023-02-20T05:17:08.806531Z","iopub.status.idle":"2023-02-20T05:17:08.815133Z","shell.execute_reply.started":"2023-02-20T05:17:08.806477Z","shell.execute_reply":"2023-02-20T05:17:08.813796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_train_cite_inputs[df_train_cite_inputs.columns.intersection(rna_list)]","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.816801Z","iopub.execute_input":"2023-02-20T05:17:08.817171Z","iopub.status.idle":"2023-02-20T05:17:08.934391Z","shell.execute_reply.started":"2023-02-20T05:17:08.817138Z","shell.execute_reply":"2023-02-20T05:17:08.933168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.939635Z","iopub.execute_input":"2023-02-20T05:17:08.940035Z","iopub.status.idle":"2023-02-20T05:17:08.970705Z","shell.execute_reply.started":"2023-02-20T05:17:08.940002Z","shell.execute_reply":"2023-02-20T05:17:08.969270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:17:08.972528Z","iopub.execute_input":"2023-02-20T05:17:08.972919Z","iopub.status.idle":"2023-02-20T05:17:08.986110Z","shell.execute_reply.started":"2023-02-20T05:17:08.972887Z","shell.execute_reply":"2023-02-20T05:17:08.984663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr = df.corr(method='pearson')","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:36:44.970466Z","iopub.execute_input":"2023-02-20T05:36:44.970977Z","iopub.status.idle":"2023-02-20T05:37:56.134608Z","shell.execute_reply.started":"2023-02-20T05:36:44.970939Z","shell.execute_reply":"2023-02-20T05:37:56.133488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:37:56.171516Z","iopub.execute_input":"2023-02-20T05:37:56.172296Z","iopub.status.idle":"2023-02-20T05:37:56.200416Z","shell.execute_reply.started":"2023-02-20T05:37:56.172247Z","shell.execute_reply":"2023-02-20T05:37:56.198968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:37:56.203795Z","iopub.execute_input":"2023-02-20T05:37:56.204340Z","iopub.status.idle":"2023-02-20T05:37:56.216345Z","shell.execute_reply.started":"2023-02-20T05:37:56.204292Z","shell.execute_reply":"2023-02-20T05:37:56.214586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr.columns[df_corr.nunique() == 0]","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:37:56.218295Z","iopub.execute_input":"2023-02-20T05:37:56.219689Z","iopub.status.idle":"2023-02-20T05:37:56.325992Z","shell.execute_reply.started":"2023-02-20T05:37:56.219633Z","shell.execute_reply":"2023-02-20T05:37:56.325079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr.index[df_corr.nunique() == 0]","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:37:56.327536Z","iopub.execute_input":"2023-02-20T05:37:56.327901Z","iopub.status.idle":"2023-02-20T05:37:56.438329Z","shell.execute_reply.started":"2023-02-20T05:37:56.327862Z","shell.execute_reply":"2023-02-20T05:37:56.437018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"to_drop = ['LILRA5']","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:42:08.451454Z","iopub.execute_input":"2023-02-20T05:42:08.452278Z","iopub.status.idle":"2023-02-20T05:42:08.458920Z","shell.execute_reply.started":"2023-02-20T05:42:08.452219Z","shell.execute_reply":"2023-02-20T05:42:08.457109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr_without_zeros = df_corr.drop(to_drop).drop(to_drop, axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:42:10.601908Z","iopub.execute_input":"2023-02-20T05:42:10.602369Z","iopub.status.idle":"2023-02-20T05:42:10.620960Z","shell.execute_reply.started":"2023-02-20T05:42:10.602335Z","shell.execute_reply":"2023-02-20T05:42:10.618104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr_without_zeros.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:42:13.639260Z","iopub.execute_input":"2023-02-20T05:42:13.639697Z","iopub.status.idle":"2023-02-20T05:42:13.649238Z","shell.execute_reply.started":"2023-02-20T05:42:13.639665Z","shell.execute_reply":"2023-02-20T05:42:13.647502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_corr = df_corr.apply(lambda x: [float(i) for i in x])","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:18:20.931137Z","iopub.execute_input":"2023-02-20T05:18:20.931730Z","iopub.status.idle":"2023-02-20T05:18:20.943031Z","shell.execute_reply.started":"2023-02-20T05:18:20.931682Z","shell.execute_reply":"2023-02-20T05:18:20.941284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nfrom scipy.spatial import distance\nfrom scipy.cluster import hierarchy\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:18:20.944721Z","iopub.execute_input":"2023-02-20T05:18:20.945069Z","iopub.status.idle":"2023-02-20T05:18:21.541259Z","shell.execute_reply.started":"2023-02-20T05:18:20.945040Z","shell.execute_reply":"2023-02-20T05:18:21.539924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlations = np.abs(df_corr_without_zeros)\ncorrelations_array = np.abs(df_corr_without_zeros)\n\nm_linkage = hierarchy.linkage(distance.pdist(correlations_array), method='average')  \n\nclusters = hierarchy.fcluster(m_linkage, 0.3*distance.pdist(correlations_array).max(), 'distance')\n\n\nclusters_list = []\n\nindex_clusters = []\n\nval, cnt = np.unique(clusters, return_counts=True)\n\nfor i,cluster in enumerate(clusters):        \n    clusters_list.append([df_corr_without_zeros.index[i], cluster])\n\n    index_cluster = f\"{df_corr_without_zeros.index[i]}_cl{cluster}_sz{int(cnt[val == cluster])}\"\n    index_clusters.append(index_cluster)\n\n\nindex_dict = dict(zip(df_corr_without_zeros.index, index_clusters))\n\n\ndf_corr_without_zeros.index = df_corr_without_zeros.index.map(index_dict.get) \n\ncorrelations_1 = np.abs(df_corr_without_zeros)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:43:07.679851Z","iopub.execute_input":"2023-02-20T05:43:07.680388Z","iopub.status.idle":"2023-02-20T05:43:07.828089Z","shell.execute_reply.started":"2023-02-20T05:43:07.680349Z","shell.execute_reply":"2023-02-20T05:43:07.826039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g =  sns.clustermap(correlations_1, row_linkage=m_linkage, col_linkage=m_linkage, method=\"average\",cmap='vlag', figsize=(25, 25)) \nplt.setp(g.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:43:56.920515Z","iopub.execute_input":"2023-02-20T05:43:56.921091Z","iopub.status.idle":"2023-02-20T05:44:02.035065Z","shell.execute_reply.started":"2023-02-20T05:43:56.921043Z","shell.execute_reply":"2023-02-20T05:44:02.034063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_clusters = pd.DataFrame(clusters_list, columns=['cd', 'cluster'])","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:44:31.212267Z","iopub.execute_input":"2023-02-20T05:44:31.213352Z","iopub.status.idle":"2023-02-20T05:44:31.219852Z","shell.execute_reply.started":"2023-02-20T05:44:31.213306Z","shell.execute_reply":"2023-02-20T05:44:31.218920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr_without_zeros_1 = df_corr.drop(to_drop).drop(to_drop, axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:45:18.850493Z","iopub.execute_input":"2023-02-20T05:45:18.851010Z","iopub.status.idle":"2023-02-20T05:45:18.861627Z","shell.execute_reply.started":"2023-02-20T05:45:18.850973Z","shell.execute_reply":"2023-02-20T05:45:18.860661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_corr_without_zeros_1.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:45:25.973041Z","iopub.execute_input":"2023-02-20T05:45:25.973501Z","iopub.status.idle":"2023-02-20T05:45:25.980008Z","shell.execute_reply.started":"2023-02-20T05:45:25.973467Z","shell.execute_reply":"2023-02-20T05:45:25.979187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pearson_v_counts = df_clusters['cluster'].value_counts().head(10)\n\npearson_df = df_clusters.merge(pearson_v_counts.to_frame(),\n                                left_on='cluster',\n                                right_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:45:51.682826Z","iopub.execute_input":"2023-02-20T05:45:51.683375Z","iopub.status.idle":"2023-02-20T05:45:51.705204Z","shell.execute_reply.started":"2023-02-20T05:45:51.683335Z","shell.execute_reply":"2023-02-20T05:45:51.703816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pearson_group = pearson_df.groupby('cluster')\n\npearson_df2 = pearson_group.apply(lambda x: x['cd'].unique())\ns = pearson_df2.str.len().sort_values(ascending=False).index\npearson_df2 = pearson_df2.reindex(s)\npearson = pearson_df2.reset_index(drop=True)\n\nfor x in pearson:\n    mask = df_corr_without_zeros_1.loc[x, x].abs()\n    result = mask.values.mean()\n    print('cluster size: ', len(x), ', mean abs correlation per cluster: ', np.round(result,3) , ', genes: ', list(x)) \n    print()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:46:14.153350Z","iopub.execute_input":"2023-02-20T05:46:14.153808Z","iopub.status.idle":"2023-02-20T05:46:14.189766Z","shell.execute_reply.started":"2023-02-20T05:46:14.153774Z","shell.execute_reply":"2023-02-20T05:46:14.188762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_list = []\nfor n in pearson:\n    mask = df_corr_without_zeros_1.loc[n, n].abs()\n    result = mask.values.mean()\n    result_list.append([len(n), result])","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:46:45.487053Z","iopub.execute_input":"2023-02-20T05:46:45.487535Z","iopub.status.idle":"2023-02-20T05:46:45.513435Z","shell.execute_reply.started":"2023-02-20T05:46:45.487493Z","shell.execute_reply":"2023-02-20T05:46:45.512049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cluster_mean_value = pd.DataFrame(result_list, columns=['cluster', 'mean_value'])","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:46:54.689795Z","iopub.execute_input":"2023-02-20T05:46:54.690340Z","iopub.status.idle":"2023-02-20T05:46:54.697527Z","shell.execute_reply.started":"2023-02-20T05:46:54.690278Z","shell.execute_reply":"2023-02-20T05:46:54.696420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cluster_mean_value = df_cluster_mean_value.sort_values(by=['mean_value'], ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:47:02.448825Z","iopub.execute_input":"2023-02-20T05:47:02.450252Z","iopub.status.idle":"2023-02-20T05:47:02.457168Z","shell.execute_reply.started":"2023-02-20T05:47:02.450178Z","shell.execute_reply":"2023-02-20T05:47:02.455751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_cluster_mean_value.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T05:47:09.703371Z","iopub.execute_input":"2023-02-20T05:47:09.703803Z","iopub.status.idle":"2023-02-20T05:47:09.717133Z","shell.execute_reply.started":"2023-02-20T05:47:09.703769Z","shell.execute_reply":"2023-02-20T05:47:09.715728Z"},"trusted":true},"execution_count":null,"outputs":[]}]}