{"metadata":{"kernelspec":{"display_name":"saturn (Python 3)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.15"},"papermill":{"default_parameters":{},"duration":1767.896818,"end_time":"2023-10-03T19:16:48.560142","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-10-03T18:47:20.663324","version":"2.4.0"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59094,"databundleVersionId":7010844,"sourceType":"competition"},{"sourceId":7123483,"sourceType":"datasetVersion","datasetId":4109071}],"dockerImageVersionId":30615,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\nt0start = time.time()\nfrom fastai.collab import *\nfrom fastai.tabular.all import *","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2023-11-25T20:02:59.483334Z","iopub.status.busy":"2023-11-25T20:02:59.483082Z","iopub.status.idle":"2023-11-25T20:03:02.631026Z","shell.execute_reply":"2023-11-25T20:03:02.630103Z","shell.execute_reply.started":"2023-11-25T20:02:59.483309Z"},"papermill":{"duration":6.509942,"end_time":"2023-10-03T18:47:30.760703","exception":false,"start_time":"2023-10-03T18:47:24.250761","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_seed = 42 #272 #582","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:02.633351Z","iopub.status.busy":"2023-11-25T20:03:02.632832Z","iopub.status.idle":"2023-11-25T20:03:02.637028Z","shell.execute_reply":"2023-11-25T20:03:02.636321Z","shell.execute_reply.started":"2023-11-25T20:03:02.633320Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading and Melting Train and Test Data","metadata":{"tags":[]}},{"cell_type":"code","source":"%%time\nfn = '/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet'\ndf_de_train = pd.read_parquet(fn)# , index_col = 0)\ntrain_df = df_de_train.melt(id_vars=['cell_type', 'sm_name'], value_vars=df_de_train.iloc[:,5:].columns, var_name='gene', value_name='value')","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:02.638235Z","iopub.status.busy":"2023-11-25T20:03:02.637988Z","iopub.status.idle":"2023-11-25T20:03:04.798393Z","shell.execute_reply":"2023-11-25T20:03:04.797699Z","shell.execute_reply.started":"2023-11-25T20:03:02.638212Z"},"papermill":{"duration":163.861162,"end_time":"2023-10-03T18:50:18.214746","exception":false,"start_time":"2023-10-03T18:47:34.353584","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = '/kaggle/input/open-problems-single-cell-perturbations/id_map.csv'\ndf_id_map = pd.read_csv(fn)\nfn = '/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv'\ndf = pd.read_csv(fn, index_col = 0)\n\ncols_to_add = df_de_train.iloc[:,5:].columns\ncols_to_add\n\ndf_zeros = pd.DataFrame(0.0, columns=cols_to_add, index=df_id_map.index)\ndf_zeros\n\ndf_id_map_preds = pd.concat([df_id_map, df_zeros], axis=1)\ntest_df = df_id_map_preds.melt(id_vars=['cell_type', 'sm_name'], value_vars=df_id_map_preds.iloc[:,3:].columns, var_name='gene', value_name='value')","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:04.799618Z","iopub.status.busy":"2023-11-25T20:03:04.799315Z","iopub.status.idle":"2023-11-25T20:03:07.017625Z","shell.execute_reply":"2023-11-25T20:03:07.016924Z","shell.execute_reply.started":"2023-11-25T20:03:04.799599Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Embeddings","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n\ndef reduce_emb_dim(data, n_comp=35, random_state=42):\n    embname = data.columns[1][:-1]\n    Y = data.iloc[:,1:]\n    scaler = StandardScaler()\n    Y_std = scaler.fit_transform(Y)\n    reducer = PCA(n_components=n_comp, random_state=random_state)\n\n    Yr = reducer.fit_transform(Y_std)\n    column_names = [f'{embname}pca{n_comp}_{i}' for i in range(n_comp)]\n    reduced_data = pd.DataFrame(Yr, columns = column_names)\n    reduced_data = pd.concat([data.iloc[:, 0], reduced_data], axis=1)\n    return reduced_data","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:07.019654Z","iopub.status.busy":"2023-11-25T20:03:07.019348Z","iopub.status.idle":"2023-11-25T20:03:07.052806Z","shell.execute_reply":"2023-11-25T20:03:07.052219Z","shell.execute_reply.started":"2023-11-25T20:03:07.019633Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_embs = pd.read_csv('/kaggle/input/op2-single-cell-perturbations-tabmodnn-embeddings/cell_embeddings_no_pca.csv', index_col = 0)\nmol_embs = pd.read_csv('/kaggle/input/op2-single-cell-perturbations-tabmodnn-embeddings/molecular_embeddings_no_pca.csv', index_col = 0)\ngene_embs = pd.read_parquet('/kaggle/input/op2-single-cell-perturbations-tabmodnn-embeddings/gene_embeddings_no_pca.parquet')","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:07.053778Z","iopub.status.busy":"2023-11-25T20:03:07.053579Z","iopub.status.idle":"2023-11-25T20:03:07.209562Z","shell.execute_reply":"2023-11-25T20:03:07.208800Z","shell.execute_reply.started":"2023-11-25T20:03:07.053761Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_embs = reduce_emb_dim(cell_embs, n_comp=2, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:07.210945Z","iopub.status.busy":"2023-11-25T20:03:07.210601Z","iopub.status.idle":"2023-11-25T20:03:07.220396Z","shell.execute_reply":"2023-11-25T20:03:07.219722Z","shell.execute_reply.started":"2023-11-25T20:03:07.210920Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(cell_embs.cembpca2_0, cell_embs.cembpca2_1)\nfor i, row in cell_embs.iterrows():\n    plt.text(row['cembpca2_0'] + 0.1, row['cembpca2_1'] + 0.1, str(row['cell_type']), fontsize=9)\nplt.title('cell_type')\nplt.xlim((-3, 3))\nplt.ylim((-3, 3))\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:07.221825Z","iopub.status.busy":"2023-11-25T20:03:07.221377Z","iopub.status.idle":"2023-11-25T20:03:07.459356Z","shell.execute_reply":"2023-11-25T20:03:07.458683Z","shell.execute_reply.started":"2023-11-25T20:03:07.221801Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gene_embs = reduce_emb_dim(gene_embs, n_comp=2, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:07.460697Z","iopub.status.busy":"2023-11-25T20:03:07.460325Z","iopub.status.idle":"2023-11-25T20:03:07.885851Z","shell.execute_reply":"2023-11-25T20:03:07.885129Z","shell.execute_reply.started":"2023-11-25T20:03:07.460674Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_data(data, labels, ax=None, figsize=(10, 4)):\n    if ax is None: _, ax = plt.subplots(figsize=figsize)\n    for i in np.unique(labels):\n        samples = data[labels==i]\n        ax.scatter(samples[:,0], samples[:,1], s=2)","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:13:04.742288Z","iopub.status.busy":"2023-11-25T20:13:04.741430Z","iopub.status.idle":"2023-11-25T20:13:04.747070Z","shell.execute_reply":"2023-11-25T20:13:04.746333Z","shell.execute_reply.started":"2023-11-25T20:13:04.742261Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.cluster import DBSCAN\ndata = gene_embs.iloc[:, 1:].values\nclustering = DBSCAN(eps=1.5, min_samples=4).fit(data)\ncluster_labels = np.unique(clustering.labels_)\ncluster_labels","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:13:05.075198Z","iopub.status.busy":"2023-11-25T20:13:05.074746Z","iopub.status.idle":"2023-11-25T20:13:05.618955Z","shell.execute_reply":"2023-11-25T20:13:05.618208Z","shell.execute_reply.started":"2023-11-25T20:13:05.075170Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_data(data, clustering.labels_, figsize=(10, 10))\nrandom_array = list(np.random.randint(0, len(gene_embs), size=100))\nfor i, row in gene_embs.iterrows():\n    if i in random_array:\n        plt.scatter(row['gembpca2_0'], row['gembpca2_1'], marker='x', color='r')\n        plt.text(row['gembpca2_0'], row['gembpca2_1'], str(row['gene']), fontsize=9)\nplt.title('Gene Embeddings PCA 2 Components')\n# plt.xlim((-3, 3))\n# plt.ylim((-3, 3))\nplt.savefig('gene_embeds.png', format='png', dpi=300)\nplt.show()\n","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:13:25.105702Z","iopub.status.busy":"2023-11-25T20:13:25.105258Z","iopub.status.idle":"2023-11-25T20:13:26.906573Z","shell.execute_reply":"2023-11-25T20:13:26.905243Z","shell.execute_reply.started":"2023-11-25T20:13:25.105676Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mol_embs = reduce_emb_dim(mol_embs, n_comp=2, random_state=random_seed)","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:10.038541Z","iopub.status.busy":"2023-11-25T20:03:10.038250Z","iopub.status.idle":"2023-11-25T20:03:10.049843Z","shell.execute_reply":"2023-11-25T20:03:10.049144Z","shell.execute_reply.started":"2023-11-25T20:03:10.038513Z"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(13,8))\nplt.scatter(mol_embs.membpca2_0, mol_embs.membpca2_1)\nrandom_array = list(np.random.randint(0, len(mol_embs), size=20))\nfor i, row in mol_embs.iterrows():\n    if i in random_array:\n        plt.scatter(row['membpca2_0'], row['membpca2_1'], marker='x', color='r')\n        plt.text(row['membpca2_0'], row['membpca2_1'], str(row['sm_name']), fontsize=9)\nplt.title('sm_name')\n# plt.xlim((-3, 3))\n# plt.ylim((-3, 3))\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2023-11-25T20:03:10.051155Z","iopub.status.busy":"2023-11-25T20:03:10.050851Z","iopub.status.idle":"2023-11-25T20:03:10.321172Z","shell.execute_reply":"2023-11-25T20:03:10.320528Z","shell.execute_reply.started":"2023-11-25T20:03:10.051134Z"},"tags":[]},"execution_count":null,"outputs":[]}]}