{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"inspired by the great EDA '[1] OP2 - EDA & LinearSVR & RegressorChain' by MEHRAN KAZEMINIA\n","metadata":{"execution":{"iopub.status.busy":"2023-10-28T05:48:48.425131Z","iopub.execute_input":"2023-10-28T05:48:48.425395Z","iopub.status.idle":"2023-10-28T05:48:48.428567Z","shell.execute_reply.started":"2023-10-28T05:48:48.425375Z","shell.execute_reply":"2023-10-28T05:48:48.427946Z"}}},{"cell_type":"markdown","source":"
\n
\n\n
\n

Open Problems Single-Cell Perturbations

\n

Exploratory Data Analysis

\n
\n
","metadata":{"execution":{"iopub.status.busy":"2023-10-28T05:51:10.470612Z","iopub.execute_input":"2023-10-28T05:51:10.470953Z","iopub.status.idle":"2023-10-28T05:51:10.475971Z","shell.execute_reply.started":"2023-10-28T05:51:10.470929Z","shell.execute_reply":"2023-10-28T05:51:10.474941Z"}}},{"cell_type":"markdown","source":"## Import Libraries","metadata":{}},{"cell_type":"code","source":"import warnings # suppress warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport glob\nimport random\nimport numpy as np \nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom scipy import stats\nfrom pathlib import Path\nfrom itertools import groupby\nfrom collections import Counter\nimport matplotlib.pyplot as plt\nimport plotly.figure_factory as ff\nimport plotly.express as px\nfrom IPython.core.display import HTML\n%matplotlib inline","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore the competition data\n### 1) de_train.parquet - Aggregated differential expression data in dense array format.\n","metadata":{}},{"cell_type":"code","source":"de_train = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet')\npd.set_option('display.max_columns', 500)\nde_train.sample(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The shape of the DE data : {de_train.shape}\\n\")\nprint(f\"The number cell_types performed in this experiment are {de_train.cell_type.nunique()} cells \\n\")\nprint(f\"The cell_types names are: {de_train.cell_type.unique()} \\n\")\nprint(f\"The number of sm_name compunds performed in this experiment are {de_train.sm_name.nunique()} compounds\\n\")\nprint(f\"The number of sm_lincs_id compunds performed in this experiment are {de_train.sm_lincs_id.nunique()} compounds\\n\")\nprint(f\"The number of SMILES: {de_train.SMILES.nunique()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt = Counter()\nfor sm_name in de_train.sm_name:\n cnt[sm_name] += 1\nprint(f\"The top most common compounds are {cnt.most_common(10)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(de_train.groupby('cell_type')['control'].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Countplot for cell types and control types","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(20, 8))\nsns.countplot(data=de_train, x='cell_type', ax=axes[0])\nsns.countplot(data=de_train, x='control', ax=axes[1])\naxes[0].set_title('Cell Types count', fontsize=15)\naxes[1].set_title('Control Types counts', fontsize=15);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ditribution for mean of gene DE","metadata":{}},{"cell_type":"code","source":"train = de_train.copy()\ntrain['DE_Mean'] = train[[x for x in train.columns[5:]]].mean(axis=1)\nsns.displot(data=train, x=f'DE_Mean', height=5, aspect=3)\nprint('Distribution of DE mean:\\n', train['DE_Mean'].describe())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.displot(data=train, x = f'DE_Mean', hue='cell_type', height=6, aspect=2)\nsns.displot(data=train, x = f'DE_Mean', hue='control', height=6, aspect=2);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install rdkit","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from rdkit import Chem\nfrom rdkit.Chem import Draw","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n# Thanks to: https://github.com/DocMinus\n\ndef element_count(data, N):\n smiles = data['SMILES'].iloc[N]\n\n pattern = \"Si|Ti|Al|Zn|Pd|Pt|Br?|Cl?|N|O|S|P|F|I|B|b|c|n|o|s|p\" \n regex = re.compile(pattern)\n elements = [token for token in regex.findall(smiles)]\n ele_length = len(elements)\n lowercase_pattern = \"b|c|n|o|s|p\"\n regex_low = re.compile(lowercase_pattern)\n for i in range(ele_length):\n if regex_low.findall(elements[i]):\n elements[i] = elements[i].upper()\n element_count = [elements.count(ele) for ele in elements]\n formula = dict(zip(elements, element_count)) \n\n print('==> Element_Count: ', str(formula))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = random.randrange(len(de_train))\ncols_sel = ['cell_type','sm_name','sm_lincs_id','SMILES']\n\nsmiles = de_train['SMILES'][N]\nmol = Chem.MolFromSmiles(smiles)\nimg = Draw.MolToImage(mol, size=(700, 300), fitImage=True) \ndisplay(pd.DataFrame(de_train[cols_sel].iloc[N]))\n\nelement_count(de_train, N)\nimg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PCA Plots","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nimport time\nx = de_train[[x for x in de_train.columns[5:]]].values\npca = PCA()\nt0 = time.time()\nr = pca.fit_transform(x.copy())\nprint(time.time()-t0, 'seconds passed for PCS')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,8))\nc=0\nfor f in ['cell_type', 'control']:\n c += 1; \n fig.add_subplot(1, 2, c)\n sns.scatterplot(x=r[:,0], y=r[:,1], hue=de_train[f])\n plt.title('colored by '+f)\n \nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## UMAP plots","metadata":{}},{"cell_type":"code","source":"import umap\nx = de_train[[x for x in de_train.columns[5:]]].values\n\nt0 = time.time()\nr = umap.UMAP().fit_transform(x.copy())\nprint(time.time()-t0, 'seconds passed for umap')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,8))\nc = 0\nfor f in ['cell_type', 'control']:\n c +=1;\n fig.add_subplot(1, 2, c)\n sns.scatterplot(x=r[:,0], y=r[:,1], hue=de_train[f])\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2) adata_train.parquet - Unaggregated count and normalized data in COO sparse-array format.\n","metadata":{}},{"cell_type":"code","source":"adata_train = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/adata_train.parquet')\npd.set_option('display.max_rows', 500)\nadata_train.sample(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The shape of the adata_train: {adata_train.shape}\\n\")\nprint(f\"The number of unique obs_id in adata_train are {adata_train.obs_id.nunique()} id \\n\")\nprint(f\"The number of unique genes in adata_train are {adata_train.gene.nunique()} genes \\n\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3) adata_obs_meta.csv - Observation metadata for adata_train.","metadata":{}},{"cell_type":"code","source":"adata_obs = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/adata_obs_meta.csv')\npd.set_option('display.max_rows', 500)\nadata_obs.sample(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The shape of the adata_obs: {adata_obs.shape}\\n\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4) id_map.csv -Identifies the cell_type / sm_name pair to be predicted for the given id.","metadata":{}},{"cell_type":"code","source":"id_map = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/id_map.csv')\nprint(f\"The shape of id_map: {id_map.shape}\\n\")\nid_map.sample(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5) multiome_var_meta.csv ","metadata":{}},{"cell_type":"code","source":"multiome_var_meta = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/multiome_var_meta.csv')\nmultiome_var_meta.sample(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"The shape of multiome_var_meta: {multiome_var_meta.shape}\\n\")\nprint(f\"The number of unique gene_id: {multiome_var_meta.gene_id.nunique()}\\n\")\nprint(f\"The feature type: {multiome_var_meta.feature_type.unique()}\\n\")\nprint(f\"The unique genome: {multiome_var_meta.genome.unique()}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=multiome_var_meta, x='feature_type');","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 6) sample_submission.csv ","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv')\nsample_submission.head()","metadata":{},"execution_count":null,"outputs":[]}]}