{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Analysis with normal distribution and ranking**\n\n# Machine learning not used.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import QuantileTransformer","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:15:31.233874Z","iopub.execute_input":"2023-09-18T14:15:31.234237Z","iopub.status.idle":"2023-09-18T14:15:31.313569Z","shell.execute_reply.started":"2023-09-18T14:15:31.234208Z","shell.execute_reply":"2023-09-18T14:15:31.312640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet')\nde_train","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:22.883776Z","iopub.execute_input":"2023-09-18T14:06:22.884317Z","iopub.status.idle":"2023-09-18T14:06:26.050719Z","shell.execute_reply.started":"2023-09-18T14:06:22.884286Z","shell.execute_reply":"2023-09-18T14:06:26.049568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train['control'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.054433Z","iopub.execute_input":"2023-09-18T14:06:26.054929Z","iopub.status.idle":"2023-09-18T14:06:26.067240Z","shell.execute_reply.started":"2023-09-18T14:06:26.054888Z","shell.execute_reply":"2023-09-18T14:06:26.066143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train.loc[de_train['control']==True]","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.070711Z","iopub.execute_input":"2023-09-18T14:06:26.071479Z","iopub.status.idle":"2023-09-18T14:06:26.119886Z","shell.execute_reply.started":"2023-09-18T14:06:26.071427Z","shell.execute_reply":"2023-09-18T14:06:26.118727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_Bcell = de_train[de_train['cell_type']=='B cells']\nde_train_Bcell","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.121238Z","iopub.execute_input":"2023-09-18T14:06:26.121624Z","iopub.status.idle":"2023-09-18T14:06:26.166060Z","shell.execute_reply.started":"2023-09-18T14:06:26.121587Z","shell.execute_reply":"2023-09-18T14:06:26.165120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_Mcell = de_train[de_train['cell_type']=='Myeloid cells']\nde_train_Mcell","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.168012Z","iopub.execute_input":"2023-09-18T14:06:26.168409Z","iopub.status.idle":"2023-09-18T14:06:26.211887Z","shell.execute_reply.started":"2023-09-18T14:06:26.168371Z","shell.execute_reply":"2023-09-18T14:06:26.210808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/id_map.csv')\nprint(id_map['cell_type'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.213487Z","iopub.execute_input":"2023-09-18T14:06:26.213916Z","iopub.status.idle":"2023-09-18T14:06:26.233026Z","shell.execute_reply.started":"2023-09-18T14:06:26.213876Z","shell.execute_reply":"2023-09-18T14:06:26.231973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.234268Z","iopub.execute_input":"2023-09-18T14:06:26.234560Z","iopub.status.idle":"2023-09-18T14:06:26.247063Z","shell.execute_reply.started":"2023-09-18T14:06:26.234534Z","shell.execute_reply":"2023-09-18T14:06:26.246221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# About B-cell expression levels","metadata":{}},{"cell_type":"code","source":"Bcell_list = id_map[id_map['cell_type']=='B cells']['sm_name']\nBcell_list","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.248300Z","iopub.execute_input":"2023-09-18T14:06:26.249204Z","iopub.status.idle":"2023-09-18T14:06:26.263181Z","shell.execute_reply.started":"2023-09-18T14:06:26.249174Z","shell.execute_reply":"2023-09-18T14:06:26.262120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Bcell_mean = []\nBcell_sd = []\nfor i in de_train_Bcell.columns[5:]:\n    BM = de_train_Bcell[i].mean()\n    BS = de_train_Bcell[i].std()\n    Bcell_mean.append(BM)\n    Bcell_sd.append(BS)","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:26.267540Z","iopub.execute_input":"2023-09-18T14:06:26.267936Z","iopub.status.idle":"2023-09-18T14:06:31.561474Z","shell.execute_reply.started":"2023-09-18T14:06:26.267906Z","shell.execute_reply":"2023-09-18T14:06:31.560332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import stats\n\nBcell_base_exp = []\nfor i in range(de_train_Bcell.iloc[:,5:].shape[1]):\n    BBE = stats.norm.rvs(loc=Bcell_mean[i], scale=Bcell_sd[i], size=128)\n    Bcell_base_exp.append(BBE)","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:31.563383Z","iopub.execute_input":"2023-09-18T14:06:31.563879Z","iopub.status.idle":"2023-09-18T14:06:32.882436Z","shell.execute_reply.started":"2023-09-18T14:06:31.563836Z","shell.execute_reply":"2023-09-18T14:06:32.881137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B = pd.DataFrame(Bcell_base_exp).T\nB.index = Bcell_list\nB.columns = de_train_Bcell.columns[5:]","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:32.884057Z","iopub.execute_input":"2023-09-18T14:06:32.884491Z","iopub.status.idle":"2023-09-18T14:06:34.281418Z","shell.execute_reply.started":"2023-09-18T14:06:32.884449Z","shell.execute_reply":"2023-09-18T14:06:34.280405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_2 = de_train.drop(\"cell_type\",axis=1).drop(\"sm_lincs_id\",axis=1).drop(\"SMILES\",axis=1).drop(\"control\",axis=1)\nde_train_3 = de_train_2.groupby('sm_name').mean().loc[Bcell_list,:]\nde_train_3","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:34.282667Z","iopub.execute_input":"2023-09-18T14:06:34.282987Z","iopub.status.idle":"2023-09-18T14:06:34.690684Z","shell.execute_reply.started":"2023-09-18T14:06:34.282959Z","shell.execute_reply":"2023-09-18T14:06:34.689326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_4 = de_train_3.rank(method='first')\nde_train_4","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:34.692282Z","iopub.execute_input":"2023-09-18T14:06:34.692610Z","iopub.status.idle":"2023-09-18T14:06:34.939281Z","shell.execute_reply.started":"2023-09-18T14:06:34.692581Z","shell.execute_reply":"2023-09-18T14:06:34.938173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B['A1BG'].sort_values(key = lambda s: s.map(de_train_4['A1BG']) )","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:34.940676Z","iopub.execute_input":"2023-09-18T14:06:34.941020Z","iopub.status.idle":"2023-09-18T14:06:34.954753Z","shell.execute_reply.started":"2023-09-18T14:06:34.940991Z","shell.execute_reply":"2023-09-18T14:06:34.953634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B2 = []\nfor i in de_train_4.columns:\n    B2.append(B[i].sort_values(key = lambda s: s.map(de_train_4[i])))\nB2 = pd.DataFrame(B2).T\nB2.index = Bcell_list\nB2.columns = de_train_Bcell.columns[5:]","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:34.956240Z","iopub.execute_input":"2023-09-18T14:06:34.957111Z","iopub.status.idle":"2023-09-18T14:06:50.606916Z","shell.execute_reply.started":"2023-09-18T14:06:34.957078Z","shell.execute_reply":"2023-09-18T14:06:50.606012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B2","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:50.608157Z","iopub.execute_input":"2023-09-18T14:06:50.608548Z","iopub.status.idle":"2023-09-18T14:06:50.649770Z","shell.execute_reply.started":"2023-09-18T14:06:50.608510Z","shell.execute_reply":"2023-09-18T14:06:50.648806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Quantile adjustment**","metadata":{}},{"cell_type":"code","source":"B3 = B2.T\nsns.histplot(B3.loc[:,\"YK 4-279\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:20:54.100922Z","iopub.execute_input":"2023-09-18T14:20:54.101293Z","iopub.status.idle":"2023-09-18T14:20:54.962748Z","shell.execute_reply.started":"2023-09-18T14:20:54.101265Z","shell.execute_reply":"2023-09-18T14:20:54.961957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trans = QuantileTransformer(n_quantiles=1000, output_distribution='normal')\nB4 = pd.DataFrame(trans.fit_transform(B3),index = B3.index, columns=B3.columns)\nsns.histplot(B4.loc[:,\"YK 4-279\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:26:53.288133Z","iopub.execute_input":"2023-09-18T14:26:53.288526Z","iopub.status.idle":"2023-09-18T14:26:54.763378Z","shell.execute_reply.started":"2023-09-18T14:26:53.288494Z","shell.execute_reply":"2023-09-18T14:26:54.762354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B5 = B4.T","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:27:33.287441Z","iopub.execute_input":"2023-09-18T14:27:33.287820Z","iopub.status.idle":"2023-09-18T14:27:33.293335Z","shell.execute_reply.started":"2023-09-18T14:27:33.287777Z","shell.execute_reply":"2023-09-18T14:27:33.292229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# About Myeloid cells expression levels","metadata":{}},{"cell_type":"code","source":"Mcell_list = id_map[id_map['cell_type']=='Myeloid cells']['sm_name']\nMcell_list","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:50.651208Z","iopub.execute_input":"2023-09-18T14:06:50.651736Z","iopub.status.idle":"2023-09-18T14:06:50.660747Z","shell.execute_reply.started":"2023-09-18T14:06:50.651708Z","shell.execute_reply":"2023-09-18T14:06:50.659610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Mcell_mean = []\nMcell_sd = []\nfor i in de_train_Mcell.columns[5:]:\n    MM = de_train_Mcell[i].mean()\n    MS = de_train_Mcell[i].std()\n    Mcell_mean.append(MM)\n    Mcell_sd.append(MS)\n    \nMcell_base_exp = []\nfor i in range(de_train_Mcell.iloc[:,5:].shape[1]):\n    MBE = stats.norm.rvs(loc=Mcell_mean[i], scale=Mcell_sd[i], size=127)\n    Mcell_base_exp.append(MBE)","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:50.662095Z","iopub.execute_input":"2023-09-18T14:06:50.662402Z","iopub.status.idle":"2023-09-18T14:06:56.928462Z","shell.execute_reply.started":"2023-09-18T14:06:50.662375Z","shell.execute_reply":"2023-09-18T14:06:56.927548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"M = pd.DataFrame(Mcell_base_exp).T\nM.index = Mcell_list\nM.columns = de_train_Mcell.columns[5:]","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:56.929880Z","iopub.execute_input":"2023-09-18T14:06:56.930181Z","iopub.status.idle":"2023-09-18T14:06:58.175306Z","shell.execute_reply.started":"2023-09-18T14:06:56.930153Z","shell.execute_reply":"2023-09-18T14:06:58.174358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train_13 = de_train_2.groupby('sm_name').mean().loc[Mcell_list,:]\nde_train_14 = de_train_13.rank(method='first')\nde_train_14","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:58.176546Z","iopub.execute_input":"2023-09-18T14:06:58.176955Z","iopub.status.idle":"2023-09-18T14:06:58.629916Z","shell.execute_reply.started":"2023-09-18T14:06:58.176917Z","shell.execute_reply":"2023-09-18T14:06:58.628844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"M2 = []\nfor i in de_train_14.columns:\n    M2.append(M[i].sort_values(key = lambda s: s.map(de_train_14[i])))\nM2 = pd.DataFrame(M2).T\nM2.index = Mcell_list\nM2.columns = de_train_Mcell.columns[5:]","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:06:58.631603Z","iopub.execute_input":"2023-09-18T14:06:58.632188Z","iopub.status.idle":"2023-09-18T14:07:14.453532Z","shell.execute_reply.started":"2023-09-18T14:06:58.632149Z","shell.execute_reply":"2023-09-18T14:07:14.452361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"M2","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:07:14.454918Z","iopub.execute_input":"2023-09-18T14:07:14.455315Z","iopub.status.idle":"2023-09-18T14:07:14.495473Z","shell.execute_reply.started":"2023-09-18T14:07:14.455277Z","shell.execute_reply":"2023-09-18T14:07:14.494275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Quantile adjustment**","metadata":{}},{"cell_type":"code","source":"M3 = M2.T\nsns.histplot(M3.loc[:,\"YK 4-279\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:28:08.372734Z","iopub.execute_input":"2023-09-18T14:28:08.373231Z","iopub.status.idle":"2023-09-18T14:28:09.203702Z","shell.execute_reply.started":"2023-09-18T14:28:08.373195Z","shell.execute_reply":"2023-09-18T14:28:09.202534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"M4 = pd.DataFrame(trans.fit_transform(M3),index = M3.index, columns=M3.columns)\nsns.histplot(M4.loc[:,\"YK 4-279\"])","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:29:05.219184Z","iopub.execute_input":"2023-09-18T14:29:05.220307Z","iopub.status.idle":"2023-09-18T14:29:06.660989Z","shell.execute_reply.started":"2023-09-18T14:29:05.220257Z","shell.execute_reply":"2023-09-18T14:29:06.660043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"M5 = M4.T","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:29:33.475037Z","iopub.execute_input":"2023-09-18T14:29:33.475424Z","iopub.status.idle":"2023-09-18T14:29:33.481493Z","shell.execute_reply.started":"2023-09-18T14:29:33.475394Z","shell.execute_reply":"2023-09-18T14:29:33.480201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"ss = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv')\nss","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:07:14.560713Z","iopub.execute_input":"2023-09-18T14:07:14.561013Z","iopub.status.idle":"2023-09-18T14:07:19.518089Z","shell.execute_reply.started":"2023-09-18T14:07:14.560987Z","shell.execute_reply":"2023-09-18T14:07:19.517035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pd.concat([B5,M5])\nresult.index = range(255)\nresult","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:35:16.448113Z","iopub.execute_input":"2023-09-18T14:35:16.448549Z","iopub.status.idle":"2023-09-18T14:35:16.496984Z","shell.execute_reply.started":"2023-09-18T14:35:16.448512Z","shell.execute_reply":"2023-09-18T14:35:16.495766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit = pd.DataFrame(result, columns = de_train.columns[5:])\ndf_submit.index.name = 'id'\nprint( df_submit.shape )\ndisplay(df_submit)\ndf_submit.to_csv('submission_vol1.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-18T14:35:25.683177Z","iopub.execute_input":"2023-09-18T14:35:25.683572Z","iopub.status.idle":"2023-09-18T14:35:39.189454Z","shell.execute_reply.started":"2023-09-18T14:35:25.683540Z","shell.execute_reply":"2023-09-18T14:35:39.188301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}