{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"****PLEASE UPVOTE  https://www.kaggle.com/code/alexandervc/op2-eda-baseline-s/notebook?scriptVersionId=142991903****","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\nimport pandas as pd\n\npd.set_option('display.max_columns', 30)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:49.467215Z","iopub.execute_input":"2023-10-26T09:32:49.467682Z","iopub.status.idle":"2023-10-26T09:32:49.899961Z","shell.execute_reply.started":"2023-10-26T09:32:49.467643Z","shell.execute_reply":"2023-10-26T09:32:49.898873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\nSEED = 42\nnp.random.seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:49.902294Z","iopub.execute_input":"2023-10-26T09:32:49.902995Z","iopub.status.idle":"2023-10-26T09:32:49.908217Z","shell.execute_reply.started":"2023-10-26T09:32:49.902953Z","shell.execute_reply":"2023-10-26T09:32:49.907147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet')\nde_train.shape\nde_train.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:49.909789Z","iopub.execute_input":"2023-10-26T09:32:49.910206Z","iopub.status.idle":"2023-10-26T09:32:51.492886Z","shell.execute_reply.started":"2023-10-26T09:32:49.910165Z","shell.execute_reply":"2023-10-26T09:32:51.491682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = de_train.columns[5:]\ntrain = de_train[features]\navg_along_cols = train.mean(axis=1)\ntrain['avg'] = avg_along_cols\navg_sort = train.sort_values(by='avg', ascending=False)\n\nsm_name_best = de_train.loc[avg_sort.index[:20], 'sm_name'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:51.495325Z","iopub.execute_input":"2023-10-26T09:32:51.495708Z","iopub.status.idle":"2023-10-26T09:32:51.621056Z","shell.execute_reply.started":"2023-10-26T09:32:51.495678Z","shell.execute_reply":"2023-10-26T09:32:51.619832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\ndel features\ndel train\ndel avg_along_cols\ndel avg_sort\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:51.622561Z","iopub.execute_input":"2023-10-26T09:32:51.622923Z","iopub.status.idle":"2023-10-26T09:32:51.697558Z","shell.execute_reply.started":"2023-10-26T09:32:51.622893Z","shell.execute_reply":"2023-10-26T09:32:51.696444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_type = de_train['cell_type']\nsm_name_non_nib = de_train['sm_name'].copy()\nsm_name_nib = de_train['sm_name'].copy()\n\nnibs = [sm for sm in de_train['sm_name'] if sm.endswith('nib')]\nisin_nibs = sm_name_non_nib.isin(nibs)\nsm_name_non_nib[~isin_nibs] = 'non_nib'\nsm_name_nib[isin_nibs] = '*nib'\n\ndel isin_nibs\ngc.collect()\n\ncontrol = de_train['control']\nisin_best = de_train['sm_name'].isin(sm_name_best)\nsm_name_best_nan = de_train['sm_name'].copy()\nsm_name_best_nan[~isin_best] = np.nan","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:51.698992Z","iopub.execute_input":"2023-10-26T09:32:51.699435Z","iopub.status.idle":"2023-10-26T09:32:51.779055Z","shell.execute_reply.started":"2023-10-26T09:32:51.699393Z","shell.execute_reply":"2023-10-26T09:32:51.777759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = de_train.iloc[:, 5:]\n\nprint(X.shape)\nX","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:51.780863Z","iopub.execute_input":"2023-10-26T09:32:51.781383Z","iopub.status.idle":"2023-10-26T09:32:51.862074Z","shell.execute_reply.started":"2023-10-26T09:32:51.781324Z","shell.execute_reply":"2023-10-26T09:32:51.860817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\npca = PCA(n_components=100)\nX_pca = pca.fit_transform(X)\nprint(X_pca.shape)\nX_pca[:2]","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:51.863497Z","iopub.execute_input":"2023-10-26T09:32:51.863894Z","iopub.status.idle":"2023-10-26T09:32:54.903830Z","shell.execute_reply.started":"2023-10-26T09:32:51.863856Z","shell.execute_reply":"2023-10-26T09:32:54.900334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nconfig = [['CELL TYPE', cell_type], ['CONTROL', control], ['SM BEST', sm_name_best_nan]]\n\nfor i,j in [[0,1],[0,2],[0,3],[20, 20],[20, 50]]:\n    plot_idx = 0\n    plt.figure(figsize=(16, 8))\n    \n    for title, hue in config:\n        \n        plot_idx += 1\n        plt.subplot(1, len(config), plot_idx)\n        sns.scatterplot(x=X_pca[:, i], y=X_pca[:, j], hue=hue, palette='hsv', s=100)\n        \n        plt.xlabel(f'PCA{str(i)}', fontsize=10)\n        plt.ylabel(f'PCA{str(j)}', fontsize=10)\n        \n        plt.title(title, fontsize=12)\n        \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:32:54.905678Z","iopub.execute_input":"2023-10-26T09:32:54.906104Z","iopub.status.idle":"2023-10-26T09:33:02.025185Z","shell.execute_reply.started":"2023-10-26T09:32:54.906062Z","shell.execute_reply":"2023-10-26T09:33:02.024384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sm_name_best","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.028940Z","iopub.execute_input":"2023-10-26T09:33:02.029668Z","iopub.status.idle":"2023-10-26T09:33:02.037040Z","shell.execute_reply.started":"2023-10-26T09:33:02.029635Z","shell.execute_reply":"2023-10-26T09:33:02.035749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_train = de_train.iloc[:, :5]\npca_train['PCA_0'] = X_pca[:, 0]\npca_train['PCA_1'] = X_pca[:, 1]\npca_train['PCA_2'] = X_pca[:, 2]\n\npca_train","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.038456Z","iopub.execute_input":"2023-10-26T09:33:02.038930Z","iopub.status.idle":"2023-10-26T09:33:02.065028Z","shell.execute_reply.started":"2023-10-26T09:33:02.038897Z","shell.execute_reply":"2023-10-26T09:33:02.063635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_0_best = pca_train.sort_values(by='PCA_0', ascending=False)\nsm_name_pca_0_best = pca_train.loc[pca_0_best.index[:20], 'sm_name'].unique()\nsm_name_pca_0_best","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.066869Z","iopub.execute_input":"2023-10-26T09:33:02.067485Z","iopub.status.idle":"2023-10-26T09:33:02.079909Z","shell.execute_reply.started":"2023-10-26T09:33:02.067444Z","shell.execute_reply":"2023-10-26T09:33:02.078653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_1_best = pca_train.sort_values(by='PCA_1', ascending=False)\nsm_name_pca_1_best = pca_train.loc[pca_1_best.index[:20], 'sm_name'].unique()\nsm_name_pca_1_best","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.081827Z","iopub.execute_input":"2023-10-26T09:33:02.082187Z","iopub.status.idle":"2023-10-26T09:33:02.100184Z","shell.execute_reply.started":"2023-10-26T09:33:02.082156Z","shell.execute_reply":"2023-10-26T09:33:02.098875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_2_best = pca_train.sort_values(by='PCA_2', ascending=False)\nsm_name_pca_2_best = pca_train.loc[pca_2_best.index[:20], 'sm_name'].unique()\nsm_name_pca_2_best","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.101462Z","iopub.execute_input":"2023-10-26T09:33:02.101822Z","iopub.status.idle":"2023-10-26T09:33:02.114479Z","shell.execute_reply.started":"2023-10-26T09:33:02.101793Z","shell.execute_reply":"2023-10-26T09:33:02.113352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sm_name_pca_best = []\nsm_name_pca_best += sm_name_pca_0_best.tolist()\nsm_name_pca_best += sm_name_pca_1_best.tolist()\nsm_name_pca_best += sm_name_pca_2_best.tolist()\n\nlist(set(sm_name_pca_best))","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.115943Z","iopub.execute_input":"2023-10-26T09:33:02.116317Z","iopub.status.idle":"2023-10-26T09:33:02.127010Z","shell.execute_reply.started":"2023-10-26T09:33:02.116288Z","shell.execute_reply":"2023-10-26T09:33:02.125732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.128280Z","iopub.execute_input":"2023-10-26T09:33:02.129052Z","iopub.status.idle":"2023-10-26T09:33:02.175606Z","shell.execute_reply.started":"2023-10-26T09:33:02.129019Z","shell.execute_reply":"2023-10-26T09:33:02.174248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.177211Z","iopub.execute_input":"2023-10-26T09:33:02.177684Z","iopub.status.idle":"2023-10-26T09:33:02.187519Z","shell.execute_reply.started":"2023-10-26T09:33:02.177645Z","shell.execute_reply":"2023-10-26T09:33:02.186198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_pca.shape)\nX_pca","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.188777Z","iopub.execute_input":"2023-10-26T09:33:02.189131Z","iopub.status.idle":"2023-10-26T09:33:02.200418Z","shell.execute_reply.started":"2023-10-26T09:33:02.189101Z","shell.execute_reply":"2023-10-26T09:33:02.199127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_pca_inverse = pca.inverse_transform(X_pca)\nX_pca_inverse_df = pd.DataFrame(X_pca_inverse)\nX_pca_inverse_df","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.204397Z","iopub.execute_input":"2023-10-26T09:33:02.204822Z","iopub.status.idle":"2023-10-26T09:33:02.403916Z","shell.execute_reply.started":"2023-10-26T09:33:02.204790Z","shell.execute_reply":"2023-10-26T09:33:02.402810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_pca_inverse_df['sm_name'] = de_train['sm_name']\nX_pca_inverse_df","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.405445Z","iopub.execute_input":"2023-10-26T09:33:02.405773Z","iopub.status.idle":"2023-10-26T09:33:02.462582Z","shell.execute_reply.started":"2023-10-26T09:33:02.405745Z","shell.execute_reply":"2023-10-26T09:33:02.461447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_pca_inverse_df_quantile = X_pca_inverse_df.groupby('sm_name').quantile(0.6)\nX_pca_inverse_df_quantile","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:02.464043Z","iopub.execute_input":"2023-10-26T09:33:02.465201Z","iopub.status.idle":"2023-10-26T09:33:04.409880Z","shell.execute_reply.started":"2023-10-26T09:33:02.465164Z","shell.execute_reply":"2023-10-26T09:33:04.409049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/sample_submission.csv', index_col=0)\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:04.411101Z","iopub.execute_input":"2023-10-26T09:33:04.411608Z","iopub.status.idle":"2023-10-26T09:33:09.203913Z","shell.execute_reply.started":"2023-10-26T09:33:04.411577Z","shell.execute_reply":"2023-10-26T09:33:09.202711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = sample_submission.columns\nfeatures","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:09.205450Z","iopub.execute_input":"2023-10-26T09:33:09.205896Z","iopub.status.idle":"2023-10-26T09:33:09.213946Z","shell.execute_reply.started":"2023-10-26T09:33:09.205853Z","shell.execute_reply":"2023-10-26T09:33:09.212835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map = pd.read_csv('/kaggle/input/open-problems-single-cell-perturbations/id_map.csv')\nid_map","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:09.215428Z","iopub.execute_input":"2023-10-26T09:33:09.215786Z","iopub.status.idle":"2023-10-26T09:33:09.240978Z","shell.execute_reply.started":"2023-10-26T09:33:09.215755Z","shell.execute_reply":"2023-10-26T09:33:09.239779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = id_map.merge(X_pca_inverse_df_quantile, how='left', on='sm_name')\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:09.242603Z","iopub.execute_input":"2023-10-26T09:33:09.242973Z","iopub.status.idle":"2023-10-26T09:33:09.322794Z","shell.execute_reply.started":"2023-10-26T09:33:09.242938Z","shell.execute_reply":"2023-10-26T09:33:09.321611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = sample_submission.set_index('id').iloc[:, 2:]\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:09.324202Z","iopub.execute_input":"2023-10-26T09:33:09.324580Z","iopub.status.idle":"2023-10-26T09:33:09.399995Z","shell.execute_reply.started":"2023-10-26T09:33:09.324549Z","shell.execute_reply":"2023-10-26T09:33:09.398840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.columns = features\nsample_submission","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:09.401179Z","iopub.execute_input":"2023-10-26T09:33:09.401513Z","iopub.status.idle":"2023-10-26T09:33:09.448788Z","shell.execute_reply.started":"2023-10-26T09:33:09.401485Z","shell.execute_reply":"2023-10-26T09:33:09.447714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:09.453913Z","iopub.execute_input":"2023-10-26T09:33:09.454271Z","iopub.status.idle":"2023-10-26T09:33:22.662396Z","shell.execute_reply.started":"2023-10-26T09:33:09.454241Z","shell.execute_reply":"2023-10-26T09:33:22.661062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('/kaggle/working/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-26T09:33:22.664495Z","iopub.execute_input":"2023-10-26T09:33:22.664980Z","iopub.status.idle":"2023-10-26T09:33:28.106211Z","shell.execute_reply.started":"2023-10-26T09:33:22.664948Z","shell.execute_reply":"2023-10-26T09:33:28.105126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}