{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"***PLEASE UPVOTE: https://www.kaggle.com/code/alexandervc/op2-models-cv-tuning?scriptVersionId=145084352***","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\n\nimport pandas as pd\n\npd.set_option('display.max_columns', 30)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:01.513063Z","iopub.execute_input":"2023-11-08T15:11:01.513420Z","iopub.status.idle":"2023-11-08T15:11:01.860284Z","shell.execute_reply.started":"2023-11-08T15:11:01.513388Z","shell.execute_reply":"2023-11-08T15:11:01.859447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\nSEED = 42\nnp.random.seed(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:01.861763Z","iopub.execute_input":"2023-11-08T15:11:01.862136Z","iopub.status.idle":"2023-11-08T15:11:01.866421Z","shell.execute_reply.started":"2023-11-08T15:11:01.862110Z","shell.execute_reply":"2023-11-08T15:11:01.865568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"de_train = pd.read_parquet('/kaggle/input/open-problems-single-cell-perturbations/de_train.parquet')\n\nprint(de_train.shape)\nde_train","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:01.867533Z","iopub.execute_input":"2023-11-08T15:11:01.867836Z","iopub.status.idle":"2023-11-08T15:11:04.658492Z","shell.execute_reply.started":"2023-11-08T15:11:01.867812Z","shell.execute_reply":"2023-11-08T15:11:04.657425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_map = pd.read_csv ('/kaggle/input/open-problems-single-cell-perturbations/id_map.csv')\nid_map","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:04.661419Z","iopub.execute_input":"2023-11-08T15:11:04.661735Z","iopub.status.idle":"2023-11-08T15:11:04.682960Z","shell.execute_reply.started":"2023-11-08T15:11:04.661708Z","shell.execute_reply":"2023-11-08T15:11:04.681983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_types_20 = de_train.groupby(['cell_type']).size() > 20\ncell_types_20","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:04.684165Z","iopub.execute_input":"2023-11-08T15:11:04.684483Z","iopub.status.idle":"2023-11-08T15:11:04.696033Z","shell.execute_reply.started":"2023-11-08T15:11:04.684454Z","shell.execute_reply":"2023-11-08T15:11:04.694850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_types_20_val = cell_types_20[cell_types_20].index\ncell_types_20_val","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:04.697506Z","iopub.execute_input":"2023-11-08T15:11:04.697823Z","iopub.status.idle":"2023-11-08T15:11:04.705604Z","shell.execute_reply.started":"2023-11-08T15:11:04.697796Z","shell.execute_reply":"2023-11-08T15:11:04.703679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"B_cells_sm = de_train.query(\"cell_type == 'B cells'\").sm_name.values\nB_cells_sm","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:04.706782Z","iopub.execute_input":"2023-11-08T15:11:04.707061Z","iopub.status.idle":"2023-11-08T15:11:05.924407Z","shell.execute_reply.started":"2023-11-08T15:11:04.707033Z","shell.execute_reply":"2023-11-08T15:11:05.923473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\nt_0 = time.time()","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:05.925661Z","iopub.execute_input":"2023-11-08T15:11:05.926030Z","iopub.status.idle":"2023-11-08T15:11:05.930453Z","shell.execute_reply.started":"2023-11-08T15:11:05.925987Z","shell.execute_reply":"2023-11-08T15:11:05.929560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport optuna\nimport lightgbm as lgb\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.metrics import r2_score\n\nN_GENES = 25\n\nY = de_train.iloc[:, 5:].values\ntsvd = TruncatedSVD(n_components=N_GENES, n_iter=7, random_state=42)\n\ntrial_count_dict = {0: 0}\n\ndef objective(trial):\n    \n    trial_count_dict[0] += 1\n    t_0 = time.time()\n    \n    params = {'num_leaves': trial.suggest_int('num_leaves', 2, 2000),\n              'max_depth': trial.suggest_int('max_depth', 1, 16),\n              'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1),\n              'n_estimators': trial.suggest_categorical('n_estimators', [1, 2, 5, 10, 20, 50, 100, 200, 500]),\n              'subsample': trial.suggest_categorical('subsample', [0.4, 0.5, 0.6, 0.7, 0.8, 1.0]),\n              'min_child_samples': trial.suggest_int('min_child_samples', 1, 300),\n              'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),\n              'reg_alpha': trial.suggest_float('reg_alpha', 1e-3, 10.0),\n              'reg_lambda': trial.suggest_float('reg_lambda', 1e-3, 10.0),\n              'random_state': 0}\n    \n    model = MultiOutputRegressor(lgb.LGBMRegressor(**params))\n    \n    mrrmse_scores = []\n    r2_scores = []\n\n    for i, cell_type_20_val in enumerate(cell_types_20_val):\n\n        mask_va = (de_train.cell_type == cell_type_20_val) & ~de_train.sm_name.isin(B_cells_sm)\n        mask_tr = ~mask_va\n\n        X_train = de_train[['cell_type', 'sm_name']][mask_tr].astype('category')\n        X_valid = de_train[['cell_type', 'sm_name']][mask_va].astype('category')\n\n        Y_tsvd_train = tsvd.fit_transform(Y[mask_tr, :])\n\n        model.fit(X_train, Y_tsvd_train)\n        Y_tsvd_pred = model.predict(X_valid)\n        Y_tsvd_inverse_valid = tsvd.inverse_transform(Y_tsvd_pred)\n\n        mrrmse = np.sqrt(np.square(Y[mask_va, :] - Y_tsvd_inverse_valid).mean(axis=1)).mean()\n        r2 = r2_score(Y[mask_va, :], Y_tsvd_inverse_valid)\n\n        mrrmse_scores.append(mrrmse)\n        r2_scores.append(r2)\n\n    mrrmse = np.array(mrrmse_scores).mean()\n    r2 = np.array(r2_scores).mean()\n    \n    if (trial_count_dict[0] in range(0, 100)) or ((trial_count_dict[0] in range(1900, 2000))):\n        print(f'TRIAL_COUNT_DICT: {trial_count_dict[0]}')\n        print(f'FINAL MRRMSE: {mrrmse}')\n        print(f'FINAL R2: {r2}')\n        print(f'PARAMS:\\n{params}')\n        print(f'TIMING: {time.time() - t_0}')\n        print(30 * '=')\n    \n    return mrrmse","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:05.932077Z","iopub.execute_input":"2023-11-08T15:11:05.932411Z","iopub.status.idle":"2023-11-08T15:11:09.635916Z","shell.execute_reply.started":"2023-11-08T15:11:05.932380Z","shell.execute_reply":"2023-11-08T15:11:09.634952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nn_trials = 2000\n\nprint(f'OPTIMIZATION TRIALS: {n_trials}')\ntrial_count_dict[0] = 0\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\n\nstudy = optuna.create_study()\nstudy.optimize(objective, n_trials=n_trials)\n\nprint(f'STUDY.BEST_PARAMS:\\n{study.best_params}')","metadata":{"execution":{"iopub.status.busy":"2023-11-08T15:11:09.638783Z","iopub.execute_input":"2023-11-08T15:11:09.639097Z","iopub.status.idle":"2023-11-08T18:56:19.098386Z","shell.execute_reply.started":"2023-11-08T15:11:09.639072Z","shell.execute_reply":"2023-11-08T18:56:19.097329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_smooth_cell_type = {0: 0.0,\n                         1: 1.0,\n                         2: 1.0, \n                         3: 1000000000000000.0,\n                         4: 10.0, \n                         5: 1000.0, \n                         6: 1000000000000000.0, \n                         7: 100.0, \n                         8: 1000000000000000.0, \n                         9: 100.0, \n                         10: 10.0, \n                         11: 100.0, \n                         12: 0.0, \n                         13: 1000.0, \n                         14: 10000000000000.0, \n                         15: 100.0, \n                         16: 100.0, \n                         17: 10.0, \n                         18: 1.0, \n                         19: 10000000000000.0, \n                         20: 100.0, \n                         21: 10.0, \n                         22: 0.0, \n                         23: 1.0, \n                         24: 0.0}","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:56:19.100007Z","iopub.execute_input":"2023-11-08T18:56:19.100357Z","iopub.status.idle":"2023-11-08T18:56:19.110446Z","shell.execute_reply.started":"2023-11-08T18:56:19.100324Z","shell.execute_reply":"2023-11-08T18:56:19.109433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_smooth_sm = {0: 1000000000000000.0, \n                  1: 1000000000000000.0,\n                  2: 1000000000000000.0, \n                  3: 1000000000000000.0,\n                  4: 10.0, \n                  5: 1000000000000000.0, \n                  6: 10000000000000.0,\n                  7: 1000000000000000.0,\n                  8: 1000000000000000.0, \n                  9: 100.0,\n                  10: 100.0, \n                  11: 100.0, \n                  12: 100.0, \n                  13: 100.0,\n                  14: 1000000000000000.0, \n                  15: 1000.0, \n                  16: 1000000000000000.0,\n                  17: 100.0, \n                  18: 100.0, \n                  19: 10000000000000.0, \n                  20: 1000000000000000.0,\n                  21: 10000000000000.0,\n                  22: 1000000000000000.0,\n                  23: 100.0, \n                  24: 1000000000000000.0}","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:56:19.112016Z","iopub.execute_input":"2023-11-08T18:56:19.112672Z","iopub.status.idle":"2023-11-08T18:56:19.125624Z","shell.execute_reply.started":"2023-11-08T18:56:19.112635Z","shell.execute_reply":"2023-11-08T18:56:19.124269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = study.best_params\nlgb_params","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:56:19.127486Z","iopub.execute_input":"2023-11-08T18:56:19.128431Z","iopub.status.idle":"2023-11-08T18:56:19.139621Z","shell.execute_reply.started":"2023-11-08T18:56:19.128380Z","shell.execute_reply":"2023-11-08T18:56:19.138797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgb_params = {'num_leaves': 835,\n#               'max_depth': 1,\n#               'learning_rate': 0.08594161349385152,\n#               'n_estimators': 20,\n#               'subsample': 0.5,\n#               'min_child_samples': 58,\n#               'colsample_bytree': 0.4,\n#               'reg_alpha': 9.853300762690996,\n#               'reg_lambda': 0.3044908287305737}","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:57:53.639314Z","iopub.execute_input":"2023-11-08T18:57:53.639707Z","iopub.status.idle":"2023-11-08T18:57:53.644139Z","shell.execute_reply.started":"2023-11-08T18:57:53.639674Z","shell.execute_reply":"2023-11-08T18:57:53.643180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_submit = np.zeros((len(id_map), len(de_train.columns[5:])))\n\nprint(Y_submit.shape)\nY_submit","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:57:54.943298Z","iopub.execute_input":"2023-11-08T18:57:54.943908Z","iopub.status.idle":"2023-11-08T18:57:54.957598Z","shell.execute_reply.started":"2023-11-08T18:57:54.943876Z","shell.execute_reply":"2023-11-08T18:57:54.956632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import TruncatedSVD\nimport category_encoders as ce\nimport lightgbm as lgb\nfrom sklearn.metrics import r2_score\n\nN_GENES = 25\nmrrmse_scores = []\n\nX_de_train = de_train[['cell_type','sm_name']]\nX_id_map = id_map[['cell_type', 'sm_name']]\n\ntsvd = TruncatedSVD(n_components=N_GENES, n_iter=7, random_state=42)\n\nY = de_train.iloc[:, 5:].values\nY_tsvd = tsvd.fit_transform(Y)\n\n\nY_pred_tsvd = np.zeros(Y_tsvd.shape)\nY_id_map_tsvd = np.zeros((len(id_map), Y_tsvd.shape[1]))\n\nfor gene in range(Y_tsvd.shape[1]):\n\n    cell_type_target_encoder = ce.TargetEncoder(smoothing=best_smooth_cell_type[gene])\n    sm_target_encoder = ce.TargetEncoder(smoothing=best_smooth_sm[gene])\n    model = lgb.LGBMRegressor(**lgb_params)\n    \n    Y_gene = Y_tsvd[:, gene]\n    X_gene_train = pd.concat([cell_type_target_encoder.fit_transform(X_de_train[['cell_type']], Y_gene),\n                              sm_target_encoder.fit_transform(X_de_train[['sm_name']], Y_gene)], axis = 1)\n\n    model.fit(X_gene_train, Y_gene)\n    \n    X_gene_valid = pd.concat([cell_type_target_encoder.transform(X_de_train[['cell_type']]),\n                              sm_target_encoder.transform(X_de_train[['sm_name']])], axis = 1)\n    \n    Y_pred = model.predict(X_gene_valid)\n    Y_pred_tsvd[:, gene] = Y_pred\n    \n    X_submit_target_encoder = pd.concat( [cell_type_target_encoder.transform(X_id_map[['cell_type']]),\n                              sm_target_encoder.transform(X_id_map[['sm_name']])], axis = 1)\n    \n    Y_pred = model.predict(X_submit_target_encoder)\n    Y_id_map_tsvd[:, gene] = Y_pred\n\nY_pred_inverse_tsvd = tsvd.inverse_transform(Y_pred_tsvd)\nY_submit = tsvd.inverse_transform(Y_id_map_tsvd)\n\n\nmrrmse = np.sqrt(np.square(Y - Y_pred_inverse_tsvd).mean(axis=1)).mean()\nr2 = r2_score( Y, Y_pred_inverse_tsvd)\nmrrmse_scores.append(mrrmse)   \n\nmrrmse = np.array(mrrmse_scores).mean()\n\nprint(f'R2: {r2}')\nprint(30 * '-')\nprint(f'MRRMSE_SCORES:\\n{mrrmse_scores}')\nprint(30 * '-')\nprint(f'MRRMSE: {mrrmse}')","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:57:57.168982Z","iopub.execute_input":"2023-11-08T18:57:57.169828Z","iopub.status.idle":"2023-11-08T18:58:01.274261Z","shell.execute_reply.started":"2023-11-08T18:57:57.169794Z","shell.execute_reply":"2023-11-08T18:58:01.273302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(Y_submit.shape)\nY_submit","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:58:11.721405Z","iopub.execute_input":"2023-11-08T18:58:11.722092Z","iopub.status.idle":"2023-11-08T18:58:11.729266Z","shell.execute_reply.started":"2023-11-08T18:58:11.722051Z","shell.execute_reply":"2023-11-08T18:58:11.728353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit = pd.DataFrame(Y_submit, columns=de_train.columns[5:])\nsubmit.index.name = 'id'\nsubmit","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:58:15.663957Z","iopub.execute_input":"2023-11-08T18:58:15.664339Z","iopub.status.idle":"2023-11-08T18:58:15.711700Z","shell.execute_reply.started":"2023-11-08T18:58:15.664308Z","shell.execute_reply":"2023-11-08T18:58:15.710858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:58:19.546183Z","iopub.execute_input":"2023-11-08T18:58:19.546574Z","iopub.status.idle":"2023-11-08T18:58:31.229274Z","shell.execute_reply.started":"2023-11-08T18:58:19.546543Z","shell.execute_reply":"2023-11-08T18:58:31.228439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('/kaggle/working/submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-08T18:58:49.257317Z","iopub.execute_input":"2023-11-08T18:58:49.257707Z","iopub.status.idle":"2023-11-08T18:58:53.906164Z","shell.execute_reply.started":"2023-11-08T18:58:49.257665Z","shell.execute_reply":"2023-11-08T18:58:53.905257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}