{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook based on \nhttps://www.kaggle.com/code/xiafire/lb0-830-lgbm-optuna-msci-citeseq\nhttps://www.kaggle.com/code/alexandervc/mmscel-crossvalidation-schemes","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"!pip install hdf5plugin~=2.0 # https://forum.hdfgroup.org/t/cant-open-directory-usr-local-hdf5-lib-plugin/9738/4\n!pip install --quiet tables","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:08.781080Z","iopub.execute_input":"2022-10-24T09:52:08.781554Z","iopub.status.idle":"2022-10-24T09:52:32.893642Z","shell.execute_reply.started":"2022-10-24T09:52:08.781518Z","shell.execute_reply":"2022-10-24T09:52:32.892007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport os\nimport sys\nimport optuna\n\nimport h5py\nimport hdf5plugin\n\nimport numpy as np \nimport pandas as pd\nimport lightgbm as lgbm\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom functools import reduce\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import r2_score\nfrom sklearn.neural_network import MLPRegressor\nfrom sklearn.linear_model import Ridge\n\nimport warnings\nwarnings.simplefilter('ignore')\n\nDATA_DIR = \"/kaggle/input/open-problems-multimodal/\"\nFP_CELL_METADATA = os.path.join(DATA_DIR,\"metadata.csv\")\n\nFP_CITE_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_cite_inputs.h5\")\nFP_CITE_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_cite_targets.h5\")\nFP_CITE_TEST_INPUTS = os.path.join(DATA_DIR,\"test_cite_inputs.h5\")\n\nFP_MULTIOME_TRAIN_INPUTS = os.path.join(DATA_DIR,\"train_multi_inputs.h5\")\nFP_MULTIOME_TRAIN_TARGETS = os.path.join(DATA_DIR,\"train_multi_targets.h5\")\nFP_MULTIOME_TEST_INPUTS = os.path.join(DATA_DIR,\"test_multi_inputs.h5\")\n\nFP_SUBMISSION = os.path.join(DATA_DIR,\"sample_submission.csv\")\nFP_EVALUATION_IDS = os.path.join(DATA_DIR,\"evaluation_ids.csv\")\n\nt0start = time.time()","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:32.899542Z","iopub.execute_input":"2022-10-24T09:52:32.899980Z","iopub.status.idle":"2022-10-24T09:52:34.701516Z","shell.execute_reply.started":"2022-10-24T09:52:32.899945Z","shell.execute_reply":"2022-10-24T09:52:34.700306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n\ndf_cell = pd.read_csv(FP_CELL_METADATA)\nprint(df_cell.head())\nprint(df_cell.info())","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:34.702938Z","iopub.execute_input":"2022-10-24T09:52:34.703278Z","iopub.status.idle":"2022-10-24T09:52:35.203303Z","shell.execute_reply.started":"2022-10-24T09:52:34.703249Z","shell.execute_reply":"2022-10-24T09:52:35.202108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correlation_score(y_true, y_pred):\n    \"\"\"Scores the predictions according to the competition rules. \n    \n    It is assumed that the predictions are not constant.\n    \n    Returns the average of each sample's Pearson correlation coefficient\"\"\"\n    if type(y_true) == pd.DataFrame: y_true = y_true.values\n    if type(y_pred) == pd.DataFrame: y_pred = y_pred.values\n    corrsum = 0\n    for i in range(len(y_true)):\n        corrsum += np.corrcoef(y_true[i], y_pred[i])[1, 0]\n    return corrsum / len(y_true)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:35.205624Z","iopub.execute_input":"2022-10-24T09:52:35.206069Z","iopub.status.idle":"2022-10-24T09:52:35.213517Z","shell.execute_reply.started":"2022-10-24T09:52:35.206036Z","shell.execute_reply":"2022-10-24T09:52:35.212190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nprint('Load prepared features for CITE-seq')\n# These files contain both train and test parts .\n# For CITEseq part - first 70988 elements - train, and later 48663 - test. Overall 119651 samples.\n# fn = '/kaggle/input/feature-shop-for-multimodal-singlecell-competition/citeseq_train_and_test_TruncatedSVD200_niter7_rs42.csv'\nfn = '/kaggle/input/feature-shop-for-multimodal-singlecell-competition/citeseq_train_and_test_PCA500.csv'\ndf_cite = pd.read_csv(fn,index_col = 0)\ndisplay(df_cite)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:35.215008Z","iopub.execute_input":"2022-10-24T09:52:35.215488Z","iopub.status.idle":"2022-10-24T09:52:54.931381Z","shell.execute_reply.started":"2022-10-24T09:52:35.215444Z","shell.execute_reply":"2022-10-24T09:52:54.930289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nif 1:\n    print('Load CITE-seq targets and ')\n    df_cite_train_y = pd.read_hdf(FP_CITE_TRAIN_TARGETS)\n    display(df_cite_train_y)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:54.933070Z","iopub.execute_input":"2022-10-24T09:52:54.933461Z","iopub.status.idle":"2022-10-24T09:52:55.744729Z","shell.execute_reply.started":"2022-10-24T09:52:54.933429Z","shell.execute_reply":"2022-10-24T09:52:55.743422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfn2 = '/kaggle/input/feature-shop-for-multimodal-singlecell-competition/_citeseq_meta_all_text_also.csv'\ndf_meta_full = pd.read_csv(fn2,index_col = 0)\ndisplay(df_meta_full)\nif 1:\n    df_meta = pd.DataFrame(index = df_cite_train_y.index) \n    df_meta = df_meta.join(df_cell.set_index('cell_id') )\n    display(df_meta)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:55.745828Z","iopub.execute_input":"2022-10-24T09:52:55.746146Z","iopub.status.idle":"2022-10-24T09:52:56.132792Z","shell.execute_reply.started":"2022-10-24T09:52:55.746118Z","shell.execute_reply":"2022-10-24T09:52:56.131326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_f = ['/kaggle/input/feature-shop-for-multimodal-singlecell-competition/_citeseq_meta_Gender_and_CellType1Hot.csv',\n            '/kaggle/input/feature-shop-for-multimodal-singlecell-competition/citeseq_KmeansFromTruncatedSVD_rs0.csv' \n#             '/kaggle/input/feature-shop-for-multimodal-singlecell-competition/_citeseq_meta_day_only.csv',\n         ]\n\nN_features = 100\nr = df_cite.values[:,:N_features]\n\nfor f in list_f:\n    d = pd.read_csv(f, index_col = 0)\n    r = np.concatenate( (r,d.values), axis = 1)\nprint(r.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:56.134733Z","iopub.execute_input":"2022-10-24T09:52:56.135192Z","iopub.status.idle":"2022-10-24T09:52:57.062296Z","shell.execute_reply.started":"2022-10-24T09:52:56.135145Z","shell.execute_reply":"2022-10-24T09:52:57.060962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train test split","metadata":{}},{"cell_type":"code","source":"# Create indices \"train_index\", \"test_index\" \nmask = ( (df_meta['day'] == 2) |  (df_meta['day'] == 3) )\ntrain_index = np.where(mask > 0 )[0]\ntest_index = np.where((~mask) > 0 )[0]\n\n# Store  as 1-fold scheme \n# Storage convention - first in tuple - is always a \nlist_folds_indices = [(train_index, test_index)]\n\n# Store \"2-fold\" scheme\ntrain_index2 , test_index2  = test_index , train_index\nlist_folds_indices2 = [(train_index, test_index), (train_index2, test_index2)]\n\n# Check\nprint( train_index.shape, test_index.shape, np.intersect1d(test_index, train_index ) )","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:57.063650Z","iopub.execute_input":"2022-10-24T09:52:57.063995Z","iopub.status.idle":"2022-10-24T09:52:57.077647Z","shell.execute_reply.started":"2022-10-24T09:52:57.063964Z","shell.execute_reply":"2022-10-24T09:52:57.076540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_folds_indices_by_days_and_donors = []\nc = 0\nfor day2exclude in [2,3,4]:\n    for donor2exclude in [32606,  31800]: # We will need to predict always MALE (not female) - like on LB. (# donor 13176 - female)\n        train_index = np.where( (df_meta['day']  != day2exclude) & ( df_meta['donor']  != donor2exclude  ) )  [0]\n        test_index1_like_private_lb = np.where( df_meta['day']  == day2exclude)[0]\n        test_index2_like_public_lb = np.where( (df_meta['day']  != day2exclude) &  (df_meta['donor']  == donor2exclude ) ) [0]\n        list_folds_indices_by_days_and_donors.append( (train_index,  test_index1_like_private_lb , test_index2_like_public_lb) )\n    \n        str_fold_inf = 'Fold ' +str(c) + ': Train: excludes Day '+str(day2exclude) + ' and Donor ' + str( donor2exclude )\n        print(str_fold_inf, 'Sizes: train:',len(train_index), 'Test Like Priv'  ,len(test_index1_like_private_lb), 'Test Like Publ',   len(test_index2_like_public_lb),  ); c+=1\n    \nprint(len(list_folds_indices_by_days_and_donors))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:57.083150Z","iopub.execute_input":"2022-10-24T09:52:57.083563Z","iopub.status.idle":"2022-10-24T09:52:57.106267Z","shell.execute_reply.started":"2022-10-24T09:52:57.083527Z","shell.execute_reply":"2022-10-24T09:52:57.104715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scol = 'donor&day&CT'\ndf_meta[scol] =df_meta['donor'].apply(lambda x:str(x)+'_') + df_meta['day'].apply(lambda x:str(x)+'_') + df_meta['cell_type']\ndf_meta\n\ny = df_meta[scol] \nskf = StratifiedKFold(n_splits=10,  shuffle=True, random_state=40)\nskf.get_n_splits(df_meta, df_meta[scol] )\nfor train_index, test_index in skf.split(df_meta, df_meta[scol]):\n    print(\"TRAIN:\", len(train_index), \"TEST:\", len(test_index) ); break\nprint(test_index)\nprint(df_meta[scol].value_counts()   )\nprint(df_meta.iloc[test_index,:][scol].value_counts()    )\n\ndf_meta['HoldOut'] = 0 \ndf_meta.loc[df_meta.index[test_index],'HoldOut']  = 1\ndf_meta","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:57.107726Z","iopub.execute_input":"2022-10-24T09:52:57.109014Z","iopub.status.idle":"2022-10-24T09:52:57.339981Z","shell.execute_reply.started":"2022-10-24T09:52:57.108965Z","shell.execute_reply":"2022-10-24T09:52:57.339090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_folds_indices_by_days_and_donors_with2holdouts = []\nc = 0\nfor day2exclude in [2,3,4]:\n    for donor2exclude in [32606,  31800]: # We will need to predict always MALE (not female) - like on LB. (# donor 13176 - female)\n        train_index = np.where( (df_meta['day']  != day2exclude) & ( df_meta['donor']  != donor2exclude  ) )  [0]\n        mask_holdout = (df_meta['HoldOut']==1)\n        test_index1_like_private_lb = np.where( (df_meta['day']  == day2exclude) & (~mask_holdout) ) [0]\n        test_index1_like_private_lb_holdout = np.where( (df_meta['day']  == day2exclude) & (mask_holdout) ) [0]\n        test_index2_like_public_lb = np.where( (df_meta['day']  != day2exclude) &  (df_meta['donor']  == donor2exclude ) & (~mask_holdout) ) [0]\n        test_index2_like_public_lb_holdout = np.where( (df_meta['day']  != day2exclude) &  (df_meta['donor']  == donor2exclude ) & mask_holdout ) [0]\n        \n        list_folds_indices_by_days_and_donors_with2holdouts.append( (train_index,  test_index1_like_private_lb , test_index2_like_public_lb, \n                                                      test_index1_like_private_lb_holdout, test_index2_like_public_lb_holdout ) )\n    \n        str_fold_inf = 'Fold ' +str(c) + ': Train: excludes Day '+str(day2exclude) + ' and Donor ' + str( donor2exclude )\n        print(str_fold_inf, 'Sizes: train:',len(train_index), 'Test Like Priv'  ,len(test_index1_like_private_lb), \n              'Test Like Publ',   len(test_index2_like_public_lb),  \n              'Test Like Priv HoldOut',   len(test_index1_like_private_lb_holdout),  \n              'Test Like Publ HoldOut',   len(test_index2_like_public_lb_holdout),  \n              \n             ); c+=1\n    \nprint(len(list_folds_indices_by_days_and_donors_with2holdouts))","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:57.341250Z","iopub.execute_input":"2022-10-24T09:52:57.341643Z","iopub.status.idle":"2022-10-24T09:52:57.378650Z","shell.execute_reply.started":"2022-10-24T09:52:57.341610Z","shell.execute_reply":"2022-10-24T09:52:57.377258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Since metric is - correlation coefficient - any rescaling aY+b will not be change it , so one can do like that:\n# it is not clear is it optimal or not (see https://www.kaggle.com/competitions/open-problems-multimodal/discussion/360253 )\nrescale_Y_to_mean0_std1 = False\nY = df_cite_train_y.values\n\nif rescale_Y_to_mean0_std1:\n    Y -= Y.mean(axis=1).reshape(-1, 1)\n    Y /= Y.std(axis=1).reshape(-1, 1)\n    print('Rescaling to mean 0 and std 1 has been done')","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:57.380028Z","iopub.execute_input":"2022-10-24T09:52:57.381002Z","iopub.status.idle":"2022-10-24T09:52:57.387562Z","shell.execute_reply.started":"2022-10-24T09:52:57.380961Z","shell.execute_reply":"2022-10-24T09:52:57.386140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finetuning with Optuna","metadata":{}},{"cell_type":"markdown","source":"Paramenters for tuning: \n\n```params = {'metric': 'mse', 'random_state': 42, 'n_estimators': 2000, 'reg_alpha': 0.03645857751758206, 'reg_lambda': 0.0025972855120393492, 'colsample_bytree': 1.0, 'subsample': 0.6, 'learning_rate': 0.013262872399411381, 'max_depth': 10, 'num_leaves': 186, 'min_child_samples': 263, 'min_data_per_groups': 46}```","metadata":{}},{"cell_type":"code","source":"list_of_features_for_tuning = {\n    \"CD31\": []\n}","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:57.389074Z","iopub.execute_input":"2022-10-24T09:52:57.389430Z","iopub.status.idle":"2022-10-24T09:52:57.405077Z","shell.execute_reply.started":"2022-10-24T09:52:57.389401Z","shell.execute_reply":"2022-10-24T09:52:57.403438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For fine tuning we will use just a train part of dataset","metadata":{}},{"cell_type":"code","source":"all_train_parts = [f[0] for f in list_folds_indices_by_days_and_donors]\nfinetune_indices = reduce(np.intersect1d, all_train_parts[:4])\nnp.random.shuffle(finetune_indices)\nfinetune_indices","metadata":{"execution":{"iopub.status.busy":"2022-10-24T09:52:57.406691Z","iopub.execute_input":"2022-10-24T09:52:57.407055Z","iopub.status.idle":"2022-10-24T09:52:57.426699Z","shell.execute_reply.started":"2022-10-24T09:52:57.407021Z","shell.execute_reply":"2022-10-24T09:52:57.425621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = r \nY = df_cite_train_y[list_of_features_for_tuning.keys()].values\nprint('X.shape:', X.shape,'Y.shape:', Y.shape)\n\nsmall_train_x, small_train_y =  X[finetune_indices][:3000,:],Y[finetune_indices][:3000,:]\nsmall_test_x, small_test_y = X[finetune_indices][3000:,:],Y[finetune_indices][3000:,:]\nprint('small_train_x:', small_train_x.shape, 'small_train_y:', small_train_y.shape)\nprint('small_test_x:', small_test_x.shape, 'small_test_y:', small_test_y.shape)\n\nfixed_params = {      \n    'random_state': 42,\n    'n_estimators': 2000, \n    'learning_rate': 0.01, \n    'max_depth': 20\n}\n\ndef objective(trial):\n    params = {\n        'metric': 'mse', \n        **fixed_params,\n#         'n_estimators': 2000, #trial.suggest_int('n_estimators', 1, 3000),\n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),\n        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),\n        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [1.0, 0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),\n        'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),\n#         'learning_rate': 0.01, #trial.suggest_loguniform('learning_rate', 0.01, 0.1),\n#         'max_depth': 20,#trial.suggest_categorical('max_depth', [10,20,100]),\n        'num_leaves' : trial.suggest_int('num_leaves', 1, 1000),\n        'min_child_samples': trial.suggest_int('min_child_samples', 1, 300),\n        'cat_smooth' : trial.suggest_int('cat_smooth', 1, 100)\n    }\n    model = lgbm.LGBMRegressor(**params)\n\n    model.fit(small_train_x, small_train_y)\n\n    y_pred = model.predict(small_test_x)\n    mse = mean_squared_error(small_test_y, y_pred)\n    return mse","metadata":{"execution":{"iopub.status.busy":"2022-10-24T10:23:32.812676Z","iopub.execute_input":"2022-10-24T10:23:32.813237Z","iopub.status.idle":"2022-10-24T10:23:32.842580Z","shell.execute_reply.started":"2022-10-24T10:23:32.813186Z","shell.execute_reply":"2022-10-24T10:23:32.840784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfind_params = True\nif find_params:\n    for feature in list_of_features_for_tuning:\n        study = optuna.create_study(\n            direction='minimize', \n            pruner=optuna.pruners.MedianPruner(n_warmup_steps=20),\n            study_name='small')\n        study.optimize(objective, n_trials=20)\n        list_of_features_for_tuning[feature] = {**fixed_params, **study.best_trial.params}\n        print('Best trial: ', study.best_trial.number, study.best_trial.params)","metadata":{"execution":{"iopub.status.busy":"2022-10-24T10:23:37.675867Z","iopub.execute_input":"2022-10-24T10:23:37.676678Z","iopub.status.idle":"2022-10-24T10:31:00.574672Z","shell.execute_reply.started":"2022-10-24T10:23:37.676626Z","shell.execute_reply":"2022-10-24T10:31:00.573733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_of_features_for_tuning","metadata":{"execution":{"iopub.status.busy":"2022-10-24T10:31:00.577136Z","iopub.execute_input":"2022-10-24T10:31:00.577561Z","iopub.status.idle":"2022-10-24T10:31:00.585106Z","shell.execute_reply.started":"2022-10-24T10:31:00.577491Z","shell.execute_reply":"2022-10-24T10:31:00.584028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross Validation","metadata":{}},{"cell_type":"code","source":"%%time\n\nuse_lgbm = True\ncc = 0\n\nX = r \nY = df_cite_train_y.values\n\nalpha4Ridge = 1\nfor list_folds_indices in  [list_folds_indices_by_days_and_donors, list_folds_indices_by_days_and_donors_with2holdouts]:\n    if cc == 0: print('CV withOUT holdout\\n' ); \n    else : print('\\n\\nCV with holdout\\n' )\n    cc += 1\n    \n    X = r \n    print('X.shape:', X.shape,'Y.shape:', Y.shape)\n\n    model_ridge = Ridge(alpha=alpha4Ridge, random_state = 42)\n\n    if use_lgbm:\n        models_lgbm = {df_cite_train_y.columns.get_loc(feature): lgbm.LGBMRegressor(**params) \n                       for feature, params in list_of_features_for_tuning.items()}\n\n    df_fold_score_stat = pd.DataFrame();df_fold_score_stat.index.name = 'Fold'\n    t0 = time.time()\n    for fold, indices_tuple  in enumerate(list_folds_indices):\n        train_index = indices_tuple[0]\n        main_test_index = indices_tuple[1]\n        print('Fold:', fold, 'Shapes of train:', train_index.shape, 'Tests: ',[t.shape for t in indices_tuple[1:] ] )\n        t1 = time.time()\n\n        # Train model: \n        model_ridge.fit(X[train_index], Y[train_index])\n        if use_lgbm:\n            for feature in models_lgbm:\n                models_lgbm[feature].fit(X[train_index], Y[train_index, feature])\n\n        # Calculate metrics on test and train folds \n        list_scores = []; list_scores_r2 = []\n        for i_loc in range(0, len(indices_tuple)):\n            indices_loc = indices_tuple[i_loc]\n            y_pred = model_ridge.predict(X[indices_loc])\n            \n            if use_lgbm:\n                for feature in models_lgbm:\n                    feature_y_pred = models_lgbm[feature].predict(X[indices_loc])\n                    y_pred[:,feature] = feature_y_pred\n            \n            s = correlation_score(y_pred ,Y[indices_loc])\n            list_scores.append(s)\n            s = r2_score(Y[indices_loc], y_pred)\n            list_scores_r2.append(s)\n\n#         Just save statistic for output\n        for i_loc in range(1, len(list_scores  )):\n             df_fold_score_stat.loc[fold, 'Score Test'+str(i_loc)] = list_scores[i_loc]\n        df_fold_score_stat.loc[fold, 'Score Train'] = list_scores[0]\n        df_fold_score_stat.loc[fold, 'Time'] = np.round( (time.time() - t1), 2) \n        for i_loc in range(1, len(list_scores  )):\n             df_fold_score_stat.loc[fold, 'R2 Score Test'+str(i_loc)] = list_scores_r2[i_loc]\n        df_fold_score_stat.loc[fold, 'R2 Score Train'] = list_scores_r2[0]\n\n        print('Correlation scores:', np.round(list_scores,4),  'time:', '%.2f'%(time.time() - t1))\n        \n    display(df_fold_score_stat)\n    display(df_fold_score_stat.describe(percentiles=[] ).iloc[1:,:])","metadata":{"execution":{"iopub.status.busy":"2022-10-24T10:46:41.584422Z","iopub.execute_input":"2022-10-24T10:46:41.585074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}