{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":162314401,"sourceType":"kernelVersion"},{"sourceId":162317063,"sourceType":"kernelVersion"}],"dockerImageVersionId":30699,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Reference \n- [1] [home-credit-baseline](https://www.kaggle.com/code/greysky/home-credit-baseline)\n- [2] reference training notebook\n  - https://www.kaggle.com/code/motono0223/home-credit-automl-training\n- [3] packages for offline installation\n  - https://www.kaggle.com/code/motono0223/autogluon-pkgs\n  - https://www.kaggle.com/code/motono0223/ray-pkgs","metadata":{}},{"cell_type":"code","source":"!python -m pip install --no-index --find-links=/kaggle/input/autogluon-pkgs autogluon > /dev/null\n!python -m pip install --no-index --find-links=/kaggle/input/ray-pkgs --upgrade --force-reinstall -q ray==2.6.3","metadata":{"execution":{"iopub.status.busy":"2024-05-15T05:31:15.357229Z","iopub.execute_input":"2024-05-15T05:31:15.357668Z","iopub.status.idle":"2024-05-15T05:35:19.559929Z","shell.execute_reply.started":"2024-05-15T05:31:15.357637Z","shell.execute_reply":"2024-05-15T05:35:19.558791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport os\nimport gc\nfrom glob import glob\nfrom pathlib import Path\nfrom datetime import datetime\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.preprocessing import LabelEncoder\n\nimport lightgbm as lgb\nfrom catboost import CatBoostClassifier, Pool\nimport joblib\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfrom autogluon.tabular import TabularDataset, TabularPredictor","metadata":{"execution":{"iopub.status.busy":"2024-05-15T05:35:38.378205Z","iopub.execute_input":"2024-05-15T05:35:38.378601Z","iopub.status.idle":"2024-05-15T05:35:43.206080Z","shell.execute_reply.started":"2024-05-15T05:35:38.378569Z","shell.execute_reply":"2024-05-15T05:35:43.205104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"ROOT            = Path(\"/kaggle/input/home-credit-credit-risk-model-stability\")\nTRAIN_DIR       = ROOT / \"parquet_files\" / \"train\"\nTEST_DIR        = ROOT / \"parquet_files\" / \"test\"","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:20:31.496221Z","iopub.execute_input":"2024-05-15T06:20:31.497225Z","iopub.status.idle":"2024-05-15T06:20:31.507347Z","shell.execute_reply.started":"2024-05-15T06:20:31.497177Z","shell.execute_reply":"2024-05-15T06:20:31.506161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Collection and Preprecessing\n\n## Pipeline","metadata":{}},{"cell_type":"code","source":"class Pipeline:\n    # DataFrame의 column type 지정 method\n    @staticmethod\n    def set_table_dtypes(df):\n        for col in df.columns:\n            if col in [\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Int64))\n            elif col in [\"date_decision\"]:\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\", \"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float64))\n            elif col[-1] in (\"M\",):\n                df = df.with_columns(pl.col(col).cast(pl.String))\n            elif col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col).cast(pl.Date))            \n\n        return df\n    \n    # date type column 처리 및 시차/일차 계산 method\n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\")) # 일차 계산\n                df = df.with_columns(pl.col(col).dt.total_days())\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n\n        return df\n    \n    # 결측 비율 및 문자형 변수 cardinality을 기준으로 변수 제거\n    @staticmethod\n    def filter_cols(df):\n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.7: # 결측 비율 70% 초과 시 제거\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200): # 문자형 변수 cardinality 기준 변수 제거\n                    df = df.drop(col)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:19:20.569899Z","iopub.execute_input":"2024-05-15T06:19:20.570299Z","iopub.status.idle":"2024-05-15T06:19:20.584091Z","shell.execute_reply.started":"2024-05-15T06:19:20.570266Z","shell.execute_reply":"2024-05-15T06:19:20.583077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Aggregation","metadata":{}},{"cell_type":"code","source":"class Aggregator:\n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\", \"A\")]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n\n        return expr_max + expr_last + expr_mean\n\n    @staticmethod\n    def date_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"D\",)]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n        expr_mean = [pl.mean(col).alias(f\"mean_{col}\") for col in cols]\n\n        return expr_max + expr_last + expr_mean\n\n    @staticmethod\n    def str_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"M\",)]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n\n        return expr_max + expr_last\n\n    @staticmethod\n    def other_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"T\", \"L\")]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n\n        return expr_max + expr_last\n    \n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n\n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n        expr_last = [pl.last(col).alias(f\"last_{col}\") for col in cols]\n\n        return expr_max + expr_last\n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.date_expr(df) + \\\n                Aggregator.str_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:19:28.109668Z","iopub.execute_input":"2024-05-15T06:19:28.110519Z","iopub.status.idle":"2024-05-15T06:19:28.124943Z","shell.execute_reply.started":"2024-05-15T06:19:28.110489Z","shell.execute_reply":"2024-05-15T06:19:28.123926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path, depth=None):\n    df = pl.read_parquet(path)\n    df = df.pipe(Pipeline.set_table_dtypes)\n    \n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    \n    return df\n\ndef read_files(regex_path, depth=None):\n    chunks = []\n    for path in glob(str(regex_path)):\n        df = pl.read_parquet(path)\n        df = df.pipe(Pipeline.set_table_dtypes)\n        \n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        \n        chunks.append(df)\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:19:37.633495Z","iopub.execute_input":"2024-05-15T06:19:37.633847Z","iopub.status.idle":"2024-05-15T06:19:37.641554Z","shell.execute_reply.started":"2024-05-15T06:19:37.633819Z","shell.execute_reply":"2024-05-15T06:19:37.640579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering","metadata":{}},{"cell_type":"code","source":"def feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\", suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(Pipeline.handle_dates)\n    \n    return df_base\n\ndef to_pandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data, cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:19:47.332056Z","iopub.execute_input":"2024-05-15T06:19:47.333075Z","iopub.status.idle":"2024-05-15T06:19:47.344420Z","shell.execute_reply.started":"2024-05-15T06:19:47.333030Z","shell.execute_reply":"2024-05-15T06:19:47.343356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" \n    Iterate through all the columns of a dataframe and modify the data type\n    to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2  # Memory usage before optimization\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            continue\n    end_mem = df.memory_usage().sum() / 1024**2  # Memory usage after optimization\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:19:53.330773Z","iopub.execute_input":"2024-05-15T06:19:53.331548Z","iopub.status.idle":"2024-05-15T06:19:53.344561Z","shell.execute_reply.started":"2024-05-15T06:19:53.331517Z","shell.execute_reply":"2024-05-15T06:19:53.343418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 상관관계 기반 그룹화\ndef group_columns_by_corr(matrix, threshold=0.8):\n    correlation_matrix = matrix.corr()\n    groups = []\n    remaining_cols = list(matrix.columns)\n    while remaining_cols:\n        col = remaining_cols.pop(0)\n        group = [col]\n        correlated_cols = [col]\n        \n        for c in remaining_cols:\n            if correlation_matrix.loc[col, c] >= threshold:\n                group.append(c)\n                correlated_cols.append(c)\n        groups.append(group)\n        remaining_cols = [c for c in remaining_cols if c not in correlated_cols]\n    return groups        \ndef reduce_groups(grps):\n    use = []\n    for g in grps:\n        mx = 0\n        vx = g[0]\n        for gg in g:\n            n = train_df[gg].nunique()\n            if n>mx:\n                mx = n\n                vx = gg\n        use.append(vx)\n    print(\"Use these\", use, \"\\n\")\n    return use","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:19:56.744173Z","iopub.execute_input":"2024-05-15T06:19:56.745053Z","iopub.status.idle":"2024-05-15T06:19:56.754366Z","shell.execute_reply.started":"2024-05-15T06:19:56.745017Z","shell.execute_reply":"2024-05-15T06:19:56.753167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Data Load","metadata":{}},{"cell_type":"code","source":"%%time\ndata_store = {\n    \"df_base\": read_file(TRAIN_DIR / \"train_base.parquet\"),\n    \"depth_0\": [\n        read_file(TRAIN_DIR / \"train_static_cb_0.parquet\"),\n        read_files(TRAIN_DIR / \"train_static_0_*.parquet\"),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR / \"train_applprev_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_a_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_tax_registry_c_1.parquet\", 1),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_1_*.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_other_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_person_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_deposit_1.parquet\", 1),\n        read_file(TRAIN_DIR / \"train_debitcard_1.parquet\", 1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR / \"train_credit_bureau_b_2.parquet\", 2),\n        read_files(TRAIN_DIR / \"train_credit_bureau_a_2_*.parquet\", 2),\n        read_file(TRAIN_DIR / \"train_applprev_2.parquet\", 2),\n        read_file(TRAIN_DIR / \"train_person_2.parquet\", 2)\n    ]\n}\n\ntrain_df = feature_eng(**data_store)\nprint(\"train data shape:\\t\", train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:20:35.071959Z","iopub.execute_input":"2024-05-15T06:20:35.072901Z","iopub.status.idle":"2024-05-15T06:23:15.596931Z","shell.execute_reply.started":"2024-05-15T06:20:35.072865Z","shell.execute_reply":"2024-05-15T06:23:15.595969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data_store\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:23:19.639773Z","iopub.execute_input":"2024-05-15T06:23:19.640596Z","iopub.status.idle":"2024-05-15T06:23:20.357081Z","shell.execute_reply.started":"2024-05-15T06:23:19.640561Z","shell.execute_reply":"2024-05-15T06:23:20.356082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.pipe(Pipeline.filter_cols)\ntrain_df, cat_cols = to_pandas(train_df)\ntrain_df = reduce_mem_usage(train_df)\nprint(\"train data shape:\\t\", train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:23:21.964768Z","iopub.execute_input":"2024-05-15T06:23:21.965478Z","iopub.status.idle":"2024-05-15T06:24:11.589187Z","shell.execute_reply.started":"2024-05-15T06:23:21.965444Z","shell.execute_reply":"2024-05-15T06:24:11.588120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nnumeric_cols = train_df.select_dtypes(exclude='category').columns\n\nfrom itertools import combinations, permutations\nnans_df = train_df[numeric_cols].isna()\n# 결측치 기반 그룹화\nnans_groups = {} # 결측치 수를 key로, 해당 결측치 수를 갖는 cols를 list로 values로 저장함\nfor col in numeric_cols:\n    cur_group = nans_df[col].sum()\n    try:\n        nans_groups[cur_group].append(col)\n    except:\n        nans_groups[cur_group]=[col]\ndel nans_df\n\nuses = []\nfor k, v in nans_groups.items():\n    print(\"####### NAN count=\", k)\n    if len(v)>1:\n        Vs = nans_groups[k]\n        grps = group_columns_by_corr(train_df[Vs], threshold=0.8)\n        use = reduce_groups(grps)\n        uses = uses + use\n    else:\n        uses = uses + v\n        print(\"\\n\")\nprint(len(uses))\nuses = uses + list(train_df.select_dtypes(include='category').columns)\nprint(len(uses))\ntrain_df = train_df[uses]","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:25:49.632259Z","iopub.execute_input":"2024-05-15T06:25:49.633116Z","iopub.status.idle":"2024-05-15T06:26:26.288468Z","shell.execute_reply.started":"2024-05-15T06:25:49.633085Z","shell.execute_reply":"2024-05-15T06:26:26.287312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport pickle\nwith open('cat_cols.pickle', 'wb') as f:\n    pickle.dump(cat_cols, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:26:26.380340Z","iopub.execute_input":"2024-05-15T06:26:26.381212Z","iopub.status.idle":"2024-05-15T06:26:26.387082Z","shell.execute_reply.started":"2024-05-15T06:26:26.381175Z","shell.execute_reply":"2024-05-15T06:26:26.386141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:26:29.299297Z","iopub.execute_input":"2024-05-15T06:26:29.299644Z","iopub.status.idle":"2024-05-15T06:26:29.450349Z","shell.execute_reply.started":"2024-05-15T06:26:29.299619Z","shell.execute_reply":"2024-05-15T06:26:29.449310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"predictor = TabularPredictor(\n    label = 'target', \n    problem_type = 'binary',\n    eval_metric = 'roc_auc',\n    path = 'predictor'\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:26:31.800089Z","iopub.execute_input":"2024-05-15T06:26:31.800824Z","iopub.status.idle":"2024-05-15T06:26:31.806365Z","shell.execute_reply.started":"2024-05-15T06:26:31.800789Z","shell.execute_reply":"2024-05-15T06:26:31.805234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weeks = train_df[\"WEEK_NUM\"]\ntrain_df = train_df.drop(columns=[\"case_id\", \"WEEK_NUM\"])","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:26:35.244581Z","iopub.execute_input":"2024-05-15T06:26:35.245347Z","iopub.status.idle":"2024-05-15T06:26:36.598727Z","shell.execute_reply.started":"2024-05-15T06:26:35.245316Z","shell.execute_reply":"2024-05-15T06:26:36.597765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedGroupKFold(n_splits=5, shuffle=False)\nfor train_idx, valid_idx in cv.split(train_df, train_df[\"target\"], groups=weeks):\n    train_fold = train_df.iloc[train_idx]\n    valid_fold = train_df.iloc[valid_idx]\n    train_data = TabularDataset(train_fold)\n    valid_data = TabularDataset(valid_fold)\n    break","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:26:39.452006Z","iopub.execute_input":"2024-05-15T06:26:39.452411Z","iopub.status.idle":"2024-05-15T06:26:44.113458Z","shell.execute_reply.started":"2024-05-15T06:26:39.452379Z","shell.execute_reply":"2024-05-15T06:26:44.112462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ndel weeks, cat_cols, train_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:26:45.424788Z","iopub.execute_input":"2024-05-15T06:26:45.425178Z","iopub.status.idle":"2024-05-15T06:26:45.570981Z","shell.execute_reply.started":"2024-05-15T06:26:45.425148Z","shell.execute_reply":"2024-05-15T06:26:45.570021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npredictor.fit(\n    train_data,\n    tuning_data=valid_data,\n    save_space=True,\n    presets=\"optimize_for_deployment\",\n    use_bag_holdout=True,\n    ag_args_fit={'num_gpus': 1},\n    # skip neural net + knn because they take too much space\n    excluded_model_types=['KNN','NN_TORCH','FASTAI', 'RF'])","metadata":{"execution":{"iopub.status.busy":"2024-05-15T06:26:47.241414Z","iopub.execute_input":"2024-05-15T06:26:47.241761Z","iopub.status.idle":"2024-05-15T06:28:10.339105Z","shell.execute_reply.started":"2024-05-15T06:26:47.241735Z","shell.execute_reply":"2024-05-15T06:28:10.338277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models leaderboard","metadata":{}},{"cell_type":"code","source":"predictor.leaderboard()","metadata":{},"execution_count":null,"outputs":[]}]}