{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":50160,"databundleVersionId":7921029,"sourceType":"competition"},{"sourceId":8207910,"sourceType":"datasetVersion","datasetId":4863667}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings,os,glob,gc,json\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-06T15:23:59.389198Z","iopub.execute_input":"2024-05-06T15:23:59.389641Z","iopub.status.idle":"2024-05-06T15:24:02.725456Z","shell.execute_reply.started":"2024-05-06T15:23:59.389606Z","shell.execute_reply":"2024-05-06T15:24:02.724273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class wrangling:\n    \n\n    @staticmethod\n    def set_datatypes(df):\n        \"\"\"\n        ends with p,a float, m string,D date\n        \"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\" as int\n        date_decision as date\n        \"\"\"\n        for col in df.columns:\n            if col in (\"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\"):\n                df = df.with_columns(pl.col(col).cast(pl.Int32))\n            elif col[-1] in ('D',) or col in (\"date_decision\"):\n                df = df.with_columns(pl.col(col).cast(pl.Date))\n            elif col[-1] in (\"P\",\"A\"):\n                df = df.with_columns(pl.col(col).cast(pl.Float32))\n        return df\n    \n    @staticmethod\n    def handle_dates(df):\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                df = df.with_columns(pl.col(col) - pl.col(\"date_decision\"))\n                df = df.with_columns(pl.col(col).dt.total_days())\n                df = df.with_columns(pl.col(col).cast(pl.Float32))\n                \n        df = df.drop(\"date_decision\", \"MONTH\")\n        return df\n    \n     \n    @staticmethod\n    def filter_cols(df):\n        \n        for col in df.columns:\n            if col not in [\"target\", \"case_id\", \"WEEK_NUM\"]:\n                isnull = df[col].is_null().mean()\n\n                if isnull > 0.95:\n                    df = df.drop(col)\n\n        for col in df.columns:\n            if (col not in [\"target\", \"case_id\", \"WEEK_NUM\"]) & (df[col].dtype == pl.String):\n                freq = df[col].n_unique()\n\n                if (freq == 1) | (freq > 200):\n                    df = df.drop(col)\n\n        return df","metadata":{"execution":{"iopub.status.busy":"2024-05-06T15:24:09.504086Z","iopub.execute_input":"2024-05-06T15:24:09.504672Z","iopub.status.idle":"2024-05-06T15:24:09.520556Z","shell.execute_reply.started":"2024-05-06T15:24:09.504635Z","shell.execute_reply":"2024-05-06T15:24:09.519372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Aggregator:\n    \"\"\"\n        ends with p,a float, m string,D date\n        \"case_id\", \"WEEK_NUM\", \"num_group1\", \"num_group2\" as int\n        date_decision as date\n    \"\"\"\n    \n    @staticmethod\n    def num_expr(df):\n        cols = [col for col in df.columns if col[-1] in (\"P\",\"A\")]\n        expr = []\n        expr.extend([pl.max(col).alias(f'max_{col}') for col in cols])\n        expr.extend([pl.min(col).alias(f'min_{col}') for col in cols])\n        expr.extend([pl.mean(col).alias(f'mean_{col}') for col in cols])\n        return expr\n    \n    @staticmethod\n    def date_expr(df):\n        expr = []\n        for col in df.columns:\n            if col[-1] in (\"D\",):\n                expr.append((pl.col(\"date_decision\") - pl.col(col)).dt.year().alias(f\"year_gap_{col}\"))\n                expr.append((pl.col(\"date_decision\") - pl.col(col)).dt.month().alias(f\"month_gap_{col}\"))\n                expr.append((pl.col(\"date_decision\") - pl.col(col)).dt.days().alias(f\"days_gap_{col}\"))\n        return expr\n    \n    @staticmethod\n    def string_expr(df):\n        expr = []\n        for col in df.columns:\n            if col[-1] in (\"M\",):\n                expr.append(pl.col(col).str.strip().str.len_chars().mean().alias(f'text_length_{col}'))\n        return expr\n    \n    @staticmethod\n    def other_expr(df):\n        expr = []\n        for col in df.columns:\n            if col[-1] in (\"T\", \"L\"):\n                expr.append(pl.max(col).alias(f'max_{col}'))\n                expr.append(pl.min(col).alias(f'min_{col}'))\n        return expr\n    \n    @staticmethod\n    def count_expr(df):\n        cols = [col for col in df.columns if \"num_group\" in col]\n        \n        expr_max = [pl.max(col).alias(f\"max_{col}\") for col in cols]\n\n        return expr_max\n\n    @staticmethod\n    def get_exprs(df):\n        exprs = Aggregator.num_expr(df) + \\\n                Aggregator.string_expr(df) + \\\n                Aggregator.other_expr(df) + \\\n                Aggregator.count_expr(df)\n\n        return exprs","metadata":{"execution":{"iopub.status.busy":"2024-05-06T15:24:10.133378Z","iopub.execute_input":"2024-05-06T15:24:10.133824Z","iopub.status.idle":"2024-05-06T15:24:10.152264Z","shell.execute_reply.started":"2024-05-06T15:24:10.133789Z","shell.execute_reply":"2024-05-06T15:24:10.15099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def topandas(df_data, cat_cols=None):\n    df_data = df_data.to_pandas()\n    \n    if cat_cols is None:\n        cat_cols = list(df_data.select_dtypes(\"object\").columns)\n    \n    df_data[cat_cols] = df_data[cat_cols].astype(\"category\")\n    \n    return df_data,cat_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-06T15:24:10.687944Z","iopub.execute_input":"2024-05-06T15:24:10.688376Z","iopub.status.idle":"2024-05-06T15:24:10.695423Z","shell.execute_reply.started":"2024-05-06T15:24:10.688336Z","shell.execute_reply":"2024-05-06T15:24:10.694144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### from https://www.kaggle.com/code/batprem/home-credit-risk-mode-utility-scripts\ndef reduce_mem_usage(df, float16_as32=True):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        if str(col_type)==\"category\":\n            continue\n        \n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    if float16_as32:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float16)                    \n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2024-05-06T15:24:11.321613Z","iopub.execute_input":"2024-05-06T15:24:11.322014Z","iopub.status.idle":"2024-05-06T15:24:11.339837Z","shell.execute_reply.started":"2024-05-06T15:24:11.321983Z","shell.execute_reply":"2024-05-06T15:24:11.338213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root = \"/kaggle/input/home-credit-credit-risk-model-stability/parquet_files/\"\nTRAIN_DIR = root + \"train/\"\nTEST_DIR = root + \"test/\"\n\ndef feature_eng(df_base, depth_0, depth_1, depth_2):\n    df_base = (\n        df_base\n        .with_columns(\n            month_decision = pl.col(\"date_decision\").dt.month(),\n            weekday_decision = pl.col(\"date_decision\").dt.weekday(),\n        )\n    )\n        \n    for i, df in enumerate(depth_0 + depth_1 + depth_2):\n        print(df_base.shape)\n        df_base = df_base.join(df, how=\"left\", on=\"case_id\",suffix=f\"_{i}\")\n        \n    df_base = df_base.pipe(wrangling.handle_dates)\n    df_base = df_base.unique(subset=[\"case_id\"])\n    return df_base\n\ndef read_file(path,non_imp_columns,depth=None):\n    df = pl.read_parquet(path).pipe(wrangling.set_datatypes)\n    \n    if depth in [1, 2]:\n        df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n    df = df.unique(subset=[\"case_id\"])\n    df = df.pipe(wrangling.filter_cols)\n    return df[list(set(df.columns)-set(non_imp_columns))]\n\ndef read_files(regex_path,non_imp_columns, depth=None):\n    chunks = []\n    for path in glob.glob(str(regex_path)):\n        df = pl.read_parquet(path).pipe(wrangling.set_datatypes)\n        \n        if depth in [1, 2]:\n            df = df.group_by(\"case_id\").agg(Aggregator.get_exprs(df))\n        \n        chunks.append(df)\n        \n    df = pl.concat(chunks, how=\"vertical_relaxed\")\n    df = df.unique(subset=[\"case_id\"])\n    df = df.pipe(wrangling.filter_cols)\n    \n    return df[list(set(df.columns)-set(non_imp_columns))]","metadata":{"execution":{"iopub.status.busy":"2024-05-06T15:24:11.96079Z","iopub.execute_input":"2024-05-06T15:24:11.961891Z","iopub.status.idle":"2024-05-06T15:24:11.976229Z","shell.execute_reply.started":"2024-05-06T15:24:11.96185Z","shell.execute_reply":"2024-05-06T15:24:11.975012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith open('/kaggle/input/credit-risk-non-imp-columns/my_data.json', 'r') as f:\n    json_string = f.read()\n    data = json.loads(json_string)\ndata_store = {\n    \"df_base\": read_file(TRAIN_DIR + \"train_base.parquet\",data[\"non_imp_columns\"]),\n    \"depth_0\": [\n        read_file(TRAIN_DIR + \"train_static_cb_0.parquet\",data[\"non_imp_columns\"]),\n        read_files(TRAIN_DIR + \"train_static_0_*.parquet\",data[\"non_imp_columns\"]),\n    ],\n    \"depth_1\": [\n        read_files(TRAIN_DIR + \"train_applprev_1_*.parquet\",data[\"non_imp_columns\"], 1),\n        read_file(TRAIN_DIR + \"train_tax_registry_a_1.parquet\",data[\"non_imp_columns\"], 1),\n        read_file(TRAIN_DIR + \"train_tax_registry_b_1.parquet\",data[\"non_imp_columns\"], 1),\n        read_file(TRAIN_DIR + \"train_tax_registry_c_1.parquet\",data[\"non_imp_columns\"], 1),\n        read_files(TRAIN_DIR + \"train_credit_bureau_a_1_*.parquet\", data[\"non_imp_columns\"],1),\n        read_file(TRAIN_DIR + \"train_credit_bureau_b_1.parquet\", data[\"non_imp_columns\"],1),\n        read_file(TRAIN_DIR + \"train_other_1.parquet\",data[\"non_imp_columns\"], 1),\n        read_file(TRAIN_DIR + \"train_person_1.parquet\",data[\"non_imp_columns\"], 1),\n        read_file(TRAIN_DIR + \"train_deposit_1.parquet\", data[\"non_imp_columns\"],1),\n        read_file(TRAIN_DIR + \"train_debitcard_1.parquet\", data[\"non_imp_columns\"],1),\n    ],\n    \"depth_2\": [\n        read_file(TRAIN_DIR + \"train_credit_bureau_b_2.parquet\",data[\"non_imp_columns\"], 2),\n        read_files(TRAIN_DIR + \"train_credit_bureau_a_2_*.parquet\",data[\"non_imp_columns\"], 2),\n    ]\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-06T15:24:12.685015Z","iopub.execute_input":"2024-05-06T15:24:12.685548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = feature_eng(**data_store)\n\nprint(\"train data shape:\\t\", df_train.shape)\n\ndel data_store\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train,cat_cols = topandas(df_train)\n\ndf_train = reduce_mem_usage(df_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.index = df_train[\"case_id\"]\ndf_train.drop(['case_id',\"target\", 'WEEK_NUM',\"month_decision\",\"weekday_decision\",\"birthdate_574D\",\"dateofbirth_337D\"], axis=1,inplace=True)\nprint(gc.collect())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbase_train = pd.read_parquet(TRAIN_DIR + \"train_base.parquet\",columns=[\"case_id\",\"WEEK_NUM\",\"target\"])\nbase_train.index = base_train[\"case_id\"]\nprint(base_train.sample(2))\nbase_train.drop(\"case_id\",axis=1,inplace=True)\nyy_train = base_train.loc[df_train.index,\"target\"]\nyy_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:20:54.634075Z","iopub.execute_input":"2024-05-06T16:20:54.634527Z","iopub.status.idle":"2024-05-06T16:20:55.078413Z","shell.execute_reply.started":"2024-05-06T16:20:54.634494Z","shell.execute_reply":"2024-05-06T16:20:55.076609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <div style=\"box-shadow: rgba(0, 0, 0, 0.16) 0px 1px 4px inset, rgb(51, 51, 51) 0px 0px 0px 3px inset; padding:20px; font-size:32px; font-family: consolas; text-align:center; display:fill; border-radius:15px; color:rgb(34, 34, 34);　background-color:rgb(255,255,255); \"> <b> Model LightGBM </b></div>","metadata":{}},{"cell_type":"code","source":"%%time\nimport optuna\nimport lightgbm as lgb\nfrom optuna.terminator.callback import TerminatorCallback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\n# Load your data (replace with your data loading logic)\nX_train, X_val, y_train, y_val = train_test_split(df_train, yy_train, test_size=0.2, random_state=42)\nprint(\"X_train : \",X_train.shape,\"X_val :\",X_val.shape)\n\n\n\ndef objective(trial):\n#     params = {\n#         \"objective\": \"binary\",  \n#         \"metric\": \"auc\", \n#         \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 256),\n#         \"max_depth\": trial.suggest_int(\"max_depth\", 3, 10),\n#         \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n#         \"random_seed\": 42,\n#         \"verbose\":-1,\n#         # Add more hyperparameters as needed (refer to LightGBM documentation)\n#     }\n\n    params = {\n        \"objective\": \"binary\",  \n        \"metric\": \"auc\", \n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 0.01, 0.03),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 20, 256),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 3, 10),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.5, 1.0),\n        \"boosting_type\": trial.suggest_categorical(\"boosting_type\", [\"gbdt\", \"dart\"]),\n        \"random_seed\": 42,\n        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.6, 1.0),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 3, 10),\n        \"n_estimators\": trial.suggest_int(\"n_estimators\",800,1000),\n        \"verbose\":-1,\n        # Add more hyperparameters as needed (refer to LightGBM documentation)\n    }\n\n    model = lgb.LGBMClassifier(**params)  # Or LGBMRegressor for regression\n    model.fit(X_train, y_train, eval_set=[(X_val, y_val)],callbacks=[lgb.log_evaluation(200), lgb.early_stopping(5)])\n\n    # Evaluate the model (replace with your evaluation metric)\n    y_pred = model.predict(X_val)\n    evaluation = roc_auc_score(y_val, y_pred)\n    return evaluation\n\n# Create Optuna study and LightGBM tuner\nstudy = optuna.create_study(direction=\"maximize\")  \nstudy.optimize(objective, n_trials=30)\n\n\n# Access best trial results\nbest_trial = study.best_trial\nbest_params = best_trial.params\nprint(\"Best parameters:\", best_params)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T15:30:03.739007Z","iopub.execute_input":"2024-05-06T15:30:03.739561Z","iopub.status.idle":"2024-05-06T15:48:24.67968Z","shell.execute_reply.started":"2024-05-06T15:30:03.739518Z","shell.execute_reply":"2024-05-06T15:48:24.677808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study.best_trial.params","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:04:08.499131Z","iopub.execute_input":"2024-05-06T16:04:08.503361Z","iopub.status.idle":"2024-05-06T16:04:08.51911Z","shell.execute_reply.started":"2024-05-06T16:04:08.503264Z","shell.execute_reply":"2024-05-06T16:04:08.51787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open(\"my_study.pkl\", \"wb\") as f:\n    pickle.dump(study, f)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:06:22.363699Z","iopub.execute_input":"2024-05-06T16:06:22.364244Z","iopub.status.idle":"2024-05-06T16:06:22.375351Z","shell.execute_reply.started":"2024-05-06T16:06:22.364206Z","shell.execute_reply":"2024-05-06T16:06:22.373959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"my_study.pkl\", \"rb\") as f:\n    loaded_study = pickle.load(f)\nprint(\"Best parameters:\", loaded_study.best_trial.params)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:06:24.6356Z","iopub.execute_input":"2024-05-06T16:06:24.636048Z","iopub.status.idle":"2024-05-06T16:06:24.653395Z","shell.execute_reply.started":"2024-05-06T16:06:24.636014Z","shell.execute_reply":"2024-05-06T16:06:24.650927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create LightGBM model with best parameters\nbest_model = lgb.LGBMClassifier(**best_params)  # Or LGBMRegressor for regression\nbest_model.fit(X_train, y_train, eval_set=[(X_val, y_val)],callbacks=[lgb.log_evaluation(200), lgb.early_stopping(5)])","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:26:45.723018Z","iopub.execute_input":"2024-05-06T16:26:45.723663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom sklearn.metrics import roc_auc_score","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef model_validation(clf):\n    rskf = RepeatedStratifiedKFold(n_splits=5,n_repeats=2,random_state=42)\n\n    scores = []\n    gini = []\n    base_train[\"score\"] = None\n    i = 0\n    for train_idx, val_idx in rskf.split(df_train, yy_train):\n        try:\n            i+=1\n            print(f\"round {i}\")\n            y_pred = clf.predict(df_train.iloc[val_idx])\n            validx = df_train.iloc[val_idx].index\n\n            base_train.loc[validx,\"score\"] = y_pred\n\n\n            # Calculate ROC AUC score\n            \n            score = roc_auc_score(yy_train[validx], y_pred)\n            print(\"AOC : \",score)\n            scores.append(score)\n            #base_test[\"score\"]=model.predict(X_test, num_iteration=gbmcv.best_iteration)\n            bt = gini_stability(base=base_train.loc[validx,:])\n            print(\"stability : \",bt)\n            gini.append(bt)\n        except Exception as e:\n            print(str(e))\n            pass\n\n    # Print average cross-validation score\n    print(f'Average ROC AUC score: {sum(scores)/len(scores)}')\n    print(f'Average Stability score: {sum(gini)/len(gini)}')\n    return sum(gini)/len(gini)\n\ndef gini_stability(base, w_fallingrate=88.0, w_resstd=-0.5):\n    gini_in_time = base.loc[:, [\"WEEK_NUM\", \"target\", \"score\"]]\\\n        .sort_values(\"WEEK_NUM\")\\\n        .groupby(\"WEEK_NUM\")[[\"target\", \"score\"]]\\\n        .apply(lambda x: 2*roc_auc_score(x[\"target\"], x[\"score\"])-1).tolist()\n    \n    x = np.arange(len(gini_in_time))\n    y = gini_in_time\n    a, b = np.polyfit(x, y, 1)\n    y_hat = a*x + b\n    residuals = y - y_hat\n    res_std = np.std(residuals)\n    avg_gini = np.mean(gini_in_time)\n    return avg_gini + w_fallingrate * min(0, a) + w_resstd * res_std","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_validation(best_model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}