{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Built from this great notebook\n\nPreprocessing and training parts based on this great notebook:\nhttps://www.kaggle.com/code/ambrosm/amex-lightgbm-quickstart","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nfrom lightgbm import LGBMClassifier, early_stopping, log_evaluation\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:03:39.474615Z","iopub.execute_input":"2022-07-20T02:03:39.475039Z","iopub.status.idle":"2022-07-20T02:03:41.649307Z","shell.execute_reply.started":"2022-07-20T02:03:39.474949Z","shell.execute_reply":"2022-07-20T02:03:41.648517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class config:\n    random_state = 4222\n    kaggle = True\n    #path = '../input/amexfeather'\n    #local_path = ''","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:03:46.329734Z","iopub.execute_input":"2022-07-20T02:03:46.330140Z","iopub.status.idle":"2022-07-20T02:03:46.334810Z","shell.execute_reply.started":"2022-07-20T02:03:46.330107Z","shell.execute_reply":"2022-07-20T02:03:46.333847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Data Preprocessing**","metadata":{}},{"cell_type":"code","source":"%%time\n\nInference = False # set to False if you only want to cross-validate\n\nfeatures_avg = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_50', 'D_51', 'D_53', 'D_54', 'D_55', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_65', 'D_66', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_75', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_86', 'D_91', 'D_92', 'D_94', 'D_96', 'D_103', 'D_104', 'D_108', 'D_112', 'D_113', 'D_114', 'D_115', 'D_117', 'D_118', 'D_119', 'D_120', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_129', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_14', 'R_15', 'R_16', 'R_17', 'R_20', 'R_21', 'R_22', 'R_24', 'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_9', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_18', 'S_22', 'S_23', 'S_25', 'S_26']\nfeatures_min = ['B_2', 'B_4', 'B_5', 'B_9', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_19', 'B_20', 'B_28', 'B_29', 'B_33', 'B_36', 'B_42', 'D_39', 'D_41', 'D_42', 'D_45', 'D_46', 'D_48', 'D_50', 'D_51', 'D_53', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_62', 'D_70', 'D_71', 'D_74', 'D_75', 'D_78', 'D_83', 'D_102', 'D_112', 'D_113', 'D_115', 'D_118', 'D_119', 'D_121', 'D_122', 'D_128', 'D_132', 'D_140', 'D_141', 'D_144', 'D_145', 'P_2', 'P_3', 'R_1', 'R_27', 'S_3', 'S_5', 'S_7', 'S_9', 'S_11', 'S_12', 'S_23', 'S_25']\nfeatures_max = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_21', 'B_23', 'B_24', 'B_25', 'B_29', 'B_30', 'B_33', 'B_37', 'B_38', 'B_39', 'B_40', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_52', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_63', 'D_64', 'D_65', 'D_70', 'D_71', 'D_72', 'D_73', 'D_74', 'D_76', 'D_77', 'D_78', 'D_80', 'D_82', 'D_84', 'D_91', 'D_102', 'D_105', 'D_107', 'D_110', 'D_111', 'D_112', 'D_115', 'D_116', 'D_117', 'D_118', 'D_119', 'D_121', 'D_122', 'D_123', 'D_124', 'D_125', 'D_126', 'D_128', 'D_131', 'D_132', 'D_133', 'D_134', 'D_135', 'D_136', 'D_138', 'D_140', 'D_141', 'D_142', 'D_144', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_3', 'R_5', 'R_6', 'R_7', 'R_8', 'R_10', 'R_11', 'R_14', 'R_17', 'R_20', 'R_26', 'R_27', 'S_3', 'S_5', 'S_7', 'S_8', 'S_11', 'S_12', 'S_13', 'S_15', 'S_16', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\nfeatures_last = ['B_1', 'B_2', 'B_3', 'B_4', 'B_5', 'B_6', 'B_7', 'B_8', 'B_9', 'B_10', 'B_11', 'B_12', 'B_13', 'B_14', 'B_15', 'B_16', 'B_17', 'B_18', 'B_19', 'B_20', 'B_21', 'B_22', 'B_23', 'B_24', 'B_25', 'B_26', 'B_28', 'B_29', 'B_30', 'B_32', 'B_33', 'B_36', 'B_37', 'B_38', 'B_39', 'B_40', 'B_41', 'B_42', 'D_39', 'D_41', 'D_42', 'D_43', 'D_44', 'D_45', 'D_46', 'D_47', 'D_48', 'D_49', 'D_50', 'D_51', 'D_52', 'D_53', 'D_54', 'D_55', 'D_56', 'D_58', 'D_59', 'D_60', 'D_61', 'D_62', 'D_63', 'D_64', 'D_65', 'D_69', 'D_70', 'D_71', 'D_72', 'D_73', 'D_75', 'D_76', 'D_77', 'D_78', 'D_79', 'D_80', 'D_81', 'D_82', 'D_83', 'D_86', 'D_91', 'D_96', 'D_105', 'D_106', 'D_112', 'D_114', 'D_119', 'D_120', 'D_121', 'D_122', 'D_124', 'D_125', 'D_126', 'D_127', 'D_130', 'D_131', 'D_132', 'D_133', 'D_134', 'D_138', 'D_140', 'D_141', 'D_142', 'D_145', 'P_2', 'P_3', 'P_4', 'R_1', 'R_2', 'R_3', 'R_4', 'R_5', 'R_6', 'R_7', 'R_8', 'R_9', 'R_10', 'R_11', 'R_12', 'R_13', 'R_14', 'R_15', 'R_19', 'R_20', 'R_26', 'R_27', 'S_3', 'S_5', 'S_6', 'S_7', 'S_8', 'S_9', 'S_11', 'S_12', 'S_13', 'S_16', 'S_19', 'S_20', 'S_22', 'S_23', 'S_24', 'S_25', 'S_26', 'S_27']\n\nfor i in ['test','train'] if Inference else ['train']:\n    df = pd.read_parquet(f'../input/amex-data-integer-dtypes-parquet-format/{i}.parquet')\n    cid = pd.Categorical(df.pop('customer_ID'), ordered = True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    \n    if 'target' in df.columns:\n        df.drop(columns=['target'], inplace=True)\n    gc.collect()\n    print('Read', i)\n    \n    df_avg = (df\n              .groupby(cid)\n              .mean()[features_avg]\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    gc.collect()\n    print('Computed avg', i)\n    \n    df_min = (df\n              .groupby(cid)\n              .min()[features_min]\n              .rename(columns={f: f\"{f}_min\" for f in features_min})\n             )\n    gc.collect()\n    print('Computed min', i)\n    \n    df_max = (df\n              .groupby(cid)\n              .max()[features_max]\n              .rename(columns={f: f\"{f}_max\" for f in features_max})\n             )\n    gc.collect()\n    print('Computed max', i)\n    \n    df = (df.loc[last, features_last]\n          .rename(columns={f: f\"{f}_last\" for f in features_last})\n          .set_index(np.asarray(cid[last]))\n         )\n    gc.collect()\n    print('Computed last', i)\n    \n    df = pd.concat([df, df_min, df_max, df_avg], axis=1)\n    \n    if i == 'train': train = df\n    else: test = df\n    print(f\"{i} shape: {df.shape}\")\n    \n    del df, df_avg, df_min, df_max, cid, last\n\ntarget = pd.read_csv('../input/amex-default-prediction/train_labels.csv').target.values\nprint(f\"target shape: {target.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:03:50.004709Z","iopub.execute_input":"2022-07-20T02:03:50.005411Z","iopub.status.idle":"2022-07-20T02:06:11.765141Z","shell.execute_reply.started":"2022-07-20T02:03:50.005376Z","shell.execute_reply":"2022-07-20T02:06:11.764147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [feat for feat in train.columns if feat != 'customer_ID' and feat != 'target']\nlen(features)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:06:21.702259Z","iopub.execute_input":"2022-07-20T02:06:21.702671Z","iopub.status.idle":"2022-07-20T02:06:21.710765Z","shell.execute_reply.started":"2022-07-20T02:06:21.702637Z","shell.execute_reply":"2022-07-20T02:06:21.709702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Model Training**","metadata":{}},{"cell_type":"code","source":"def amex_metric(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n\n    if isinstance(y_true, np.ndarray):\n            y_true = pd.DataFrame(y_true, columns = [\"target\"])\n    \n    if isinstance(y_pred, np.ndarray):\n            y_pred = pd.DataFrame(y_pred, columns = [\"prediction\"])\n            #y_pred[\"prediction\"] = y_pred\n    \n    def top_four_percent_captured(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n      \n        df['weight'] = df[\"target\"].apply(lambda x: 20 if x==0 else 1)\n        four_pct_cutoff = int(0.04 * df['weight'].sum())\n        df['weight_cumsum'] = df['weight'].cumsum()\n        df_cutoff = df.loc[df['weight_cumsum'] <= four_pct_cutoff]\n        return (df_cutoff['target'] == 1).sum() / (df[\"target\"] == 1).sum()\n        \n    def weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        df = (pd.concat([y_true, y_pred], axis='columns')\n              .sort_values('prediction', ascending=False))\n        df['weight'] = df[\"target\"].apply(lambda x: 20 if x==0 else 1)\n        df['random'] = (df['weight'] / df['weight'].sum()).cumsum()\n        total_pos = (df[\"target\"] * df['weight']).sum()\n        df['cum_pos_found'] = (df[\"target\"] * df['weight']).cumsum()\n        df['lorentz'] = df['cum_pos_found'] / total_pos\n        df['gini'] = (df['lorentz'] - df['random']) * df['weight']\n        return df['gini'].sum()\n\n    def normalized_weighted_gini(y_true: pd.DataFrame, y_pred: pd.DataFrame) -> float:\n        y_true_pred = y_true.rename(columns={'target': 'prediction'})\n        return weighted_gini(y_true, y_pred) / weighted_gini(y_true, y_true_pred)\n\n    d = top_four_percent_captured(y_true, y_pred)\n    g = normalized_weighted_gini(y_true, y_pred)\n\n    return 0.5 * (g + d)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:06:25.127603Z","iopub.execute_input":"2022-07-20T02:06:25.127987Z","iopub.status.idle":"2022-07-20T02:06:25.139541Z","shell.execute_reply.started":"2022-07-20T02:06:25.127954Z","shell.execute_reply":"2022-07-20T02:06:25.138606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lgb_amex_metric(y_true, y_pred):\n    \"\"\"The competition metric with lightgbm's calling convention\"\"\"\n    return ('amex',\n            amex_metric(y_true, y_pred),\n            True)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:06:29.436504Z","iopub.execute_input":"2022-07-20T02:06:29.436953Z","iopub.status.idle":"2022-07-20T02:06:29.443048Z","shell.execute_reply.started":"2022-07-20T02:06:29.436916Z","shell.execute_reply":"2022-07-20T02:06:29.441732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"search_params = { \n    'learning_rate' : 0.065,\n    'lambda_l1': 0, #3.673178365035792e-06,\n    'lambda_l2': 9.61643267449367,\n    'num_leaves': 220, #36,\n    'feature_fraction': 0.6, #0.4,\n    'bagging_fraction': 1.0,\n    'bagging_freq': 0,\n    'min_child_samples':  255 #5\n}\n\nfixed_params={\n    'objective': 'binary',\n    'metric': 'custom', #'binay_logloss',\n    'boosting_type' : 'gbdt',\n    #'force_row_wise' : True,\n    #'device': 'gpu',\n    'random_state' : config.random_state,\n    #'extra_trees' : True,\n    #'feature_pre_filter': False,\n    'n_estimators': 600,\n    'early_stopping_round': 30\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:22:23.910763Z","iopub.execute_input":"2022-07-20T02:22:23.911179Z","iopub.status.idle":"2022-07-20T02:22:23.917140Z","shell.execute_reply.started":"2022-07-20T02:22:23.911148Z","shell.execute_reply":"2022-07-20T02:22:23.916316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_modelo(df,target,features):\n    \n    x = df[features]\n    y = pd.Series(target)\n    \n    #enc = OrdinalEncoder()\n    #x[cat_features] = enc.fit_transform(x[cat_features])\n\n    X_train, X_test, y_train, y_test = train_test_split(x,y,test_size = 0.3,\n                                random_state = config.random_state, stratify = y)\n    \n    model = LGBMClassifier(**fixed_params, **search_params)\n    \n    model.fit(\n        X_train, y_train, \n        eval_set=[(X_test,y_test)],\n        eval_metric= lgb_amex_metric,\n        callbacks=[log_evaluation(100)]\n    )\n    \n    del x,y,X_train, y_train\n    \n    return model, X_test, y_test","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:22:28.926561Z","iopub.execute_input":"2022-07-20T02:22:28.927041Z","iopub.status.idle":"2022-07-20T02:22:28.934434Z","shell.execute_reply.started":"2022-07-20T02:22:28.927003Z","shell.execute_reply":"2022-07-20T02:22:28.933270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel, X_test, y_test = train_modelo(train,target,features)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:22:31.971806Z","iopub.execute_input":"2022-07-20T02:22:31.972177Z","iopub.status.idle":"2022-07-20T02:26:10.213179Z","shell.execute_reply.started":"2022-07-20T02:22:31.972146Z","shell.execute_reply":"2022-07-20T02:26:10.212094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = pd.DataFrame(y_test, columns = [\"target\"])\ny_pred = pd.DataFrame(y_test.copy(), columns = [\"prediction\"])\n\ny_pred[\"prediction\"] = model.predict_proba(X_test)[:,1]\namex_metric(y_test, y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T02:26:20.713087Z","iopub.execute_input":"2022-07-20T02:26:20.713536Z","iopub.status.idle":"2022-07-20T02:26:25.431512Z","shell.execute_reply.started":"2022-07-20T02:26:20.713498Z","shell.execute_reply":"2022-07-20T02:26:25.430548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, target, X_test, y_test, y_pred\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.booster_.save_model(\"./amex-model.txt\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Submission**\n\nRead the test file in chunks. Idea from this great notebook:\nhttps://www.kaggle.com/code/kunheekimkr/amex-lgbm-gpu-starter-0-795/comments","metadata":{}},{"cell_type":"code","source":"NAN_VALUE = -127","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(path = '', usecols = None):\n    if usecols is not None: df = pd.read_parquet(path,columns = usecols)\n    else: df = pd.read_parquet(path)\n   \n    #df['customer_ID'] = df['customer_ID'].str[-16:].str.hex_to_int().astype('int64')\n    df.S_2 = pd.to_datetime( df.S_2 )\n    df = df.fillna(NAN_VALUE) \n    print('shape of data:', df.shape)\n    \n    return df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate size of each separate test part\n\ndef get_rows(customers, test, NUM_PARTS = 4, verbose = ''):\n    chunk = len(customers)//NUM_PARTS\n    if verbose != '':\n        print(f'We will process {verbose} data as {NUM_PARTS} separate parts.')\n        print(f'There will be {chunk} customers in each part (except the last part).')\n        print('Below are number of rows in each part:')\n    rows = []\n\n    for k in range(NUM_PARTS):\n        if k == NUM_PARTS-1: cc = customers[k*chunk:]\n        else: cc = customers[k*chunk:(k+1)*chunk]\n        s = test.loc[test.customer_ID.isin(cc)].shape[0]\n        rows.append(s)\n    \n    if verbose != '': print( rows )\n    \n    return rows,chunk","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compute size of parts for test data\nNUM_PARTS = 4\nTEST_PATH =  '../input/amex-data-integer-dtypes-parquet-format/test.parquet'\n\nprint(f'Reading test data...')\ntest = read_file(path = TEST_PATH, usecols = ['customer_ID','S_2'])\n\ncustomers = test[['customer_ID']].drop_duplicates().sort_index().values.flatten()\n\nrows,num_cust = get_rows(customers,test[['customer_ID']], NUM_PARTS = NUM_PARTS, verbose = 'test')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test\ngc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocessing_te (df):\n    j = 'test'\n    #df = cudf.read_parquet(f'../input/amex-data-integer-dtypes-parquet-format/{j}.parquet')\n    cid = pd.Categorical(df.pop('customer_ID'), ordered = True)\n    last = (cid != np.roll(cid, -1)) # mask for last statement of every customer\n    \n    if 'target' in df.columns:\n        df.drop(columns=['target'], inplace=True)\n    gc.collect()\n    print('Read', j)\n    \n    df_avg = (df\n              .groupby(cid)\n              .mean()[features_avg]\n              .rename(columns={f: f\"{f}_avg\" for f in features_avg})\n             )\n    gc.collect()\n    print('Computed avg', j)\n    \n    df_min = (df\n              .groupby(cid)\n              .min()[features_min]\n              .rename(columns={f: f\"{f}_min\" for f in features_min})\n             )\n    gc.collect()\n    print('Computed min', j)\n    \n    df_max = (df\n              .groupby(cid)\n              .max()[features_max]\n              .rename(columns={f: f\"{f}_max\" for f in features_max})\n             )\n    gc.collect()\n    print('Computed max', j)\n    \n    df = (df.loc[last, features_last]\n          .rename(columns={f: f\"{f}_last\" for f in features_last})\n          .set_index(np.asarray(cid[last]))\n         )\n    gc.collect()\n    print('Computed last', j)\n    \n    df = pd.concat([df, df_min, df_max, df_avg], axis=1)\n    \n    if j == 'train': train = df\n    else: test = df\n    print(f\"{j} shape: {df.shape}\")\n    \n    del df, df_avg, df_min, df_max, cid, last\n    \n    return test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# INFER TEST DATA IN PARTS\nskip_rows = 0\nskip_cust = 0\ntest_preds = []\n\nfor k in range(NUM_PARTS):\n    print(f'\\nReading test data...')\n    test = read_file(path = TEST_PATH)\n    test = test.iloc[skip_rows:skip_rows + rows[k]]\n    skip_rows += rows[k]\n    print(f'=> Test part {k+1} has shape', test.shape )\n    \n    test = preprocessing_te(test)\n    test = test.fillna(NAN_VALUE)\n    #if k == 0: \n    #    features = [feat for feat in test.columns if feat != 'customer_ID' and feat != 'target']\n    \n    if k == NUM_PARTS - 1: test = test.loc[customers[skip_cust:]]\n    else: test = test.loc[customers[skip_cust:skip_cust+num_cust]]\n    skip_cust += num_cust\n    \n    #dtest = test[features].as_gpu_matrix()\n    #del test \n    gc.collect()\n    \n    preds = model.predict_proba(test[features])[:,1]\n    print(\"1=\",preds[:3])\n    test_preds.append(preds)\n\n# CLEAN MEMORY\ndel test, model\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = np.concatenate(test_preds)\n\nsubmission = pd.read_csv(\"../input/amex-default-prediction/sample_submission.csv\")\nsubmission.loc[:, \"prediction\"] = test_predictions\n\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}