{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### **Credits and References:**\n* https://www.kaggle.com/code/adaubas/tps-jul22-lgbm-extratree-qda-soft-voting  \n* https://www.kaggle.com/code/pourchot/simple-soft-voting  \n* https://www.kaggle.com/code/ricopue/tps-jul22-clusters-and-lgb  \n* https://www.kaggle.com/code/ambrosm/tpsjul22-gaussian-mixture-cluster-analysis  \n* https://www.kaggle.com/code/thedevastator/how-to-ensemble-clustering-algorithms-updated  \n* https://www.kaggle.com/code/eduus710/getting-cluster-ensembles-to-work  \n* https://www.kaggle.com/code/plarmuseau/bruteforce-clustering  \n* https://www.kaggle.com/code/thedevastator/bruteforce-clustering  \n\nThank you everyone for great ideas.","metadata":{}},{"cell_type":"markdown","source":"# Librairies","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport random\nimport os\nimport gc\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import PowerTransformer\nfrom sklearn.metrics import silhouette_score, calinski_harabasz_score, davies_bouldin_score\nfrom sklearn.metrics import balanced_accuracy_score, roc_auc_score\n\nfrom sklearn.mixture import BayesianGaussianMixture\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoost, Pool\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.linear_model import LogisticRegression\n\nfrom cuml.svm import SVC\nfrom cuml.neighbors import KNeighborsClassifier\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\npd.options.display.max_rows = 100\npd.options.display.max_columns = 100","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:09:38.164738Z","iopub.execute_input":"2022-07-20T17:09:38.165481Z","iopub.status.idle":"2022-07-20T17:09:41.104413Z","shell.execute_reply.started":"2022-07-20T17:09:38.165384Z","shell.execute_reply":"2022-07-20T17:09:41.103311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{}},{"cell_type":"code","source":"class CFG:\n    seed_bgm = 1\n    seed = 42\n    n_splits = 10\n    n_clusters = 7\n    threshold = 0.7","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:09:41.107780Z","iopub.execute_input":"2022-07-20T17:09:41.108518Z","iopub.status.idle":"2022-07-20T17:09:41.113663Z","shell.execute_reply.started":"2022-07-20T17:09:41.108479Z","shell.execute_reply":"2022-07-20T17:09:41.112776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:09:41.115315Z","iopub.execute_input":"2022-07-20T17:09:41.115666Z","iopub.status.idle":"2022-07-20T17:09:41.123450Z","shell.execute_reply.started":"2022-07-20T17:09:41.115631Z","shell.execute_reply":"2022-07-20T17:09:41.122510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/tabular-playground-series-jul-2022/data.csv\").drop('id', axis=1)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:09:41.128047Z","iopub.execute_input":"2022-07-20T17:09:41.128618Z","iopub.status.idle":"2022-07-20T17:09:42.165185Z","shell.execute_reply.started":"2022-07-20T17:09:41.128591Z","shell.execute_reply":"2022-07-20T17:09:42.164177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions","metadata":{}},{"cell_type":"code","source":"all_scores = []\nbest_features = [f\"f_{i:02d}\" for i in list(range(7, 14)) + list(range(22, 29))]\nfeatures = df.columns\n\ndef scores(preds, lib, df=df[best_features], verbose=True, compute_silhouette=True): \n    # Silhouette is very slow\n    sil = 0\n    if compute_silhouette:\n        sil = silhouette_score(df, preds, metric='euclidean')\n    \n    s = (lib,\n         sil, \n         calinski_harabasz_score(df, preds), \n         davies_bouldin_score(df, preds))\n    \n    if verbose:\n        print(f\"{s[0]} : Silhouette : {s[1]:.1%} | Calinski Harabasz : {s[2]:.1f} | Davis Bouldin : {s[3]:.3f}\")\n        \n    return s","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:09:42.167181Z","iopub.execute_input":"2022-07-20T17:09:42.167595Z","iopub.status.idle":"2022-07-20T17:09:42.178948Z","shell.execute_reply.started":"2022-07-20T17:09:42.167548Z","shell.execute_reply":"2022-07-20T17:09:42.177826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_score(labels, preds, probas):\n    s = (balanced_accuracy_score(labels, preds),\n        roc_auc_score(labels, probas, average=\"weighted\", multi_class=\"ovo\"))\n    return s","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:09:42.180771Z","iopub.execute_input":"2022-07-20T17:09:42.181453Z","iopub.status.idle":"2022-07-20T17:09:42.188202Z","shell.execute_reply.started":"2022-07-20T17:09:42.181413Z","shell.execute_reply":"2022-07-20T17:09:42.187236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Unsupervosed learning","metadata":{}},{"cell_type":"code","source":"seed_everything(CFG.seed_bgm)\ndf_scaled = pd.DataFrame(PowerTransformer().fit_transform(df[features]), columns=features)\n\nBGM = BayesianGaussianMixture(n_components=CFG.n_clusters, covariance_type='full', random_state=CFG.seed_bgm, max_iter=300, n_init=1, tol=1e-3)\nBGM.fit(df_scaled[best_features])\n\nBGM_predict_proba = BGM.predict_proba(df_scaled[best_features])\nBGM_predict = np.argmax(BGM_predict_proba, axis=1)\n\nall_scores.append(scores(BGM_predict, lib=\"BayesianGaussianMixture after powertransformer\"))","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-20T17:09:42.190541Z","iopub.execute_input":"2022-07-20T17:09:42.191335Z","iopub.status.idle":"2022-07-20T17:12:05.830821Z","shell.execute_reply.started":"2022-07-20T17:09:42.191296Z","shell.execute_reply":"2022-07-20T17:12:05.829838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trusted Data","metadata":{}},{"cell_type":"code","source":"proba_threshold = CFG.threshold\n\ndf_scaled['predict'] = BGM_predict\ndf_scaled['predict_proba'] = 0\nfor n in range(CFG.n_clusters):\n    df_scaled[f'predict_proba_{n}'] = BGM_predict_proba[:, n]\n    df_scaled.loc[df_scaled['predict']==n, 'predict_proba'] = df_scaled[f'predict_proba_{n}']\n    \n    \nidxs = np.array([])\nfor n in range(CFG.n_clusters):\n    median = df_scaled[df_scaled.predict==n]['predict_proba'].median()\n    idx = df_scaled[(df_scaled.predict==n) & (df_scaled.predict_proba > proba_threshold)].index\n    idxs = np.concatenate((idxs, idx))\n    print(f'Class n{n}  |  Median : {median:.4f}  |  Training data : {len(idx)/len(df_scaled[(df_scaled.predict==n)]):.1%}')\n    \nX = df_scaled.loc[idxs][best_features].reset_index(drop=True)\ny = df_scaled.loc[idxs]['predict'].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:12:05.832316Z","iopub.execute_input":"2022-07-20T17:12:05.832890Z","iopub.status.idle":"2022-07-20T17:12:06.038462Z","shell.execute_reply.started":"2022-07-20T17:12:05.832851Z","shell.execute_reply":"2022-07-20T17:12:06.037544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(X.shape)\ndisplay(X)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:12:06.040036Z","iopub.execute_input":"2022-07-20T17:12:06.040378Z","iopub.status.idle":"2022-07-20T17:12:06.065881Z","shell.execute_reply.started":"2022-07-20T17:12:06.040349Z","shell.execute_reply":"2022-07-20T17:12:06.064887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Supervised learning","metadata":{}},{"cell_type":"code","source":"params_lgb = {\n    'objective': 'multiclass',\n    'boosting': 'gbdt',\n    'learning_rate': 4e-2,\n    'verbosity': -1,\n    'n_jobs': -1,\n    'num_classes': CFG.n_clusters,\n    'random_state': CFG.seed\n}\n\nparams_xgb = {\n    'booster': 'gbtree',\n    'objective': 'multi:softprob',\n    'learning_rate': 4e-2,\n    'num_class': CFG.n_clusters,\n    'seed': CFG.seed,\n    'gpu_id': 0,\n    'tree_method': 'gpu_hist',\n    'predictor': 'gpu_predictor'\n    }\n\nparams_ctb = {\n    'objective': 'MultiClass',\n    'bootstrap_type': 'Poisson',\n    #'boosting_type': 'Ordered',  # or 'Plain'\n    'classes_count': CFG.n_clusters,\n    'num_boost_round': 20000,\n    'learning_rate': 4e-1,\n    'random_seed': CFG.seed,\n    'task_type': 'GPU'\n    \n}","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:12:06.069153Z","iopub.execute_input":"2022-07-20T17:12:06.069505Z","iopub.status.idle":"2022-07-20T17:12:06.076847Z","shell.execute_reply.started":"2022-07-20T17:12:06.069476Z","shell.execute_reply":"2022-07-20T17:12:06.075208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed_everything(CFG.seed)\n\nlgb_predict_proba = 0\nxgb_predict_proba = 0\nctb_predict_proba = 0\netc_predict_proba = 0\nqda_predict_proba = 0\nlda_predict_proba = 0\ngnb_predict_proba = 0\nlrg_predict_proba = 0\nsvc_predict_proba = 0\nknc_predict_proba = 0\nclassif_scores = []\n\nskf = StratifiedKFold(CFG.n_splits, shuffle=True, random_state=CFG.seed)\n\nfor fold, (trn_idx, val_idx) in enumerate(skf.split(X, y)):\n    print(f\"===== fold{fold} =====\")\n    X_train, y_train = X.iloc[trn_idx], y.iloc[trn_idx]\n    X_valid, y_valid = X.iloc[val_idx], y.iloc[val_idx]\n\n    # LightGBM\n    lgb_train = lgb.Dataset(X_train, y_train)\n    lgb_valid = lgb.Dataset(X_valid, y_valid)\n\n    ES = lgb.early_stopping(stopping_rounds=500, verbose=False)\n    model = lgb.train(params=params_lgb, \n                      train_set=lgb_train,\n                      valid_sets=lgb_valid, \n                      num_boost_round = 20000, \n                      callbacks = [ES])\n\n    y_pred_proba = model.predict(X_valid)\n    y_pred = np.argmax(y_pred_proba, axis=1)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"LightGBM   AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    lgb_predict_proba += model.predict(df_scaled[best_features]) / CFG.n_splits\n\n    del lgb_train, lgb_valid, model, s, y_pred, y_pred_proba\n    gc.collect()\n\n    # XGBoost\n    xgb_train = xgb.DMatrix(X_train, label=y_train)\n    xgb_valid = xgb.DMatrix(X_valid, label=y_valid)\n\n    model = xgb.train(params_xgb,\n                      dtrain=xgb_train,\n                      evals=[(xgb_train, 'train'),(xgb_valid, 'eval')],\n                      verbose_eval=False,\n                      num_boost_round=20000,\n                      early_stopping_rounds=500,\n                     )\n\n    y_pred_proba = model.predict(xgb_valid, iteration_range=(0, model.best_ntree_limit))\n    y_pred = np.argmax(y_pred_proba, axis=1)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"XGBoost    AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    xgb_predict_proba += model.predict(\n        xgb.DMatrix(df_scaled[best_features]),\n        iteration_range=(0, model.best_ntree_limit)\n    ) / CFG.n_splits\n\n    del xgb_train, xgb_valid, model, s, y_pred, y_pred_proba\n    gc.collect()\n    \n    # catboost\n    ctb_train = Pool(X_train, y_train)\n    ctb_valid = Pool(X_valid, y_valid)\n    \n    model = CatBoost(params_ctb)\n    model.fit(ctb_train,\n              eval_set=[ctb_valid],\n              verbose_eval=False,\n              early_stopping_rounds=500,\n              use_best_model=True\n             )\n    \n    y_pred_proba = model.predict(ctb_valid, prediction_type='Probability')\n    y_pred = np.argmax(y_pred_proba, axis=1)\n    \n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"CatBoost   AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    ctb_predict_proba += model.predict(Pool(df_scaled[best_features]),  prediction_type='Probability') / CFG.n_splits\n    \n    del ctb_train, ctb_valid, model, s, y_pred, y_pred_proba\n    gc.collect()\n    \n    # ExtraTreesClassifier\n    model = ExtraTreesClassifier(n_estimators=300, random_state=CFG.seed)\n    model.fit(X_train, y_train)\n\n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"ExtraTree  AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    etc_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\n    # QuadraticDiscriminantAnalysis\n    model = QuadraticDiscriminantAnalysis(priors=CFG.n_clusters)\n    model.fit(X_train, y_train)\n\n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"QDA        AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    qda_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\n    # Linear Discriminant Analysis\n    model = LinearDiscriminantAnalysis()\n    model.fit(X_train, y_train)\n\n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"LDA        AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    lda_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\n    # Gaussian Naïve Bayes\n    model = GaussianNB(var_smoothing=.1)\n    model.fit(X_train, y_train)\n\n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"GaussianNB AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    gnb_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\n    # Logistic Regression\n    model = LogisticRegression(random_state=CFG.seed, n_jobs=-1)\n    model.fit(X_train, y_train)\n\n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"LogisticR  AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    lrg_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\n    # SVC\n    model = SVC(probability=True)\n    model.fit(X_train, y_train)\n\n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"SVC        AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    svc_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\n    # KNeighborsClassifier\n    model = KNeighborsClassifier(n_neighbors=20)\n    model.fit(X_train, y_train)\n\n    y_pred = model.predict(X_valid)\n    y_pred_proba = model.predict_proba(X_valid)\n\n    s = get_score(y_valid, y_pred, y_pred_proba)\n    print(f\"KNeighbors AUC : {s[1]:.3f} | Accuracy : {s[0]:.1%}\")\n    classif_scores.append(s)\n\n    knc_predict_proba += model.predict_proba(df_scaled[best_features]) / CFG.n_splits\n\n    del model, s, y_pred, y_pred_proba\n    gc.collect()\n\nall_scores.append(scores(np.argmax(lgb_predict_proba, axis=1), lib=\"LightGBM\"))\nall_scores.append(scores(np.argmax(xgb_predict_proba, axis=1), lib=\"XGBoost\"))\nall_scores.append(scores(np.argmax(ctb_predict_proba, axis=1), lib=\"CatBoost\"))\nall_scores.append(scores(np.argmax(etc_predict_proba, axis=1), lib=\"ExtraTrees\"))\nall_scores.append(scores(np.argmax(qda_predict_proba, axis=1), lib=\"QuadraticDiscriminantAnalysis\"))\nall_scores.append(scores(np.argmax(lda_predict_proba, axis=1), lib=\"LinearDiscriminantAnalysis\"))\nall_scores.append(scores(np.argmax(gnb_predict_proba, axis=1), lib=\"GaussianNaïveBayes\"))\nall_scores.append(scores(np.argmax(lrg_predict_proba, axis=1), lib=\"LogisticRegression\"))\nall_scores.append(scores(np.argmax(svc_predict_proba, axis=1), lib=\"SVC\"))\nall_scores.append(scores(np.argmax(knc_predict_proba, axis=1), lib=\"KNeighbors\"))\n\npd.DataFrame(classif_scores, columns = [\"balanced_accuracy_score\", \"roc_auc_score\"]).mean(0)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T17:12:06.078427Z","iopub.execute_input":"2022-07-20T17:12:06.079224Z","iopub.status.idle":"2022-07-20T18:29:59.538053Z","shell.execute_reply.started":"2022-07-20T17:12:06.079186Z","shell.execute_reply":"2022-07-20T18:29:59.537040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Soft voting","metadata":{}},{"cell_type":"code","source":"def soft_voting(preds_probas):\n    pred_test = np.zeros((df.shape[0], CFG.n_clusters))\n    \n    for i, p in enumerate(preds_probas):\n        pred_test += p[0] * p[1]\n    \n    return np.argmax(pred_test, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T18:29:59.539820Z","iopub.execute_input":"2022-07-20T18:29:59.540493Z","iopub.status.idle":"2022-07-20T18:29:59.550581Z","shell.execute_reply.started":"2022-07-20T18:29:59.540455Z","shell.execute_reply":"2022-07-20T18:29:59.549558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sv_predict = soft_voting((\n    (lgb_predict_proba, 0.0),\n    (xgb_predict_proba, 0.75),\n    (ctb_predict_proba, 0.0),\n    (etc_predict_proba, 1.0),\n    (qda_predict_proba, 1.0),\n    (lda_predict_proba, 0.0),\n    (gnb_predict_proba, 0.0),\n    (lrg_predict_proba, 0.0),\n    (svc_predict_proba, 1.0),\n    (knc_predict_proba, 1.0),\n))\nall_scores.append(scores(sv_predict, lib=\"Soft voting\"))","metadata":{"execution":{"iopub.status.busy":"2022-07-20T18:29:59.553218Z","iopub.execute_input":"2022-07-20T18:29:59.554925Z","iopub.status.idle":"2022-07-20T18:31:36.052367Z","shell.execute_reply.started":"2022-07-20T18:29:59.554887Z","shell.execute_reply":"2022-07-20T18:31:36.051399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_scores_df = pd.DataFrame(all_scores, columns=[\"Model\", \"silhouette\", \"Calinski_Harabasz\", \"Davis_Bouldin\"])\nall_scores_df.to_csv(\"scores.csv\", index=False)\nall_scores_df","metadata":{"execution":{"iopub.status.busy":"2022-07-20T18:31:36.053986Z","iopub.execute_input":"2022-07-20T18:31:36.054670Z","iopub.status.idle":"2022-07-20T18:31:36.079073Z","shell.execute_reply.started":"2022-07-20T18:31:36.054631Z","shell.execute_reply":"2022-07-20T18:31:36.078055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/tabular-playground-series-jul-2022/sample_submission.csv\")\nsub['Predicted'] = sv_predict\nsub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T18:31:36.080721Z","iopub.execute_input":"2022-07-20T18:31:36.081378Z","iopub.status.idle":"2022-07-20T18:31:36.289679Z","shell.execute_reply.started":"2022-07-20T18:31:36.081342Z","shell.execute_reply":"2022-07-20T18:31:36.288711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}