{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\n\nThis is a copy of https://www.kaggle.com/code/vickeytomer/catboost-classifier (public score: **0.765**)\n\nUsed the original features, and CoLES embeddings added.\nCoLES made with unsupervised contrastive learning technic in https://www.kaggle.com/code/ivkireev/amex-contrastive-embeddings-with-ptls-coles\n\nNew public score is: **0.778**","metadata":{}},{"cell_type":"code","source":"import gc\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%config Completer.use_jedi = False\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\npd.set_option('display.max_colwidth', None)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-16T07:46:01.028539Z","iopub.execute_input":"2022-08-16T07:46:01.029852Z","iopub.status.idle":"2022-08-16T07:46:02.242547Z","shell.execute_reply.started":"2022-08-16T07:46:01.029724Z","shell.execute_reply":"2022-08-16T07:46:02.241556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**The dataset contains aggregated profile features for each customer at each statement date. Features are anonymized and normalized, and fall into the following general categories:**\n\n* D_* = Delinquency variables\n* S_* = Spend variables\n* P_* = Payment variables\n* B_* = Balance variables\n* R_* = Risk variables\n**with the following features being categorical:**\n\n**['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']**\n\nYour task is to predict, for each customer_ID, the probability of a future payment default (target = 1).","metadata":{}},{"cell_type":"code","source":"# test=pd.read_parquet('../input/amex-parquet/test_data.parquet')\ntrain=pd.read_parquet('../input/amex-parquet/train_data.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:02.244326Z","iopub.execute_input":"2022-08-16T07:46:02.245558Z","iopub.status.idle":"2022-08-16T07:46:40.281947Z","shell.execute_reply.started":"2022-08-16T07:46:02.245518Z","shell.execute_reply":"2022-08-16T07:46:40.280790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape of training data:\",train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:40.284304Z","iopub.execute_input":"2022-08-16T07:46:40.285402Z","iopub.status.idle":"2022-08-16T07:46:40.295108Z","shell.execute_reply.started":"2022-08-16T07:46:40.285340Z","shell.execute_reply":"2022-08-16T07:46:40.293906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking the null's in data","metadata":{}},{"cell_type":"code","source":"percent_missing = train.isnull().sum() * 100 / len(train)\nmissing_value_df = pd.DataFrame({'column_name': train.columns,\n                                 'percent_missing': percent_missing})\nmissing_value_df.sort_values('percent_missing',ascending=False, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:40.297917Z","iopub.execute_input":"2022-08-16T07:46:40.298814Z","iopub.status.idle":"2022-08-16T07:46:43.285986Z","shell.execute_reply.started":"2022-08-16T07:46:40.298646Z","shell.execute_reply":"2022-08-16T07:46:43.284674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11.7,8.27)})\nax = sns.barplot(x=\"percent_missing\", y=\"column_name\", data=missing_value_df[missing_value_df['percent_missing']>5]).set_title('Graphs showing more then 5 percente of null value columns')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:43.287529Z","iopub.execute_input":"2022-08-16T07:46:43.287915Z","iopub.status.idle":"2022-08-16T07:46:43.886000Z","shell.execute_reply.started":"2022-08-16T07:46:43.287881Z","shell.execute_reply":"2022-08-16T07:46:43.884174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"count of unique customers:\",train.customer_ID.nunique())","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:43.888582Z","iopub.execute_input":"2022-08-16T07:46:43.889025Z","iopub.status.idle":"2022-08-16T07:46:44.838041Z","shell.execute_reply.started":"2022-08-16T07:46:43.888984Z","shell.execute_reply":"2022-08-16T07:46:44.836750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Filtering data for one customer lets see how many records it has.","metadata":{}},{"cell_type":"code","source":"train[train.customer_ID=='0000099d6bd597052cdcda90ffabf56573fe9d7c79be5fbac11a8ed792feb62a']","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:44.839959Z","iopub.execute_input":"2022-08-16T07:46:44.840378Z","iopub.status.idle":"2022-08-16T07:46:45.443586Z","shell.execute_reply.started":"2022-08-16T07:46:44.840340Z","shell.execute_reply":"2022-08-16T07:46:45.442111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# As we have no of records lets pick the latest one only\ntrain_data=train.groupby('customer_ID').tail(1)\ntrain_data=train_data.set_index(['customer_ID'])\n#Drop date column since it is no longer relevant\ntrain_data.drop(['S_2'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:45.445378Z","iopub.execute_input":"2022-08-16T07:46:45.445921Z","iopub.status.idle":"2022-08-16T07:46:48.186462Z","shell.execute_reply.started":"2022-08-16T07:46:45.445871Z","shell.execute_reply":"2022-08-16T07:46:48.185229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"checking for the above member again for latest record\")\ntrain_data[train_data.index=='0000099d6bd597052cdcda90ffabf56573fe9d7c79be5fbac11a8ed792feb62a']","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:48.188236Z","iopub.execute_input":"2022-08-16T07:46:48.188897Z","iopub.status.idle":"2022-08-16T07:46:48.347412Z","shell.execute_reply.started":"2022-08-16T07:46:48.188845Z","shell.execute_reply":"2022-08-16T07:46:48.346538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"shape of new data frame : \",train_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:48.350782Z","iopub.execute_input":"2022-08-16T07:46:48.351890Z","iopub.status.idle":"2022-08-16T07:46:48.358125Z","shell.execute_reply.started":"2022-08-16T07:46:48.351849Z","shell.execute_reply":"2022-08-16T07:46:48.356443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Identify columns which are not numeric\n# features being categorical:*\n# ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n# 'D_63', 'D_64' are of object must be categorical\ntrain_data.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:48.359643Z","iopub.execute_input":"2022-08-16T07:46:48.360757Z","iopub.status.idle":"2022-08-16T07:46:48.377445Z","shell.execute_reply.started":"2022-08-16T07:46:48.360708Z","shell.execute_reply":"2022-08-16T07:46:48.375820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform one-hot encoding for D_63 and D_64\n# Drop columns D_63 and D_64 subsequently\ntrain_D63 = pd.get_dummies(train_data[['D_63']])\ntrain_data = pd.concat([train_data, train_D63], axis=1)\ntrain_data = train_data.drop(['D_63'], axis=1)\n\ntrain_D64 = pd.get_dummies(train_data[['D_64']])\ntrain_data = pd.concat([train_data, train_D64], axis=1)\ntrain_data = train_data.drop(['D_64'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:48.379667Z","iopub.execute_input":"2022-08-16T07:46:48.380185Z","iopub.status.idle":"2022-08-16T07:46:49.781215Z","shell.execute_reply.started":"2022-08-16T07:46:48.380134Z","shell.execute_reply":"2022-08-16T07:46:49.779841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:49.783105Z","iopub.execute_input":"2022-08-16T07:46:49.786925Z","iopub.status.idle":"2022-08-16T07:46:49.801953Z","shell.execute_reply.started":"2022-08-16T07:46:49.786878Z","shell.execute_reply":"2022-08-16T07:46:49.800900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets check for the columns\ntrain_data.columns\n# We now have 197 columns including target\n# We need to reduce the dimensionality of the data\n# We shall remove highly correlated features.","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:49.803318Z","iopub.execute_input":"2022-08-16T07:46:49.804202Z","iopub.status.idle":"2022-08-16T07:46:49.815647Z","shell.execute_reply.started":"2022-08-16T07:46:49.804163Z","shell.execute_reply":"2022-08-16T07:46:49.814193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We shall remove highly correlated features.\ntrain_without_target=train_data.drop(['target'],axis=1)\ncor_matrix = train_without_target.corr().abs()\nupper_tri = cor_matrix.where((np.triu(np.ones(cor_matrix.shape), k=1) + np.tril(np.ones(cor_matrix.shape), k=-1)).astype(bool))\n#Drop out columns with absolute correlation of more than 85%\nto_drop = [column for column in upper_tri.columns if any(upper_tri[column] > 0.85)]\ntrain_drop_highcorr=train_without_target.drop(to_drop,axis=1)\ntrain_drop_highcorr.shape\n#We are now left with 159 columns (excluding target), which is still significant","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:46:49.817220Z","iopub.execute_input":"2022-08-16T07:46:49.817632Z","iopub.status.idle":"2022-08-16T07:47:33.760558Z","shell.execute_reply.started":"2022-08-16T07:46:49.817595Z","shell.execute_reply":"2022-08-16T07:47:33.759313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lets remove columns with low variance=0.06. Keep only columns with high variance\nfrom sklearn.feature_selection import VarianceThreshold\nfrom itertools import compress\ndef fs_variance(df, threshold:float=0.06):\n    \"\"\"\n    Return a list of selected variables based on the threshold.\n    \"\"\"\n    # The list of columns in the data frame\n    features = list(df.columns)\n    \n    # Initialize and fit the method\n    vt = VarianceThreshold(threshold = threshold)\n    _ = vt.fit(df)\n    \n    # Get which column names which pass the threshold\n    feat_select = list(compress(features, vt.get_support()))\n    \n    return feat_select\ncolumns_to_keep=fs_variance(train_drop_highcorr)\n# columns_to_keep.extend(cat_features)\n# We are left with 74 columns (excluding target), which passed the threshold.\ntrain_final=train_data[columns_to_keep]\nlen(columns_to_keep)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:47:33.761940Z","iopub.execute_input":"2022-08-16T07:47:33.762418Z","iopub.status.idle":"2022-08-16T07:47:35.262115Z","shell.execute_reply.started":"2022-08-16T07:47:33.762385Z","shell.execute_reply":"2022-08-16T07:47:35.260756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Concat target onto train_final\ntrain_final1=train_final.join(train_data['target'])\nx_train=train_final1.drop(['target'],axis=1)\ny_train=train_final1['target']","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:47:35.264368Z","iopub.execute_input":"2022-08-16T07:47:35.265077Z","iopub.status.idle":"2022-08-16T07:47:35.510548Z","shell.execute_reply.started":"2022-08-16T07:47:35.265022Z","shell.execute_reply":"2022-08-16T07:47:35.509131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add ColES embeddings\ndf_train_emb = pd.read_parquet('../input/amex-coles-embeddings/df_emb_train.parquet')\nx_train = pd.merge(x_train, df_train_emb.drop(columns='target').set_index('customer_ID'), left_index=True, right_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:47:54.177705Z","iopub.execute_input":"2022-08-16T07:47:54.178400Z","iopub.status.idle":"2022-08-16T07:47:56.264771Z","shell.execute_reply.started":"2022-08-16T07:47:54.178356Z","shell.execute_reply":"2022-08-16T07:47:56.262644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split train data into training and testing sets\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score  \nfrom sklearn.metrics import precision_score                         \nfrom sklearn.metrics import recall_score\nx_train_split, x_test_split, y_train_split, y_test_split = train_test_split(x_train, y_train, test_size=0.30, random_state=20)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:47:56.266976Z","iopub.execute_input":"2022-08-16T07:47:56.268024Z","iopub.status.idle":"2022-08-16T07:47:58.027813Z","shell.execute_reply.started":"2022-08-16T07:47:56.267977Z","shell.execute_reply":"2022-08-16T07:47:58.026643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_train\ndel train_final\ndel train\ndel train_final1\ndel train_data\ndel df_train_emb\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:51:53.187999Z","iopub.execute_input":"2022-08-16T07:51:53.188466Z","iopub.status.idle":"2022-08-16T07:51:53.195962Z","shell.execute_reply.started":"2022-08-16T07:51:53.188431Z","shell.execute_reply":"2022-08-16T07:51:53.194379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostClassifier\nmodel = CatBoostClassifier()\n# Fit model\nmodel.fit(x_train_split,y_train_split,verbose=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:52:06.035979Z","iopub.execute_input":"2022-08-16T07:52:06.036492Z","iopub.status.idle":"2022-08-16T07:58:09.370732Z","shell.execute_reply.started":"2022-08-16T07:52:06.036440Z","shell.execute_reply":"2022-08-16T07:58:09.367926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Test the model\ny_predict=model.predict(x_test_split)\nprint('\\n CatBoost Classifier Accuracy: {:.3f}'.format(accuracy_score(y_test_split, y_predict)))\n# Achieved 89.3% accuracy, 0.896","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:58:09.377532Z","iopub.execute_input":"2022-08-16T07:58:09.378471Z","iopub.status.idle":"2022-08-16T07:58:09.882162Z","shell.execute_reply.started":"2022-08-16T07:58:09.378419Z","shell.execute_reply":"2022-08-16T07:58:09.879593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\n CatBoost Classifier Precision: {:.3f}'.format(precision_score (y_test_split, y_predict)))\n# Achieved Precision Score of 0.792, 0.802","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:58:09.885326Z","iopub.execute_input":"2022-08-16T07:58:09.885796Z","iopub.status.idle":"2022-08-16T07:58:09.959556Z","shell.execute_reply.started":"2022-08-16T07:58:09.885754Z","shell.execute_reply":"2022-08-16T07:58:09.958192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\n CatBoost Classifier Recall: {:.3f}'.format(recall_score (y_test_split, y_predict)))\n#Achieved Recall Score of 0.791, 0.796","metadata":{"execution":{"iopub.status.busy":"2022-08-16T07:58:09.961981Z","iopub.execute_input":"2022-08-16T07:58:09.962394Z","iopub.status.idle":"2022-08-16T07:58:10.037740Z","shell.execute_reply.started":"2022-08-16T07:58:09.962358Z","shell.execute_reply":"2022-08-16T07:58:10.036512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_load=list(columns_to_keep)\ncolumns_to_load=columns_to_load+['D_63','D_64','customer_ID','S_2']\ncolumns_to_load.remove('D_63_CO')\ncolumns_to_load.remove('D_63_CR')\ncolumns_to_load.remove('D_63_CL')\ncolumns_to_load.remove('D_64_O')\ncolumns_to_load.remove('D_64_R')\ncolumns_to_load.remove('D_64_U')","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:03:53.269768Z","iopub.execute_input":"2022-08-16T08:03:53.270459Z","iopub.status.idle":"2022-08-16T08:03:53.279319Z","shell.execute_reply.started":"2022-08-16T08:03:53.270401Z","shell.execute_reply":"2022-08-16T08:03:53.277601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_parquet('../input/amex-parquet/test_data.parquet',columns=columns_to_load)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:03:56.548944Z","iopub.execute_input":"2022-08-16T08:03:56.549794Z","iopub.status.idle":"2022-08-16T08:04:36.650124Z","shell.execute_reply.started":"2022-08-16T08:03:56.549749Z","shell.execute_reply":"2022-08-16T08:04:36.648761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test_data.groupby('customer_ID').tail(1)\ntest=test.set_index(['customer_ID'])\n\n#Drop date column since it is no longer relevant\ntest.drop(['S_2'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:04:36.653188Z","iopub.execute_input":"2022-08-16T08:04:36.653840Z","iopub.status.idle":"2022-08-16T08:04:38.132115Z","shell.execute_reply.started":"2022-08-16T08:04:36.653801Z","shell.execute_reply":"2022-08-16T08:04:38.130967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_D63 = pd.get_dummies(test[['D_63']])\ntest = pd.concat([test, test_D63], axis=1)\ntest = test.drop(['D_63'], axis=1)\n\ntest_D64 = pd.get_dummies(test[['D_64']])\ntest = pd.concat([test, test_D64], axis=1)\ntest = test.drop(['D_64'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:04:38.133753Z","iopub.execute_input":"2022-08-16T08:04:38.134167Z","iopub.status.idle":"2022-08-16T08:04:39.196548Z","shell.execute_reply.started":"2022-08-16T08:04:38.134123Z","shell.execute_reply":"2022-08-16T08:04:39.194574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_final=test[columns_to_keep]","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:04:39.199621Z","iopub.execute_input":"2022-08-16T08:04:39.200071Z","iopub.status.idle":"2022-08-16T08:04:39.308675Z","shell.execute_reply.started":"2022-08-16T08:04:39.200030Z","shell.execute_reply":"2022-08-16T08:04:39.307386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add CoLES Embeddings\ndf_test_emb = pd.read_parquet('../input/amex-coles-embeddings/df_emb_test.parquet')\ntest_final = pd.merge(test_final, df_test_emb.set_index('customer_ID'), left_index=True, right_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:04:39.310616Z","iopub.execute_input":"2022-08-16T08:04:39.311109Z","iopub.status.idle":"2022-08-16T08:04:45.456263Z","shell.execute_reply.started":"2022-08-16T08:04:39.311068Z","shell.execute_reply":"2022-08-16T08:04:45.454685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_data\ndel test\ndel df_test_emb\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:05:30.127917Z","iopub.execute_input":"2022-08-16T08:05:30.128355Z","iopub.status.idle":"2022-08-16T08:05:30.363456Z","shell.execute_reply.started":"2022-08-16T08:05:30.128319Z","shell.execute_reply":"2022-08-16T08:05:30.361580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_predict=model.predict_proba(test_final)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:05:46.120634Z","iopub.execute_input":"2022-08-16T08:05:46.121171Z","iopub.status.idle":"2022-08-16T08:05:48.375610Z","shell.execute_reply.started":"2022-08-16T08:05:46.121128Z","shell.execute_reply":"2022-08-16T08:05:48.374139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Retrieve the probability of default\ny_predict_final=y_test_predict[:,1]\n\n# Merge the prediction and customer_ID into submission dataframe\nsubmission = pd.DataFrame({\"customer_ID\":test_final.index,\"prediction\":y_predict_final})\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T08:05:49.909038Z","iopub.execute_input":"2022-08-16T08:05:49.909614Z","iopub.status.idle":"2022-08-16T08:05:53.409293Z","shell.execute_reply.started":"2022-08-16T08:05:49.909571Z","shell.execute_reply":"2022-08-16T08:05:53.407828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}