{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_csv('../input/contradictory-my-dear-watson/train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:45:37.526097Z","iopub.execute_input":"2022-08-10T02:45:37.526421Z","iopub.status.idle":"2022-08-10T02:45:37.684531Z","shell.execute_reply.started":"2022-08-10T02:45:37.526349Z","shell.execute_reply":"2022-08-10T02:45:37.683615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForMaskedLM\n\nmodel_checkpoint = \"xlm-roberta-base\"\nmodel = AutoModelForMaskedLM.from_pretrained(model_checkpoint)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:45:37.686456Z","iopub.execute_input":"2022-08-10T02:45:37.687053Z","iopub.status.idle":"2022-08-10T02:46:34.642538Z","shell.execute_reply.started":"2022-08-10T02:45:37.687015Z","shell.execute_reply":"2022-08-10T02:46:34.641476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roberta_num_parameters = model.num_parameters() / 1_000_000\nprint(f\"'>>> DistilBERT number of parameters: {round(roberta_num_parameters)}M'\")\nprint(f\"'>>> BERT number of parameters: 110M'\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:34.644049Z","iopub.execute_input":"2022-08-10T02:46:34.645225Z","iopub.status.idle":"2022-08-10T02:46:34.652815Z","shell.execute_reply.started":"2022-08-10T02:46:34.645181Z","shell.execute_reply":"2022-08-10T02:46:34.651080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \"This is a great <mask>.\"","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:14:59.835503Z","iopub.execute_input":"2022-08-10T03:14:59.836144Z","iopub.status.idle":"2022-08-10T03:14:59.846398Z","shell.execute_reply.started":"2022-08-10T03:14:59.836102Z","shell.execute_reply":"2022-08-10T03:14:59.845147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(model_checkpoint)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:34.663148Z","iopub.execute_input":"2022-08-10T02:46:34.663759Z","iopub.status.idle":"2022-08-10T02:46:40.698068Z","shell.execute_reply.started":"2022-08-10T02:46:34.663724Z","shell.execute_reply":"2022-08-10T02:46:40.696880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\ninputs = tokenizer(text, return_tensors=\"pt\")\ntoken_logits = model(**inputs).logits\n# Find the location of [MASK] and extract its logits\nmask_token_index = torch.where(inputs[\"input_ids\"] == tokenizer.mask_token_id)[1]\nmask_token_logits = token_logits[0, mask_token_index, :]\n# Pick the [MASK] candidates with the highest logits\ntop_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist()\n\nfor token in top_5_tokens:\n    print(f\"'>>> {text.replace(tokenizer.mask_token, tokenizer.decode([token]))}'\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:40.699719Z","iopub.execute_input":"2022-08-10T02:46:40.700156Z","iopub.status.idle":"2022-08-10T02:46:46.687530Z","shell.execute_reply.started":"2022-08-10T02:46:40.700104Z","shell.execute_reply":"2022-08-10T02:46:46.686406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset\n\nds = Dataset.from_pandas(df)\nds = ds.train_test_split(test_size = 0.1, shuffle=True, seed = 2022)\nds = ds.remove_columns(['id','lang_abv', 'language'])\nds","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:46.688939Z","iopub.execute_input":"2022-08-10T02:46:46.689745Z","iopub.status.idle":"2022-08-10T02:46:47.271739Z","shell.execute_reply.started":"2022-08-10T02:46:46.689705Z","shell.execute_reply":"2022-08-10T02:46:47.270701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = ds['train'].shuffle(seed=2022).select(range(3))\nfor i in range(3):\n    print('Premise :', sample['premise'][i])\n    print('hypothesis :', sample['hypothesis'][i])\n    print('label :', sample['label'][i])\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:47.273314Z","iopub.execute_input":"2022-08-10T02:46:47.274083Z","iopub.status.idle":"2022-08-10T02:46:47.294159Z","shell.execute_reply.started":"2022-08-10T02:46:47.274044Z","shell.execute_reply":"2022-08-10T02:46:47.292869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize_function(examples):\n    result = tokenizer(examples[\"premise\"], examples['hypothesis'])\n    if tokenizer.is_fast:\n        result[\"word_ids\"] = [result.word_ids(i) for i in range(len(result[\"input_ids\"]))]\n    return result\n\n\n# Use batched=True to activate fast multithreading!\ntokenized_datasets = ds.map(\n    tokenize_function, batched=True, remove_columns=[\"premise\", \"hypothesis\", \"label\"]\n)\ntokenized_datasets","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:47.296227Z","iopub.execute_input":"2022-08-10T02:46:47.296701Z","iopub.status.idle":"2022-08-10T02:46:50.052383Z","shell.execute_reply.started":"2022-08-10T02:46:47.296664Z","shell.execute_reply":"2022-08-10T02:46:50.051420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.model_max_length\nchunk_size = 128","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:50.056612Z","iopub.execute_input":"2022-08-10T02:46:50.058825Z","iopub.status.idle":"2022-08-10T02:46:50.063975Z","shell.execute_reply.started":"2022-08-10T02:46:50.058796Z","shell.execute_reply":"2022-08-10T02:46:50.061963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenized_samples = tokenized_datasets['train'].shuffle(seed = 2020).select(range(3))\n\nfor i in range(3):\n    print(len(tokenized_samples['input_ids'][i]))\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:50.067519Z","iopub.execute_input":"2022-08-10T02:46:50.068136Z","iopub.status.idle":"2022-08-10T02:46:51.166265Z","shell.execute_reply.started":"2022-08-10T02:46:50.068098Z","shell.execute_reply":"2022-08-10T02:46:51.165227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenized_samples.features\n\nfeatures = ['input_ids', 'attention_mask', 'word_ids']","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:51.167765Z","iopub.execute_input":"2022-08-10T02:46:51.168439Z","iopub.status.idle":"2022-08-10T02:46:51.174595Z","shell.execute_reply.started":"2022-08-10T02:46:51.168404Z","shell.execute_reply":"2022-08-10T02:46:51.172864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concatenated_examples = {\n    k: sum(tokenized_samples[k], []) for k in tokenized_samples.features.keys()\n}\ntotal_length = len(concatenated_examples[\"input_ids\"])\nprint(f\"'>>> Concatenated reviews length: {total_length}'\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:51.176271Z","iopub.execute_input":"2022-08-10T02:46:51.177432Z","iopub.status.idle":"2022-08-10T02:46:51.187679Z","shell.execute_reply.started":"2022-08-10T02:46:51.177404Z","shell.execute_reply":"2022-08-10T02:46:51.186618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunks = {\n    k: [t[i : i + chunk_size] for i in range(0, total_length, chunk_size)]\n    for k, t in concatenated_examples.items()\n}\n\nfor chunk in chunks[\"input_ids\"]:\n    print(f\"'>>> Chunk length: {len(chunk)}'\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:51.189132Z","iopub.execute_input":"2022-08-10T02:46:51.189772Z","iopub.status.idle":"2022-08-10T02:46:51.198725Z","shell.execute_reply.started":"2022-08-10T02:46:51.189738Z","shell.execute_reply":"2022-08-10T02:46:51.197381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def group_texts(examples):\n    # Concatenate all texts\n    concatenated_examples = {k: sum(examples[k], []) for k in features}\n    # Compute length of concatenated texts\n    total_length = len(concatenated_examples[features[0]])\n    # We drop the last chunk if it's smaller than chunk_size\n    total_length = (total_length // chunk_size) * chunk_size\n    # Split by chunks of max_len\n    result = {\n        k: [t[i : i + chunk_size] for i in range(0, total_length, chunk_size)]\n        for k, t in concatenated_examples.items()\n    }\n    # Create a new labels column\n    result[\"labels\"] = result[\"input_ids\"].copy()\n    return result","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:51.200288Z","iopub.execute_input":"2022-08-10T02:46:51.200848Z","iopub.status.idle":"2022-08-10T02:46:51.209469Z","shell.execute_reply.started":"2022-08-10T02:46:51.200814Z","shell.execute_reply":"2022-08-10T02:46:51.208479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lm_datasets = tokenized_datasets.map(group_texts, batched=True)\nlm_datasets","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:51.211052Z","iopub.execute_input":"2022-08-10T02:46:51.211746Z","iopub.status.idle":"2022-08-10T02:46:56.596332Z","shell.execute_reply.started":"2022-08-10T02:46:51.211712Z","shell.execute_reply":"2022-08-10T02:46:56.595373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.decode(lm_datasets[\"train\"][1][\"input_ids\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.597839Z","iopub.execute_input":"2022-08-10T02:46:56.598430Z","iopub.status.idle":"2022-08-10T02:46:56.608007Z","shell.execute_reply.started":"2022-08-10T02:46:56.598395Z","shell.execute_reply":"2022-08-10T02:46:56.607000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.decode(lm_datasets[\"train\"][1][\"labels\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.609548Z","iopub.execute_input":"2022-08-10T02:46:56.610288Z","iopub.status.idle":"2022-08-10T02:46:56.622357Z","shell.execute_reply.started":"2022-08-10T02:46:56.610251Z","shell.execute_reply":"2022-08-10T02:46:56.621450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import DataCollatorForLanguageModeling\n\ndata_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.624648Z","iopub.execute_input":"2022-08-10T02:46:56.625290Z","iopub.status.idle":"2022-08-10T02:46:56.726676Z","shell.execute_reply.started":"2022-08-10T02:46:56.625255Z","shell.execute_reply":"2022-08-10T02:46:56.725823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples = [lm_datasets[\"train\"][i] for i in range(2)]\nfor sample in samples:\n    _ = sample.pop(\"word_ids\")\n\nfor chunk in data_collator(samples)[\"input_ids\"]:\n    print(f\"\\n'>>> {tokenizer.decode(chunk)}'\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.727824Z","iopub.execute_input":"2022-08-10T02:46:56.729652Z","iopub.status.idle":"2022-08-10T02:46:56.739479Z","shell.execute_reply.started":"2022-08-10T02:46:56.729594Z","shell.execute_reply":"2022-08-10T02:46:56.738445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# text = 'terör karşıtı aracı olarak finansal takibe güveniyordu.</s><s> Секторът на въздушната отбрана в югоизточна Африка беше уведомен за събитието в 9:55, 28 минути по-късно.</s></s> 28 минути след факта Югоизточният въздушен отбранителен сектор получи известие.</s><s> (A bigger contribution may or may not mean, I really, really support Candidate X.) Freedom of association is an even bigger stretch--one that Justice Thomas would laugh out of court if some liberal proposed it.</s></s> There were some liberals opposed to it.</s><s> The'\ntext = [lm_datasets[\"train\"][12]]\ntext[0].pop('word_ids')\ntokenizer.decode(data_collator(text)['input_ids'][0])","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.741430Z","iopub.execute_input":"2022-08-10T02:46:56.742201Z","iopub.status.idle":"2022-08-10T02:46:56.751466Z","shell.execute_reply.started":"2022-08-10T02:46:56.742166Z","shell.execute_reply":"2022-08-10T02:46:56.750389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\nimport numpy as np\n\nfrom transformers import default_data_collator\n\nwwm_probability = 0.2\n\n\ndef whole_word_masking_data_collator(features):\n    for feature in features:\n        word_ids = feature.pop(\"word_ids\")\n\n        # Create a map between words and corresponding token indices\n        mapping = collections.defaultdict(list)\n        current_word_index = -1\n        current_word = None\n        for idx, word_id in enumerate(word_ids):\n            if word_id is not None:\n                if word_id != current_word:\n                    current_word = word_id\n                    current_word_index += 1\n                mapping[current_word_index].append(idx)\n\n        # Randomly mask words\n        mask = np.random.binomial(1, wwm_probability, (len(mapping),))\n        input_ids = feature[\"input_ids\"]\n        labels = feature[\"labels\"]\n        new_labels = [-100] * len(labels)\n        for word_id in np.where(mask)[0]:\n            word_id = word_id.item()\n            for idx in mapping[word_id]:\n                new_labels[idx] = labels[idx]\n                input_ids[idx] = tokenizer.mask_token_id\n\n    return default_data_collator(features)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.755106Z","iopub.execute_input":"2022-08-10T02:46:56.755756Z","iopub.status.idle":"2022-08-10T02:46:56.765525Z","shell.execute_reply.started":"2022-08-10T02:46:56.755724Z","shell.execute_reply":"2022-08-10T02:46:56.764619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples = [lm_datasets[\"train\"][i] for i in range(2)]\nbatch = whole_word_masking_data_collator(samples)\n\nfor chunk in batch[\"input_ids\"]:\n    print(f\"\\n'>>> {tokenizer.decode(chunk)}'\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.767073Z","iopub.execute_input":"2022-08-10T02:46:56.767721Z","iopub.status.idle":"2022-08-10T02:46:56.777486Z","shell.execute_reply.started":"2022-08-10T02:46:56.767687Z","shell.execute_reply":"2022-08-10T02:46:56.776458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lm_datasets","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.780928Z","iopub.execute_input":"2022-08-10T02:46:56.781578Z","iopub.status.idle":"2022-08-10T02:46:56.792353Z","shell.execute_reply.started":"2022-08-10T02:46:56.781526Z","shell.execute_reply":"2022-08-10T02:46:56.791219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from huggingface_hub import notebook_login\n\nnotebook_login()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:46:56.793463Z","iopub.execute_input":"2022-08-10T02:46:56.796831Z","iopub.status.idle":"2022-08-10T02:46:56.839342Z","shell.execute_reply.started":"2022-08-10T02:46:56.796797Z","shell.execute_reply":"2022-08-10T02:46:56.838416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import TrainingArguments\n\nbatch_size = 16\n# Show the training loss with every epoch\nlogging_steps = len(lm_datasets[\"train\"]) // batch_size\nmodel_name = model_checkpoint.split(\"/\")[-1]\n\ntraining_args = TrainingArguments(\n    output_dir=f\"{model_name}-finetuned-my_dear_watson2\",\n    overwrite_output_dir=True,\n    evaluation_strategy=\"epoch\",\n    learning_rate=2e-5,\n    weight_decay=0.01,\n    per_device_train_batch_size=batch_size,\n    per_device_eval_batch_size=batch_size,\n    push_to_hub=True,\n    fp16=True,\n    logging_steps=logging_steps,\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:49:30.743218Z","iopub.execute_input":"2022-08-10T02:49:30.743608Z","iopub.status.idle":"2022-08-10T02:49:30.756141Z","shell.execute_reply.started":"2022-08-10T02:49:30.743566Z","shell.execute_reply":"2022-08-10T02:49:30.754744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!apt install git-lfs\n!git config --global user.email \"samarthgarg92001@gmail.com\"\n!git config --global user.name \"SamarthGarg09\"","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:48:31.694327Z","iopub.execute_input":"2022-08-10T02:48:31.694731Z","iopub.status.idle":"2022-08-10T02:48:37.150545Z","shell.execute_reply.started":"2022-08-10T02:48:31.694696Z","shell.execute_reply":"2022-08-10T02:48:37.149112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Trainer\n\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    tokenizer=tokenizer,\n    train_dataset=lm_datasets[\"train\"],\n    eval_dataset=lm_datasets[\"test\"],\n    data_collator=data_collator,\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:09:21.891981Z","iopub.execute_input":"2022-08-10T03:09:21.892810Z","iopub.status.idle":"2022-08-10T03:09:26.429840Z","shell.execute_reply.started":"2022-08-10T03:09:21.892769Z","shell.execute_reply":"2022-08-10T03:09:26.428670Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\n\neval_results = trainer.evaluate()\nprint(f\">>> Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:49:47.891469Z","iopub.execute_input":"2022-08-10T02:49:47.892019Z","iopub.status.idle":"2022-08-10T02:50:01.619148Z","shell.execute_reply.started":"2022-08-10T02:49:47.891971Z","shell.execute_reply":"2022-08-10T02:50:01.618204Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:50:04.584316Z","iopub.execute_input":"2022-08-10T02:50:04.584708Z","iopub.status.idle":"2022-08-10T02:58:21.511827Z","shell.execute_reply.started":"2022-08-10T02:50:04.584671Z","shell.execute_reply":"2022-08-10T02:58:21.510640Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_results = trainer.evaluate()\nprint(f\">>> Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:03:46.928922Z","iopub.execute_input":"2022-08-10T03:03:46.929549Z","iopub.status.idle":"2022-08-10T03:03:52.282001Z","shell.execute_reply.started":"2022-08-10T03:03:46.929511Z","shell.execute_reply":"2022-08-10T03:03:52.281073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.push_to_hub()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:09:40.313689Z","iopub.execute_input":"2022-08-10T03:09:40.314264Z","iopub.status.idle":"2022-08-10T03:10:17.273241Z","shell.execute_reply.started":"2022-08-10T03:09:40.314227Z","shell.execute_reply":"2022-08-10T03:10:17.272059Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import pipeline\n\nmask_filler = pipeline(\n    \"fill-mask\", model=\"SmartPy/xlm-roberta-base-finetuned-my_dear_watson2\"\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:12:43.980063Z","iopub.execute_input":"2022-08-10T03:12:43.980808Z","iopub.status.idle":"2022-08-10T03:13:52.853918Z","shell.execute_reply.started":"2022-08-10T03:12:43.980771Z","shell.execute_reply":"2022-08-10T03:13:52.852888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !rm -rf ./xlm-roberta-base-finetuned-my_dear_watson \n!ls ./","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:47:11.216679Z","iopub.status.idle":"2022-08-10T02:47:11.217477Z","shell.execute_reply.started":"2022-08-10T02:47:11.217207Z","shell.execute_reply":"2022-08-10T02:47:11.217233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = mask_filler(text)\n\nfor pred in preds:\n    print(f\">>> {pred['sequence']}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T03:15:07.868454Z","iopub.execute_input":"2022-08-10T03:15:07.869195Z","iopub.status.idle":"2022-08-10T03:15:08.218150Z","shell.execute_reply.started":"2022-08-10T03:15:07.869159Z","shell.execute_reply":"2022-08-10T03:15:08.216995Z"},"trusted":true},"execution_count":null,"outputs":[]}]}