{"cells":[{"metadata":{"id":"28Bk-1zNiytK","colab_type":"text"},"cell_type":"markdown","source":"### Import"},{"metadata":{"id":"0a80kPfPi-bP","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"!pip install transformers -q","execution_count":null,"outputs":[]},{"metadata":{"id":"C-8MDk3Hh3pC","colab_type":"code","outputId":"0ae8958c-970e-4051-a053-833dee2081cb","executionInfo":{"status":"ok","timestamp":1576302372809,"user_tz":-480,"elapsed":61158,"user":{"displayName":"Stephen Lau","photoUrl":"https://lh6.googleusercontent.com/-XObJ68l__TM/AAAAAAAAAAI/AAAAAAAANgg/JsLvBRYBE80/s64/photo.jpg","userId":"17894701130798590588"}},"colab":{"base_uri":"https://localhost:8080/","height":64},"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport os\nimport torch\nimport torch.optim as optim\nimport random\n\nimport fastai\nfrom fastai import *\nfrom fastai.text import *\nfrom fastai.callbacks import *\n\nimport transformers\nfrom transformers import PreTrainedModel, PreTrainedTokenizer, PretrainedConfig\nfrom transformers import BertForSequenceClassification, BertTokenizer, BertConfig","execution_count":null,"outputs":[]},{"metadata":{"id":"w0bKRq4Dh3pr","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"def seed_all(seed_value):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n        \nseed_all(42)","execution_count":null,"outputs":[]},{"metadata":{"id":"ZMVmRZH1h3pY","colab_type":"text"},"cell_type":"markdown","source":"### Prepare everything"},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH = Path('../input/jigsaw-multilingual-toxic-comment-classification')\ndf_train = pd.read_csv(PATH/'jigsaw-toxic-comment-train.csv')\ndf_train = df_train.drop(['severe_toxic','obscene','threat','insult','identity_hate'], axis=1)\ndf_train['valid'] = False\n\ndf_valid = pd.read_csv(PATH/'validation.csv')\ndf_valid = df_valid.drop(['lang'], axis=1)\ndf_valid['valid'] = True\n\ndf_test =  pd.read_csv(PATH/'test.csv')\ndf_train_all = pd.concat([df_train, df_valid], axis=0)","execution_count":null,"outputs":[]},{"metadata":{"id":"aOpV70mAh3pn","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"model_class     = BertForSequenceClassification\ntokenizer_class = BertTokenizer\nconfig_class    = BertConfig","execution_count":null,"outputs":[]},{"metadata":{"id":"Ri87ls_9h3px","colab_type":"text"},"cell_type":"markdown","source":"### **Data pre-processing**\n"},{"metadata":{"id":"yLuLctfoh3p2","colab_type":"text"},"cell_type":"markdown","source":"#### **Custom Tokenizer**"},{"metadata":{"scrolled":true,"id":"nXDFaCGkh3p7","colab_type":"code","outputId":"370b2e24-0a2a-4553-9f83-3eec74823cdc","executionInfo":{"status":"ok","timestamp":1576302375727,"user_tz":-480,"elapsed":64010,"user":{"displayName":"Stephen Lau","photoUrl":"https://lh6.googleusercontent.com/-XObJ68l__TM/AAAAAAAAAAI/AAAAAAAANgg/JsLvBRYBE80/s64/photo.jpg","userId":"17894701130798590588"}},"colab":{"base_uri":"https://localhost:8080/","height":67,"referenced_widgets":["96fb71cd317445cdb3babe958522bc28","a45f084740fc48a788542f7c1800c797","3feca0bcecf44784a91a7806bd288b3d","a6f45548439b4c73b6651949bc69f69b","e183125886b747e78e19d42f36c553f3","f961fc0a3111435ea4a889024910d05f","73d3e7e57701403b91edcf09f5dbf3bf","1f6b7d96d18b42ccbef0ea189467d166"]},"trusted":true},"cell_type":"code","source":"class TransformersBaseTokenizer(BaseTokenizer):\n    \"\"\"\n    Wrapper around PreTrainedTokenizer to be compatible with fastai.\n    \"\"\"\n    \n    def __init__(self, pretrained_tokenizer, **kwargs):\n        self._pretrained_tokenizer = pretrained_tokenizer\n        self.max_seq_len = pretrained_tokenizer.max_len\n        \n    def __call__(self, *args, **kwargs):\n        return self\n    \n    def tokenizer(self, t:str) -> List[str]:  # replace the `tokenizer` function in the parent class\n        \"\"\"\n        Limits the maximum sequence length and add the special tokens\n        \"\"\"\n        # Get the CLS and SEP tokens\n        CLS = self._pretrained_tokenizer.cls_token  # method found in `pretrained_tokenizer`\n        SEP = self._pretrained_tokenizer.sep_token\n        \n        # tokenize the string `t` pass into this function\n        tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]\n        \n        return [CLS] + tokens + [SEP]\n    \ntransformer_tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-uncased')\ntransformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer=transformer_tokenizer)\nfastai_tokenizer = Tokenizer(tok_func=transformer_base_tokenizer, pre_rules=[], post_rules=[])","execution_count":null,"outputs":[]},{"metadata":{"id":"FcnL9yzPh3qC","colab_type":"text"},"cell_type":"markdown","source":"#### Custom Numericalizer"},{"metadata":{"id":"xg3tbOQMh3qH","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class TransformersVocab(Vocab):\n    def __init__(self, tokenizer: PreTrainedTokenizer):\n        super().__init__(itos=[])\n        self.tokenizer = tokenizer\n        \n    def numericalize(self, t:Collection[str]) -> List[int]:\n        \"Convert a list of tokens `t` into their ids.\"\n        return self.tokenizer.convert_tokens_to_ids(t)\n    \n    def textify(self, nums:Collection[int], sep=' ') -> List[str]:\n        \"Convert a list of `nums` to their tokens.\"\n        nums = np.array(nums).tolist()\n        return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums) \n    \n    def __getstate__(self):\n        return {'itos':self.itos, 'tokenizer':self.tokenizer}\n\n    def __setstate__(self, state:dict):\n        self.itos = state['itos']\n        self.tokenizer = state['tokenizer']\n        self.stoi = collections.defaultdict(int,{v:k for k,v in enumerate(self.itos)})","execution_count":null,"outputs":[]},{"metadata":{"id":"jUKPpwZQh3qO","colab_type":"text"},"cell_type":"markdown","source":"#### Custom processor"},{"metadata":{"id":"kV-fpXCgh3qU","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"transformer_vocab      = TransformersVocab(tokenizer=transformer_tokenizer)\nnumericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)   # fastai\ntokenize_processor     = TokenizeProcessor(tokenizer=fastai_tokenizer,\n                                           include_bos=False,\n                                           include_eos=False)\ntransformers_processor = [tokenize_processor, numericalize_processor]","execution_count":null,"outputs":[]},{"metadata":{"id":"1HI5DEsSh3qZ","colab_type":"text"},"cell_type":"markdown","source":"### Setting up the DataBunch"},{"metadata":{"trusted":true},"cell_type":"code","source":"databunch = load_data(path='', file='../input/fastai-dataloaders/jigsaw_data.pkl', \n                      bs=32, pad_first=False, pad_idx=transformer_tokenizer.pad_token_id)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"id":"P6h4sJCbh3qr","colab_type":"code","outputId":"361568cd-705f-4494-f554-876dd772ab64","executionInfo":{"status":"ok","timestamp":1576302425677,"user_tz":-480,"elapsed":113930,"user":{"displayName":"Stephen Lau","photoUrl":"https://lh6.googleusercontent.com/-XObJ68l__TM/AAAAAAAAAAI/AAAAAAAANgg/JsLvBRYBE80/s64/photo.jpg","userId":"17894701130798590588"}},"colab":{"base_uri":"https://localhost:8080/","height":17},"trusted":true},"cell_type":"code","source":"# databunch = (TextList.from_df(df_train_all, cols='comment_text', processor=transformers_processor)\n#              .split_from_df(col='valid')\n#              .label_from_df(cols=['toxic'])\n#              .add_test(df_test)\n#              .databunch(bs=32, pad_first=False, pad_idx=transformer_tokenizer.pad_token_id))","execution_count":null,"outputs":[]},{"metadata":{"id":"h8JAn8Xxh3qy","colab_type":"text"},"cell_type":"markdown","source":"### Custom Model"},{"metadata":{"id":"P07lrL8eh3q4","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"class CustomTransformerModel(nn.Module):\n    def __init__(self, transformer_model: PreTrainedModel):\n        super().__init__()\n        self.transformer = transformer_model\n        \n    def forward(self, input_ids):\n        logits = self.transformer(input_ids)[0] # first element of the output tuple: `last_hidden_state` - from the docs\n        return logits","execution_count":null,"outputs":[]},{"metadata":{"id":"iuBKVYtKh3rM","colab_type":"code","outputId":"139da00a-a767-4fc1-8704-dc422941c113","executionInfo":{"status":"ok","timestamp":1576302445088,"user_tz":-480,"elapsed":133303,"user":{"displayName":"Stephen Lau","photoUrl":"https://lh6.googleusercontent.com/-XObJ68l__TM/AAAAAAAAAAI/AAAAAAAANgg/JsLvBRYBE80/s64/photo.jpg","userId":"17894701130798590588"}},"colab":{"base_uri":"https://localhost:8080/","height":67,"referenced_widgets":["697699b6aaa642a3acdb6164b9deb984","4bd586e25c76496b9cd4bff737f6230d","04ce9bead16e4e72bf7438dddb2fb384","891deb05dcc640cd880e1cffcdfccfa8","d19116e507364a2cb50d39701ce10150","2b2cbba87e8a4f0ebe586a007b54de01","61ed9af4e4ee4b969f389e19c3472ea5","0993c4bde5834adc91dc1a8fd5575c28"]},"trusted":true},"cell_type":"code","source":"config = BertConfig.from_pretrained('bert-base-multilingual-uncased')\nconfig.num_labels = 2\ntransformer_model = BertForSequenceClassification.from_pretrained('bert-base-multilingual-uncased', config=config)\ncustom_transformer_model = CustomTransformerModel(transformer_model)","execution_count":null,"outputs":[]},{"metadata":{"id":"KRnMZlmHh3rb","colab_type":"text"},"cell_type":"markdown","source":"### Learner : Custom Optimizer / Custom Metric"},{"metadata":{"id":"A7dJYB_Ih3rf","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from transformers import AdamW\nlearner = Learner(    data = databunch, \n                     model = custom_transformer_model, \n                  opt_func = lambda input:AdamW(input, correct_bias=False),\n                   metrics = AUROC())\nlearner.model_dir = '../input/models/'\nlearner.load('bert_fastai_unfrozen_last_12')","execution_count":null,"outputs":[]},{"metadata":{"id":"RASvo-8Bh3sN","colab_type":"text"},"cell_type":"markdown","source":"### Prediction"},{"metadata":{"id":"SJNgQA5Qh3sQ","colab_type":"code","outputId":"e1f2204b-ebed-4963-f778-c3b7be582644","executionInfo":{"status":"ok","timestamp":1576315148629,"user_tz":-480,"elapsed":87656,"user":{"displayName":"Stephen Lau","photoUrl":"https://lh6.googleusercontent.com/-XObJ68l__TM/AAAAAAAAAAI/AAAAAAAANgg/JsLvBRYBE80/s64/photo.jpg","userId":"17894701130798590588"}},"colab":{"base_uri":"https://localhost:8080/","height":17},"trusted":true},"cell_type":"code","source":"def get_preds_as_nparray(ds_type) -> np.ndarray:\n    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n    sampler = [i for i in databunch.dl(ds_type).sampler]\n    reverse_sampler = np.argsort(sampler)\n    return preds[reverse_sampler, :]\n\ntest_preds = get_preds_as_nparray(DatasetType.Test)","execution_count":null,"outputs":[]},{"metadata":{"id":"IwtExsaRh3sY","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(PATH/'sample_submission.csv')\nsample_submission['toxic'] = test_preds[:,1] \nsample_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false},"colab":{"name":"BERT with fastai.ipynb","provenance":[],"collapsed_sections":[]},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"96fb71cd317445cdb3babe958522bc28":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_a45f084740fc48a788542f7c1800c797","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_3feca0bcecf44784a91a7806bd288b3d","IPY_MODEL_a6f45548439b4c73b6651949bc69f69b"]}},"a45f084740fc48a788542f7c1800c797":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"3feca0bcecf44784a91a7806bd288b3d":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_e183125886b747e78e19d42f36c553f3","_dom_classes":[],"description":"Downloading","_model_name":"IntProgressModel","bar_style":"success","max":231508,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":231508,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_f961fc0a3111435ea4a889024910d05f"}},"a6f45548439b4c73b6651949bc69f69b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_73d3e7e57701403b91edcf09f5dbf3bf","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100% 232k/232k [00:00<00:00, 1.28MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_1f6b7d96d18b42ccbef0ea189467d166"}},"e183125886b747e78e19d42f36c553f3":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"f961fc0a3111435ea4a889024910d05f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"73d3e7e57701403b91edcf09f5dbf3bf":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"1f6b7d96d18b42ccbef0ea189467d166":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"e1d4211982db4dd5ac080c5985c723ef":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_0082d94a18f7445297146c9693f79210","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_ca2d27da6ede475896d4c95e5a6848c4","IPY_MODEL_5931213da7cf4058acfb805ce9f67cfa"]}},"0082d94a18f7445297146c9693f79210":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ca2d27da6ede475896d4c95e5a6848c4":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_661c9fb87bdf48a2a3ddf23e25be38e4","_dom_classes":[],"description":"Downloading","_model_name":"IntProgressModel","bar_style":"success","max":313,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":313,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_fa4bfc13b7c84c9f8a9a57123a1619a8"}},"5931213da7cf4058acfb805ce9f67cfa":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_0edb025668a447b1b9452807cb14836d","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100% 313/313 [00:00<00:00, 13.8kB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_51c5bc42949d428981fece24d7e13f97"}},"661c9fb87bdf48a2a3ddf23e25be38e4":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"fa4bfc13b7c84c9f8a9a57123a1619a8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"0edb025668a447b1b9452807cb14836d":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"51c5bc42949d428981fece24d7e13f97":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"697699b6aaa642a3acdb6164b9deb984":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_4bd586e25c76496b9cd4bff737f6230d","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_04ce9bead16e4e72bf7438dddb2fb384","IPY_MODEL_891deb05dcc640cd880e1cffcdfccfa8"]}},"4bd586e25c76496b9cd4bff737f6230d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"04ce9bead16e4e72bf7438dddb2fb384":{"model_module":"@jupyter-widgets/controls","model_name":"IntProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_d19116e507364a2cb50d39701ce10150","_dom_classes":[],"description":"Downloading","_model_name":"IntProgressModel","bar_style":"success","max":440473133,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":440473133,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2b2cbba87e8a4f0ebe586a007b54de01"}},"891deb05dcc640cd880e1cffcdfccfa8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_61ed9af4e4ee4b969f389e19c3472ea5","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"​","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":"100% 440M/440M [00:13<00:00, 33.0MB/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_0993c4bde5834adc91dc1a8fd5575c28"}},"d19116e507364a2cb50d39701ce10150":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"2b2cbba87e8a4f0ebe586a007b54de01":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"61ed9af4e4ee4b969f389e19c3472ea5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"0993c4bde5834adc91dc1a8fd5575c28":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"nbformat":4,"nbformat_minor":4}