{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\nWikipedia Image/Caption Starter
\n
","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"This Notebook tries to provide a trainable approach by training Efficientnet b0 for generating image embeddings and Xlm-roberta for generating text embeddings","metadata":{}},{"cell_type":"markdown","source":"# Install Required Libraries
","metadata":{}},{"cell_type":"code","source":"!pip install git+https://github.com/rwightman/pytorch-image-models\n!pip install --upgrade wandb","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:52:08.065002Z","iopub.execute_input":"2021-11-03T15:52:08.066271Z","iopub.status.idle":"2021-11-03T15:52:42.048857Z","shell.execute_reply.started":"2021-11-03T15:52:08.066094Z","shell.execute_reply":"2021-11-03T15:52:42.047752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Required Libraries 📚
","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nfrom PIL import Image\nfrom PIL import ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\nimport base64\nimport pickle\n\n# For downloading images\nfrom io import BytesIO\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, KFold\n\n# For Image Models\nimport timm\n\n# For Transformer Models\nfrom transformers import AutoTokenizer, AutoModel\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:52:42.051805Z","iopub.execute_input":"2021-11-03T15:52:42.052189Z","iopub.status.idle":"2021-11-03T15:52:54.869811Z","shell.execute_reply.started":"2021-11-03T15:52:42.052137Z","shell.execute_reply":"2021-11-03T15:52:54.868411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"![\"Weights](\"https://i.imgur.com/gb6B4ig.png\")
\n\n Weights & Biases (W&B) is a set of machine learning tools that helps you build better models faster. Kaggle competitions require fast-paced model development and evaluation. There are a lot of components: exploring the training data, training different models, combining trained models in different combinations (ensembling), and so on.\n\n> ⏳ Lots of components = Lots of places to go wrong = Lots of time spent debugging\n\nW&B can be useful for Kaggle competition with it's lightweight and interoperable tools:\n\n* Quickly track experiments,
\n* Version and iterate on datasets,
\n* Evaluate model performance,
\n* Reproduce models,
\n* Visualize results and spot regressions,
\n* Share findings with colleagues.\n\nTo learn more about Weights and Biases check out this kernel.","metadata":{}},{"cell_type":"code","source":"import wandb\n\ntry:\n from kaggle_secrets import UserSecretsClient\n user_secrets = UserSecretsClient()\n api_key = user_secrets.get_secret(\"wandb_api\")\n wandb.login(key=api_key)\n anony = None\nexcept:\n anony = \"must\"\n print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:52:54.876728Z","iopub.execute_input":"2021-11-03T15:52:54.880405Z","iopub.status.idle":"2021-11-03T15:52:56.770259Z","shell.execute_reply.started":"2021-11-03T15:52:54.880336Z","shell.execute_reply":"2021-11-03T15:52:56.769171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Configuration ⚙️
","metadata":{}},{"cell_type":"code","source":"CONFIG = {\"seed\": 2021,\n \"epochs\": 5,\n \"img_size\": 256,\n \"image_model_name\": \"tf_efficientnet_b0\",\n \"text_model_name\": \"xlm-roberta-base\",\n \"embedding_size\": 256,\n \"train_batch_size\": 32,\n \"valid_batch_size\": 64,\n \"learning_rate\": 1e-4,\n \"scheduler\": 'CosineAnnealingLR',\n \"min_lr\": 1e-6,\n \"T_max\": 500,\n \"weight_decay\": 1e-6,\n \"max_length\": 32,\n \"n_accumulate\": 1,\n \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n }\n\nCONFIG[\"tokenizer\"] = AutoTokenizer.from_pretrained(CONFIG['text_model_name'])","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:52:56.771839Z","iopub.execute_input":"2021-11-03T15:52:56.772193Z","iopub.status.idle":"2021-11-03T15:53:03.529559Z","shell.execute_reply.started":"2021-11-03T15:52:56.772146Z","shell.execute_reply":"2021-11-03T15:53:03.528472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:03.533504Z","iopub.execute_input":"2021-11-03T15:53:03.533986Z","iopub.status.idle":"2021-11-03T15:53:03.548425Z","shell.execute_reply.started":"2021-11-03T15:53:03.533931Z","shell.execute_reply":"2021-11-03T15:53:03.545692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Set Seed for Reproducibility
","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:03.550899Z","iopub.execute_input":"2021-11-03T15:53:03.551700Z","iopub.status.idle":"2021-11-03T15:53:03.565160Z","shell.execute_reply.started":"2021-11-03T15:53:03.551623Z","shell.execute_reply":"2021-11-03T15:53:03.563922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Download Dataset
","metadata":{}},{"cell_type":"markdown","source":"Please check this discussion to know about the dataset creation procedure","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project=\"Wikipedia\", \n anonymous=\"must\")\nartifact = run.use_artifact('dchanda/Wikipedia/Wiki-data:latest', type='dataset')\nartifact_dir = artifact.download()\nrun.finish()\n\nfor file in os.listdir(artifact_dir):\n filepath = os.path.join(artifact_dir, file)\n with open(filepath, \"rb\") as fp:\n data = pickle.load(fp)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:03.566686Z","iopub.execute_input":"2021-11-03T15:53:03.571030Z","iopub.status.idle":"2021-11-03T15:53:37.224198Z","shell.execute_reply.started":"2021-11-03T15:53:03.570966Z","shell.execute_reply":"2021-11-03T15:53:37.222913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dataset Structure\n\n\n data: list of dictionaries\n \n b64_bytes: base64 encoded bytes of the image file at a 300px resolutioncaption_title_and_reference_description: list of captionstarget: 1 for positive samples and -1 for negative samples
","metadata":{}},{"cell_type":"code","source":"random.shuffle(data)\n\ntrain_data = data[:45000]\nvalid_data = data[45000:]\nprint(f\"Number of training samples: {len(train_data)}\")\nprint(f\"Number of validation samples: {len(valid_data)}\")","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:37.226460Z","iopub.execute_input":"2021-11-03T15:53:37.226853Z","iopub.status.idle":"2021-11-03T15:53:37.303534Z","shell.execute_reply.started":"2021-11-03T15:53:37.226802Z","shell.execute_reply":"2021-11-03T15:53:37.302391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualize Images
","metadata":{}},{"cell_type":"code","source":"# run = wandb.init(project='Wikipedia',\n# job_type='Visualization',\n# anonymous='must')\n\n# preview_table = wandb.Table(columns=['Image', 'Captions'])\n# for content in json_content[:1000]:\n# out = base64.b64decode(content['b64_bytes'])\n# img = Image.open(BytesIO(out)).convert(\"RGB\")\n# preview_table.add_data(wandb.Image(img), \n# content['caption_title_and_reference_description'])\n\n# wandb.log({'Visualization': preview_table})\n# run.finish()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:37.305828Z","iopub.execute_input":"2021-11-03T15:53:37.306608Z","iopub.status.idle":"2021-11-03T15:53:37.312439Z","shell.execute_reply.started":"2021-11-03T15:53:37.306557Z","shell.execute_reply":"2021-11-03T15:53:37.311098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"View the Complete Table Here ⮕","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Dataset Class
","metadata":{}},{"cell_type":"code","source":"class WikipediaDataset(Dataset):\n def __init__(self, data, tokenizer, max_length, transforms=None):\n self.data = data\n self.max_len = max_length\n self.tokenizer = tokenizer\n self.transforms = transforms\n \n def __len__(self):\n return len(self.data)\n \n def __getitem__(self, index):\n image_bytes = base64.b64decode(self.data[index][\"b64_bytes\"])\n img = np.asarray(Image.open(BytesIO(image_bytes)).convert(\"RGB\"))\n caption = random.choice(self.data[index][\"caption_title_and_reference_description\"])\n caption = caption.replace(\"[SEP]\", \"\") # sep token for xlm-roberta\n inputs = self.tokenizer.encode_plus(\n caption,\n truncation=True,\n add_special_tokens=True,\n max_length=self.max_len,\n padding='max_length'\n )\n target = self.data[index]['target']\n \n ids = inputs['input_ids']\n mask = inputs['attention_mask']\n \n if self.transforms:\n img = self.transforms(image=img)[\"image\"]\n \n return {\n 'ids': torch.tensor(ids, dtype=torch.long),\n 'mask': torch.tensor(mask, dtype=torch.long),\n 'image': img,\n 'target': torch.tensor(target, dtype=torch.long)\n }","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:37.314644Z","iopub.execute_input":"2021-11-03T15:53:37.315063Z","iopub.status.idle":"2021-11-03T15:53:37.337699Z","shell.execute_reply.started":"2021-11-03T15:53:37.315012Z","shell.execute_reply":"2021-11-03T15:53:37.336664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations
","metadata":{}},{"cell_type":"code","source":"data_transforms = {\n \"train\": A.Compose([\n A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n A.HorizontalFlip(p=0.5),\n A.Normalize(\n mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225], \n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.),\n \n \"valid\": A.Compose([\n A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n A.Normalize(\n mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225], \n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.)\n}","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:37.339803Z","iopub.execute_input":"2021-11-03T15:53:37.340533Z","iopub.status.idle":"2021-11-03T15:53:37.351876Z","shell.execute_reply.started":"2021-11-03T15:53:37.340483Z","shell.execute_reply":"2021-11-03T15:53:37.350739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = WikipediaDataset(train_data, CONFIG[\"tokenizer\"], CONFIG[\"max_length\"], \n transforms=data_transforms[\"train\"])\ntrain_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n num_workers=4, shuffle=True, pin_memory=True, drop_last=True)\n\nvalid_dataset = WikipediaDataset(valid_data, CONFIG[\"tokenizer\"], CONFIG[\"max_length\"], \n transforms=data_transforms[\"valid\"])\nvalid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n num_workers=4, shuffle=False, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:37.354002Z","iopub.execute_input":"2021-11-03T15:53:37.354791Z","iopub.status.idle":"2021-11-03T15:53:37.371283Z","shell.execute_reply.started":"2021-11-03T15:53:37.354715Z","shell.execute_reply":"2021-11-03T15:53:37.370090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Model
","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"class WikipediaModel(nn.Module):\n def __init__(self, image_model, text_model, embedding_size):\n super(WikipediaModel, self).__init__()\n self.image_model = timm.create_model(image_model, pretrained=True)\n self.n_features = self.image_model.classifier.in_features\n self.image_model.reset_classifier(0)\n self.image_drop = nn.Dropout(p=0.2)\n self.image_fc = nn.Linear(self.n_features, embedding_size)\n \n self.text_model = AutoModel.from_pretrained(text_model)\n self.text_drop = nn.Dropout(p=0.2)\n self.text_fc = nn.Linear(768, embedding_size)\n \n self.freeze_backbone()\n \n def forward(self, images, ids, mask):\n image_features = self.image_model(images)\n image_embeddings = self.image_fc(self.image_drop(image_features))\n \n out = self.text_model(input_ids=ids,attention_mask=mask,\n output_hidden_states=False)\n out = self.text_drop(out[1])\n text_embeddings = self.text_fc(out)\n\n return image_embeddings, text_embeddings\n \n def freeze_backbone(self):\n for params in self.image_model.parameters():\n params.requires_grad = False\n # Only finetune final layer\n self.image_fc.weight.requires_grad = True\n self.image_fc.bias.requires_grad = True\n \n for params in self.text_model.parameters():\n params.requires_grad = False\n # Only finetune final layer\n self.text_fc.weight.requires_grad = True\n self.text_fc.bias.requires_grad = True\n \n\nmodel = WikipediaModel(CONFIG['image_model_name'], CONFIG['text_model_name'], CONFIG['embedding_size'])\nmodel.to(CONFIG['device']);","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:53:37.372992Z","iopub.execute_input":"2021-11-03T15:53:37.373537Z","iopub.status.idle":"2021-11-03T15:54:31.357595Z","shell.execute_reply.started":"2021-11-03T15:53:37.373485Z","shell.execute_reply":"2021-11-03T15:54:31.356518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss Function
","metadata":{}},{"cell_type":"code","source":"def criterion(outputs1, outputs2, targets):\n return nn.CosineEmbeddingLoss()(outputs1, outputs2, targets)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:31.362722Z","iopub.execute_input":"2021-11-03T15:54:31.363152Z","iopub.status.idle":"2021-11-03T15:54:31.368876Z","shell.execute_reply.started":"2021-11-03T15:54:31.363101Z","shell.execute_reply":"2021-11-03T15:54:31.367608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Function
","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n model.train()\n \n dataset_size = 0\n running_loss = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar:\n ids = data['ids'].to(device, dtype = torch.long)\n mask = data['mask'].to(device, dtype = torch.long)\n images = data['image'].to(device, dtype=torch.float)\n targets = data['target'].to(device, dtype=torch.long)\n \n batch_size = ids.size(0)\n\n image_outputs, text_outputs = model(images, ids, mask)\n loss = criterion(image_outputs, text_outputs, targets)\n loss = loss / CONFIG['n_accumulate']\n loss.backward()\n \n if (step + 1) % CONFIG['n_accumulate'] == 0:\n optimizer.step()\n\n # zero the parameter gradients\n optimizer.zero_grad()\n\n if scheduler is not None:\n scheduler.step()\n \n running_loss += (loss.item() * batch_size)\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n \n bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n LR=optimizer.param_groups[0]['lr'])\n gc.collect()\n \n return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:31.371135Z","iopub.execute_input":"2021-11-03T15:54:31.371843Z","iopub.status.idle":"2021-11-03T15:54:31.388019Z","shell.execute_reply.started":"2021-11-03T15:54:31.371783Z","shell.execute_reply":"2021-11-03T15:54:31.386756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Function
","metadata":{}},{"cell_type":"code","source":"@torch.no_grad()\ndef valid_one_epoch(model, dataloader, device, epoch):\n model.eval()\n \n dataset_size = 0\n running_loss = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar: \n ids = data['ids'].to(device, dtype = torch.long)\n mask = data['mask'].to(device, dtype = torch.long)\n images = data['image'].to(device, dtype=torch.float)\n targets = data['target'].to(device, dtype=torch.long)\n \n batch_size = ids.size(0)\n \n image_outputs, text_outputs = model(images, ids, mask)\n loss = criterion(image_outputs, text_outputs, targets)\n \n running_loss += (loss.item() * batch_size)\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n \n bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss,\n LR=optimizer.param_groups[0]['lr']) \n \n gc.collect()\n \n return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:31.390531Z","iopub.execute_input":"2021-11-03T15:54:31.391115Z","iopub.status.idle":"2021-11-03T15:54:31.409170Z","shell.execute_reply.started":"2021-11-03T15:54:31.391059Z","shell.execute_reply":"2021-11-03T15:54:31.407752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Run Training
","metadata":{}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n # To automatically log gradients\n wandb.watch(model, log_freq=100)\n \n if torch.cuda.is_available():\n print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n \n start = time.time()\n best_model_wts = copy.deepcopy(model.state_dict())\n best_epoch_loss = np.inf\n history = defaultdict(list)\n \n for epoch in range(1, num_epochs + 1): \n gc.collect()\n train_epoch_loss = train_one_epoch(model, optimizer, scheduler, \n dataloader=train_loader, \n device=CONFIG['device'], epoch=epoch)\n \n val_epoch_loss = valid_one_epoch(model, valid_loader, device=CONFIG['device'], \n epoch=epoch)\n \n history['Train Loss'].append(train_epoch_loss)\n history['Valid Loss'].append(val_epoch_loss)\n \n # Log the metrics\n wandb.log({\"Train Loss\": train_epoch_loss})\n wandb.log({\"Valid Loss\": val_epoch_loss})\n \n # deep copy the model\n if val_epoch_loss <= best_epoch_loss:\n print(f\"{b_}Validation Loss Improved ({best_epoch_loss} ---> {val_epoch_loss})\")\n best_epoch_loss = val_epoch_loss\n run.summary[\"Best Loss\"] = best_epoch_loss\n best_model_wts = copy.deepcopy(model.state_dict())\n PATH = \"Loss{:.4f}_epoch{:.0f}.bin\".format(best_epoch_loss, epoch)\n torch.save(model.state_dict(), PATH)\n # Save a model file from the current directory\n print(f\"Model Saved{sr_}\")\n \n print()\n \n end = time.time()\n time_elapsed = end - start\n print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n print(\"Best Loss: {:.4f}\".format(best_epoch_loss))\n \n # load best model weights\n model.load_state_dict(best_model_wts)\n \n return model, history","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:31.411290Z","iopub.execute_input":"2021-11-03T15:54:31.412236Z","iopub.status.idle":"2021-11-03T15:54:31.431551Z","shell.execute_reply.started":"2021-11-03T15:54:31.412179Z","shell.execute_reply":"2021-11-03T15:54:31.430413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n if CONFIG['scheduler'] == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == None:\n return None\n \n return scheduler","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:31.434022Z","iopub.execute_input":"2021-11-03T15:54:31.434457Z","iopub.status.idle":"2021-11-03T15:54:31.450688Z","shell.execute_reply.started":"2021-11-03T15:54:31.434393Z","shell.execute_reply":"2021-11-03T15:54:31.449460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define Optimizer and Scheduler","metadata":{}},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:31.452374Z","iopub.execute_input":"2021-11-03T15:54:31.453724Z","iopub.status.idle":"2021-11-03T15:54:31.466741Z","shell.execute_reply.started":"2021-11-03T15:54:31.453638Z","shell.execute_reply":"2021-11-03T15:54:31.465536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Start Training","metadata":{}},{"cell_type":"code","source":"run = wandb.init(project='Wikipedia', \n config=CONFIG,\n job_type='Train',\n anonymous='must')","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:31.468418Z","iopub.execute_input":"2021-11-03T15:54:31.468829Z","iopub.status.idle":"2021-11-03T15:54:39.131573Z","shell.execute_reply.started":"2021-11-03T15:54:31.468778Z","shell.execute_reply":"2021-11-03T15:54:39.130246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler, \n device=CONFIG['device'],\n num_epochs=CONFIG['epochs'])","metadata":{"execution":{"iopub.status.busy":"2021-11-03T15:54:39.136041Z","iopub.execute_input":"2021-11-03T15:54:39.136627Z","iopub.status.idle":"2021-11-03T16:25:26.371138Z","shell.execute_reply.started":"2021-11-03T15:54:39.136568Z","shell.execute_reply":"2021-11-03T16:25:26.369994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{"execution":{"iopub.status.busy":"2021-11-03T16:25:26.373339Z","iopub.execute_input":"2021-11-03T16:25:26.373696Z","iopub.status.idle":"2021-11-03T16:25:32.050773Z","shell.execute_reply.started":"2021-11-03T16:25:26.373634Z","shell.execute_reply":"2021-11-03T16:25:32.049720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizations
","metadata":{}},{"cell_type":"markdown","source":"View the Complete Dashboard Here ⮕","metadata":{}},{"cell_type":"code","source":"# Code taken from https://www.kaggle.com/ayuraj/interactive-eda-using-w-b-tables\n\n# This is just to display the W&B run page in this interactive session.\nfrom IPython import display\n\n# we create an IFrame and set the width and height\niF = display.IFrame(run.url, width=1080, height=720)\niF","metadata":{"execution":{"iopub.status.busy":"2021-11-03T16:25:32.052291Z","iopub.execute_input":"2021-11-03T16:25:32.052736Z","iopub.status.idle":"2021-11-03T16:25:32.067127Z","shell.execute_reply.started":"2021-11-03T16:25:32.052692Z","shell.execute_reply":"2021-11-03T16:25:32.065836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}