{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm==0.6.2dev0 wandb --upgrade","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport wandb\nfrom fastai.vision.all import *\nfrom fastai.vision.learner import _update_first_layer\nfrom fastai.callback.wandb import WandbCallback\nimport timm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing Batch Sizes\n\nBatch sizes don't need to be 32, 64 or 128. The fixation on powers of 2 for efficient GPU utilization might just be an urban myth.\n\nThis notebook explores the effect of different batch sizes on training runtimes.","metadata":{}},{"cell_type":"markdown","source":"# Setting up W&B","metadata":{}},{"cell_type":"code","source":"# set your Weights & Biases Key in Kaggle via the menu Add-Ons > Secrets\nfrom kaggle_secrets import UserSecretsClient\nWANDB_API_KEY = UserSecretsClient().get_secret(\"WANDB_API_KEY\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WANDB_PROJECT_NAME = 'Testing-Batch-Sizes'\nwandb.login(key=WANDB_API_KEY)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"markdown","source":"The data set consists of images of baby clothes for boys and girls.","metadata":{}},{"cell_type":"code","source":"ROOT_IMAGE_PATH = '../input/hm-fashion-images-squared-224/images_224x224'\n\ndef image_path_for_article(article_id):\n    id = f'{article_id:010d}'\n    return(f'{ROOT_IMAGE_PATH}/{id[:3]}/{id}.jpg')   ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.read_parquet('../input/hm-fashion-recommendation-parquet/articles.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"baby_boy_articles = articles[(articles.index_group=='Baby/Children') & (articles.department_name.str.contains('Baby Boy'))].article_id\nbaby_girl_articles = articles[(articles.index_group=='Baby/Children') & (articles.department_name.str.contains('Baby Girl'))].article_id\n\nbaby_girl_articles = list(filter(lambda article_id: os.path.isfile(image_path_for_article(article_id)), baby_girl_articles))\nbaby_boy_articles = list(filter(lambda article_id: os.path.isfile(image_path_for_article(article_id)), baby_boy_articles))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing Data Set","metadata":{}},{"cell_type":"code","source":"article_imgs = DataBlock(blocks=(ImageBlock, CategoryBlock),\n                 get_items=lambda source: baby_girl_articles + baby_boy_articles,\n                 splitter=RandomSplitter(0.2),\n                 get_y=lambda x: x in baby_girl_articles,\n                 get_x=image_path_for_article,\n                 item_tfms=None,\n                 batch_tfms=aug_transforms(flip_vert=True) + [Normalize.from_stats(*imagenet_stats)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"article_imgs.dataloaders(None, bs=16).show_batch(max_n=9, figsize=(6,7))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def train(cfg, lr = None):\n    wandb.init(project=WANDB_PROJECT_NAME, config=cfg)\n    dls = article_imgs.dataloaders(None, bs=cfg['batch_size'])\n    learn = vision_learner(dls, cfg['model_name'], pretrained=True, metrics=error_rate, cbs=WandbCallback()).to_fp16()\n    if not 'lr' in cfg:\n        lrs = learn.lr_find()\n        cfg['lr'] = lrs.valley\n        wandb.config.update(cfg)\n    learn.fine_tune(cfg['epochs'], base_lr=cfg['lr'])\n    wandb.config.update({\"model\": learn})\n    wandb.finish()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create some learners just to download the model files\nvision_learner(article_imgs.dataloaders(None, bs=1), 'convnext_base_in22k', pretrained=True, metrics=error_rate).to_fp16()\nvision_learner(article_imgs.dataloaders(None, bs=1), 'deit_base_patch16_224', pretrained=True, metrics=error_rate).to_fp16()\nvision_learner(article_imgs.dataloaders(None, bs=1), 'resnet50', pretrained=True, metrics=error_rate).to_fp16()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ResNet50","metadata":{}},{"cell_type":"code","source":"cfg={'batch_size': 128, 'model_name': 'resnet50', 'epochs': 80}\ntrain(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg={'batch_size': 129, 'model_name': 'resnet50', 'epochs': 80}\ntrain(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ConvNeXt","metadata":{}},{"cell_type":"code","source":"cfg={'batch_size': 64, 'model_name': 'convnext_base_in22k', 'epochs': 80}\ntrain(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg={'batch_size': 65, 'model_name': 'convnext_base_in22k', 'epochs': 80}\ntrain(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DeiT","metadata":{}},{"cell_type":"code","source":"cfg={'batch_size': 64, 'model_name': 'deit_base_patch16_224', 'epochs': 80}\ntrain(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg={'batch_size': 65, 'model_name': 'deit_base_patch16_224', 'epochs': 80}\ntrain(cfg)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Batch Sizes one by one","metadata":{}},{"cell_type":"code","source":"for bs in range(8, 67):\n    cfg={'batch_size': bs, 'model_name': 'convnext_base_in22k', 'epochs': 4, 'lr': 0.001}\n    train(cfg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}