{"cells":[{"metadata":{},"cell_type":"markdown","source":"# iMet Collection 2019 - FGVC6\n**Simple baseline for iMet Collection 2019 competition using fastai v1**\n* Model: resnet152\n* Loss: Focal loss\n* Metric: $F_{2}$ score\n\n**What to try next?**\n* Different models\n* Optimize hyperparameter choice\n* Few-shot learning to improve score on classes with very few samples\n\nRefereed from original kernel [here](https://www.kaggle.com/mnpinto/imet-fastai-starter)"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import fastai\nfrom fastai.vision import *\nfastai.__version__","execution_count":1,"outputs":[{"output_type":"execute_result","execution_count":1,"data":{"text/plain":"'1.0.51'"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH = 32\nSIZE = 250","execution_count":2,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\nseed_everything()","execution_count":3,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initial setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = Path('../input/imet-2019-fgvc6/') # iMet data path","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nos.listdir('../input/')","execution_count":5,"outputs":[{"output_type":"execute_result","execution_count":5,"data":{"text/plain":"['resnet15216epochs',\n 'imet-2019-fgvc6',\n 'resnet152',\n 'densenet201',\n 'densenet121',\n 'resnet101']"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making pretrained weights work without needing to find the default filename\nfrom torch.utils import model_zoo\nPath('models').mkdir(exist_ok=True)\n!cp '../input/resnet15216epochs/stage-1.pth' 'models/resnet152.pth'\ndef load_url(*args, **kwargs):\n model_dir = Path('models')\n filename = 'resnet152.pth'\n if not (model_dir/filename).is_file(): raise FileNotFoundError\n return torch.load(model_dir/filename)\nmodel_zoo.load_url = load_url","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load train dataframe\ntrain_df = pd.read_csv(path/'train.csv')\ntrain_df.head()","execution_count":7,"outputs":[{"output_type":"execute_result","execution_count":7,"data":{"text/plain":" id attribute_ids\n0 1000483014d91860 147 616 813\n1 1000fe2e667721fe 51 616 734 813\n2 1001614cb89646ee 776\n3 10041eb49b297c08 51 671 698 813 1092\n4 100501c227f8beea 13 404 492 903 1093","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
idattribute_ids
01000483014d91860147 616 813
11000fe2e667721fe51 616 734 813
21001614cb89646ee776
310041eb49b297c0851 671 698 813 1092
4100501c227f8beea13 404 492 903 1093
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load labels dataframe\nlabels_df = pd.read_csv(path/'labels.csv')\nlabels_df.head()","execution_count":8,"outputs":[{"output_type":"execute_result","execution_count":8,"data":{"text/plain":" attribute_id attribute_name\n0 0 culture::abruzzi\n1 1 culture::achaemenid\n2 2 culture::aegean\n3 3 culture::afghan\n4 4 culture::after british","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
attribute_idattribute_name
00culture::abruzzi
11culture::achaemenid
22culture::aegean
33culture::afghan
44culture::after british
\n
"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load sample submission\ntest_df = pd.read_csv(path/'sample_submission.csv')\ntest_df.head()","execution_count":9,"outputs":[{"output_type":"execute_result","execution_count":9,"data":{"text/plain":" id attribute_ids\n0 10023b2cc4ed5f68 0 1 2\n1 100fbe75ed8fd887 0 1 2\n2 101b627524a04f19 0 1 2\n3 10234480c41284c6 0 1 2\n4 1023b0e2636dcea8 0 1 2","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
idattribute_ids
010023b2cc4ed5f680 1 2
1100fbe75ed8fd8870 1 2
2101b627524a04f190 1 2
310234480c41284c60 1 2
41023b0e2636dcea80 1 2
\n
"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"# Create data object using datablock API"},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = get_transforms(do_flip=True, flip_vert=False, max_rotate=0.10, max_zoom=1.5, max_warp=0.2, max_lighting=0.2,\n xtra_tfms=[(symmetric_warp(magnitude=(-0,0), p=0)),])\n\ntrain, test = [ImageList.from_df(df, path=path, cols='id', folder=folder, suffix='.png') \n for df, folder in zip([train_df, test_df], ['train', 'test'])]\ndata = (train.split_by_rand_pct(0.1, seed=42)\n .label_from_df(cols='attribute_ids', label_delim=' ')\n .add_test(test)\n .transform(tfms, size=SIZE, resize_method=ResizeMethod.PAD, padding_mode='border',)\n .databunch(path=Path('.'), bs=BATCH).normalize(imagenet_stats))","execution_count":10,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create learner with resnet152 and FocalLoss\nFor problems with high class imbalance Focal Loss is usually a better choice than the usual Cross Entropy Loss."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Source: https://www.kaggle.com/c/human-protein-atlas-image-classification/discussion/78109\nclass FocalLoss(nn.Module):\n def __init__(self, gamma=2):\n super().__init__()\n self.gamma = gamma\n\n def forward(self, logit, target):\n target = target.float()\n max_val = (-logit).clamp(min=0)\n loss = logit - logit * target + max_val + \\\n ((-max_val).exp() + (-logit - max_val).exp()).log()\n\n invprobs = F.logsigmoid(-logit * (target * 2.0 - 1.0))\n loss = (invprobs * self.gamma).exp() * loss\n if len(loss.size())==2:\n loss = loss.sum(dim=1)\n return loss.mean()","execution_count":11,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(data, base_arch=models.resnet152, loss_func=FocalLoss(), metrics=fbeta, pretrained=False)\nlearn.load('resnet152')","execution_count":12,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/torch/serialization.py:251: UserWarning: Couldn't retrieve source code for container of type FocalLoss. It won't be checked for correctness upon loading.\n \"type \" + obj.__name__ + \". It won't be checked \"\n","name":"stderr"},{"output_type":"execute_result","execution_count":12,"data":{"text/plain":"Learner(data=ImageDataBunch;\n\nTrain: LabelList (98314 items)\nx: ImageList\nImage (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250)\ny: MultiCategoryList\n147;616;813,51;616;734;813,776,51;671;698;813;1092,13;404;492;903;1093\nPath: ../input/imet-2019-fgvc6;\n\nValid: LabelList (10923 items)\nx: ImageList\nImage (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250)\ny: MultiCategoryList\n872,147;542;733;813;1092,51;393;584;671;746;784;954,13;813;896,498;637;704\nPath: ../input/imet-2019-fgvc6;\n\nTest: LabelList (7443 items)\nx: ImageList\nImage (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250),Image (3, 250, 250)\ny: EmptyLabelList\n,,,,\nPath: ../input/imet-2019-fgvc6, model=Sequential(\n (0): Sequential(\n (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (2): ReLU(inplace)\n (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n (4): Sequential(\n (0): Bottleneck(\n (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n (downsample): Sequential(\n (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )\n )\n (1): Bottleneck(\n (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (2): Bottleneck(\n (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n )\n (5): Sequential(\n (0): Bottleneck(\n (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n (downsample): Sequential(\n (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )\n )\n (1): Bottleneck(\n (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (2): Bottleneck(\n (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (3): Bottleneck(\n (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (4): Bottleneck(\n (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (5): Bottleneck(\n (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (6): Bottleneck(\n (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (7): Bottleneck(\n (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n )\n (6): Sequential(\n (0): Bottleneck(\n (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n (downsample): Sequential(\n (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )\n )\n (1): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (2): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (3): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (4): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (5): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (6): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (7): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (8): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (9): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (10): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (11): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (12): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (13): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (14): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (15): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (16): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (17): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (18): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (19): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (20): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (21): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (22): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (23): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (24): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (25): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (26): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (27): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (28): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (29): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (30): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (31): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (32): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (33): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (34): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (35): Bottleneck(\n (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n )\n (7): Sequential(\n (0): Bottleneck(\n (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n (downsample): Sequential(\n (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n )\n )\n (1): Bottleneck(\n (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n (2): Bottleneck(\n (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (relu): ReLU(inplace)\n )\n )\n )\n (1): Sequential(\n (0): AdaptiveConcatPool2d(\n (ap): AdaptiveAvgPool2d(output_size=1)\n (mp): AdaptiveMaxPool2d(output_size=1)\n )\n (1): Flatten()\n (2): BatchNorm1d(4096, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (3): Dropout(p=0.25)\n (4): Linear(in_features=4096, out_features=512, bias=True)\n (5): ReLU(inplace)\n (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (7): Dropout(p=0.5)\n (8): Linear(in_features=512, out_features=1103, bias=True)\n )\n), opt_func=functools.partial(, betas=(0.9, 0.99)), loss_func=FocalLoss(), metrics=[], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=PosixPath('.'), model_dir='models', callback_fns=[functools.partial(, add_time=True)], callbacks=[], layer_groups=[Sequential(\n (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (2): ReLU(inplace)\n (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n (4): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (5): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (6): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (8): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (9): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (10): ReLU(inplace)\n (11): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (12): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (13): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (14): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (15): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (16): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (17): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (18): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (19): ReLU(inplace)\n (20): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (21): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (22): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (23): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (24): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (25): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (26): ReLU(inplace)\n (27): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (28): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (29): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n (30): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (31): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (32): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (33): ReLU(inplace)\n (34): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n (35): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (36): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (37): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (38): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (39): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (40): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (41): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (42): ReLU(inplace)\n (43): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (44): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (45): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (46): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (47): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (48): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (49): ReLU(inplace)\n (50): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (51): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (52): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (53): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (54): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (55): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (56): ReLU(inplace)\n (57): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (58): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (59): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (60): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (61): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (62): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (63): ReLU(inplace)\n (64): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (65): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (66): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (67): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (68): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (69): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (70): ReLU(inplace)\n (71): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (72): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (73): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (74): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (75): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (76): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (77): ReLU(inplace)\n (78): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (79): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (80): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (81): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (82): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (83): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (84): ReLU(inplace)\n), Sequential(\n (0): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n (3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (4): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (5): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (6): ReLU(inplace)\n (7): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n (8): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (9): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (10): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (11): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (12): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (13): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (14): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (15): ReLU(inplace)\n (16): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (17): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (18): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (19): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (20): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (21): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (22): ReLU(inplace)\n (23): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (24): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (25): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (26): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (27): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (28): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (29): ReLU(inplace)\n (30): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (31): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (32): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (33): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (34): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (35): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (36): ReLU(inplace)\n (37): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (38): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (39): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (40): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (41): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (42): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (43): ReLU(inplace)\n (44): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (45): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (46): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (47): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (48): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (49): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (50): ReLU(inplace)\n (51): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (52): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (53): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (54): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (55): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (56): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (57): ReLU(inplace)\n (58): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (59): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (60): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (61): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (62): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (63): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (64): ReLU(inplace)\n (65): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (66): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (67): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (68): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (69): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (70): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (71): ReLU(inplace)\n (72): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (73): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (74): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (75): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (76): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (77): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (78): ReLU(inplace)\n (79): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (80): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (81): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (82): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (83): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (84): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (85): ReLU(inplace)\n (86): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (87): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (88): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (89): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (90): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (91): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (92): ReLU(inplace)\n (93): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (94): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (95): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (96): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (97): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (98): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (99): ReLU(inplace)\n (100): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (101): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (102): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (103): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (104): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (105): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (106): ReLU(inplace)\n (107): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (108): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (109): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (110): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (111): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (112): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (113): ReLU(inplace)\n (114): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (115): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (116): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (117): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (118): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (119): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (120): ReLU(inplace)\n (121): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (122): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (123): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (124): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (125): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (126): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (127): ReLU(inplace)\n (128): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (129): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (130): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (131): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (132): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (133): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (134): ReLU(inplace)\n (135): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (136): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (137): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (138): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (139): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (140): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (141): ReLU(inplace)\n (142): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (143): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (144): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (145): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (146): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (147): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (148): ReLU(inplace)\n (149): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (150): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (151): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (152): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (153): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (154): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (155): ReLU(inplace)\n (156): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (157): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (158): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (159): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (160): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (161): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (162): ReLU(inplace)\n (163): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (164): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (165): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (166): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (167): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (168): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (169): ReLU(inplace)\n (170): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (171): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (172): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (173): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (174): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (175): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (176): ReLU(inplace)\n (177): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (178): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (179): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (180): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (181): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (182): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (183): ReLU(inplace)\n (184): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (185): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (186): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (187): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (188): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (189): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (190): ReLU(inplace)\n (191): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (192): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (193): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (194): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (195): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (196): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (197): ReLU(inplace)\n (198): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (199): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (200): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (201): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (202): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (203): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (204): ReLU(inplace)\n (205): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (206): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (207): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (208): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (209): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (210): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (211): ReLU(inplace)\n (212): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (213): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (214): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (215): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (216): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (217): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (218): ReLU(inplace)\n (219): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (220): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (221): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (222): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (223): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (224): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (225): ReLU(inplace)\n (226): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (227): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (228): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (229): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (230): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (231): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (232): ReLU(inplace)\n (233): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (234): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (235): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (236): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (237): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (238): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (239): ReLU(inplace)\n (240): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (241): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (242): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (243): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (244): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (245): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (246): ReLU(inplace)\n (247): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (248): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (249): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (250): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (251): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (252): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (253): ReLU(inplace)\n (254): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (255): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (256): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n (257): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (258): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (259): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (260): ReLU(inplace)\n (261): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n (262): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (263): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (264): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (265): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (266): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (267): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (268): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (269): ReLU(inplace)\n (270): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (271): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (272): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n (273): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (274): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n (275): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (276): ReLU(inplace)\n), Sequential(\n (0): AdaptiveAvgPool2d(output_size=1)\n (1): AdaptiveMaxPool2d(output_size=1)\n (2): Flatten()\n (3): BatchNorm1d(4096, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (4): Dropout(p=0.25)\n (5): Linear(in_features=4096, out_features=512, bias=True)\n (6): ReLU(inplace)\n (7): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n (8): Dropout(p=0.5)\n (9): Linear(in_features=512, out_features=1103, bias=True)\n)], add_time=True)"},"metadata":{}}]},{"metadata":{},"cell_type":"markdown","source":"# Train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find a good learning rate\nlearn.lr_find()\nlearn.recorder.plot()","execution_count":13,"outputs":[{"output_type":"display_data","data":{"text/plain":"","text/html":""},"metadata":{}},{"output_type":"stream","text":"LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n","name":"stdout"},{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAAYYAAAEKCAYAAAAW8vJGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xt8XVWd9/HPLzm5J02aJr2mN9qC5Q4NpYAXLsojqCCKioqCih0cFR2d8TWOz4MjzjhenlF0fBQr4oCC3BRFR7mMDCLSFlLa0gKlTculTdvc2tzvOb/nj7MDSUzapM0+Z5/k+369ziv77L3O3r+Vk5zfWXuvvZa5OyIiIgMyUh2AiIhEixKDiIgMocQgIiJDKDGIiMgQSgwiIjKEEoOIiAyhxCAiIkMoMYiIyBBKDCIiMkQs1QGMV1lZmS9atCjVYYiIpJUNGzY0uHv5WMqmXWJYtGgRVVVVqQ5DRCStmNnLYy2rU0kiIjKEEoOIiAyhxCAiIkMoMYiIyBBKDCIiMoQSg4iIDKHEICIiQygxiIikge/+9w6eqG5IyrGUGEREIu5gew83/nE7VS8fTMrxlBhERCJu7a5G3OGcpWVJOZ4Sg4hIxD1e3UBhToxTKoqTcjwlBhGRiHuiuoFVx5QSy0zOR3aoRzGzEjO718y2mdnzZnbWsO1mZt8zs2oze8bMTg8zHhGRdLP7QAcvNXYk7TQShD+66neBB9z9cjPLBvKHbb8IWBY8zgR+GPwUERHgiZ2JnkivT2JiCK3FYGbFwBuBnwC4e4+7Nw0rdilwmyesA0rMbE5YMYmIpJvHqxuZWZTD0pmFSTtmmKeSFgP1wE/NbKOZ3WxmBcPKzAN2D3q+J1gnIjLlxePOE9UNnLO0DDNL2nHDTAwx4HTgh+5+GtAO/OOR7MjMVptZlZlV1dfXT2SMIiKR9UJtK43tPUm9vgDhJoY9wB53Xx88v5dEohisBpg/6HlFsG4Id1/j7pXuXllePqaZ6URE0t5fgjudz1k6I6nHDS0xuPt+YLeZHResugB4blix+4EPB72TVgHN7r4vrJhERNLJ49UNLCkvYE5xXlKPG3avpE8Dtwc9knYBHzGzawHc/Sbg98DFQDXQAXwk5HhERNJCT1+c9bsO8J7KiqQfO9TE4O6bgMphq28atN2BT4YZg4hIOtr4ykE6e/uTfn0BdOeziEgk/WVnIxkGq45J7vUFUGIQEYmkv1Q3cHJFCcV5WUk/thKDiEjEtHb1sml3U9J7Iw1QYhARiZj1uw7QH/eUXF8AJQYRkch5vLqB3KwMTl8wPSXHV2IQEYmYrTXNnDyvhNyszJQcX4lBRCRi9jZ1UlGa3JvaBlNiEBGJkL7+OPtbuphXosQgIiJAbWs3cYe5SgwiIgKJ00igxCAiIoGBxKBTSSIiAkDNqy2G3JTFoMQgIhIhNQc7mZ6fRX522INfj06JQUQkQvY2dab0+gIoMYiIRMrepi4lBhERec3eps6UXngGJQYRkcho6eqltbsvpReeQYlBRCQyonAPA4Q8taeZvQS0Av1An7tXDtt+LvAb4MVg1a/c/YYwYxIRiaopkRgC57l7wyG2/9nd356EOEREIq2mqQtI7c1toFNJIiKRsbepk6xMo7wwJ6VxhJ0YHHjIzDaY2epRypxlZpvN7A9mdkLI8YiIRNbepk7mFOeRkWEpjSPsU0mvd/caM5sJPGxm29z9sUHbnwYWunubmV0M/BpYNnwnQVJZDbBgwYKQQxYRSY3EzW2p7ZEEIbcY3L0m+FkH3AesHLa9xd3bguXfA1lm9leTnLr7GnevdPfK8vLyMEMWEUmZKNzcBiEmBjMrMLOigWXgQmDrsDKzzcyC5ZVBPI1hxSQiElVRmKBnQJinkmYB9wWf+zHgDnd/wMyuBXD3m4DLgU+YWR/QCVzh7h5iTCIikVTb2k1/3CPRYggtMbj7LuCUEdbfNGj5+8D3w4pBRCRdROUeBlB3VRGRSHhtgp5JfvFZRETGZmCCnjnFajGIiAiJFkNJfhYFOamboGeAEoOISATsbYpGjyRQYhARiYQozNw2QIlBRCQCaiIwQc8AJQYRkRRr6eqltSv1E/QMUGIQEUmxfcFw2zqVJCIiQLRubgMlBhGRlNvz6s1tSgwiIkJ0JugZoMQgIpJie5s6mV2cm/IJegYoMYiIpNjepk7mRmAojAFKDCIiKRalu55BiUFEJKVenaBnuhKDiIgAdRGaoGeAEoOISApF7R4GUGIQEUmpmghN0DMg1MRgZi+Z2RYz22RmVSNsNzP7nplVm9kzZnZ6mPGIiETNnoPRmaBnQDJmhDjP3RtG2XYRsCx4nAn8MPgpIjIl7KxvY/a03EhM0DMg1aeSLgVu84R1QImZzUlxTCIiSVNd18ayWYWpDmOIsBODAw+Z2QYzWz3C9nnA7kHP9wTrREQmvXjc2VHbxrKZRakOZYiw2y6vd/caM5sJPGxm29z9sfHuJEgqqwEWLFgw0TGKiKRETVMnnb39U6vF4O41wc864D5g5bAiNcD8Qc8rgnXD97PG3SvdvbK8vDyscEVEkmpHXSsAx06VxGBmBWZWNLAMXAhsHVbsfuDDQe+kVUCzu+8LKyYRkSjZUdsGwNLyqXMqaRZwn5kNHOcOd3/AzK4FcPebgN8DFwPVQAfwkRDjERGJlO21bcwsyqE4PyvVoQwRWmJw913AKSOsv2nQsgOfDCsGEZEoq65r5dhZ0WotQOq7q4qITEnxuLOjro2lM6N1fQGUGEREUmJvcycdPf1qMYiISMLAheeodVUFJQYRkZQY6Kq6TKeSREQEEj2SyotyKMnPTnUof0WJQUQkBXbUtUXuxrYBSgwiIknm7lTXtkZujKQBSgwiIkm2t7mL9p7+SHZVBSUGEZGk21E7MEaSWgwiIsKgrqpqMYiICCS6qpYV5jC9IHo9kkCJQUQk6bbXtkW2tQBKDCIiSeXuVEe4qyooMYiIJNW+5i7auvtYGtELz6DEICKSVDvqEheej9WpJBERgde6qi5Ti0FERCDRVXVGQTalEe2RBEoMIiJJtaOuNZJDbQ+mxCAikiTuzo7atsiOkTQg9MRgZplmttHMfjfCtqvNrN7MNgWPa8KOR0QkVWpbumnt7ot0V1WAWBKO8RngeWDaKNvvcvdPJSEOEZGU2lrTDMBxs0f7OIyGUFsMZlYBvA24OczjiIikg7W7GsmOZXByRXGqQzmksE8l3Qh8AYgfosy7zewZM7vXzOaPVMDMVptZlZlV1dfXhxKoiEjYntjZSOXC6eRmZaY6lEMaU2IwsyVmlhMsn2tm15lZyWFe83agzt03HKLYb4FF7n4y8DBw60iF3H2Nu1e6e2V5eflYQhYRiZSD7T08v6+Fs46ZkepQDmusLYZfAv1mthRYA8wH7jjMa84BLjGzl4A7gfPN7OeDC7h7o7t3B09vBlaMNXARkXSy/sVGAM5aMnkSQ9zd+4DLgP9w938A5hzqBe7+RXevcPdFwBXAI+5+5eAyZjZ4H5eQuEgtIjLpPLGzkfzsTE6uOOTJlkgYa6+kXjN7P3AV8I5gXdaRHNDMbgCq3P1+4DozuwToAw4AVx/JPkVEom7tzkYqF5WSHYv+7WNjTQwfAa4F/tXdXzSzxcDPxnoQd38UeDRYvn7Q+i8CXxzrfkRE0lF9azc76tp41+kVqQ5lTMaUGNz9OeA6ADObDhS5+zfCDExEZLJYtytxfeHsNLi+AGPvlfSomU0zs1LgaeDHZvbtcEMTEZkcntjZSFFOjBPmRvvGtgFjPdlV7O4twLuA29z9TODN4YUlIjJ5rNvVyMrFpcQyo399AcaeGGJBD6L3An815pGIiIxsX3MnLza0p0U31QFjTQw3AA8CO939KTM7BtgRXlgiIpPD2p3pc//CgLFefL4HuGfQ813Au8MKSkRksli7s5GS/CyWR3zgvMHGevG5wszuM7O64PHLYIA8ERE5hLW7Glm1eAYZGZbqUMZsrKeSfgrcD8wNHr8N1omIyCh2H+hgz8HOtDqNBGNPDOXu/lN37wse/wloNDsRkUNIx+sLMPbE0GhmVwazsWWa2ZVAY5iBiYiku7W7GikrzGbZzGjP2DbcWBPDR0l0Vd0P7AMuR+MaiYiMyt15YmcDq46ZgVn6XF+AMSYGd3/Z3S9x93J3n+nu70S9kkRERrX+xQPUtnRz/utmpjqUcTua2/A+N2FRiIhMMnc/tZuinBgXnXjIGQoi6WgSQ3q1jUREkqS5s5f/2rKPS0+bS152tKfxHMnRJAafsChERCaR+zfvpbsvzvsqF6Q6lCNyyDufzayVkROAAXmhRCQikubueuoVjp8zjRPnpc/dzoMdMjG4e1GyAhERmQy21jSztaaFr1xyQtr1RhqQHmPAioikiburdpMdy+Cdp85LdShHLPTEENwQt9HM/mq4bjPLMbO7zKzazNab2aKw4xERCUtXbz+/3ljDRSfOpjg/K9XhHLFktBg+Azw/yraPAQfdfSnwHUDThYpI2npg635auvp43xnzUx3KUQk1MQQjsL4NuHmUIpcCtwbL9wIXWLqelBORKe+up3azoDSfVYvTa2yk4cJuMdwIfAGIj7J9HrAbwN37gGbgr36jZrbazKrMrKq+vj6sWEVEjtjLje2s3dXI+86Yn1ZDbI8ktMRgZm8H6tx9w9Huy93XuHulu1eWl2tQVxGJnrurdpNh8O7T03+qmjBbDOcAl5jZS8CdwPlm9vNhZWqA+QBmFgOK0aitIpJmOnr6uGP9K5z/ulnMLs5NdThHLbTE4O5fdPcKd18EXAE84u5XDit2P3BVsHx5UEZ3VItIWrlj/Ssc7OjlE+cuSXUoE2JMcz5PJDO7Aahy9/uBnwA/M7Nq4ACJBCIikja6evtZ89guzl4ygxULp6c6nAmRlMTg7o8CjwbL1w9a3wW8JxkxiIiE4Z4Ne6hr7ebGK05NdSgTRnc+i4gcod7+ODc9upMVC6dz1jHp3UV1MCUGEZEj9OuNNdQ0dfKp85am7bhII1FiEBE5Av1x5weP7uSEudM497jJ1Y1eiUFE5Aj8fss+Xmxon3StBVBiEBEZt3jc+f4j1SydWcj/OmF2qsOZcEoMIiLj9KuNNbxQ28qnzlua9sNfjCTp9zGIiKSreNz57h938N0/7uC0BSW8/eQ5qQ4pFEoMIiJj0NzRy9/dvYlHttVx+YoK/uWdJxLLnJwnXZQYREQOY9v+Fv7mZxvY29TJV995IleeuWDSXXAeTIlBROQQHti6n7+7axNFuTHuXL2KFQtLUx1S6JQYRERG4O6seWwXX39gG6dUlLDmQyuYOS39R04dCyUGEZFhevvjXP+brfziyd287aQ5/Pt7TyE3KzPVYSWNEoOIyCDNnb188vaneby6gU+et4TPv+W4Sdkl9VCUGEREAi82tPPx26p4qaGdb15+Mu+tnJ/qkFJCiUFEBHhkWy2fuXMTsQzjto+t5OwlZakOKWWUGERkSovHnf94pJob/7id5bOn8aMPrWB+aX6qw0opJQYRmbJau3r53N2befi5Wi47bR5fu+wk8rKnzkXm0SgxiMiUVNfSxYd+8iTV9W18+R3Hc/XZiyb1TWvjEdr93GaWa2ZPmtlmM3vWzL4yQpmrzazezDYFj2vCikdEZMDuAx1cftNadh/s4NaPrOQj5yxWUhgkzBZDN3C+u7eZWRbwuJn9wd3XDSt3l7t/KsQ4REReVV3XypU3P0lnbz+3X3Mmpy2YnuqQIie0xODuDrQFT7OCh4d1PBGRw9myp5kP37KezIwM7ly9iuVzpqU6pEgKdWhAM8s0s01AHfCwu68fodi7zewZM7vXzKZmp2ERCd1TLx3gAz9eR352jHuvPUtJ4RBCTQzu3u/upwIVwEozO3FYkd8Ci9z9ZOBh4NaR9mNmq82sysyq6uvrwwxZRCahdbsaueqWJykvyuGea89iUVlBqkOKtKQMJu7uTcD/AG8dtr7R3buDpzcDK0Z5/Rp3r3T3yvLyyTXptoiE64nqBq7+6ZPMLcnjzr9ZxdySvFSHFHlh9koqN7OSYDkPeAuwbViZwdMfXQI8H1Y8IjL1PL6jgY/851MsKM3nFx9fxcyiqTE66tEKs1fSHOBWM8skkYDudvffmdkNQJW73w9cZ2aXAH3AAeDqEOMRkSnkT9vrWX1bFYvLCrj9mjOZUZiT6pDShiU6D6WPyspKr6qqSnUYIhJR7s49VXv437/ZypLyQm6/5kxKC7JTHVbKmdkGd68cS1nd+Swik0ZzZy//dN8W/uuZfZy9ZAb/7wOnM11JYdyUGERkUtjw8gGu+8Umalu6+MJbj+Nv3riEzCk2j8JEUWIQkbTm7vzg0Z18++HtzC3J5Z5rz9LdzEdJiUFE0tovn67hWw++wDtOmcvXLjuRotysVIeU9pQYRCRt7apv4/rfbOXMxaXc+L5TdepogiTlBjcRkYnW0xfnujs3kh3L4MYrlBQmkloMIpKWvvXgNrbWtLDmQyuYU6y7mSeSWgwiknb+tL2eH//5RT60aiEXnjA71eFMOkoMIpJW6lu7+fzdmzhuVhFfetvyVIczKelUkoikjXjc+fw9m2nt6uOOj68iN0vzM4dBLQYRSRs3P76Lx7bX83/efjzHzipKdTiTlhKDiKSFzbub+OYDL3DRibP54JkLUh3OpKbEICKR19rVy6d/sZFZ03L5+rtOxkxdU8OkawwiEmnuzv/+9VZqmjq5a/UqivN1Z3PY1GIQkUi7d8MefrNpL5+9YBmVi0pTHc6UoMQgIpFVXdfGl+9/llXHlPK35y1NdThThk4liUgk7aht5cqfrCc3K5Mb33eahrxIIrUYRCRyNu9u4j0/Wkvc4Y6Pn8nsYs3VnEyhJQYzyzWzJ81ss5k9a2ZfGaFMjpndZWbVZrbezBaFFY+IpIcndjbwgR+voyg3xr3XnsXrZk9LdUhTTpgthm7gfHc/BTgVeKuZrRpW5mPAQXdfCnwH+EaI8YhIxD307H6u/ulTzJuex73Xns3CGQWpDmlKCi0xeEJb8DQrePiwYpcCtwbL9wIXmDooi0xJD2zdxyduf5rlc6Zx1+qzmDVNp49SJdRrDGaWaWabgDrgYXdfP6zIPGA3gLv3Ac3AjDBjEpHoWbuzket+sYmTK4q5/ZozmV6QneqQprRQE4O797v7qUAFsNLMTjyS/ZjZajOrMrOq+vr6iQ1SRFLqub0trL6tigUz8rnlqjMozFFnyVRLSq8kd28C/gd467BNNcB8ADOLAcVA4wivX+Pule5eWV5eHna4IpIkuw90cNVPn6QwN8ZtH12plkJEhNkrqdzMSoLlPOAtwLZhxe4HrgqWLwcecffh1yFEZBJqbOvmw7c8SU9fnFs/upK5JZqFLSrCbLPNAW41s0wSCehud/+dmd0AVLn7/cBPgJ+ZWTVwALgixHhEJCJebmznk3c8zb7mTm6/5kwNoR0xoSUGd38GOG2E9dcPWu4C3hNWDCISLX39cW75y4t8++HtxDIy+OEHV7BiocY/ihpd5RGRpNha08w//uoZtta08JbjZ/HVS0/UHc0RpcQgIqGqa+nipj/t4ta1LzE9P5sffPB0LjpxtuZUiDAlBhEJxZ6DHfzoT7u4q2o3ff1x3ls5ny9etFzzKaQBJQYRmVA1TZ185+Ht/HpjDWZw+YoKrn3TEg1vkUaUGERkwqzb1cgnfr6Bzt5+rly1kNVvPEbdUNOQEoOITIg71r/C9b/ZyoIZ+fzqqjNYXKYWQrpSYhCRo9LXH+df/ut5/vOJl3jTseV87/2nUZyn6wjpTIlBRI5IPO5sqWnmWw++wOPVDVzz+sV88eLlmmltElBiEJEx6+mLs3ZXIw8/t5+Hn6ultqWb7MwMvvnuk3nvGfNTHZ5MECUGERlRc0cvz+1rYdv+Frbta2VbbSsv7G+hqzdOXlYmbzy2jAuPn835r5upwe8mGSUGEaGpo4dt+1vZsqeZzXua2FLTzMuNHa9uLy3I5nWzi/jAyoWcs3QG5ywtIzcrM4URS5iUGESmmFcaO3jg2X1U17Wxq76dXQ3tHGjveXX7vJI8Tq4o5n1nzOeEucUsn1NEeWGO7lSeQpQYRKaAvv44j2yr4+frX+Gx7YnJrsoKszmmrJALj5/FMeUFLJtZxEkVxZQV5qQ4Wkk1JQaRNNfc2csrjR28fKCdVw500NnTj5lhQIYZHT193L95L/uau5g1LYfPvnkZ7ztjPnOKdeOZjEyJQSRNxOPOroZ2ttY0s6WmmS17mtle10pTR++QcmYweLorM3j90jK+/I4TePPymcQykzJxo6QxJQaRCeDu1LV2U5KfRU5sbBdl3Z3W7j7qWrqob+3hQHsPB9q7aWxPLDd19NLU2UtzZy/NHT3UtXbT0dMPQG5WBsfPmcbFJ81h0Yx8FpQWsKA0nwUz8l+dM9ndcQcH3Vsg46LEkCSNbd3ct7GGP2zdz8LSfC48YTZvOracvOzo9uzo7Y/zwv5WttY080zwDbW+tZtjZxdxwtxpwaOYiul5xDIs9IuT7k53X5zu3jjZsQxyYhlkBB948bjz8oEOttY0s3VvM8/WtNDU2UNJXjbF+VmU5GVRkp9FLCODuDv9cSfuvLrcH3f64nH6406GGWWFOcyalsvMosTP0sJsCnNiFObEXv2QbWjr5i/VDTy+o4G/VDewt7mLzAxj4Yx8ls0s5NhZRcwryaOtu4+DHcEHfUcvDW3d1LV2U9vS9eoH/XDFeVlMz8+iOD+b4rwsFpbmU1qQzQlzp3FSRTFLywsP+83fzND1YjkSlm5TLFdWVnpVVVVo++/rj1PX2s3+li5qm7uobemitrWbzp5+evrj9PQlHpkZxukLp/OGpWUsnJE/4odif9x5bEc9dz+1m/9+vpbefuf4OdPY29xJU0cvuVkZnHvsTM5fPpPywhxyszLJy84kLyuT/OxMCoIPouxYuE1/d6enP051XRtb9iROU2ytaeb5/a309MUBKMqNcXJFMeWFOWzb30p1XRt98df+dswgKzODnMwMsmMZePC7THzgJj54PTjWwKtyY5kc92qSKeb4udPIiWWwvTax/4GfTR29dPb209nbz/A/15xYBnnZmfT2xWkPPmSzMzM4bnYRZYXZNHcG37qDb9+JD/7EufeMDCPTjFhGYnngZ3/cOdjR81fHGpCfnXh/GtoSPXmK87I4e8kMKheV0tTRw/baVnbUtfFyYwf9we8ow6AkP5uS/CxK87OZVZzLrKJcZhfnMLMol/KiHEoLsplRmM30/GyydLpHJpiZbXD3yjGVDSsxmNl84DZgFonW7Bp3/+6wMucCvwFeDFb9yt1vONR+JzIxdPb0U/XyAbbta+X5/S28sD/xDz3wYTggK9PIy8okO5ZJdqaRHcugs7ef2pZuACqm5/GGZWUsKS9kz8FOdh/o4JXg0d0Xp7Qgm3edNo/3VM7nuNlF9PbHefLFAzywdT8PPrufutbuQ8aZHcugMCfGzKIclswsZGl5IUtmFrKkvIC8rEz64k5ff+LDt6c/Tlt3H61dvbR2JX42d/ZyoL2HhrYeGtsSpyrauvpeS3T98SEfgkW5MU6cW8xJFcWcNC/xGJ78uvv62VHbxrN7m6lr6R6yr56+OGYQy8gglmFkZiY+gDOC15uBAS1dfTy/r4Xn9rXQ2tU3pM6Dv3mXF+WQl5VImHnZiUTZ2x+ns6efrr5+uoKLrcvnFHHC3GKOnVU0YjId+FsfS8umtz9OQ1s3tS3d1LV0caC9h7buvsSjq4/2nj4qpufz+qVlnDiveMRTNd19/dS3dlOUm0VRTuzV1o1IKkQlMcwB5rj702ZWBGwA3unuzw0qcy7w9+7+9rHudyISQ0dPHz9b+zJrHttFY9B/e9a0HI6bPY3ls4tYOKOA2cWJUwizp+VSWpD9Vx8m7s5LjR38eUc9f97RwLqdjbR295GfncmC0nwWzshnQWk+KxZO5/zXzRr1W3/igmIbbd39iQ+64Jtxe3cf7QMfRN39tHb1sq+5i+q6NnYf7Bj12+xIMjMs8W00+EY6oyCHwtwY2ZmJ0zHZsQyyMzNYWFaQSAKl+Un9EHN3dh/o5Nm9zfTGnWUzCzmmvGDM5+pF5PDGkxhCu8bg7vuAfcFyq5k9D8wDnjvkC0PU3t3HbWtf5sd/3sWB9h7esKyMj71+MSdXlFA6zlv6zYzFZQUsLivgw2ctorc/TmtXH9Pzs8Z1rj0jw1g6s2hcx+7q7efFhnZebGintz9xWiuWYWRmZBDLNIpyYolvqbkxinJjFGRH+9uqmbFgRuLCqYikXlIuPpvZIuA0YP0Im88ys83AXhKth2fDiOGRbbV8/u7NHOzo5Y3HlvOZC5axYuH0Cdt/VmbGuJPLkcrNymT5nGksnzMtKccTkakl9MRgZoXAL4HPunvLsM1PAwvdvc3MLgZ+DSwbYR+rgdUACxYsOKI4FpcVcur8Ej59wTJOXzBxCUFEZLIJtVeSmWUBvwMedPdvj6H8S0CluzeMVibsXkkiIpPReK4xhNYnzhIn2n8CPD9aUjCz2UE5zGxlEE9jWDGJiMjhhXkq6RzgQ8AWM9sUrPsnYAGAu98EXA58wsz6gE7gCk+3GytERCaZMHslPU6iu/qhynwf+H5YMYiIyPjp9koRERlCiUFERIZQYhARkSGUGEREZAglBhERGSLtht02s3rg5UGrioHmYcXGsu5QzweWy4BRb7Ybh5HiOZKyo20bb31HW052fQ9XbiLe26jU9XBlx/reHsnfMUTrvdXf8cjrw35vF7p7+RjKDczylL4PEsN5j3vdoZ4PLANVYcV4JGVH2zbe+h5iOan1PVy5iXhvo1LXiXpvj+TvOGrvrf6Oo/neDn5MhlNJvz3CdYd6PtLrj8Z49neosqNtG299w6zrePZ5uHIT8d5Gpa6HKzvW9zaVf8fj2af+jg+/PWrv7avS7lRSMplZlY9xbJHJYCrVdyrVFaZWfadSXSGc+k6GFkOY1qQ6gCSbSvWdSnWFqVXfqVRXCKG+ajGIiMgQajGIiMgQUyYxmNktZlZnZluP4LUrzGyLmVWb2fcGhgoPtn3azLaZ2bNm9s2JjfrIhVFfM/tnM6sxs03B4+Jk2EUiAAAGVUlEQVSJj3z8wnpvg+2fNzM3s7KJi/jIhfS+ftXMngne04fMbO7ER35kQqrvt4L/2WfM7D4zK5n4yMcvpLq+J/hsipvZ2K9DTHQ3p6g+gDcCpwNbj+C1TwKrSIwW+wfgomD9ecB/AznB85mprmfI9f1nEtOvprx+Ydc12DYfeJDEfTNlqa5niO/rtEFlrgNuSnU9Q67vhUAsWP4G8I1U1zPEui4HjgMeJTEJ2pj2N2VaDO7+GHBg8DozW2JmD5jZBjP7s5m9bvjrzGwOiX+cdZ74Td8GvDPY/Ang6+7eHRyjLtxajF1I9Y2kEOv6HeALQGQuxIVRVx865W4Bk7++D7l7X1B0HVARbi3GJqS6Pu/uL4w3limTGEaxBvi0u68A/h74wQhl5gF7Bj3fE6wDOBZ4g5mtN7M/mdkZoUZ79I62vgCfCprgt5hZlCfPPqq6mtmlQI27bw470Alw1O+rmf2rme0GPghcH2KsE2Ei/o4HfJTEN+yomsi6jlmYM7hFmpkVAmcD9ww6rZwzzt3EgFISTbgzgLvN7Jgga0fKBNX3h8BXSXyj/Crw7yT+sSLlaOtqZvkkZhu8cOKjm1gT9L7i7l8CvmRmXwQ+BXx5woKcQBNV32BfXwL6gNsnJrqJNZF1Ha8pmxhItJaa3P3UwSvNLBPYEDy9n8SH4eCmZgVQEyzvAX4VJIInzSxOYtyS+jADP0JHXV93rx30uh8Dvwsz4KNwtHVdAiwGNgf/kBXA02a20t33hxz7eE3E3/FgtwO/J6KJgQmqr5ldDbwduCCKX+QCE/3ejl2qL7gk8wEsYtCFHeAJ4D3BsgGnjPK64Rd2Lg7WXwvcECwfC+wmuDckCo8Q6jtnUJm/A+5MdR3DquuwMi8RkYvPIb2vywaV+TRwb6rrGHJ93wo8B5Snum5h13XQ9kcZx8XnlP8ikvgL/wWwD+gl8U3/YyS+FT4AbA7+UK4f5bWVwFZgJ4k5qgduDMwGfh5sexo4P9X1DLm+PwO2AM+Q+KYyJ1n1SXZdh5WJTGII6X39ZbD+GRJj8MxLdT1Drm81iS9xm4JHJHphhVTXy4J9dQO1wINjiUV3PouIyBBTvVeSiIgMo8QgIiJDKDGIiMgQSgwiIjKEEoOIiAyhxCCTgpm1Jfl4N5vZ8RO0r/5gZNOtZvbbw432aWYlZva3E3FskZGou6pMCmbW5u6FE7i/mL820FqoBsduZrcC2939Xw9RfhHwO3c/MRnxydSjFoNMWmZWbma/NLOngsc5wfqVZrbWzDaa2RNmdlyw/mozu9/MHgH+aGbnmtmjZnZvMH7/7YPGuX90YHx7M2sLBqHbbGbrzGxWsH5J8HyLmf3LGFs1a3ltIL9CM/ujmT0d7OPSoMzXgSVBK+NbQdl/COr4jJl9ZQJ/jTIFKTHIZPZd4DvufgbwbuDmYP024A3ufhqJkUS/Nug1pwOXu/ubguenAZ8FjgeOAc4Z4TgFwDp3PwV4DPj4oON/191PYujolyMKxsC5gMRd5QBdwGXufjqJuT/+PUhM/wjsdPdT3f0fzOxCYBmwEjgVWGFmbzzc8URGM5UH0ZPJ783A8YNGppwWjFhZDNxqZstIjBSbNeg1D7v74DHxn3T3PQBmtonEWDaPDztOD68NKLgBeEuwfBavze9wB/B/R4kzL9j3POB54OFgvQFfCz7k48H2WSO8/sLgsTF4XkgiUTw2yvFEDkmJQSazDGCVu3cNXmlm3wf+x90vC87XPzpoc/uwfXQPWu5n5P+ZXn/tYt1oZQ6l091PDYb7fhD4JPA9EnMjlAMr3L3XzF4Cckd4vQH/5u4/GudxRUakU0kymT1EYrRQAMxsYPjiYl4blvjqEI+/jsQpLIArDlfY3TtITK35eTOLkYizLkgK5wELg6KtQNGglz4IfDRoDWFm88xs5gTVQaYgJQaZLPLNbM+gx+dIfMhWBhdknyMxTDrAN4F/M7ONhNtq/izwOTN7BlgKNB/uBe6+kcQop+8nMTdCpZltAT5M4toI7t4I/CXo3votd3+IxKmqtUHZexmaOETGRd1VRUISnBrqdHc3syuA97v7pYd7nUiq6RqDSHhWAN8PehI1EcFpUEVGohaDiIgMoWsMIiIyhBKDiIgMocQgIiJDKDGIiMgQSgwiIjKEEoOIiAzx/wFga0KxLcLGPgAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.unfreeze()","execution_count":14,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-2\nlearn.fit_one_cycle(10, slice(1e-5, 1e-3))","execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":"","text/html":"\n
\n \n \n 0.00% [0/16 00:00<00:00]\n
\n \n\n \n \n \n \n \n \n \n \n \n \n \n
epochtrain_lossvalid_lossfbetatime

\n\n

\n \n \n 6.15% [189/3072 02:07<32:32 2.5571]\n
\n "},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.save('stage-1', return_path=True)\nlearn.export()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Get predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_best_fixed_threshold(preds, targs, do_plot=True):\n score = []\n thrs = np.arange(0, 0.5, 0.01)\n for thr in progress_bar(thrs):\n score.append(fbeta(valid_preds[0],valid_preds[1], thresh=thr))\n score = np.array(score)\n pm = score.argmax()\n best_thr, best_score = thrs[pm], score[pm].item()\n print(f'thr={best_thr:.3f}', f'F2={best_score:.3f}')\n if do_plot:\n plt.plot(thrs, score)\n plt.vlines(x=best_thr, ymin=score.min(), ymax=score.max())\n plt.text(best_thr+0.03, best_score-0.01, f'$F_{2}=${best_score:.3f}', fontsize=14);\n plt.show()\n return best_thr\n\ni2c = np.array([[i, c] for c, i in learn.data.train_ds.y.c2i.items()]).astype(int) # indices to class number correspondence\n\ndef join_preds(preds, thr):\n return [' '.join(i2c[np.where(t==1)[0],1].astype(str)) for t in (preds[0].sigmoid()>thr).long()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Validation predictions\nvalid_preds = learn.get_preds(DatasetType.Valid)\nbest_thr = find_best_fixed_threshold(*valid_preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test predictions\ntest_preds = learn.TTA(ds_type=DatasetType.Test)\ntest_df.attribute_ids = join_preds(test_preds, best_thr)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}