{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport os\nimport random\n\nimport librosa\nimport librosa.display\nimport cv2\nimport torch\nimport torch.nn as nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport transformers\nfrom transformers import AdamW, get_linear_schedule_with_warmup\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2 \nfrom os import path, listdir\n\nfrom sklearn.metrics import f1_score\n\nSEED = 42\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root_dir = \"../input/birdsong-recognition\"\nfake_test_dir = \"../input/birdcall-check\"\n\nresampled_dirs = [f\"birdsong-resampled-train-audio-0{i}\" for i in range(5)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_audio_path = path.join(root_dir, \"test_audio\")\nif path.exists(test_audio_path):\n    test = pd.read_csv(path.join(root_dir, \"test.csv\"))\nelse:\n    test_audio_path = path.join(fake_test_dir, \"test_audio\")\n    test = pd.read_csv(path.join(fake_test_dir, \"test.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# config\nsr = 32000 # because test audios will be of 32k sr, gives better generalization\neffective_length = 5 * 32000 # we take random 5 sec audio\nTRAIN_BATCH_SIZE = 16\nEPOCHS = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_range(img, max_, min_):\n    img[img > max_] = max_\n    img[img < min_] = min_\n    return img\ndef min_max(img):\n    img = (img - img.min())/(img.max() - img.min())\n    img = img * 255\n    \n    return img.astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataSet Class --> returns image in 3 channels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"BIRD_CODE = {\n    'aldfly': 0, 'ameavo': 1, 'amebit': 2, 'amecro': 3, 'amegfi': 4,\n    'amekes': 5, 'amepip': 6, 'amered': 7, 'amerob': 8, 'amewig': 9,\n    'amewoo': 10, 'amtspa': 11, 'annhum': 12, 'astfly': 13, 'baisan': 14,\n    'baleag': 15, 'balori': 16, 'banswa': 17, 'barswa': 18, 'bawwar': 19,\n    'belkin1': 20, 'belspa2': 21, 'bewwre': 22, 'bkbcuc': 23, 'bkbmag1': 24,\n    'bkbwar': 25, 'bkcchi': 26, 'bkchum': 27, 'bkhgro': 28, 'bkpwar': 29,\n    'bktspa': 30, 'blkpho': 31, 'blugrb1': 32, 'blujay': 33, 'bnhcow': 34,\n    'boboli': 35, 'bongul': 36, 'brdowl': 37, 'brebla': 38, 'brespa': 39,\n    'brncre': 40, 'brnthr': 41, 'brthum': 42, 'brwhaw': 43, 'btbwar': 44,\n    'btnwar': 45, 'btywar': 46, 'buffle': 47, 'buggna': 48, 'buhvir': 49,\n    'bulori': 50, 'bushti': 51, 'buwtea': 52, 'buwwar': 53, 'cacwre': 54,\n    'calgul': 55, 'calqua': 56, 'camwar': 57, 'cangoo': 58, 'canwar': 59,\n    'canwre': 60, 'carwre': 61, 'casfin': 62, 'caster1': 63, 'casvir': 64,\n    'cedwax': 65, 'chispa': 66, 'chiswi': 67, 'chswar': 68, 'chukar': 69,\n    'clanut': 70, 'cliswa': 71, 'comgol': 72, 'comgra': 73, 'comloo': 74,\n    'commer': 75, 'comnig': 76, 'comrav': 77, 'comred': 78, 'comter': 79,\n    'comyel': 80, 'coohaw': 81, 'coshum': 82, 'cowscj1': 83, 'daejun': 84,\n    'doccor': 85, 'dowwoo': 86, 'dusfly': 87, 'eargre': 88, 'easblu': 89,\n    'easkin': 90, 'easmea': 91, 'easpho': 92, 'eastow': 93, 'eawpew': 94,\n    'eucdov': 95, 'eursta': 96, 'evegro': 97, 'fiespa': 98, 'fiscro': 99,\n    'foxspa': 100, 'gadwal': 101, 'gcrfin': 102, 'gnttow': 103, 'gnwtea': 104,\n    'gockin': 105, 'gocspa': 106, 'goleag': 107, 'grbher3': 108, 'grcfly': 109,\n    'greegr': 110, 'greroa': 111, 'greyel': 112, 'grhowl': 113, 'grnher': 114,\n    'grtgra': 115, 'grycat': 116, 'gryfly': 117, 'haiwoo': 118, 'hamfly': 119,\n    'hergul': 120, 'herthr': 121, 'hoomer': 122, 'hoowar': 123, 'horgre': 124,\n    'horlar': 125, 'houfin': 126, 'houspa': 127, 'houwre': 128, 'indbun': 129,\n    'juntit1': 130, 'killde': 131, 'labwoo': 132, 'larspa': 133, 'lazbun': 134,\n    'leabit': 135, 'leafly': 136, 'leasan': 137, 'lecthr': 138, 'lesgol': 139,\n    'lesnig': 140, 'lesyel': 141, 'lewwoo': 142, 'linspa': 143, 'lobcur': 144,\n    'lobdow': 145, 'logshr': 146, 'lotduc': 147, 'louwat': 148, 'macwar': 149,\n    'magwar': 150, 'mallar3': 151, 'marwre': 152, 'merlin': 153, 'moublu': 154,\n    'mouchi': 155, 'moudov': 156, 'norcar': 157, 'norfli': 158, 'norhar2': 159,\n    'normoc': 160, 'norpar': 161, 'norpin': 162, 'norsho': 163, 'norwat': 164,\n    'nrwswa': 165, 'nutwoo': 166, 'olsfly': 167, 'orcwar': 168, 'osprey': 169,\n    'ovenbi1': 170, 'palwar': 171, 'pasfly': 172, 'pecsan': 173, 'perfal': 174,\n    'phaino': 175, 'pibgre': 176, 'pilwoo': 177, 'pingro': 178, 'pinjay': 179,\n    'pinsis': 180, 'pinwar': 181, 'plsvir': 182, 'prawar': 183, 'purfin': 184,\n    'pygnut': 185, 'rebmer': 186, 'rebnut': 187, 'rebsap': 188, 'rebwoo': 189,\n    'redcro': 190, 'redhea': 191, 'reevir1': 192, 'renpha': 193, 'reshaw': 194,\n    'rethaw': 195, 'rewbla': 196, 'ribgul': 197, 'rinduc': 198, 'robgro': 199,\n    'rocpig': 200, 'rocwre': 201, 'rthhum': 202, 'ruckin': 203, 'rudduc': 204,\n    'rufgro': 205, 'rufhum': 206, 'rusbla': 207, 'sagspa1': 208, 'sagthr': 209,\n    'savspa': 210, 'saypho': 211, 'scatan': 212, 'scoori': 213, 'semplo': 214,\n    'semsan': 215, 'sheowl': 216, 'shshaw': 217, 'snobun': 218, 'snogoo': 219,\n    'solsan': 220, 'sonspa': 221, 'sora': 222, 'sposan': 223, 'spotow': 224,\n    'stejay': 225, 'swahaw': 226, 'swaspa': 227, 'swathr': 228, 'treswa': 229,\n    'truswa': 230, 'tuftit': 231, 'tunswa': 232, 'veery': 233, 'vesspa': 234,\n    'vigswa': 235, 'warvir': 236, 'wesblu': 237, 'wesgre': 238, 'weskin': 239,\n    'wesmea': 240, 'wessan': 241, 'westan': 242, 'wewpew': 243, 'whbnut': 244,\n    'whcspa': 245, 'whfibi': 246, 'whtspa': 247, 'whtswi': 248, 'wilfly': 249,\n    'wilsni1': 250, 'wiltur': 251, 'winwre3': 252, 'wlswar': 253, 'wooduc': 254,\n    'wooscj2': 255, 'woothr': 256, 'y00475': 257, 'yebfly': 258, 'yebsap': 259,\n    'yehbla': 260, 'yelwar': 261, 'yerwar': 262, 'yetvir': 263\n}\n\ndef one_hot(label):\n    identity_matrix = np.eye(264)\n    lbl_int = BIRD_CODE[label]\n    lbl_enc = identity_matrix[lbl_int]\n    return torch.from_numpy(lbl_enc)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_train_transforms():\n    return A.Compose([\n#         A.HorizontalFlip(p = 0.5),\n#         A.VerticalFlip(p = 0.5),\n#         A.Resize(height = 512, width = 512, p = 1.0),\n        ToTensorV2(p = 1.0),\n    ], p = 1.0)\n\n# def get_valid_transforms():\n#     return A.Compose([\n#         A.Resize(height = 512, width = 512, p = 1.0),\n#         ToTensorV2(p = 1.0)\n#     ], p = 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BirdcallDataset(Dataset):\n    def __init__(self, data, resampled_dirs, effective_length, transforms = None):\n        self.data = data\n        self.audiofilename = self.data[\"resampled_filename\"].values\n        self.ebird_code = self.data[\"ebird_code\"]\n        self.resampled_dirs = resampled_dirs\n        self.effective_length = effective_length\n        self.transforms = transforms\n        \n        \n    def __getitem__(self, index: int):\n        audiofilename = self.audiofilename[index]\n        ebird_code = self.ebird_code[index]\n        for dir_ in self.resampled_dirs:\n            if path.exists(path.join(\"../input\", dir_, ebird_code, audiofilename)):\n                x, sr = librosa.load(path.join(\"../input\", dir_, ebird_code, audiofilename),\n                                     sr = 32000)\n                # Randomly taking 5 sec audio\n                if len(x) > effective_length:\n                    start_index = np.random.randint(len(x) - effective_length)\n                    end_index = start_index + self.effective_length\n                    x = x[start_index: end_index]\n                elif self.effective_length > len(x):\n                    temp_ = np.zeros(effective_length)\n                    start_index = np.random.randint(effective_length - len(x))\n                    end_index = start_index + len(x)\n                    temp_[start_index: end_index] = x\n                    x = temp_\n                else:\n                    x = x\n                    \n                # convert audio to image mel-spectrogram\n                # https://github.com/librosa/librosa/blob/main/examples/LibROSA%20demo.ipynb\n                S = librosa.feature.melspectrogram(x, sr=sr, n_mels= 128,\n                        fmin = 20,\n                        fmax = 16000)\n                image = librosa.power_to_db(S)\n                # https://medium.com/@manivannan_data/resize-image-using-opencv-python-d2cdbbc480f0\n                image = cv2.resize(image,(360,360))\n                \n                \n                \n                # Scaling image between 0-255\n                image = (image - image.mean())/image.std()\n                image = set_range(image, max_ = 4.0, min_ = -4.0)\n                imgray = min_max(image)\n                \n                \n                # Changing 1 channel to three channel image (so that resnest can work)\n                # https://stackoverflow.com/questions/14786179/how-to-convert-a-1-channel-image-into-a-3-channel-with-opencv2\n                image = cv2.merge((imgray,imgray,imgray)).astype(np.float32)\n                \n                if self.transforms:\n                    sample = {\"image\": image}\n                    sample = self.transforms(**sample)\n                    image = sample[\"image\"]    \n                ebird_code = one_hot(ebird_code)\n                return (image, ebird_code)\n                break\n                \n    def __len__(self):\n        return len(self.audiofilename)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(path.join(\"../input\", resampled_dirs[0], \"train_mod.csv\"))\ndata = BirdcallDataset(df, resampled_dirs, effective_length, transforms = get_train_transforms())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data[0][0].shape\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import OrderedDict\n\nimport torch\nfrom torch import nn\nfrom torch.jit.annotations import Dict\n\n\nclass IntermediateLayerGetter(nn.ModuleDict):\n    _version = 2\n    __annotations__ = {\n        \"return_layers\": Dict[str, str],\n    }\n\n    def __init__(self, model, return_layers):\n        if not set(return_layers).issubset([name for name, _ in model.named_children()]):\n            raise ValueError(\"return_layers are not present in model\")\n        orig_return_layers = return_layers\n        return_layers = {str(k): str(v) for k, v in return_layers.items()}\n        layers = OrderedDict()\n        for name, module in model.named_children():\n            layers[name] = module\n            if name in return_layers:\n                del return_layers[name]\n            if not return_layers:\n                break\n\n        super(IntermediateLayerGetter, self).__init__(layers)\n        self.return_layers = orig_return_layers\n\n    def forward(self, x):\n        out = OrderedDict()\n        for name, module in self.items():\n            x = module(x)\n            if name in self.return_layers:\n                out_name = self.return_layers[name]\n                out[out_name] = x\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    from torch.hub import load_state_dict_from_url\nexcept ImportError:\n    from torch.utils.model_zoo import load_url as load_state_dict_from_url","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\n\n__all__ = ['resnet50']\n\n\nmodel_urls = {\n    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n}\n\n\ndef conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):\n    \"\"\"3x3 convolution with padding\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n                     padding=dilation, groups=groups, bias=False, dilation=dilation)\n\n\ndef conv1x1(in_planes, out_planes, stride=1):\n    \"\"\"1x1 convolution\"\"\"\n    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)\n\n\nclass BasicBlock(nn.Module):\n    expansion = 1\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(BasicBlock, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        if groups != 1 or base_width != 64:\n            raise ValueError('BasicBlock only supports groups=1 and base_width=64')\n        if dilation > 1:\n            raise NotImplementedError(\"Dilation > 1 not supported in BasicBlock\")\n        # Both self.conv1 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv3x3(inplanes, planes, stride)\n        self.bn1 = norm_layer(planes)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = conv3x3(planes, planes)\n        self.bn2 = norm_layer(planes)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass Bottleneck(nn.Module):\n\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,\n                 base_width=64, dilation=1, norm_layer=None):\n        super(Bottleneck, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        width = int(planes * (base_width / 64.)) * groups\n        # Both self.conv2 and self.downsample layers downsample the input when stride != 1\n        self.conv1 = conv1x1(inplanes, width)\n        self.bn1 = norm_layer(width)\n        self.conv2 = conv3x3(width, width, stride, groups, dilation)\n        self.bn2 = norm_layer(width)\n        self.conv3 = conv1x1(width, planes * self.expansion)\n        self.bn3 = norm_layer(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        identity = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            identity = self.downsample(x)\n\n        out += identity\n        out = self.relu(out)\n\n        return out\n\n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,\n                 groups=1, width_per_group=64, replace_stride_with_dilation=None,\n                 norm_layer=None):\n        super(ResNet, self).__init__()\n        if norm_layer is None:\n            norm_layer = nn.BatchNorm2d\n        self._norm_layer = norm_layer\n\n        self.inplanes = 64\n        self.dilation = 1\n        if replace_stride_with_dilation is None:\n            # each element in the tuple indicates if we should replace\n            # the 2x2 stride with a dilated convolution instead\n            replace_stride_with_dilation = [False, False, False]\n        if len(replace_stride_with_dilation) != 3:\n            raise ValueError(\"replace_stride_with_dilation should be None \"\n                             \"or a 3-element tuple, got {}\".format(replace_stride_with_dilation))\n        self.groups = groups\n        self.base_width = width_per_group\n        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = norm_layer(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2,\n                                       dilate=replace_stride_with_dilation[0])\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2,\n                                       dilate=replace_stride_with_dilation[1])\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2,\n                                       dilate=replace_stride_with_dilation[2])\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n        # Zero-initialize the last BN in each residual branch,\n        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n        if zero_init_residual:\n            for m in self.modules():\n                if isinstance(m, Bottleneck):\n                    nn.init.constant_(m.bn3.weight, 0)\n                elif isinstance(m, BasicBlock):\n                    nn.init.constant_(m.bn2.weight, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):\n        norm_layer = self._norm_layer\n        downsample = None\n        previous_dilation = self.dilation\n        if dilate:\n            self.dilation *= stride\n            stride = 1\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                conv1x1(self.inplanes, planes * block.expansion, stride),\n                norm_layer(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,\n                            self.base_width, previous_dilation, norm_layer))\n        self.inplanes = planes * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups=self.groups,\n                                base_width=self.base_width, dilation=self.dilation,\n                                norm_layer=norm_layer))\n\n        return nn.Sequential(*layers)\n\n    def _forward_impl(self, x):\n        # See note [TorchScript super()]\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1)\n        x = self.fc(x)\n\n        return x\n\n    def forward(self, x):\n        return self._forward_impl(x)\n\n\ndef _resnet(arch, block, layers, pretrained, progress, **kwargs):\n    model = ResNet(block, layers, **kwargs)\n    if pretrained:\n        state_dict = load_state_dict_from_url(model_urls[arch],\n                                              progress=progress)\n        model.load_state_dict(state_dict)\n    return model\n\n\n\ndef resnet50(pretrained=False, progress=True, **kwargs):\n    \n    return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress,\n                   **kwargs)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = resnet50(pretrained = True)\nnet.fc = nn.Linear(in_features=2048, out_features=264, bias=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Eval Metric","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def micro_precision(ytrue, ypred, micro_recall = None):\n    \"\"\"\n    Function for calculating micro-precision:\n    y_true: list of one hot\n    y_pred: list of one hot\n \n    \"\"\"\n    \n    # Taking 0.5 as threshold\n    m = nn.Sigmoid()\n    ypred = m(ypred.detach()).cpu().numpy()\n    ypred[ypred > 0.5] = 1    \n    ypred[ypred < 0.5] = 0\n\n    ytrue = ytrue.detach().cpu().numpy()\n    score = f1_score(ytrue, ypred, average=\"samples\")\n    return score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataLoader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_loader = DataLoader(\n            data,\n            batch_size=TRAIN_BATCH_SIZE,\n            drop_last=True,  # take care of last batch\n            num_workers=2\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _run():\n    device = torch.device(\"cuda: 0\")\n    model = net.to(device)\n    \n    def training_loop(train_data_loader, model, scheduler = None):\n        for batch, (data, target) in enumerate(train_data_loader):\n            inputs = data.to(device, dtype = torch.float32)\n            targets = target.to(device, dtype = torch.float32)\n            # clear optimizer\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = loss_fn(outputs, targets)\n            \n            micro_prec = micro_precision(targets, outputs)\n            if batch != 0 and batch % 300 == 0:\n                print(f\"Batch = {batch}, Loss = {loss}, Micro precision = {micro_prec}\")\n            # gradients calculations\n            loss.backward()\n            # Update weights\n            optimizer.step()\n            if scheduler is not None:\n                scheduler.step()\n            \n            \n    lr = 0.001\n    num_train_steps = (len(data) / TRAIN_BATCH_SIZE) * EPOCHS\n    loss_fn = nn.BCEWithLogitsLoss()\n    optimizer = AdamW(model.parameters(), lr=lr)\n    scheduler = get_linear_schedule_with_warmup(\n                optimizer,\n                num_warmup_steps=0,\n                num_training_steps=num_train_steps\n            )\n    for e in range(EPOCHS):\n        print(\"#\" * 25)\n        print(f\"Epoch no: {e}\")\n        print(\"#\" * 25)\n\n        training_loop(train_data_loader, model, scheduler = scheduler)\n    torch.save(model.state_dict(), f\"model{e}.bin\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_run()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}