{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nimport os\n!pip install timm\n#os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_gb:15\"\n\nimport gc\nimport time\nimport cv2\nimport math\nimport copy\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\n\nimport timm\n\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nimport timm\nimport torch.optim as optim\nfrom torch.cuda import amp\n\nfrom PIL import Image\n\nfrom albumentations.pytorch import ToTensorV2\nimport albumentations as A\n\nfrom sklearn.preprocessing import LabelEncoder\n\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\n#from transformers import *","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:47.955151Z","iopub.execute_input":"2022-10-19T09:59:47.955527Z","iopub.status.idle":"2022-10-19T09:59:57.492078Z","shell.execute_reply.started":"2022-10-19T09:59:47.955495Z","shell.execute_reply":"2022-10-19T09:59:57.490891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.496021Z","iopub.execute_input":"2022-10-19T09:59:57.496573Z","iopub.status.idle":"2022-10-19T09:59:57.500772Z","shell.execute_reply.started":"2022-10-19T09:59:57.496540Z","shell.execute_reply":"2022-10-19T09:59:57.499759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_DIR = '../input/humpback-whale-identification/test'\nTRAIN_DIR = '../input/humpback-whale-identification/train'","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.502250Z","iopub.execute_input":"2022-10-19T09:59:57.502843Z","iopub.status.idle":"2022-10-19T09:59:57.512787Z","shell.execute_reply.started":"2022-10-19T09:59:57.502807Z","shell.execute_reply":"2022-10-19T09:59:57.511581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/humpback-whale-identification/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.515810Z","iopub.execute_input":"2022-10-19T09:59:57.516475Z","iopub.status.idle":"2022-10-19T09:59:57.554096Z","shell.execute_reply.started":"2022-10-19T09:59:57.516439Z","shell.execute_reply":"2022-10-19T09:59:57.553088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CONFIG:\n    seed= 2022\n    img_height = 224\n    img_width = 224\n    embedding_size = 512\n    #model_name = 'efficientnet-b0'\n    model_name = 'resnet50'\n    embedding_size = 128\n    num_class = 5005\n    lr = 1e-4\n    batch_size = 128\n    n_workers = 1\n    n_accumulate = 28","metadata":{"execution":{"iopub.status.busy":"2022-10-19T10:29:02.774995Z","iopub.execute_input":"2022-10-19T10:29:02.775354Z","iopub.status.idle":"2022-10-19T10:29:02.781677Z","shell.execute_reply.started":"2022-10-19T10:29:02.775325Z","shell.execute_reply":"2022-10-19T10:29:02.780588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.565294Z","iopub.execute_input":"2022-10-19T09:59:57.566415Z","iopub.status.idle":"2022-10-19T09:59:57.573736Z","shell.execute_reply.started":"2022-10-19T09:59:57.566376Z","shell.execute_reply":"2022-10-19T09:59:57.572788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HumpBackWhaleData(Dataset):\n    def __init__(self, data, image_folder, transforms = None, train = True):\n        self.data = data\n        self.image_folder = image_folder\n        self.transforms = transforms\n        self.train = train\n        self.label_list = self.data.Id_code.values\n        self.img_list = self.data.Image.values\n        \n    def __len__(self):\n        return len(self.data)\n        \n    def __getitem__(self, index):\n        if self.train:\n            img_name, labels = self.img_list[index], self.label_list[index]\n        else:\n            img_name, labels = self.img_list[index], -1\n        \n        image = cv2.imread(os.path.join(self.image_folder, img_name))\n        if self.transforms:\n            img = self.transforms(image = image)['image']\n        return img, labels","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.575339Z","iopub.execute_input":"2022-10-19T09:59:57.576042Z","iopub.status.idle":"2022-10-19T09:59:57.585093Z","shell.execute_reply.started":"2022-10-19T09:59:57.576006Z","shell.execute_reply":"2022-10-19T09:59:57.584064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = cv2.imread('../input/humpback-whale-identification/train/0000e88ab.jpg')\nimg.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.586484Z","iopub.execute_input":"2022-10-19T09:59:57.587461Z","iopub.status.idle":"2022-10-19T09:59:57.614506Z","shell.execute_reply.started":"2022-10-19T09:59:57.587425Z","shell.execute_reply":"2022-10-19T09:59:57.613571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = A.Compose([\n        A.Resize(CONFIG.img_height, CONFIG.img_width),\n        A.ShiftScaleRotate(shift_limit=0.1, \n                           scale_limit=0.15, \n                           rotate_limit=60, \n                           p=0.5),\n        A.HueSaturationValue(\n                hue_shift_limit=0.2, \n                sat_shift_limit=0.2, \n                val_shift_limit=0.2, \n                p=0.5\n            ),\n        A.RandomBrightnessContrast(\n                brightness_limit=(-0.1,0.1), \n                contrast_limit=(-0.1, 0.1), \n                p=0.5\n            ),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.616070Z","iopub.execute_input":"2022-10-19T09:59:57.616669Z","iopub.status.idle":"2022-10-19T09:59:57.624275Z","shell.execute_reply.started":"2022-10-19T09:59:57.616633Z","shell.execute_reply":"2022-10-19T09:59:57.623373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n    def __init__(self, in_features, out_features, s=30.0, \n                 m=0.50, easy_margin=False, ls_eps=0.0):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------\n        #one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.shape, device = 'cuda')\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.629186Z","iopub.execute_input":"2022-10-19T09:59:57.630031Z","iopub.status.idle":"2022-10-19T09:59:57.642302Z","shell.execute_reply.started":"2022-10-19T09:59:57.630003Z","shell.execute_reply":"2022-10-19T09:59:57.641343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HumpBackModel(nn.Module):\n    def __init__(self, model, embedding_size, pretrained = True):\n        super(HumpBackModel, self).__init__()\n        #self.model = efficientnet_pytorch.EfficientNet.from_name(model)\n        self.model = timm.create_model(model)\n        #self.model.load_state_dict(torch.load(checkpoint))\n        in_features = self.model.fc.out_features\n        self.bn1 = nn.BatchNorm1d(in_features)\n        self.dropout = nn.Dropout(0.3)\n        self.fc1 = nn.Linear(in_features, embedding_size)\n        self.bn2 = nn.BatchNorm1d(embedding_size)\n        self.margin = ArcMarginProduct(embedding_size, CONFIG.num_class)\n    \n    def forward(self, images, labels):\n        x = self.model(images)\n        x =  self.bn1(x)\n        x = self.dropout(x)\n        x = self.fc1(x)\n        x = self.bn2(x)\n        output = self.margin(x, labels)\n        return output\n\n#checkpoint = '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth'\nmodel = HumpBackModel(CONFIG.model_name, CONFIG.embedding_size)\nmodel.cuda()","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:57.644624Z","iopub.execute_input":"2022-10-19T09:59:57.644968Z","iopub.status.idle":"2022-10-19T09:59:58.084422Z","shell.execute_reply.started":"2022-10-19T09:59:57.644933Z","shell.execute_reply":"2022-10-19T09:59:58.083320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr = 0.0001)\nscheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode = 'min', factor = 0.1, patience = 3, min_lr = 0.1)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:58.085723Z","iopub.execute_input":"2022-10-19T09:59:58.087124Z","iopub.status.idle":"2022-10-19T09:59:58.094686Z","shell.execute_reply.started":"2022-10-19T09:59:58.087086Z","shell.execute_reply":"2022-10-19T09:59:58.093595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain_df['Id_code'] = encoder.fit_transform(train_df['Id'])\ntrain, valid = train_test_split(train_df, test_size = 0.3, shuffle = True)\ntrain_data = HumpBackWhaleData(train, TRAIN_DIR, transforms = transforms)\nval_data = HumpBackWhaleData(valid, TRAIN_DIR, transforms = transforms)\n\ntrain_loader, val_loader = DataLoader(train_data, batch_size = 2), DataLoader(val_data, batch_size = 2)\ndataloader = {'train':train_loader, \n              'valid':val_loader}","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:58.096408Z","iopub.execute_input":"2022-10-19T09:59:58.096841Z","iopub.status.idle":"2022-10-19T09:59:58.127352Z","shell.execute_reply.started":"2022-10-19T09:59:58.096804Z","shell.execute_reply":"2022-10-19T09:59:58.126265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, dataloader, criterion, optimizer):\n    model.train()\n    \n    scaler = amp.GradScaler()\n    dataset_size = 0.0\n    running_loss = 0.0\n    \n    bar = tqdm(dataloader, total = len(dataloader))\n    for step, (images, labels) in enumerate(bar):\n        images = images.cuda().float()\n        labels = labels.cuda()\n        \n        batch = images.size(0)\n        \n        output = model(images, labels)\n        loss = criterion(output, labels)\n            \n        loss.backward()\n        \n        if (step + 1) % CONFIG.n_accumulate == 0:\n            optimizer.step()\n\n            # zero the parameter gradients\n            optimizer.zero_grad()                            \n        \n        \n        running_loss += loss.item()*batch\n        dataset_size += batch\n        \n        epoch_loss = running_loss/dataset_size\n        \n        scheduler.step(epoch_loss)\n        \n        bar.set_postfix(Epoch = step, Train_loss = epoch_loss, LR = optimizer.param_groups[0]['lr'])\n    \n    gc.collect()\n    return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:58.128650Z","iopub.execute_input":"2022-10-19T09:59:58.129905Z","iopub.status.idle":"2022-10-19T09:59:58.138924Z","shell.execute_reply.started":"2022-10-19T09:59:58.129842Z","shell.execute_reply":"2022-10-19T09:59:58.137913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_one_epoch(model, dataloader, optimizer, criterion):\n    model.eval()\n    \n    dataset_size = 0.0\n    running_loss = 0.0\n    \n    bar = tqdm(dataloader, total = len(dataloader))\n    for step, (images, labels) in enumerate(bar):\n        images = images.cuda().float()\n        labels = labels.cuda()\n        \n        batch = images.size(0)\n        \n        output = model(images, labels)\n        loss = criterion(output, labels)\n        \n        running_loss += loss.item()*batch\n        dataset_size += batch\n        \n        epoch_loss = running_loss/dataset_size\n        \n        bar.set_postfix(Epoch=step, Valid_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])\n    gc.collect()\n    return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:58.141301Z","iopub.execute_input":"2022-10-19T09:59:58.142007Z","iopub.status.idle":"2022-10-19T09:59:58.154779Z","shell.execute_reply.started":"2022-10-19T09:59:58.141966Z","shell.execute_reply":"2022-10-19T09:59:58.153801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(model, dataloader, criterion, optimizer, scheduler, epoch):\n\n    start = time.time()\n    best_weights = copy.deepcopy(model.state_dict())\n    best_loss = np.inf\n    history = defaultdict(list)\n    \n    for epoch in range(1, epoch+1):\n        gc.collect()\n        train_loss = train_one_epoch(model, dataloader['train'], criterion, optimizer)\n        valid_loss = valid_one_epoch(model, dataloader['valid'],optimizer, criterion)\n        \n        history['Train Loss'].append(train_loss)\n        history['Valid Loss'].append(valid_loss)\n        \n        if valid_loss <= best_loss:\n            print(f\"{b_}Validation Loss Improved ({best_loss}-------->{valid_loss})\")\n            best_loss = valid_loss\n            best_model_wts = copy.deepcopy(model.state_dict())\n            PATH = \"Loss{:.4f}_epoch{:.0f}.bin\".format(best_loss, epoch)\n            torch.save(model.state_dict(), PATH)\n            print(f\"Model Saved{sr_}\")\n        print()\n    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best Loss: {:.4f}\".format(best_loss))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2022-10-19T09:59:58.156536Z","iopub.execute_input":"2022-10-19T09:59:58.156957Z","iopub.status.idle":"2022-10-19T09:59:58.169739Z","shell.execute_reply.started":"2022-10-19T09:59:58.156922Z","shell.execute_reply":"2022-10-19T09:59:58.168674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, history = run(model, dataloader, criterion, optimizer, scheduler, epoch = 15)","metadata":{"execution":{"iopub.status.busy":"2022-10-19T10:29:05.810565Z","iopub.execute_input":"2022-10-19T10:29:05.811299Z","iopub.status.idle":"2022-10-19T10:44:43.669895Z","shell.execute_reply.started":"2022-10-19T10:29:05.811258Z","shell.execute_reply":"2022-10-19T10:44:43.668841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}