{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":6799,"databundleVersionId":4225553,"sourceType":"competition"},{"sourceId":269359,"sourceType":"datasetVersion","datasetId":111880}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport time\nimport torch\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nfrom timm.models.layers import trunc_normal_, DropPath\nimport torch.nn.functional as F","metadata":{"execution":{"iopub.status.busy":"2023-12-06T04:30:51.013301Z","iopub.execute_input":"2023-12-06T04:30:51.014054Z","iopub.status.idle":"2023-12-06T04:30:53.06091Z","shell.execute_reply.started":"2023-12-06T04:30:51.014023Z","shell.execute_reply":"2023-12-06T04:30:53.059778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 1234\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2023-12-06T04:30:53.062977Z","iopub.execute_input":"2023-12-06T04:30:53.063348Z","iopub.status.idle":"2023-12-06T04:30:53.073132Z","shell.execute_reply.started":"2023-12-06T04:30:53.063315Z","shell.execute_reply":"2023-12-06T04:30:53.072133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose(\n    [transforms.ToTensor(), transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5)), \n    transforms.Resize((224,224))]\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T04:30:53.074558Z","iopub.execute_input":"2023-12-06T04:30:53.074893Z","iopub.status.idle":"2023-12-06T04:30:53.081684Z","shell.execute_reply.started":"2023-12-06T04:30:53.074863Z","shell.execute_reply":"2023-12-06T04:30:53.08072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = datasets.ImageFolder(\"/kaggle/input/intel-image-classification/seg_train/seg_train\",\n                              transform = transform)\nvalid_data = datasets.ImageFolder(\"/kaggle/input/intel-image-classification/seg_test/seg_test\",\n                             transform = transform)","metadata":{"execution":{"iopub.status.busy":"2023-12-06T04:30:53.084314Z","iopub.execute_input":"2023-12-06T04:30:53.084872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndef plot_images(images, labels, classes, normalize = False):\n\n    n_images = len(images)\n\n    rows = int(np.sqrt(n_images))\n    cols = int(np.sqrt(n_images))\n\n    fig = plt.figure(figsize = (10, 10))\n\n    for i in range(rows*cols):\n\n        ax = fig.add_subplot(rows, cols, i+1)\n\n        image = images[i]\n\n        if normalize:\n            image_min = image.min()\n            image_max = image.max()\n            image.clamp_(min = image_min, max = image_max)\n            image.add_(-image_min).div_(image_max - image_min + 1e-5)\n\n        ax.imshow(image.permute(1, 2, 0).cpu().numpy())\n        ax.set_title(classes[labels[i]])\n        ax.axis('off')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_IMAGES = 25\n\nimages, labels = zip(*[(image, label) for image, label in\n                           [train_data[i] for i in range(N_IMAGES)]])\n\nclasses = train_data.classes\n\nplot_images(images, labels, classes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_augument_1 = transforms.Compose(\n    [transforms.ToTensor(),    \n     transforms.RandomResizedCrop(size=(224, 224), antialias=True),\n     transforms.RandomHorizontalFlip(p=0.5),\n     transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))]\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_augument_2 = transforms.Compose([\n    transforms.RandomResizedCrop(size=(224, 224), antialias=True),\n    transforms.RandomRotation(degrees=(30, 70)),\n    transforms.ToTensor(),\n    transforms.Normalize(\n        mean=[0.5, 0.5, 0.5],\n        std=[0.5, 0.5, 0.5]\n    )\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augument_1 = datasets.ImageFolder(\"/kaggle/input/intel-image-classification/seg_train/seg_train\",\n                              transform = transform_augument_1)\ndata_augument_2 = datasets.ImageFolder(\"/kaggle/input/intel-image-classification/seg_train/seg_train\",\n                              transform = transform_augument_2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = (train_data) + (data_augument_1) + (data_augument_2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.utils.data as data\nBATCH_SIZE = 64\n\ntrain_iterator = data.DataLoader(train_data,\n                                 shuffle = True,\n                                 batch_size = BATCH_SIZE,\n                                num_workers=2)\n\nvalid_iterator = data.DataLoader(valid_data,\n                                 shuffle = True,\n                                 batch_size = BATCH_SIZE,\n                                num_workers=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nclass ConvNeXt(nn.Module):\n    def __init__(self, in_chans=3, num_classes=1000, \n             depths=[3, 3, 9, 3], dims=[96, 192, 384, 768], drop_path_rate=0., \n             layer_scale_init_value=1e-6, head_init_scale=1.,\n             ):\n        super().__init__()\n\n        self.downsample_layers = nn.ModuleList() # stem and 3 intermediate downsampling conv layers\n        stem = nn.Sequential(\n            nn.Conv2d(in_chans, dims[0], kernel_size=4, stride=4),\n            LayerNorm(dims[0], eps=1e-6, data_format=\"channels_first\")\n        )\n        self.downsample_layers.append(stem)\n        for i in range(3):\n            downsample_layer = nn.Sequential(\n                    LayerNorm(dims[i], eps=1e-6, data_format=\"channels_first\"),\n                    nn.Conv2d(dims[i], dims[i+1], kernel_size=2, stride=2),\n            )\n            self.downsample_layers.append(downsample_layer)\n\n        self.stages = nn.ModuleList() # 4 feature resolution stages, each consisting of multiple residual blocks\n        dp_rates=[x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] \n        cur = 0\n        for i in range(4):\n            stage = nn.Sequential(\n                *[Block(dim=dims[i], drop_path=dp_rates[cur + j], \n                layer_scale_init_value=layer_scale_init_value) for j in range(depths[i])]\n            )\n            self.stages.append(stage)\n            cur += depths[i]\n\n        self.norm = nn.LayerNorm(dims[-1], eps=1e-6) # final norm layer\n        self.head = nn.Linear(dims[-1], num_classes)\n\n        self.apply(self._init_weights)\n        self.head.weight.data.mul_(head_init_scale)\n        self.head.bias.data.mul_(head_init_scale)\n\n    def _init_weights(self, m):\n        if isinstance(m, (nn.Conv2d, nn.Linear)):\n            trunc_normal_(m.weight, std=.02)\n            nn.init.constant_(m.bias, 0)\n\n    def forward_features(self, x):\n        for i in range(4):\n            x = self.downsample_layers[i](x)\n            x = self.stages[i](x)\n        return self.norm(x.mean([-2, -1])) # global average pooling, (N, C, H, W) -> (N, C)\n\n    def forward(self, x):\n        x = self.forward_features(x)\n        x = self.head(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Block(nn.Module):\n    def __init__(self, dim, drop_path=0., layer_scale_init_value=1e-6):\n        super().__init__()\n        self.dwconv = nn.Conv2d(dim, dim, kernel_size=7, padding=3, groups=dim) # depthwise conv\n        self.norm = LayerNorm(dim, eps=1e-6)\n        self.pwconv1 = nn.Linear(dim, 4 * dim) # pointwise/1x1 convs, implemented with linear layers\n        self.act = nn.GELU()\n        self.pwconv2 = nn.Linear(4 * dim, dim)\n        self.gamma = nn.Parameter(layer_scale_init_value * torch.ones((dim)), \n                                    requires_grad=True) if layer_scale_init_value > 0 else None\n        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n\n    def forward(self, x):\n        input = x\n        x = self.dwconv(x)\n        x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C)\n        x = self.norm(x)\n        x = self.pwconv1(x)\n        x = self.act(x)\n        x = self.pwconv2(x)\n        if self.gamma is not None:\n            x = self.gamma * x\n        x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W)\n\n        x = input + self.drop_path(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LayerNorm(nn.Module):\n    def __init__(self, normalized_shape, eps=1e-6, data_format=\"channels_last\"):\n        super().__init__()\n        self.weight = nn.Parameter(torch.ones(normalized_shape))\n        self.bias = nn.Parameter(torch.zeros(normalized_shape))\n        self.eps = eps\n        self.data_format = data_format\n        if self.data_format not in [\"channels_last\", \"channels_first\"]:\n            raise NotImplementedError \n        self.normalized_shape = (normalized_shape, )\n    \n    def forward(self, x):\n        if self.data_format == \"channels_last\":\n            return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)\n        elif self.data_format == \"channels_first\":\n            u = x.mean(1, keepdim=True)\n            s = (x - u).pow(2).mean(1, keepdim=True)\n            x = (x - u) / torch.sqrt(s + self.eps)\n            x = self.weight[:, None, None] * x + self.bias[:, None, None]\n            return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"OUTPUT_DIM = 6\nnum_classes = 6\nchannels = 3\n\nmodel = ConvNeXt(channels, num_classes)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\nFOUND_LR = 1e-4\noptimizer = optim.Adam(model.parameters(), lr=FOUND_LR)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_accuracy(y_pred, y):\n    top_pred = y_pred.argmax(1, keepdim = True)\n    correct = top_pred.eq(y.view_as(top_pred)).sum()\n    acc = correct.float() / y.shape[0]\n    return acc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, device):\n\n    epoch_loss = 0\n    epoch_acc = 0\n\n    model.train()\n\n    for (x, y) in iterator:\n\n        x = x.to(device)\n        y = y.to(device)\n\n        optimizer.zero_grad()\n\n        y_pred = model(x)\n\n        loss = criterion(y_pred, y)\n\n        acc = calculate_accuracy(y_pred, y)\n\n        loss.backward()\n\n        optimizer.step()\n\n        epoch_loss += loss.item()\n\n        epoch_acc += acc.item()\n\n    return epoch_loss / len(iterator), epoch_acc / len(iterator)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, iterator, criterion, device):\n\n    epoch_loss = 0\n    epoch_acc = 0\n\n    model.eval()\n\n    with torch.no_grad():\n\n        for (x, y) in iterator:\n\n            x = x.to(device)\n            y = y.to(device)\n\n            y_pred = model(x)\n\n            loss = criterion(y_pred, y)\n\n            acc = calculate_accuracy(y_pred, y)\n\n            epoch_loss += loss.item()\n            epoch_acc += acc.item()\n\n    return epoch_loss / len(iterator), epoch_acc / len(iterator)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def epoch_time(start_time, end_time):\n    elapsed_time = end_time - start_time\n    elapsed_mins = int(elapsed_time / 60)\n    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n    return elapsed_mins, elapsed_secs","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ncriterion = nn.CrossEntropyLoss()\n\nmodel = model.to(device)\ncriterion = criterion.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 30\n\nbest_valid_loss = float('inf')\n\nfor epoch in range(EPOCHS):\n\n    start_time = time.time()\n\n    train_loss, train_acc = train(model, train_iterator, optimizer, criterion, device)\n    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, device)\n\n    if valid_loss < best_valid_loss:\n        best_valid_loss = valid_loss\n        torch.save(model, 'convnext-model.pt')\n\n    end_time = time.time()\n\n    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n\n    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score, ConfusionMatrixDisplay\nmodel_saved = torch.load('convnext-model.pt')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_result(model, data, device):\n    predict = None\n    gold = None\n    epoch_loss = 0\n    epoch_acc = 0\n\n    model.eval()\n\n    with torch.no_grad():\n\n        for (x, y) in data:\n\n            x = x.to(device)\n            y = y.to(device)\n    \n            y_pred = model(x)\n            top_pred = y_pred.argmax(1, keepdim = True)\n            if predict == None:\n                predict = top_pred\n                gold = y\n            else:\n                predict = torch.cat([predict, top_pred])\n                gold = torch.cat([gold, y])\n    predict = predict.cpu()\n    gold = gold.cpu()\n    precision_macro = precision_score(gold, predict, average='macro')\n    recall_macro = recall_score(gold, predict, average='macro')\n    f1_macro = f1_score(gold, predict, average='macro')\n    accuracy = accuracy_score(gold, predict)\n    ConfusionMatrixDisplay.from_predictions(gold, predict, display_labels = valid_data.classes,cmap=plt.cm.Blues)\n    plt.show()\n    return {\"precision macro\":precision_macro,\"recall macro\": recall_macro ,\"f1 macro\": f1_macro, \"accuracy\": accuracy}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_result(model_saved, valid_iterator, device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}