{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Libraries and configuration","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport tensorflow_datasets as tfds\nimport random\nfrom PIL import Image\nimport cv2\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.optim as optim\nfrom torchinfo import summary\nfrom torch.optim import lr_scheduler\n\n!pip install tfrecord\nimport tfrecord","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:16:46.590502Z","iopub.execute_input":"2023-09-17T10:16:46.591472Z","iopub.status.idle":"2023-09-17T10:16:58.614456Z","shell.execute_reply.started":"2023-09-17T10:16:46.591434Z","shell.execute_reply":"2023-09-17T10:16:58.613218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 256\nNUM_EPOCHS = 30\nBASE_DIR = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224'\nIMAGE_SIZE = (224, 224)\n\ndevice = ('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:16:58.618085Z","iopub.execute_input":"2023-09-17T10:16:58.618497Z","iopub.status.idle":"2023-09-17T10:16:58.625483Z","shell.execute_reply.started":"2023-09-17T10:16:58.618459Z","shell.execute_reply":"2023-09-17T10:16:58.624551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading data","metadata":{}},{"cell_type":"code","source":"def load_data(subset):\n    df = pd.DataFrame({\n        'id': pd.Series(dtype='str'), \n        'image': pd.Series(dtype='object'),\n        'class': pd.Series(dtype='int')\n    })    \n    files = [f'{BASE_DIR}/{subset}/{file}' for file in os.listdir(os.path.join(BASE_DIR, subset))]\n    \n    for file in files:\n        columns = ['id', 'image'] if subset == 'test' else ['id', 'image', 'class']\n        loader = tfrecord.tfrecord_loader(file, None, {key : 'byte' if key != 'class' else 'int' for key in columns})\n        \n        for record in loader:\n            id = record['id'].decode('utf-8')\n            label = record['class'][0].item() if subset != 'test' else None\n            img_bytes = np.frombuffer(record['image'], dtype=np.uint8)\n            img = cv2.imdecode(img_bytes, cv2.IMREAD_COLOR)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            df.loc[len(df.index)] = [id, img, label]\n            \n    return df\n\ntrain_dataset = load_data('train')\nvalidation_dataset = load_data('val')\ntest_dataset = load_data('test')","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:16:58.627664Z","iopub.execute_input":"2023-09-17T10:16:58.628641Z","iopub.status.idle":"2023-09-17T10:19:05.009615Z","shell.execute_reply.started":"2023-09-17T10:16:58.628608Z","shell.execute_reply":"2023-09-17T10:19:05.002903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Concise EDA","metadata":{}},{"cell_type":"markdown","source":"Sample images","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 6, figsize=(40, 20))\n\ntrain_dataset['image'].head(6)\nfor i, image in enumerate(train_dataset['image'].head(6)):\n    axes[i].imshow(image, cmap='gray')\n    axes[i].axis('off')\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:05.023885Z","iopub.execute_input":"2023-09-17T10:19:05.024718Z","iopub.status.idle":"2023-09-17T10:19:07.610905Z","shell.execute_reply.started":"2023-09-17T10:19:05.024669Z","shell.execute_reply":"2023-09-17T10:19:07.606888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Datasets lengths","metadata":{}},{"cell_type":"code","source":"print(f'Train set length: {len(train_dataset)}')\nprint(f'Validation set length: {len(validation_dataset)}')\nprint(f'Test set length: {len(test_dataset)}')","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:07.618742Z","iopub.execute_input":"2023-09-17T10:19:07.619194Z","iopub.status.idle":"2023-09-17T10:19:07.631397Z","shell.execute_reply.started":"2023-09-17T10:19:07.619158Z","shell.execute_reply":"2023-09-17T10:19:07.629103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Classes distribution in train set\n","metadata":{}},{"cell_type":"code","source":"counts = train_dataset['class'].value_counts()\ncounts.index = [label for label in counts.index]\nvalues = [i for i in range(1, 104)]\noccurances = [counts.get(i, 0) for i in range(1, 104)]\n\nplt.figure(figsize=(8, 6))\nplt.bar(values, occurances)\nplt.xlabel(\"Classes\")\nplt.ylabel(\"Occurrences\")\nplt.title(\"Classes distribution - train set\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:07.633434Z","iopub.execute_input":"2023-09-17T10:19:07.634448Z","iopub.status.idle":"2023-09-17T10:19:08.188787Z","shell.execute_reply.started":"2023-09-17T10:19:07.634414Z","shell.execute_reply":"2023-09-17T10:19:08.187620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And for validation set","metadata":{}},{"cell_type":"code","source":"counts = validation_dataset['class'].value_counts()\ncounts.index = [label for label in counts.index]\nvalues = [i for i in range(1, 104)]\noccurances = [counts.get(i, 0) for i in range(1, 104)]\n\nplt.figure(figsize=(8, 6))\nplt.bar(values, occurances)\nplt.xlabel(\"Classes\")\nplt.ylabel(\"Occurrences\")\nplt.title(\"Classes distribution - validation set\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:08.190782Z","iopub.execute_input":"2023-09-17T10:19:08.191177Z","iopub.status.idle":"2023-09-17T10:19:08.766733Z","shell.execute_reply.started":"2023-09-17T10:19:08.191140Z","shell.execute_reply":"2023-09-17T10:19:08.765715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, classes are highly imbalanced, it will be a good idea to use data augmentation","metadata":{}},{"cell_type":"markdown","source":"## Dataset and Dataloader","metadata":{}},{"cell_type":"markdown","source":"We will wrap our data in torch dataset and dataloader classes, it will allow us to pass data to model easily and perform some augmentation","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n    transforms.ColorJitter(brightness=[0.5, 1.5], contrast=[0.8, 1.2], saturation=[0.8, 1.2]),\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    transforms.RandomCrop(200), \n    transforms.RandomHorizontalFlip(),\n])\n\ntest_val_transforms = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nclass Dataset(Dataset):\n    \n    def __init__(self, data, transform=None, test=False):\n        self.data = data\n        self.test = test\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n        \n    def __getitem__(self, idx):\n        image = self.data['image'][idx]\n        image = Image.fromarray(image)\n        image = self.transform(image)\n        if not self.test:\n            label = self.data['class'][idx]\n            label = torch.tensor([1.0 if i == label else 0.0 for i in range(104)]).float()\n            return image, label\n        else:\n            idx = self.data['id'][idx]\n            return idx, image\n        \ntrain_ds = Dataset(train_dataset, transform=train_transforms)\nvalidation_ds = Dataset(validation_dataset, transform=test_val_transforms)\ntest_ds = Dataset(test_dataset, test=True, transform=test_val_transforms)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:08.768067Z","iopub.execute_input":"2023-09-17T10:19:08.768477Z","iopub.status.idle":"2023-09-17T10:19:08.813290Z","shell.execute_reply.started":"2023-09-17T10:19:08.768445Z","shell.execute_reply":"2023-09-17T10:19:08.812332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(\n    train_ds,\n    batch_size = BATCH_SIZE,\n)\n\nvalidation_dataloader = DataLoader(\n    validation_ds,\n    batch_size = BATCH_SIZE,\n)\n\ntest_dataloader = DataLoader(\n    test_ds,\n    batch_size = BATCH_SIZE,\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:08.815050Z","iopub.execute_input":"2023-09-17T10:19:08.815829Z","iopub.status.idle":"2023-09-17T10:19:08.866521Z","shell.execute_reply.started":"2023-09-17T10:19:08.815796Z","shell.execute_reply":"2023-09-17T10:19:08.865548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ResNet50 is used as a base for a model, documentation [here](https://pytorch.org/vision/master/models/generated/torchvision.models.resnet50.html#torchvision.models.resnet50).\nModel head is 512-neurons dense connected layer with relu activation and 104-neurons output layer with softmax activation - one neuron per class.\nOptimizer Adam and cross entropy as loss function.","metadata":{}},{"cell_type":"code","source":"class Model(torch.nn.Module):\n    \n    def __init__(self):\n        super().__init__()\n        \n        self.base = torch.hub.load('pytorch/vision:v0.10.0', 'shufflenet_v2_x1_0', pretrained=True)\n        self.linear1 = torch.nn.Linear(1000, 512)\n        self.bn1 = torch.nn.BatchNorm1d(512)\n        self.dropout1 = torch.nn.Dropout(0.5) \n        self.linear2 = torch.nn.Linear(512, 256)\n        self.bn2 = torch.nn.BatchNorm1d(256)\n        self.dropout2 = torch.nn.Dropout(0.5)\n        self.linear3 = torch.nn.Linear(256, 128)\n        self.bn3 = torch.nn.BatchNorm1d(128)\n        self.dropout3 = torch.nn.Dropout(0.5)\n        self.linear4 = torch.nn.Linear(128, 104)\n        \n    def forward(self, x):\n        x = self.base(x)\n        x = torch.nn.ReLU()(self.linear1(x)) \n        x = self.bn1(x)\n        if self.training:\n            x = self.dropout1(x)\n        x = torch.nn.ReLU()(self.linear2(x))\n        x = self.bn2(x)\n        if self.training:\n            x = self.dropout2(x)\n        x = torch.nn.ReLU()(self.linear3(x))\n        x = self.bn3(x)\n        if self.training:\n            x = self.dropout3(x)\n        x = self.linear4(x)\n        \n        return x\n        \n    \nmodel = Model()\n\nmodel = model.to(device)\nmodel = torch.nn.DataParallel(model)\n# summary(model, (3, 224, 224))","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:20:15.585586Z","iopub.execute_input":"2023-09-17T10:20:15.586337Z","iopub.status.idle":"2023-09-17T10:20:15.678389Z","shell.execute_reply.started":"2023-09-17T10:20:15.586302Z","shell.execute_reply":"2023-09-17T10:20:15.677396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters())","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:20:15.755076Z","iopub.execute_input":"2023-09-17T10:20:15.755393Z","iopub.status.idle":"2023-09-17T10:20:15.760977Z","shell.execute_reply.started":"2023-09-17T10:20:15.755360Z","shell.execute_reply":"2023-09-17T10:20:15.759962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scheduler = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:20:15.897210Z","iopub.execute_input":"2023-09-17T10:20:15.897990Z","iopub.status.idle":"2023-09-17T10:20:15.904650Z","shell.execute_reply.started":"2023-09-17T10:20:15.897954Z","shell.execute_reply":"2023-09-17T10:20:15.903118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion =  torch.nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:20:16.026359Z","iopub.execute_input":"2023-09-17T10:20:16.026734Z","iopub.status.idle":"2023-09-17T10:20:16.031832Z","shell.execute_reply.started":"2023-09-17T10:20:16.026669Z","shell.execute_reply":"2023-09-17T10:20:16.030535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"train_losses = []\nvalidation_losses = []\ntrain_accs = []\nval_accs = []\n\nfor epoch in range(NUM_EPOCHS):\n    print('------------------------------------------------')\n    print(f'EPOCH: {epoch + 1}/{NUM_EPOCHS}')\n    train_loss = 0.0\n    validation_loss = 0.0\n    \n    ### TRAINING\n    model.train()\n    \n    train_correct = 0\n    train_samples = 0\n    \n    for batch_idx, data in enumerate(train_dataloader):    # For each batch in train_dataloader\n\n        image, label = data\n        image, label = image.to(device), label.to(device)\n        output = model(image)                  # propagate input images forward\n\n        loss = criterion(output, label)        # Loss value for current batch\n        train_loss += loss.item()\n        \n        predicted = torch.max(output, 1)\n        label = torch.max(label, 1)\n        train_correct += (predicted[1] == label[1]).sum().item()\n        train_samples += label[1].size(0)\n        \n        optimizer.zero_grad()                    # Set all gradients to zero\n        \n        loss.backward()                        # Compute gradients\n        \n        optimizer.step()                      # Perform backpropagation\n        \n    train_loss = train_loss/(batch_idx+1)  \n    print(f'Mean train loss: {train_loss}')\n    train_losses.append(train_loss)\n    \n    train_acc = train_correct/train_samples * 100\n    print(f'Train accuracy: {train_acc} %')\n    print()\n    train_accs.append(train_acc)\n    \n    ### VALIDATION\n    model.eval()\n    \n    val_correct = 0\n    val_samples = 0\n    \n    with torch.no_grad():          # disable gradient calculation, we only want to evaluate model\n        for batch_idx, data in enumerate(validation_dataloader): # For each batch in validation_dataloader\n            image, label = data\n            image, label = image.to(device), label.to(device)\n            output = model(image)                           # propagate input images forward\n                        \n            loss = criterion(output, label)                # And compute loss\n            validation_loss += loss.item()\n            \n            predicted = torch.max(output, 1)\n            label = torch.max(label, 1)\n            val_correct += (predicted[1] == label[1]).sum().item()\n            val_samples += label[1].size(0)\n\n    validation_loss = validation_loss/(batch_idx+1) \n    print(f'Mean validation loss: {validation_loss}')\n    validation_losses.append(validation_loss)\n    \n    val_acc = val_correct/val_samples * 100\n    print(f'Validation accuracy: {val_acc} %')\n    print('------------------------------------------------')\n    val_accs.append(val_acc)\n    \n    scheduler.step()\n    ","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:20:16.397819Z","iopub.execute_input":"2023-09-17T10:20:16.398155Z","iopub.status.idle":"2023-09-17T10:21:39.679667Z","shell.execute_reply.started":"2023-09-17T10:20:16.398125Z","shell.execute_reply":"2023-09-17T10:21:39.677660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Training curves","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Training Loss')\nplt.plot(validation_losses, label='Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:12.242511Z","iopub.status.idle":"2023-09-17T10:19:12.243278Z","shell.execute_reply.started":"2023-09-17T10:19:12.243002Z","shell.execute_reply":"2023-09-17T10:19:12.243029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(train_accs, label='Train accuracy')\nplt.plot(val_accs, label='Validation accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:12.244712Z","iopub.status.idle":"2023-09-17T10:19:12.245484Z","shell.execute_reply.started":"2023-09-17T10:19:12.245240Z","shell.execute_reply":"2023-09-17T10:19:12.245264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predictions","metadata":{}},{"cell_type":"code","source":"predictions = []\nids = []\nfor idx, image in test_dataloader:\n    image = image.to(device)\n    output = model(image)  \n    predictions.extend(torch.max(output, 1)[1])   \n    ids.extend(idx)\n\npredictions = [x.cpu().numpy() for x in predictions]\n    \nsubmission = pd.DataFrame({\n    'id': ids,\n    'label': predictions\n})\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-09-17T10:19:12.246817Z","iopub.status.idle":"2023-09-17T10:19:12.247645Z","shell.execute_reply.started":"2023-09-17T10:19:12.247366Z","shell.execute_reply":"2023-09-17T10:19:12.247397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}