{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-10T10:30:41.327746Z","iopub.execute_input":"2021-09-10T10:30:41.328084Z","iopub.status.idle":"2021-09-10T10:30:41.454327Z","shell.execute_reply.started":"2021-09-10T10:30:41.328016Z","shell.execute_reply":"2021-09-10T10:30:41.453519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-09-10T05:38:48.833875Z","iopub.execute_input":"2021-09-10T05:38:48.834145Z","iopub.status.idle":"2021-09-10T05:38:48.839909Z","shell.execute_reply.started":"2021-09-10T05:38:48.834117Z","shell.execute_reply":"2021-09-10T05:38:48.839078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\nimport glob\nimport tensorflow as tf\n# import timm\nimport random\nimport time\nimport copy\nfrom operator import itemgetter\n\nfrom collections import OrderedDict, namedtuple\nimport joblib\n\nimport logging\nimport sys\n\nimport PIL\nimport cv2\nimport albumentations as A\nimport io\nimport IPython.display as display\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nimport torch.optim as optim\n# import torch_xla\n# import torch_xla.core.xla_model as xm\n# import torch_xla.debug.metrics as met\n# import torch_xla.distributed.parallel_loader as pl\n# import torch_xla.distributed.xla_multiprocessing as xmp\n# import torch_xla.utils.utils as xu\n\nimport torchvision\n# from torchvision import datasets, transforms\nfrom torch.utils.data import Dataset, DataLoader, ConcatDataset\nimport torchvision.transforms as transforms\nfrom torchvision.utils import make_grid\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics, model_selection\n\nimport warnings\nwarnings.filterwarnings(\"ignore\");","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:30:41.457134Z","iopub.execute_input":"2021-09-10T10:30:41.457377Z","iopub.status.idle":"2021-09-10T10:30:48.919149Z","shell.execute_reply.started":"2021-09-10T10:30:41.457354Z","shell.execute_reply":"2021-09-10T10:30:48.918258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = glob.glob('../input/tpu-getting-started/*224/train/*.tfrec')\nval_files = glob.glob('../input/tpu-getting-started/*224/val/*.tfrec')\ntest_files = glob.glob('../input/tpu-getting-started/*224/test/*.tfrec')","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:30:48.920925Z","iopub.execute_input":"2021-09-10T10:30:48.921284Z","iopub.status.idle":"2021-09-10T10:30:48.931238Z","shell.execute_reply.started":"2021-09-10T10:30:48.921248Z","shell.execute_reply":"2021-09-10T10:30:48.930306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_tfrec_data(files, test=False):\n    if not test: \n        feature_description = {\n            'class': tf.io.FixedLenFeature([], tf.int64),\n            'id': tf.io.FixedLenFeature([], tf.string),\n            'image': tf.io.FixedLenFeature([], tf.string),\n        }\n    else:\n        feature_description = {\n        'id': tf.io.FixedLenFeature([], tf.string),\n        'image': tf.io.FixedLenFeature([], tf.string),\n    }\n    parse_image_f = lambda x: tf.io.parse_single_example(x, feature_description)\n\n    ids = []\n    images = []\n    if not test:\n        cl = []\n\n    for i in tqdm(files):\n        image_dataset = tf.data.TFRecordDataset(i)\n        image_dataset = image_dataset.map(parse_image_f)\n\n        ids_ = [str(id_features['id'].numpy())[2:-1] for id_features in image_dataset] # [2:-1] is done to remove b' from 1st and 'from last in train id names\n        ids = ids + ids_\n\n        images_ = [image_features['image'].numpy() for image_features in image_dataset]\n        images = images + images_\n\n        if not test:\n                cl_ = [int(class_features['class'].numpy()) for class_features in image_dataset]\n                cl = cl + cl_\n    if test:\n        return ids, images\n    else:\n        return ids, cl, images","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:30:48.932800Z","iopub.execute_input":"2021-09-10T10:30:48.933280Z","iopub.status.idle":"2021-09-10T10:30:48.943214Z","shell.execute_reply.started":"2021-09-10T10:30:48.933243Z","shell.execute_reply":"2021-09-10T10:30:48.941992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids, train_cl, train_images = parse_tfrec_data(train_files)\nval_ids, val_cl, val_images = parse_tfrec_data(val_files)\ntest_ids, test_images = parse_tfrec_data(test_files, test=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:30:48.944934Z","iopub.execute_input":"2021-09-10T10:30:48.945311Z","iopub.status.idle":"2021-09-10T10:31:31.937488Z","shell.execute_reply.started":"2021-09-10T10:30:48.945274Z","shell.execute_reply":"2021-09-10T10:31:31.936278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#add more samples to the train dataset from the eval dataset\ntrain_ids, train_cl, train_images = train_ids + val_ids[:1500], train_cl + val_cl[:1500], train_images + val_images[:1500]\nval_ids, val_cl, val_images = val_ids[1500:], val_cl[1500:], val_images[1500:]","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:31:31.939175Z","iopub.execute_input":"2021-09-10T10:31:31.939499Z","iopub.status.idle":"2021-09-10T10:31:31.945437Z","shell.execute_reply.started":"2021-09-10T10:31:31.939461Z","shell.execute_reply":"2021-09-10T10:31:31.944621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'''# train images: {len(train_ids)}\n# val images: {len(val_ids)}\n# test images {len(test_ids)}''' )","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:31:31.946871Z","iopub.execute_input":"2021-09-10T10:31:31.947385Z","iopub.status.idle":"2021-09-10T10:31:31.955953Z","shell.execute_reply.started":"2021-09-10T10:31:31.947347Z","shell.execute_reply":"2021-09-10T10:31:31.955014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_ids[0])\nprint(train_cl[0])\nsample_img = PIL.Image.open(io.BytesIO(train_images[0]))\nsample_img","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:31:31.957352Z","iopub.execute_input":"2021-09-10T10:31:31.957704Z","iopub.status.idle":"2021-09-10T10:31:31.997542Z","shell.execute_reply.started":"2021-09-10T10:31:31.957671Z","shell.execute_reply":"2021-09-10T10:31:31.996782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize(image):\n    plt.figure(figsize=(10, 10))\n    plt.axis('off')\n    plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:31:31.998450Z","iopub.execute_input":"2021-09-10T10:31:31.998720Z","iopub.status.idle":"2021-09-10T10:31:32.003451Z","shell.execute_reply.started":"2021-09-10T10:31:31.998694Z","shell.execute_reply":"2021-09-10T10:31:32.002307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FlowerDataset(Dataset):\n    \n    def __init__(self, ids, images, cl, transforms, test=False):\n        \n        self.ids = ids\n        self.images = images\n        if not test:\n            self.cl = cl\n        self.transforms = transforms\n        self.is_test = test\n    \n    def __len__(self):\n        return len(self.ids)\n        \n    def __getitem__(self, idx):\n        \n        img = self.images[idx]\n        img = PIL.Image.open(io.BytesIO(img))\n        img = np.array(img)\n        img = self.transforms(image=img)['image']\n        \n        if self.is_test:\n            return img, -1, self.ids[idx]\n        return img, int(self.cl[idx]), self.ids[idx]\n     ","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:31:32.005791Z","iopub.execute_input":"2021-09-10T10:31:32.006358Z","iopub.status.idle":"2021-09-10T10:31:32.015432Z","shell.execute_reply.started":"2021-09-10T10:31:32.006320Z","shell.execute_reply":"2021-09-10T10:31:32.014333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from albumentations.pytorch import ToTensorV2\n\ntrain_transforms = A.Compose([A.HorizontalFlip(p=0.3),\n                           A.VerticalFlip(p=0.3),\n                           A.ShiftScaleRotate(p=0.1),\n                           A.OneOf([A.RandomShadow(p=1.), A.RandomSunFlare(src_radius=100, p=1.),], p=0.25),\n                            A.RandomFog(p=0.1),\n                              A.Blur(blur_limit=3, p=0.2),\n                           A.GaussNoise(var_limit=200. ,p=0.1),\n                           A.RandomBrightness(limit=0.2, p=0.1),\n                           A.CLAHE(clip_limit=3, p=0.1),\n                            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n                           ToTensorV2()\n                              \n    ])\n\nval_transforms = A.Compose([A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2() ])\ntest_transforms = A.Compose([A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ToTensorV2()])","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:32:39.101281Z","iopub.execute_input":"2021-09-10T10:32:39.101618Z","iopub.status.idle":"2021-09-10T10:32:39.110188Z","shell.execute_reply.started":"2021-09-10T10:32:39.101588Z","shell.execute_reply":"2021-09-10T10:32:39.109280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = FlowerDataset(train_ids, train_images, train_cl, transforms=train_transforms)\nval_dataset = FlowerDataset(val_ids, val_images, val_cl, transforms=val_transforms)\ntest_dataset = FlowerDataset(test_ids, test_images, None, transforms=test_transforms, test=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=50, num_workers=4, shuffle=True, pin_memory=True)\nval_loader = DataLoader(val_dataset, batch_size=50, num_workers=4,  shuffle=True, pin_memory=True)\ntest_loader = DataLoader(test_dataset, batch_size=50, num_workers=4,  shuffle=True, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:32:40.362130Z","iopub.execute_input":"2021-09-10T10:32:40.362457Z","iopub.status.idle":"2021-09-10T10:32:40.368220Z","shell.execute_reply.started":"2021-09-10T10:32:40.362426Z","shell.execute_reply":"2021-09-10T10:32:40.367384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_iter = iter(train_loader)\nimg = next(train_iter)\nfor img2show in img[0][:10].permute(0, 2, 3, 1):\n    visualize(img2show.numpy())","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:32:41.286939Z","iopub.execute_input":"2021-09-10T10:32:41.287246Z","iopub.status.idle":"2021-09-10T10:32:45.680728Z","shell.execute_reply.started":"2021-09-10T10:32:41.287219Z","shell.execute_reply":"2021-09-10T10:32:45.679599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = torchvision.models.resnet50(pretrained=True)\nmodel","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:33:16.569331Z","iopub.execute_input":"2021-09-10T10:33:16.569710Z","iopub.status.idle":"2021-09-10T10:33:18.002138Z","shell.execute_reply.started":"2021-09-10T10:33:16.569670Z","shell.execute_reply":"2021-09-10T10:33:18.000961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n, p in model.named_parameters():\n    if n.split('.')[0] not in ['layer4']:\n        p.requires_grad = False\nmodel.fc = nn.Linear(2048, 104)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:33:50.641713Z","iopub.execute_input":"2021-09-10T10:33:50.642071Z","iopub.status.idle":"2021-09-10T10:33:50.651780Z","shell.execute_reply.started":"2021-09-10T10:33:50.642038Z","shell.execute_reply":"2021-09-10T10:33:50.649614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k, p in model.named_parameters():\n    if p.requires_grad:\n        print(k)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:33:57.108238Z","iopub.execute_input":"2021-09-10T10:33:57.108580Z","iopub.status.idle":"2021-09-10T10:33:57.124041Z","shell.execute_reply.started":"2021-09-10T10:33:57.108551Z","shell.execute_reply":"2021-09-10T10:33:57.123181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"smpl_input = torch.ones(1, 3, 224, 224)\nout = model(smpl_input)\nout.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:34:02.391678Z","iopub.execute_input":"2021-09-10T10:34:02.392014Z","iopub.status.idle":"2021-09-10T10:34:02.701315Z","shell.execute_reply.started":"2021-09-10T10:34:02.391983Z","shell.execute_reply":"2021-09-10T10:34:02.700516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_model(model, criterion, test_loader, cuda=True):\n    \n\n    model = model.cuda()\n    model = model.eval()\n    \n    running_loss = 0.\n    num_correct = 0.\n    num_total = 0.\n    \n    for batch, labels, _ in test_loader:\n        \n\n        batch = batch.cuda()\n        labels = labels.cuda()\n        \n        out = model(batch)\n        pred_labels = out.argmax(dim=1)\n        num_correct += float((pred_labels == labels).sum())\n        \n        loss = criterion(out, labels)\n        running_loss += loss.data.cpu()\n        \n        num_total += labels.shape[0]\n    \n    mean_loss = running_loss / num_total\n    accuracy = num_correct / num_total\n    \n    return mean_loss, accuracy\n\ndef train_model(n_epochs, model, optimizer, scheduler, criterion, train_loader, test_loader, cuda=True):\n    \n    if cuda:\n        model = model.cuda()\n    \n    model = model.train()\n    \n    train_loss, train_acc = [], []\n    test_loss, test_acc = [], []\n    \n    for epoch in range(n_epochs):\n        t0 = time.perf_counter()\n        \n        running_loss = 0.\n        num_correct = 0.\n        num_total = 0.\n        \n        for batch, labels, _ in train_loader:\n            if cuda:\n                batch = batch.cuda()\n                labels = labels.cuda()\n            \n            optimizer.zero_grad()\n            \n            out = model(batch)\n            pred_labels = out.argmax(dim=1)\n            num_correct += float((pred_labels == labels).sum())\n            num_total += labels.shape[0]\n            \n            \n            \n            loss = criterion(out, labels)\n            running_loss += loss\n            loss.backward()\n            optimizer.step()\n        \n        epoch_loss = running_loss / num_total\n        epoch_acc = num_correct / num_total\n        \n        train_loss.append(epoch_loss.data.cpu())\n        train_acc.append(epoch_acc)\n        \n        t_loss, t_acc = eval_model(model, criterion, test_loader, cuda=cuda)\n        \n        test_loss.append(t_loss.data.cpu())\n        test_acc.append(t_acc)\n        \n        if scheduler is not None:\n            scheduler.step(t_loss.data.cpu())\n        \n        t1 = time.perf_counter()\n        \n        delta_t = t1 - t0\n        print(f\"EPOCH {epoch} ({round(delta_t, 4)} s.): train loss - {epoch_loss}, train accuracy - {epoch_acc}; test loss - {t_loss}, test accuracy - {t_acc}\")\n        \n    \n    return model, train_loss, train_acc, test_loss, test_acc \n","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:34:04.726672Z","iopub.execute_input":"2021-09-10T10:34:04.727034Z","iopub.status.idle":"2021-09-10T10:34:04.741091Z","shell.execute_reply.started":"2021-09-10T10:34:04.726985Z","shell.execute_reply":"2021-09-10T10:34:04.739857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_learning_curve(train_loss, train_acc, test_loss, test_acc, figsize=(12,6)):\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=figsize)\n    ax1.plot(train_loss, marker='.')\n    ax1.plot(test_loss, marker='.')\n    ax1.legend(['train', 'test'])\n    ax1.set_title('Loss')\n    ax2.plot(train_acc, marker='.')\n    ax2.plot(test_acc, marker='.')\n    ax2.legend(['train', 'test'])\n    ax2.set_title('Accuracy')","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:34:05.908195Z","iopub.execute_input":"2021-09-10T10:34:05.908527Z","iopub.status.idle":"2021-09-10T10:34:05.916387Z","shell.execute_reply.started":"2021-09-10T10:34:05.908497Z","shell.execute_reply":"2021-09-10T10:34:05.913852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=1e-3)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5)\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:35:52.431099Z","iopub.execute_input":"2021-09-10T10:35:52.431441Z","iopub.status.idle":"2021-09-10T10:35:52.438587Z","shell.execute_reply.started":"2021-09-10T10:35:52.431412Z","shell.execute_reply":"2021-09-10T10:35:52.437627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, train_loss, train_acc, test_loss, test_acc = train_model(40, model, optimizer, scheduler, criterion, train_loader, val_loader, cuda=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T10:36:00.532622Z","iopub.execute_input":"2021-09-10T10:36:00.532964Z","iopub.status.idle":"2021-09-10T11:20:24.735497Z","shell.execute_reply.started":"2021-09-10T10:36:00.532931Z","shell.execute_reply":"2021-09-10T11:20:24.733959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_learning_curve(train_loss, train_acc, test_loss, test_acc)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T11:21:39.604979Z","iopub.execute_input":"2021-09-10T11:21:39.605312Z","iopub.status.idle":"2021-09-10T11:21:39.963980Z","shell.execute_reply.started":"2021-09-10T11:21:39.605281Z","shell.execute_reply":"2021-09-10T11:21:39.963168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_submission(model, test_loader, cuda=True):\n\n    results=[]\n    for batch, _, ids in tqdm(test_loader):\n        with torch.no_grad():\n            if cuda:\n                batch = batch.cuda()\n            model.eval()\n            out = model(batch)\n            pred_labels = torch.argmax(out.data.cpu(), dim=1)\n            rows = list(\n                zip(\n                    list(ids), list(pred_labels.numpy().tolist())\n                )\n            )\n            results.append(pd.DataFrame(rows, columns=['id', 'label']))\n    result_df = pd.concat(results)\n    result_df['label'] = result_df['label'].astype(int)\n    return result_df\n","metadata":{"execution":{"iopub.status.busy":"2021-09-10T11:21:59.773117Z","iopub.execute_input":"2021-09-10T11:21:59.773476Z","iopub.status.idle":"2021-09-10T11:21:59.780293Z","shell.execute_reply.started":"2021-09-10T11:21:59.773444Z","shell.execute_reply":"2021-09-10T11:21:59.779440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = make_submission(model, test_loader)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T11:22:02.560379Z","iopub.execute_input":"2021-09-10T11:22:02.560723Z","iopub.status.idle":"2021-09-10T11:22:24.114478Z","shell.execute_reply.started":"2021-09-10T11:22:02.560692Z","shell.execute_reply":"2021-09-10T11:22:24.113529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-10T11:22:29.072236Z","iopub.execute_input":"2021-09-10T11:22:29.072578Z","iopub.status.idle":"2021-09-10T11:22:29.096314Z","shell.execute_reply.started":"2021-09-10T11:22:29.072544Z","shell.execute_reply":"2021-09-10T11:22:29.095315Z"},"trusted":true},"execution_count":null,"outputs":[]}]}