{"cells":[{"metadata":{"_uuid":"e0b357f742e5a2c3d79b2848ade285d09f82dddc"},"cell_type":"markdown","source":"Pretrained Neural Networks like VGG16/VGG19/ResNet/DenseNet are trained on ImageNet which contains 1000-class images. This competition just contains two classes: cat and dog. In this kernel, I want to demonstrate how to build a model with **Pytorch** to classify dog or cat to **Finetuning the convnet**, and then **fix ConvNet to extract image features**. \n\nThis include four steps:\n\n- Build Dog/Cat classify dataset for supervised training.\n- Prepare dataset for Pytorch.\n- Fintune pretrained ResNet-18 model.\n- Fixed ConvNet to extract image features.\n\nReference:\n\n- [Extract Image features from pretrained NN](https://www.kaggle.com/christofhenkel/extract-image-features-from-pretrained-nn)\n- [Transfer Learning Using Pytorch](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html)\n\n**Please UPVOTE if you find it useful** :)"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import os\nfrom PIL import Image\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport numpy as np\nimport torchvision\nfrom torch.autograd import Variable\nfrom torchvision import datasets, models, transforms\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\nimport time\nimport copy\nplt.ion()   # interactive mode\n\nimport time\nfrom tqdm import tqdm, trange\ntqdm.pandas()\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport os\nfrom shutil import copyfile\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train_df = pd.read_csv('../input/train/train.csv')\ntest_df = pd.read_csv('../input/test/test.csv')\ntest_df['AdoptionSpeed'] = [-1] * len(test_df)\ndata_df = pd.concat([train_df, test_df], axis=0).reset_index()\nprint(train_df.shape[0], test_df.shape[0], data_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b393ae8b2f6c31643b7ff8f795f35d46b86c479c","trusted":false},"cell_type":"code","source":"data_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e759c2154f4d1467d252a4b104edd5c72b92b665"},"cell_type":"markdown","source":"## Split Dog/Cat images for supervised training"},{"metadata":{"_uuid":"3e1eb524fd1f1119059275e076789ed16e909169","trusted":false},"cell_type":"code","source":"def pil_loader(path):\n    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)\n    with open(path, 'rb') as f:\n        img = Image.open(f)\n        return img.convert('RGB')\n\n\ndef accimage_loader(path):\n    import accimage\n    try:\n        return accimage.Image(path)\n    except IOError:\n        # Potentially a decoding problem, fall back to PIL.Image\n        return pil_loader(path)\n\n\ndef default_loader(path):\n    from torchvision import get_image_backend\n    if get_image_backend() == 'accimage':\n        return accimage_loader(path)\n    else:\n        return pil_loader(path)\n\nclass DogCatDataset(Dataset):\n    \"\"\"Dog Cat classify dataset.\"\"\"\n    \n    def __init__(self, data_df, root_dir='../input/', train_or_valid='train', transform=None):\n        super(DogCatDataset, self).__init__()\n        self.classes = ['dog', 'cat']\n        self.class_to_idx = {'dog':0, 'cat':1}\n        \n        self.transform = transform\n        self.img_list = [] # read train/valid image path\n        petids = data_df['PetID'].values\n        for petid in tqdm(petids):\n            row = data_df.loc[data_df['PetID'] == petid, :]\n            anim_type = 'cat' if row['Type'].values[0] == 2 else 'dog'\n            photo_amt = row['PhotoAmt'].values[0]\n            img_type = 'train' if row['AdoptionSpeed'].values[0] >= 0 else 'test'\n            \n            if train_or_valid == 'train':\n                for i in range(2, int(photo_amt) + 1):\n                    img_path = f'{root_dir}{img_type}_images/{petid}-{i}.jpg'\n                    if not os.path.exists(img_path): continue\n                    self.img_list.append((img_path, self.class_to_idx[anim_type]))\n            else:  # valid\n                img_path = f'{root_dir}{img_type}_images/{petid}-1.jpg'\n                if not os.path.exists(img_path): continue\n                self.img_list.append((img_path, self.class_to_idx[anim_type]))\n    \n    def __len__(self):\n        return len(self.img_list)\n    \n    def __getitem__(self, index):\n        path, target = self.img_list[index]\n        image = default_loader(path)\n        if self.transform is not None:\n            image = self.transform(image)\n        return image, target\n        ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4927072e1dd117f4db748faabad33cb1759041b"},"cell_type":"markdown","source":"Here we use `PetID-1.jpg`(default profile) image for valid image per PetId, and finally we have 68350 training images and 5000 valid images, the `cat : dog = 1 : 1`"},{"metadata":{"_uuid":"ef7a7e046aedf63b8838b265e59a95d4ab72b0c0","trusted":false},"cell_type":"code","source":"batch_size = 64\n\nimage_transforms = {\n    'train': transforms.Compose([\n        transforms.Resize(256),\n        transforms.CenterCrop(224),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ]),\n    'valid': transforms.Compose([\n        transforms.Resize(256),\n        transforms.CenterCrop(224),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ]),\n}\n\nimage_datasets = {x: DogCatDataset(data_df, train_or_valid=x, transform=image_transforms[x])\n                  for x in ['train', 'valid']}\n\ndataloaders = {x: torch.utils.data.dataloader.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True)#, num_workers=4)\n               for x in ['train', 'valid']}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c875fe469e058283c08b8250801cc1a6120d94ed","trusted":false},"cell_type":"code","source":"print('Train:', len(image_datasets['train']), ', Valid:', len(image_datasets['valid']))\ndataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'valid']}\nclass_names = image_datasets['train'].classes\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint('class:', class_names)\nprint('device:', device)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8356817b2b496ca2f7eccf4d36f4e61d23d843c0"},"cell_type":"markdown","source":"## Visualize a few images"},{"metadata":{"_uuid":"3bbccd498f170239611aada59cc299552510fbe2","trusted":false},"cell_type":"code","source":"def imshow(inp, title=None):\n    \"\"\"Imshow for Tensor.\"\"\"\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    inp = np.clip(inp, 0, 1)\n    plt.figure(figsize=(16, 6))\n    plt.imshow(inp)\n    if title is not None:\n        plt.title(title)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69575e21f4a748e701bc5191105b380e4e09e122","trusted":false},"cell_type":"code","source":"# Get a batch of training data\ninputs, classes = next(iter(dataloaders['train']))\n# Make a grid from batch\nout = torchvision.utils.make_grid(inputs[:4])\nimshow(out, title=[class_names[x] for x in classes[:4]])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac7fc71141daae06f61303f5ed118cdb65ed704a"},"cell_type":"markdown","source":"## Finetuning the pretrained model"},{"metadata":{"_uuid":"cb3079752949ce78905380c71af570064419e5de","trusted":false},"cell_type":"code","source":"model = models.resnet18(pretrained=True)\nfc_in_features = model.fc.in_features\nmodel.fc = nn.Linear(fc_in_features, 2)\nmodel = model.to(device)\n\nloss_fn = nn.CrossEntropyLoss()\n\n# Observe that all parameters are being optimized\noptimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c95ebe099503c7910cd9762ed4c2f84f1f276605","trusted":false},"cell_type":"code","source":"epochs = 6\n\nbest_valid_loss = np.inf\nbest_valid_acc = 0.\nbest_model_wts = copy.deepcopy(model.state_dict())\ndidnt_improve_count = 0\n\nfor epoch in range(epochs):\n    start_time = time.time()\n    # set train mode\n    model.train()\n    avg_train_loss = 0.\n    train_corrects = 0.\n    \n    for x_batch, y_batch in dataloaders['train']:\n        x_batch = x_batch.to(device)\n        y_batch = y_batch.to(device)\n        \n        y_pred = model(x_batch)\n        loss = loss_fn(y_pred, y_batch)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        avg_train_loss += loss.item() / len(dataloaders['train'])\n        \n        _, y_pred = torch.max(y_pred, 1)\n        train_corrects += torch.sum(y_pred == y_batch.data).double()\n    \n    train_acc = train_corrects / dataset_sizes['train']\n    \n    torch.cuda.empty_cache()\n    \n    model.eval()\n    avg_val_loss = 0.\n    valid_corrects = 0.\n    for x_batch, y_batch in dataloaders['valid']:\n        with torch.no_grad():\n            x_batch = x_batch.to(device)\n            y_batch = y_batch.to(device)\n        \n            y_pred = model(x_batch)\n            loss = loss_fn(y_pred, y_batch)\n            avg_val_loss += loss.item() / len(dataloaders['valid'])\n        \n            _, y_pred = torch.max(y_pred, 1)\n            valid_corrects += torch.sum(y_pred == y_batch.data).double()\n    \n    valid_acc = valid_corrects / dataset_sizes['valid']\n    \n    elapsed_time = time.time() - start_time \n    print('Epoch {}/{}  train-loss={:.4f}  train-acc={:.4f}  val_loss={:.4f}  valid-acc={:.4f}  time={:.2f}s'.format(\n        epoch + 1, epochs, avg_train_loss, train_acc, avg_val_loss, valid_acc, elapsed_time))\n    \n    # deep copy the model\n    if avg_val_loss < best_valid_loss:\n        best_valid_loss = avg_val_loss\n        best_valid_acc = valid_acc\n        didnt_improve_count = 0\n        best_model_wts = copy.deepcopy(model.state_dict())\n    else:\n        didnt_improve_count += 1\n        if didnt_improve_count > 2:\n            break\n    \nprint('Best valid-loss={:.4f} \\t valid-acc={:.4f}'.format(best_valid_loss, best_valid_acc))\nprint('save and load best model weights')\nmodel.load_state_dict(best_model_wts)\ntorch.save(model.state_dict(), 'best_resnet18_weights.model')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e006257e3b5e316b15c3c02f500cf49ae99b3915","scrolled":true,"trusted":false},"cell_type":"code","source":"# Get a batch of training data\ninputs, classes = next(iter(dataloaders['train']))\ninputs, classes = inputs[:4], classes[:4]\nground_truth = [class_names[i] for i in classes]\n# Make a grid from batch\nout = torchvision.utils.make_grid(inputs)\n\ninputs = inputs.cuda()\npreds = model(inputs)\n_, preds = torch.max(preds, 1)\npredict_class = [class_names[i] for i in preds]\nimshow(out, title=f\"Truth : {ground_truth}\\nPredict: {predict_class}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0b21bbecfa2f781687b4e1c6a19a8e99d7ea9c25"},"cell_type":"code","source":"image_features = []\ndef hook_feature(module, input, output):\n    # hook the feature extractor\n    image_features.append(np.squeeze(output.data.cpu().numpy()))\n\nmodel._modules.get('avgpool').register_forward_hook(hook_feature)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0219cba5be481398bfd01aaf6c05373422dd6b83"},"cell_type":"markdown","source":"## Extract Train Image Features"},{"metadata":{"trusted":false,"_uuid":"1d27668502ae8f6e5eeab943e2ad2b44f5258fc9"},"cell_type":"code","source":"extract_transform = image_transforms['valid']","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"1a9165d985d0bfa92364178167fa3009f28d5f4b"},"cell_type":"code","source":"train_pids = train_df.PetID.values\ninput_tensor = torch.zeros(1, 3, 224, 224)\n\ntrain_image_features = {}\nfor petid in tqdm(train_pids):\n    train_img = f\"../input/train_images/{petid}-1.jpg\"\n    if not os.path.exists(train_img): continue\n    \n    train_img = Image.open(train_img)\n    train_img = extract_transform(train_img)\n    input_tensor[0, :, :, :] = train_img\n    input_tensor = input_tensor.cuda()\n    model(input_tensor)\n    train_image_features[petid] = image_features[0]\n    image_features.clear()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"86fcffba85a2ad17bddcc7c48da708c947fe08c3"},"cell_type":"code","source":"train_image_features = pd.DataFrame.from_dict(train_image_features, orient='index')\ntrain_image_features.columns = [f'img_nn_feat{idx}' for idx in train_image_features.columns.values]\ntrain_image_features = train_image_features.reset_index().rename(columns={'index':'PetID'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a9ac88a0ac235db84bef4f8df3003f0ca28dfbfd"},"cell_type":"code","source":"train_image_features.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e47ff767fbedab5a3f24f8145a6dde0e09fa7701"},"cell_type":"code","source":"train_image_features.to_csv('train_image_features.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d85f010c92a2c77adbe89b738f364761f1061a3"},"cell_type":"markdown","source":"## Extract Test Image Features"},{"metadata":{"trusted":false,"_uuid":"4361b3a6e853039cef4563dbd1c6074b906c24b0"},"cell_type":"code","source":"test_pids = test_df.PetID.values\ninput_tensor = torch.zeros(1, 3, 224, 224)\n\ntest_image_features = {}\nfor petid in tqdm(test_pids):\n    test_img = f\"../input/test_images/{petid}-1.jpg\"\n    if not os.path.exists(test_img): continue\n    \n    test_img = Image.open(test_img)\n    test_img = extract_transform(test_img)\n    input_tensor[0, :, :, :] = test_img\n    input_tensor = input_tensor.cuda()\n    model(input_tensor)\n    test_image_features[petid] = image_features[0]\n    image_features.clear()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d4940a419c57141c936c62320c4f060ddf0b998b"},"cell_type":"code","source":"test_image_features = pd.DataFrame.from_dict(test_image_features, orient='index')\ntest_image_features.columns = [f'img_nn_feat{idx}' for idx in test_image_features.columns.values]\ntest_image_features = test_image_features.reset_index().rename(columns={'index':'PetID'})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2e19a3c5b0032e5f5ea02298acec41b2e5edd86e"},"cell_type":"code","source":"test_image_features.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7e792e50a78042cee48c6d5813d01566ef727dac"},"cell_type":"code","source":"test_image_features.to_csv('test_image_features.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0a3f60a0a35379869f4b27bf2385f9e76f599d1"},"cell_type":"markdown","source":"We save the features as a csv to disk, so others can link and join the data frame with their train.csv and test.csv"},{"metadata":{"trusted":false,"_uuid":"baf8b4294611bf96c709ead0b91f5ebf4846fd8f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}