{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA Visualizations for Image Recognition (Conv Filter Edition)","metadata":{}},{"cell_type":"markdown","source":"## Dependencies and Imports","metadata":{}},{"cell_type":"code","source":"!pip install -q timm\n!pip install -q torch==1.10.0 torchvision==0.11.1 torchaudio===0.10.0","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:24.807054Z","iopub.execute_input":"2021-12-03T13:02:24.807525Z","iopub.status.idle":"2021-12-03T13:02:42.584882Z","shell.execute_reply.started":"2021-12-03T13:02:24.807419Z","shell.execute_reply":"2021-12-03T13:02:42.583806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import Dict\n\nimport matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport timm\nimport torch\nimport torchvision\nfrom torchvision.models.feature_extraction import (create_feature_extractor,\n get_graph_node_names)\n\n%matplotlib inline\nimport glob\nimport os\nfrom math import ceil\nimport random\n\nimport cv2\nimport PIL\nfrom IPython.core.interactiveshell import InteractiveShell\n\nInteractiveShell.ast_node_interactivity = \"all\"\n\nfrom typing import *","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:42.587394Z","iopub.execute_input":"2021-12-03T13:02:42.587759Z","iopub.status.idle":"2021-12-03T13:02:44.227041Z","shell.execute_reply.started":"2021-12-03T13:02:42.587704Z","shell.execute_reply":"2021-12-03T13:02:44.226023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config and Logging","metadata":{}},{"cell_type":"code","source":"import logging\nfrom logging import INFO, FileHandler, Formatter, StreamHandler, getLogger\n\ndef init_logger(log_file: str = \"info.log\") -> logging.Logger:\n \"\"\"Initialize logger and save to file.\n\n Consider having more log_file paths to save, eg: debug.log, error.log, etc.\n\n Args:\n log_file (str, optional): [description]. Defaults to Path(LOGS_DIR, \"info.log\").\n\n Returns:\n logging.Logger: [description]\n \"\"\"\n logger = getLogger(__name__)\n logger.setLevel(INFO)\n stream_handler = StreamHandler()\n stream_handler.setFormatter(\n Formatter(\"%(asctime)s: %(message)s\", \"%Y-%m-%d %H:%M:%S\")\n )\n file_handler = FileHandler(filename=log_file)\n file_handler.setFormatter(\n Formatter(\"%(asctime)s: %(message)s\", \"%Y-%m-%d %H:%M:%S\")\n )\n logger.addHandler(stream_handler)\n logger.addHandler(file_handler)\n\n return logger\n\nlogger = init_logger()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:44.22844Z","iopub.execute_input":"2021-12-03T13:02:44.228688Z","iopub.status.idle":"2021-12-03T13:02:44.237025Z","shell.execute_reply.started":"2021-12-03T13:02:44.228653Z","shell.execute_reply":"2021-12-03T13:02:44.23608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utils","metadata":{}},{"cell_type":"code","source":"def plot_multiple_img(img_matrix_list, title_list, ncols, main_title=\"\"):\n fig, myaxes = plt.subplots(\n figsize=(20, 15),\n nrows=ceil(len(img_matrix_list) / ncols),\n ncols=ncols,\n squeeze=False,\n )\n fig.suptitle(main_title, fontsize=30)\n fig.subplots_adjust(wspace=0.3)\n fig.subplots_adjust(hspace=0.3)\n for i, (img, title) in enumerate(zip(img_matrix_list, title_list)):\n myaxes[i // ncols][i % ncols].imshow(img)\n myaxes[i // ncols][i % ncols].set_title(title, fontsize=15)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:44.238466Z","iopub.execute_input":"2021-12-03T13:02:44.23911Z","iopub.status.idle":"2021-12-03T13:02:44.251457Z","shell.execute_reply.started":"2021-12-03T13:02:44.239064Z","shell.execute_reply":"2021-12-03T13:02:44.250483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Seeding","metadata":{}},{"cell_type":"code","source":"def seed_all(seed: int = 1992) -> None:\n \"\"\"Seed all random number generators.\"\"\"\n print(f\"Using Seed Number {seed}\")\n\n os.environ[\"PYTHONHASHSEED\"] = str(seed) # set PYTHONHASHSEED env var at fixed value\n torch.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n torch.cuda.manual_seed(seed) # pytorch (both CPU and CUDA)\n np.random.seed(seed) # for numpy pseudo-random generator\n # set fixed value for python built-in pseudo-random generator\n random.seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n torch.backends.cudnn.enabled = True\n\n\ndef seed_worker(_worker_id) -> None:\n \"\"\"Seed a worker with the given ID.\"\"\"\n worker_seed = torch.initial_seed() % 2 ** 32\n np.random.seed(worker_seed)\n random.seed(worker_seed)\n \n \nseed_all()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transforms Params","metadata":{}},{"cell_type":"code","source":"mean: List[float] = [0.485, 0.456, 0.406]\nstd: List[float] = [0.229, 0.224, 0.225]\nimage_size: int = 224\n\ntransform = torchvision.transforms.Compose(\n [\n torchvision.transforms.Resize((image_size, image_size)),\n torchvision.transforms.ToTensor(),\n torchvision.transforms.Normalize(mean=mean, std=std),\n ]\n)\n\npre_normalize_transform = torchvision.transforms.Compose(\n [\n torchvision.transforms.Resize((image_size, image_size)),\n torchvision.transforms.ToTensor(),\n ]\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:44.253848Z","iopub.execute_input":"2021-12-03T13:02:44.254105Z","iopub.status.idle":"2021-12-03T13:02:44.267949Z","shell.execute_reply.started":"2021-12-03T13:02:44.254075Z","shell.execute_reply":"2021-12-03T13:02:44.267287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizations","metadata":{}},{"cell_type":"code","source":"cat_p = \"../input/petfinder-pawpularity-score/train/0042bc5bada6d1cf8951f8f9f0d399fa.jpg\"\ndog_p = \"../input/petfinder-pawpularity-score/train/86a71a412f662212fe8dcd40fdaee8e6.jpg\"","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:44.269222Z","iopub.execute_input":"2021-12-03T13:02:44.270189Z","iopub.status.idle":"2021-12-03T13:02:44.280858Z","shell.execute_reply.started":"2021-12-03T13:02:44.270126Z","shell.execute_reply":"2021-12-03T13:02:44.280025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot cat and dog with title using PIL\nplt.figure(figsize=(10, 10))\nplt.subplot(1, 2, 1)\ncat = PIL.Image.open(cat_p)\nplt.imshow(cat)\nplt.title(\"Cat\")\nplt.subplot(1, 2, 2)\ndog = PIL.Image.open(dog_p)\nplt.imshow(dog)\nplt.title(\"Dog\")\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:44.282671Z","iopub.execute_input":"2021-12-03T13:02:44.283011Z","iopub.status.idle":"2021-12-03T13:02:44.913755Z","shell.execute_reply.started":"2021-12-03T13:02:44.282969Z","shell.execute_reply":"2021-12-03T13:02:44.913087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convolution Layers \n\nCourtesy of [https://www.kaggle.com/tarunpaparaju/plant-pathology-2020-eda-models](https://www.kaggle.com/tarunpaparaju/plant-pathology-2020-eda-models).\n\n---\n\nConvolution is a rather simple algorithm which involves a kernel (a 2D matrix) which moves over the entire image, calculating dot products with each window along the way. The GIF below demonstrates convolution in action.\n\n
\n\nThe above process can be summarized with an equation, where *f* is the image and *h* is the kernel. The dimensions of *f* are *(m, n)* and the kernel is a square matrix with dimensions smaller than *f*:\n\n
\n
\n\nIn the above equation, the kernel *h* is moving across the length and breadth of the image. The dot product of *h* with a sub-matrix or window of matrix *f* is taken at each step, hence the double summation (rows and columns). ","metadata":{}},{"cell_type":"markdown","source":"I have always remembered from the revered Andrew Ng about how he taught us about what convolutional layers do.\n\n> In the beginning, the conv layers are of low level abstraction, detailing a image's features such as shapes and sizes. In particular, he described to us the horizontal and vertical conv filters. As the conv layers go later, it will pick up on many abstract features, which is not really easily distinguished by human eyes.\n\nBelow, we see an example of horizontal and vertical filters.","metadata":{}},{"cell_type":"code","source":"def conv_horizontal(image: np.ndarray) -> None:\n \"\"\"Plot the horizontal convolution of the image.\n\n Args:\n image (torch.Tensor): [description]\n \"\"\"\n fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(20, 20))\n kernel = np.ones((3, 3), np.float32)\n kernel[1] = np.array([0, 0, 0], np.float32)\n kernel[2] = np.array([-1, -1, -1], np.float32)\n conv = cv2.filter2D(image, -1, kernel)\n ax[0].imshow(image)\n ax[0].set_title(\"Original Image\", fontsize=24)\n ax[1].imshow(conv)\n ax[1].set_title(\"Convolved Image with horizontal edges\", fontsize=24)\n plt.show()\n\n\ndef conv_vertical(image: np.ndarray) -> None:\n \"\"\"Plot the vertical convolution of the image.\n\n Args:\n image (torch.Tensor): [description]\n \"\"\"\n fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(20, 20))\n kernel = np.ones((3, 3), np.float32)\n kernel[0] = np.array([1, 0, -1])\n kernel[1] = np.array([1, 0, -1])\n kernel[2] = np.array([1, 0, -1])\n conv = cv2.filter2D(image, -1, kernel)\n ax[0].imshow(image)\n ax[0].set_title(\"Original Image\", fontsize=24)\n ax[1].imshow(conv)\n ax[1].set_title(\"Convolved Image with vertical edges\", fontsize=24)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:44.91501Z","iopub.execute_input":"2021-12-03T13:02:44.915405Z","iopub.status.idle":"2021-12-03T13:02:44.927823Z","shell.execute_reply.started":"2021-12-03T13:02:44.915359Z","shell.execute_reply":"2021-12-03T13:02:44.926893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Well, I can easily make out the horizontal and vertical edges from the cat image! ","metadata":{}},{"cell_type":"code","source":"conv_horizontal(np.asarray(cat))\nconv_vertical(np.asarray(cat))","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:44.92932Z","iopub.execute_input":"2021-12-03T13:02:44.929726Z","iopub.status.idle":"2021-12-03T13:02:47.092588Z","shell.execute_reply.started":"2021-12-03T13:02:44.929673Z","shell.execute_reply":"2021-12-03T13:02:47.0915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The issue is, I want to visualize what our models' conv layers are seeing, like for example, the first conv layer usually has 64 filters, that is a whooping 64 different combinations of filters, each doing a slightly different thing. A mental model that I have for the first conv layer looks something like the following.\n\n```python\nconv_1_filters = [\"vertical edge detector\", \"horizontal edge detector\",\n \"slanted 45 degrees detector\", \"slanted 180 degrees detector\",\n ...]\n```","metadata":{}},{"cell_type":"markdown","source":"### Feature Extractor using PyTorch's native Feature Extraction Module\n\nIn order to visualize properly, I made use of **PyTorch's** newest `feature_extraction` module to do so. Note that the new feature is still in development, but it does make my life easier and reduces overhead. I no longer need use `hooks` or what not to plot layer information!\n\nWe just need to import\n```python\nfrom torchvision.models.feature_extraction import (create_feature_extractor,\n get_graph_node_names)\n```","metadata":{}},{"cell_type":"code","source":"def get_conv_layers(model: torchvision.models) -> Dict[str, str]:\n \"\"\"Create a function that give me the conv layers of PyTorch model.\n\n Args:\n model (Union[torchvision.models, timm.models]): A PyTorch model.\n\n Returns:\n conv_layers (Dict[str, str]): {\"layer1.0.conv1\": layer1.0.conv1, ...}\n \"\"\"\n conv_layers = {}\n for name, layer in model.named_modules():\n if isinstance(layer, torch.nn.Conv2d):\n conv_layers[name] = name\n return conv_layers\n\n\ndef get_feature_maps(\n model_name: str, image: torch.Tensor, reduction: str = \"mean\",\n pretrained: bool = True\n) -> Union[Dict[str, torch.Tensor], List[torch.Tensor], List[str]]:\n \"\"\"Function to plot feature maps from PyTorch models.\n\n Args:\n model_name (str): Name of the model to use.\n image (torch.Tensor): image should be a tensor of shape (1, 3, H, W)\n reduction (str, optional): Defaults to \"mean\". One of [\"mean\", \"max\", \"sum\"]\n pretrained (bool): whether the model is pretrained or not\n\n Raises:\n ValueError: Must use Torchvision models.\n\n Returns:\n model_feature_maps (Dict[str, torch.Tensor]): {\"conv_1\": conv_1_feature_map, ...}\n processed_feature_maps (List[torch.Tensor]): [conv_1_feature_map, ...] processed using a reduction method.\n feature_map_names (List[str]): [conv_1, ...]\n\n Example:\n >>> from torchvision.models.vgg import vgg16\n >>> model = vgg16(pretrained=True)\n >>> image = torch.rand(1, 3, 224, 224)\n >>> feature_maps = get_feature_maps(model, image, reduction=\"mean\")\n\n Reduction:\n If a feature map has 4 filters, in the shape of (4, H, W) = (4, 32, 32), then the reduction can be done as follows:\n >>> reduction = \"mean\": There are 4 filters in this feature map, you can imagine it as 4 32x32 images.\n We sum up all 4 filters element-wise and get a single 32x32 image.\n Then we take the mean of all 32x32 images by dividing by num of kernels to get a single 32x32 image, which is reduction=\"mean\".\n \"\"\"\n\n try:\n model = getattr(torchvision.models, model_name)(pretrained=pretrained)\n except AttributeError:\n raise ValueError(f\"Model {model_name} not found.\")\n\n train_nodes, eval_nodes = get_graph_node_names(model)\n logger.info(f\"The train nodes of the model graph is:\\n\\n{train_nodes}\")\n\n return_conv_nodes = get_conv_layers(model)\n feature_extractor = create_feature_extractor(model, return_nodes=return_conv_nodes)\n\n # `model_feature_maps` will be a dict of Tensors, each representing a feature map\n model_feature_maps = feature_extractor(image)\n\n processed_feature_maps = []\n feature_map_names = []\n\n for conv_name, conv_feature_map in model_feature_maps.items():\n\n conv_feature_map = conv_feature_map.squeeze(dim=0)\n num_filters = conv_feature_map.shape[0]\n\n if reduction == \"mean\":\n gray_scale = torch.sum(conv_feature_map, dim=0) / num_filters\n elif reduction == \"max\":\n gray_scale = torch.max(conv_feature_map, dim=0)\n elif reduction == \"sum\":\n gray_scale = torch.sum(conv_feature_map, dim=0)\n\n processed_feature_maps.append(gray_scale.data.cpu().numpy())\n feature_map_names.append(conv_name)\n\n return model_feature_maps, processed_feature_maps, feature_map_names","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:47.09408Z","iopub.execute_input":"2021-12-03T13:02:47.094368Z","iopub.status.idle":"2021-12-03T13:02:47.109441Z","shell.execute_reply.started":"2021-12-03T13:02:47.094336Z","shell.execute_reply":"2021-12-03T13:02:47.108381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing VGG16 and ResNet18","metadata":{}},{"cell_type":"markdown","source":"#### Step 1: Initialize the models.\n\nAs of now, I recommend using `torchvision`'s models. Ideally, I will want to use `timm` library for a more detailed list, but there are some bugs that is not easily integrated with the module.","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nimport torchvision.models as models\n\nvgg16_pretrained_true = models.vgg16(pretrained=True)\nvgg16_pretrained_true = vgg16_pretrained_true.to(device)\n\nresnet18_pretrained_true = models.resnet18(pretrained=True)\nresnet18_pretrained_true = resnet18_pretrained_true.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:47.110902Z","iopub.execute_input":"2021-12-03T13:02:47.111324Z","iopub.status.idle":"2021-12-03T13:02:49.63458Z","shell.execute_reply.started":"2021-12-03T13:02:47.111277Z","shell.execute_reply":"2021-12-03T13:02:49.63381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get node names\ntrain_nodes, eval_nodes = get_graph_node_names(vgg16_pretrained_true)\nlogger.info(f\"Train nodes of VGG16:\\n\\n{train_nodes}\")\n\ntrain_nodes, eval_nodes = get_graph_node_names(resnet18_pretrained_true)\nlogger.info(f\"Train nodes of ResNet18:\\n\\n{train_nodes}\")","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:49.635915Z","iopub.execute_input":"2021-12-03T13:02:49.636295Z","iopub.status.idle":"2021-12-03T13:02:49.717043Z","shell.execute_reply.started":"2021-12-03T13:02:49.63625Z","shell.execute_reply":"2021-12-03T13:02:49.716199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Good God!** When I saw the layer names from `vgg16`, I nearly fainted, I see no easy way to know which layer belongs to a Conv layer. I understand that `get_graph_node_names` will get all the nodes on the model's graph, but it is difficult to map the node names to a layer if it is named as such, seeing `resnet18`'s node names is much easier for one to identify which is conv layer or not.\n\n```python\ntrain_nodes, eval_nodes = get_graph_node_names(model)\nlogger.info(f\"The train nodes of the model graph is:\\n\\n{train_nodes}\")\n```\n\nThus I wrote a small function `get_conv_layers` to get the conv layer names. It is not perfect, as downsample layers (1x1 conv layers) are tagged under `Conv2d` but we may not really need to use them to visualize our feature maps. One can tweak a bit if need be, but for now, I will get all layers that use the `Conv2d` blocks.\n\nIf the feature names in vgg16 are named with conv, then we can simply use a small loop below to find the conv layer names.\n\n```python\nconv_layers = []\nfor node in nodes:\n if \"conv\" in node:\n conv_layers.append(node)\n```","metadata":{}},{"cell_type":"markdown","source":"I actually thought ResNet18 has 18 conv layers, but even minusing to 3 downsample layers, it's 17 conv layers, wonder why?","metadata":{}},{"cell_type":"markdown","source":"#### Step 2: Transform the Tensors\n\nThe PyTorch `feature_extraction` expects the image input to be of shape `[B,C,H,W]`. \n\n```python\n# We use torchvision's transform to transform the cat image to channels first.\ncat_tensor = transform(cat)\n\n# Now feature_extractor expects batch_size x C x H x W, so we expand one dimension in the 0th dim\ncat_tensor = cat_tensor.unsqueeze(dim=0).to(device)\n```","metadata":{}},{"cell_type":"code","source":"# We use torchvision's transform to transform the cat image with resize and normalization.\n# Conveniently, also making it channel first!\ncat_tensor = transform(cat)\ndog_tensor = transform(dog)\nassert cat_tensor.shape[0] == dog_tensor.shape[0] == 3, \"PyTorch expects Channel First!\"\n\n# Now feature_extractor expects batch_size x C x H x W, so we expand one dimension in the 0th dim\ncat_tensor = cat_tensor.unsqueeze(dim=0).to(device)\ndog_tensor = dog_tensor.unsqueeze(dim=0).to(device)\n\nlogger.info(f\"\\n\\ncat_tensor's shape:\\n{cat_tensor.shape}\\n\\ndog_tensor's shape:\\n{dog_tensor.shape}\")","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:49.718614Z","iopub.execute_input":"2021-12-03T13:02:49.718921Z","iopub.status.idle":"2021-12-03T13:02:49.740674Z","shell.execute_reply.started":"2021-12-03T13:02:49.718878Z","shell.execute_reply":"2021-12-03T13:02:49.740059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Step 3: Plotting the Feature Maps\n\nWe first walk through `get_feature_maps` and see what my function is doing.","metadata":{}},{"cell_type":"markdown","source":"```python\n# Get node names\ntrain_nodes, eval_nodes = get_graph_node_names(model)\n\n# Since get node names do not indicate properly which is a conv layer or not,\n# we use get_conv_layer instead to do the job, which returns a dict {\"conv_layer_name\": \"conv_layer_name\"}\nreturn_conv_nodes = get_conv_layers(model)\n\n# call create_feature_extractor on the model and its corresponding conv layer names.\nfeature_extractor = create_feature_extractor(model, return_nodes=return_conv_nodes)\n\n# `model_feature_maps` will be a dict of Tensors, each representing a feature map\n# {\"conv_layer_1\": output filter map,...}\nmodel_feature_maps = feature_extractor(image)\n\n# we need to further process the feature maps\nprocessed_feature_maps, feature_map_names = [], []\n\n\nfor conv_name, conv_feature_map in model_feature_maps.items():\n # Squeeze the dimension from [1, 64, 32, 32] to [64, 32, 32]\n # This means we have 64 filters of 32x32 \"images\" or kernels\n conv_feature_map = conv_feature_map.squeeze(dim=0)\n # get number of feature/kernels in this layer\n num_filters = conv_feature_map.shape[0]\n \n\n # If a feature map has 4 filters, in the shape of (4, H, W) = (4, 32, 32), then the reduction mean can be done as follows: There are 4 filters in this feature map, you can imagine it as 4 32x32 images.\n # Step 1: We sum up all 4 filters element-wise and get a single 32x32 image.\n # Step 2: Then we take the mean of all 32x32 images to get a single 32x32 image, which is reduction=\"mean\".\n if reduction == \"mean\":\n gray_scale = torch.sum(conv_feature_map, dim=0) / num_filters\n elif reduction == \"max\":\n gray_scale = torch.max(conv_feature_map, dim=0)\n elif reduction == \"sum\":\n gray_scale = torch.sum(conv_feature_map, dim=0)\n\n processed_feature_maps.append(gray_scale.data.cpu().numpy())\n feature_map_names.append(conv_name)\n```","metadata":{}},{"cell_type":"code","source":"_, vgg16_processed_feature_maps, vgg16_feature_map_names = get_feature_maps(\n model_name=\"vgg16\", image=cat_tensor, reduction=\"mean\", pretrained=True\n)\n_, resnet18_processed_feature_maps, resnet18_feature_map_names = get_feature_maps(\n model_name=\"resnet18\", image=cat_tensor, reduction=\"mean\", pretrained=True\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:49.743034Z","iopub.execute_input":"2021-12-03T13:02:49.743747Z","iopub.status.idle":"2021-12-03T13:02:52.266198Z","shell.execute_reply.started":"2021-12-03T13:02:49.743712Z","shell.execute_reply":"2021-12-03T13:02:52.265155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then we create a simple `plot_feature_maps` that take in the `processed_feature_maps` and `feature_map_names` to plot them.","metadata":{}},{"cell_type":"code","source":"def plot_feature_maps(\n processed_feature_maps: List[torch.Tensor], feature_map_names: List[str], nrows: int,\n title: str = None\n) -> None:\n \"\"\"Plot the feature maps.\n\n Args:\n processed_feature_maps (List[torch.Tensor]): [description]\n feature_map_names (List[str]): [description]\n nrows (int): [description]\n \"\"\"\n fig = plt.figure(figsize=(30, 50))\n ncols = len(processed_feature_maps) // nrows + 1\n for i in range(len(processed_feature_maps)):\n a = fig.add_subplot(nrows, ncols, i + 1)\n imgplot = plt.imshow(processed_feature_maps[i])\n a.axis(\"off\")\n a.set_title(feature_map_names[i].split(\"(\")[0], fontsize=30)\n\n fig.suptitle(title, fontsize=50)\n\n fig.tight_layout()\n fig.subplots_adjust(top=0.95)\n plt.savefig(title, bbox_inches='tight')\n plt.show();","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:52.267685Z","iopub.execute_input":"2021-12-03T13:02:52.267935Z","iopub.status.idle":"2021-12-03T13:02:52.277743Z","shell.execute_reply.started":"2021-12-03T13:02:52.267905Z","shell.execute_reply":"2021-12-03T13:02:52.276648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_maps(\n vgg16_processed_feature_maps,\n vgg16_feature_map_names,\n nrows=5,\n title=\"VGG16 Pretrained Feature Maps\",\n)\n\nplot_feature_maps(\n resnet18_processed_feature_maps,\n resnet18_feature_map_names,\n nrows=5,\n title=\"ResNet18 Pretrained Feature Maps\",\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T13:02:52.279175Z","iopub.execute_input":"2021-12-03T13:02:52.279432Z","iopub.status.idle":"2021-12-03T13:02:56.28219Z","shell.execute_reply.started":"2021-12-03T13:02:52.279402Z","shell.execute_reply":"2021-12-03T13:02:56.281224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Comparison with Randomly Initialized Weights\n\nWe know that if the model is not pretrained, it will initialize with random weights using weight initialization methods such as Kaimin or Xavier. I expect the edges to be not so \"smooth\" as the ones that are pretrained! This is logical, as the filters in the conv layers are mostly random, and we have not trained any epochs yet, so let's see what it gives us.","metadata":{}},{"cell_type":"code","source":"_, vgg16_processed_feature_maps, vgg16_feature_map_names = get_feature_maps(\n model_name=\"vgg16\", image=cat_tensor, reduction=\"mean\", pretrained=False\n)\n_, resnet18_processed_feature_maps, resnet18_feature_map_names = get_feature_maps(\n model_name=\"resnet18\", image=cat_tensor, reduction=\"mean\", pretrained=False\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_feature_maps(\n vgg16_processed_feature_maps,\n vgg16_feature_map_names,\n nrows=5,\n title=\"VGG16 NOT Pretrained Feature Maps\",\n)\nplot_feature_maps(\n resnet18_processed_feature_maps,\n resnet18_feature_map_names,\n nrows=5,\n title=\"ResNet18 NOT Pretrained Feature Maps\",\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"References:\n\n- [https://pytorch.org/vision/stable/feature_extraction.html](https://pytorch.org/vision/stable/feature_extraction.html)\n- [https://ravivaishnav20.medium.com/visualizing-feature-maps-using-pytorch](https://ravivaishnav20.medium.com/visualizing-feature-maps-using-pytorch-12a48cd1e573)\n- [https://pytorch.org/blog/FX-feature-extraction-torchvision/](https://pytorch.org/blog/FX-feature-extraction-torchvision/)\n- [https://www.kaggle.com/tarunpaparaju/plant-pathology-2020-eda-models](https://www.kaggle.com/tarunpaparaju/plant-pathology-2020-eda-models)","metadata":{}}]}