{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"In this notebook, I will try to implement the Inception-v3 model architecture.\n\nArchitecture inspired from the paper \"Rethinking the Inception Architecture for Computer Vision\" by Szegedy et al. (2015).\nLink: <http://arxiv.org/abs/1512.00567>.\n\nFurther help in code has been taken from the source code: https://github.com/pytorch/vision/blob/master/torchvision/models/inception.py","metadata":{"_uuid":"0889c818-cba3-46b7-b6af-d774a010a860","_cell_guid":"dc7390fc-0d5a-402c-a124-63643cd360ee","trusted":true}},{"cell_type":"markdown","source":"Model architecture:","metadata":{"_uuid":"15908659-d691-4eeb-baa8-864ac6f6e32f","_cell_guid":"b92b1ffc-f0bd-405b-8f4b-18716cea19ee","trusted":true}},{"cell_type":"code","source":"\"\"\" Inception-V3\nInspired from the torchvision Inception3 model\nLicensed BSD-Clause 3 https://github.com/pytorch/vision/blob/master/LICENSE\n\"\"\"\n\n#Importing the libraries\nfrom collections import namedtuple\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F","metadata":{"_uuid":"784e30e1-6469-4140-97a1-30fe9ea3e216","_cell_guid":"76e1be05-177a-4882-ae36-1b35438c766d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:47.819187Z","iopub.execute_input":"2021-08-20T16:46:47.819529Z","iopub.status.idle":"2021-08-20T16:46:47.824436Z","shell.execute_reply.started":"2021-08-20T16:46:47.819495Z","shell.execute_reply":"2021-08-20T16:46:47.823314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"__all__ = ['Inception3', 'inception_v3']\n_InceptionOutputs = namedtuple('InceptionOutputs', ['logits', 'aux_logits'])","metadata":{"_uuid":"a661b79e-67b6-4fda-bf28-014d36695988","_cell_guid":"2caff3c2-983c-4757-a9cf-e7d449ceb21d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:48.114163Z","iopub.execute_input":"2021-08-20T16:46:48.114512Z","iopub.status.idle":"2021-08-20T16:46:48.119084Z","shell.execute_reply.started":"2021-08-20T16:46:48.11448Z","shell.execute_reply":"2021-08-20T16:46:48.118073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inception_v3(pretrained=False, **kwargs):\n    \n    \"\"\"Inception v3 model architecture from\n    `\"Rethinking the Inception Architecture for Computer Vision\" <http://arxiv.org/abs/1512.00567>`_.\n    \n    The model implementation is for experimentation on the CIFAR-10 and MNIST datasets.\n    \n    .. note::\n        **Important**: In contrast to the other models the inception_v3 expects tensors with a size of\n        N x 3 x 299 x 299, so ensure your images are sized accordingly.\n        \n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n        progress (bool): If True, displays a progress bar of the download to stderr\n        aux_logits (bool): If True, add an auxiliary branch that can improve training.\n            Default: *True*\n        transform_input (bool): If True, preprocesses the input according to the method with which it\n            was trained on ImageNet. Default: *False*\n    \"\"\"\n  \n    if pretrained:\n        if 'transform_input' not in kwargs:\n            kwargs['transform_input'] = True\n        if 'aux_logits' in kwargs:\n            original_aux_logits = kwargs['aux_logits']\n            kwargs['aux_logits'] = True\n        else:\n            original_aux_logits = True\n        model = Inception3(**kwargs)\n        if not original_aux_logits:\n            model.aux_logits = False\n        return model\n\n    return Inception3(**kwargs)","metadata":{"_uuid":"740ac1bd-a932-4f01-b066-a57b48486a43","_cell_guid":"ef882d71-97ea-4183-84ec-788180e697a2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:48.334382Z","iopub.execute_input":"2021-08-20T16:46:48.334744Z","iopub.status.idle":"2021-08-20T16:46:48.34237Z","shell.execute_reply.started":"2021-08-20T16:46:48.334697Z","shell.execute_reply":"2021-08-20T16:46:48.341244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#parent class\nclass Inception3(nn.Module):\n\n  def __init__(self, num_classes=10, aux_logits=True, transform_input=True):\n    super(Inception3, self).__init__()\n    self.aux_logits = aux_logits\n    self.transform_input = transform_input\n    self.Conv2d_4a_3x3 = BasicConv2d(3, 32, kernel_size=3,padding=1)\n    self.Mixed_5b = InceptionA(32, pool_features=8)\n    self.Mixed_5c = InceptionA(64, pool_features=72)\n    self.Mixed_6a = InceptionB(128)\n    self.Mixed_6b = InceptionC(256, channels_7x7=64)\n    if aux_logits:\n      self.AuxLogits = InceptionAux(512, num_classes)\n    self.Mixed_7a = InceptionD(512)\n    self.fc = nn.Linear(768, num_classes)\n\n    for m in self.modules():\n      if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):\n        import scipy.stats as stats\n        stddev = m.stddev if hasattr(m, 'stddev') else 0.1\n        X = stats.truncnorm(-2, 2, scale=stddev)\n        values = torch.as_tensor(X.rvs(m.weight.numel()), dtype=m.weight.dtype)\n        values = values.view(m.weight.size())\n        with torch.no_grad():\n          m.weight.copy_(values)\n      elif isinstance(m, nn.BatchNorm2d):\n        nn.init.constant_(m.weight, 1)\n        nn.init.constant_(m.bias, 0)\n\n  def forward(self, x):\n    global aux\n    print(x.shape) \n    x = self.Conv2d_4a_3x3(x)\n    x = self.Mixed_5b(x)\n    x = self.Mixed_5c(x)\n    x = self.Mixed_6a(x)\n    x = self.Mixed_6b(x)\n    if self.training and self.aux_logits:\n      aux = self.AuxLogits(x)\n    x = self.Mixed_7a(x)\n    x = F.adaptive_avg_pool2d(x, (1, 1))\n    x = F.dropout(x, training=self.training)\n    x = torch.flatten(x, 1)\n    x = self.fc(x)\n    print(x.shape)\n    if self.training and self.aux_logits:\n        return _InceptionOutputs(x, aux)\n    return x","metadata":{"_uuid":"5d11956a-0d9f-4120-be11-5bd87e55df5f","_cell_guid":"7579eff9-8693-4ddf-beb5-df4288743e35","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:48.530621Z","iopub.execute_input":"2021-08-20T16:46:48.531015Z","iopub.status.idle":"2021-08-20T16:46:48.547779Z","shell.execute_reply.started":"2021-08-20T16:46:48.53098Z","shell.execute_reply":"2021-08-20T16:46:48.546681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InceptionA(nn.Module):\n\n  def __init__(self, in_channels, pool_features):\n    super(InceptionA, self).__init__()\n    self.branch1x1 = BasicConv2d(in_channels, 8, kernel_size=1)\n\n    self.branch5x5_1 = BasicConv2d(in_channels, 8, kernel_size=1)\n    self.branch5x5_2 = BasicConv2d(8, 16, kernel_size=5, padding=2)\n\n    self.branch3x3dbl_1 = BasicConv2d(in_channels, 8, kernel_size=1)\n    self.branch3x3dbl_2 = BasicConv2d(8, 16, kernel_size=3, padding=1)\n    self.branch3x3dbl_3 = BasicConv2d(16, 32, kernel_size=3, padding=1)\n\n    self.branch_pool = BasicConv2d(in_channels, pool_features, kernel_size=1)\n\n  def forward(self, x):\n    branch1x1 = self.branch1x1(x)\n\n    branch5x5 = self.branch5x5_1(x)\n    branch5x5 = self.branch5x5_2(branch5x5)\n\n    branch3x3dbl = self.branch3x3dbl_1(x)\n    branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n    branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)\n\n    branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n    branch_pool = self.branch_pool(branch_pool)\n\n    outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool]\n    return torch.cat(outputs, 1)","metadata":{"_uuid":"3cb10faa-35fe-422f-9ee2-b400e289d0e4","_cell_guid":"602e9d07-341c-4b1f-aaa7-a4161235e288","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:48.735371Z","iopub.execute_input":"2021-08-20T16:46:48.735705Z","iopub.status.idle":"2021-08-20T16:46:48.746115Z","shell.execute_reply.started":"2021-08-20T16:46:48.735675Z","shell.execute_reply":"2021-08-20T16:46:48.744639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InceptionB(nn.Module):\n\n  def __init__(self, in_channels):\n    super(InceptionB, self).__init__()\n    self.branch3x3 = BasicConv2d(in_channels, 32, kernel_size=3, stride=2)\n\n    self.branch3x3dbl_1 = BasicConv2d(in_channels, 32, kernel_size=1)\n    self.branch3x3dbl_2 = BasicConv2d(32, 64, kernel_size=3, padding=1)\n    self.branch3x3dbl_3 = BasicConv2d(64, 96, kernel_size=3, stride=2)\n\n  def forward(self, x):\n    branch3x3 = self.branch3x3(x)\n\n    branch3x3dbl = self.branch3x3dbl_1(x)\n    branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n    branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)\n\n    branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)\n\n    outputs = [branch3x3, branch3x3dbl, branch_pool]\n    return torch.cat(outputs, 1)","metadata":{"_uuid":"10432731-2c28-42dc-929b-92abefe1712e","_cell_guid":"aaab72e3-26ea-4f41-adde-3954b7fc30f5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:48.935404Z","iopub.execute_input":"2021-08-20T16:46:48.935776Z","iopub.status.idle":"2021-08-20T16:46:48.94369Z","shell.execute_reply.started":"2021-08-20T16:46:48.935719Z","shell.execute_reply":"2021-08-20T16:46:48.9425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InceptionC(nn.Module):\n\n  def __init__(self, in_channels, channels_7x7):\n    super(InceptionC, self).__init__()\n    self.branch1x1 = BasicConv2d(in_channels, 128, kernel_size=1)\n\n    c7 = channels_7x7\n    self.branch7x7_1 = BasicConv2d(in_channels, c7, kernel_size=1)\n    self.branch7x7_2 = BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3))\n    self.branch7x7_3 = BasicConv2d(c7, 128, kernel_size=(7, 1), padding=(3, 0))\n\n    self.branch7x7dbl_1 = BasicConv2d(in_channels, c7, kernel_size=1)\n    self.branch7x7dbl_2 = BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0))\n    self.branch7x7dbl_3 = BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3))\n    self.branch7x7dbl_4 = BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0))\n    self.branch7x7dbl_5 = BasicConv2d(c7, 128, kernel_size=(1, 7), padding=(0, 3))\n\n    self.branch_pool = BasicConv2d(in_channels, 128, kernel_size=1)\n\n  def forward(self, x):\n    branch1x1 = self.branch1x1(x)\n\n    branch7x7 = self.branch7x7_1(x)\n    branch7x7 = self.branch7x7_2(branch7x7)\n    branch7x7 = self.branch7x7_3(branch7x7)\n\n    branch7x7dbl = self.branch7x7dbl_1(x)\n    branch7x7dbl = self.branch7x7dbl_2(branch7x7dbl)\n    branch7x7dbl = self.branch7x7dbl_3(branch7x7dbl)\n    branch7x7dbl = self.branch7x7dbl_4(branch7x7dbl)\n    branch7x7dbl = self.branch7x7dbl_5(branch7x7dbl)\n\n    branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n    branch_pool = self.branch_pool(branch_pool)\n\n    outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool]\n    return torch.cat(outputs, 1)","metadata":{"_uuid":"f05f3afe-84a6-4d44-b192-88baa4b03e82","_cell_guid":"6a0acdc2-dc4d-4362-b91f-0b0d49df35ce","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:49.21931Z","iopub.execute_input":"2021-08-20T16:46:49.219799Z","iopub.status.idle":"2021-08-20T16:46:49.232676Z","shell.execute_reply.started":"2021-08-20T16:46:49.219759Z","shell.execute_reply":"2021-08-20T16:46:49.231784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InceptionD(nn.Module):\n\n  def __init__(self, in_channels):\n    super(InceptionD, self).__init__()\n    self.branch3x3_1 = BasicConv2d(in_channels, 32, kernel_size=1)\n    self.branch3x3_2 = BasicConv2d(32,64, kernel_size=3, stride=2)\n\n    self.branch7x7x3_1 = BasicConv2d(in_channels, 32, kernel_size=1)\n    self.branch7x7x3_2 = BasicConv2d(32,64, kernel_size=(1, 7), padding=(0, 3))\n    self.branch7x7x3_3 = BasicConv2d(64, 128, kernel_size=(7, 1), padding=(3, 0))\n    self.branch7x7x3_4 = BasicConv2d(128,192, kernel_size=3, stride=2)\n\n  def forward(self, x):\n    branch3x3 = self.branch3x3_1(x)\n    branch3x3 = self.branch3x3_2(branch3x3)\n\n    branch7x7x3 = self.branch7x7x3_1(x)\n    branch7x7x3 = self.branch7x7x3_2(branch7x7x3)\n    branch7x7x3 = self.branch7x7x3_3(branch7x7x3)\n    branch7x7x3 = self.branch7x7x3_4(branch7x7x3)\n\n    branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)\n    outputs = [branch3x3, branch7x7x3, branch_pool]\n    return torch.cat(outputs, 1)","metadata":{"_uuid":"17de854a-a1c8-44e3-9aa7-ead339d4f3c9","_cell_guid":"ef12cf30-8786-44a9-a7be-32fb7ebf9cd0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:49.475332Z","iopub.execute_input":"2021-08-20T16:46:49.475846Z","iopub.status.idle":"2021-08-20T16:46:49.486414Z","shell.execute_reply.started":"2021-08-20T16:46:49.475809Z","shell.execute_reply":"2021-08-20T16:46:49.485194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class InceptionAux(nn.Module):\n\n  def __init__(self, in_channels, num_classes):\n    super(InceptionAux, self).__init__()\n    self.conv0 = BasicConv2d(in_channels, 128, kernel_size=1)\n    self.conv1 = BasicConv2d(128, 512, kernel_size=5)\n    self.conv1.stddev = 0.01\n    self.fc = nn.Linear(512, num_classes)\n    self.fc.stddev = 0.001\n\n  def forward(self, x):\n    x = F.avg_pool2d(x, kernel_size=3, stride=3)\n    x = self.conv0(x)\n    x = self.conv1(x)\n    x = F.adaptive_avg_pool2d(x, (1, 1))\n    x = torch.flatten(x, 1)\n    x = self.fc(x)\n    return x","metadata":{"_uuid":"8b050e6a-994e-4350-8fa3-e49814a56687","_cell_guid":"1fc89e4c-ab57-48aa-8eda-19b559ad4730","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:49.701405Z","iopub.execute_input":"2021-08-20T16:46:49.701783Z","iopub.status.idle":"2021-08-20T16:46:49.709939Z","shell.execute_reply.started":"2021-08-20T16:46:49.701748Z","shell.execute_reply":"2021-08-20T16:46:49.708713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BasicConv2d(nn.Module):\n\n  def __init__(self, in_channels, out_channels, **kwargs):\n    super(BasicConv2d, self).__init__()\n    self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)\n    self.bn = nn.BatchNorm2d(out_channels, eps=0.001)\n\n  def forward(self, x):\n    x = self.conv(x)\n    x = self.bn(x)\n    return F.relu(x, inplace=True)","metadata":{"_uuid":"744724d1-cef5-4a4f-8cbe-ef1038f0980c","_cell_guid":"3ef9a8b4-097b-4eb0-be31-590f209302ad","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-20T16:46:49.921471Z","iopub.execute_input":"2021-08-20T16:46:49.922079Z","iopub.status.idle":"2021-08-20T16:46:49.929164Z","shell.execute_reply.started":"2021-08-20T16:46:49.922028Z","shell.execute_reply":"2021-08-20T16:46:49.928194Z"},"trusted":true},"execution_count":null,"outputs":[]}]}