{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Environment"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom pathlib import Path\nimport pandas as pd\nfrom PIL import Image\nimport random\nimport seaborn as sns\nfrom tqdm import tqdm\nimport torch\nimport torchvision","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set(style=\"darkgrid\", context=\"notebook\", palette=\"muted\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 7\nrandom.seed(seed)\nnp.random.seed(seed)\ntorch.manual_seed(seed)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Load"},{"metadata":{"trusted":true},"cell_type":"code","source":"input_path = Path(\"../input/kuzushiji-recognition\")\ntrain_imgs_path = input_path / \"train_images\"\nprint(\"Train Images:%d\" % len(list(train_imgs_path.glob(\"*jpg\"))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(input_path / \"train.csv\")\nuc_trans = pd.read_csv(input_path / \"unicode_translation.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Feature Engineering"},{"metadata":{},"cell_type":"markdown","source":"## Check Training Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some `labels` seem to contain `NaN` in `train`.  \nWe check showing top-6 images containing `NaN` at `labels`."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_nan_labels = train[train[\"labels\"].isnull()]\ntrain_nan_labels.head(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_nan_labels.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(20, 80))\nfor i in range(6):\n    image_id = train_nan_labels[\"image_id\"].iloc[i]\n    file_name = image_id + \".jpg\"\n    train_img_path = train_imgs_path / file_name\n    train_img = np.asarray(Image.open(train_img_path))\n    fig.add_subplot(1, 6, i+1, title=file_name)\n    plt.axis(\"off\")\n    plt.imshow(train_img)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It seem to contain no characters in `NaN` label's images.  \nTherefore, we can delete them all and reset the index."},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.dropna()\ntrain = train.reset_index(drop=True)\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We use **3605** training images."},{"metadata":{},"cell_type":"markdown","source":"## Create Characters Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"From [Data Description](https://www.kaggle.com/c/kuzushiji-recognition/data),\n\n> The string should be read as space separated series of values where `Unicode character`, `X`, `Y`, `Width`, and `Height` are repeated as many times as necessary.\n\nWe create a dictionary `train_chars` where the key is `image_id` and the value is a dictionary containing `Unicode character`, `X`, `Y`, `Width` and `Height`."},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"train_chars = {}\ntrain_chars_num = 0\nfor i in tqdm(range(train.shape[0])):\n    image_id = train.iloc[i][\"image_id\"]\n    labels = train.iloc[i][\"labels\"].split(\" \")\n    values = {\"Unicode\" : [],\n              \"X\" : [],\n              \"Y\" : [],\n              \"Width\" : [],\n              \"Height\" : []}\n    for j in range(0, len(labels), 5):\n        uc = labels[j]\n        x = int(labels[j+1])\n        y = int(labels[j+2])\n        w = int(labels[j+3])\n        h = int(labels[j+4])\n        values[\"Unicode\"].append(uc)\n        values[\"X\"].append(x)\n        values[\"Y\"].append(y)\n        values[\"Width\"].append(w)\n        values[\"Height\"].append(h)\n        train_chars_num += 1\n    train_chars[image_id] = values\ntrain_chars_num","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We get **683464** character images(seems to be too large).  \nWe check showing top-6 characters at 1st `image_id` and its images."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(20, 80))\nimage_id_1st = train.iloc[0][\"image_id\"]\nimg_1st = Image.open(train_imgs_path/(image_id_1st+\".jpg\"))\nfor i in range(6):\n    uc = train_chars[image_id_1st][\"Unicode\"][i]\n    x = train_chars[image_id_1st][\"X\"][i]\n    y = train_chars[image_id_1st][\"Y\"][i]\n    w = train_chars[image_id_1st][\"Width\"][i]\n    h = train_chars[image_id_1st][\"Height\"][i]\n    img = img_1st.crop((x, y, x+w, y+h))\n    args = (uc, x, y, w, h)\n    print(\"Unicode:%s,X:%d,Y:%d,Width:%d,Height:%d\" % args)\n    fig.add_subplot(1, 6, i+1, title=\"Unicode:%s\" % uc)\n    plt.axis(\"off\")\n    plt.imshow(np.asarray(img))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Each image has diffrent width and height.  \nWe have to consider resizing images while the training."},{"metadata":{},"cell_type":"markdown","source":"## Decide Resizing scale"},{"metadata":{},"cell_type":"markdown","source":"We check histgrams of width and height."},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_data = []\nfor train_chars_value in train_chars.values():\n    plot_data.extend(train_chars_value[\"Width\"])\nsns.distplot(plot_data, kde=False, rug=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_data = []\nfor train_chars_value in train_chars.values():\n    plot_data.extend(train_chars_value[\"Height\"])\nsns.distplot(plot_data, kde=False, rug=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some width or height are too large.  \nFor the time being, we decide the resizing scale by fixed values(=48)."},{"metadata":{"trusted":true},"cell_type":"code","source":"w_resize = 48\nh_resize = 48","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check Unicodes"},{"metadata":{"trusted":true},"cell_type":"code","source":"uc_trans.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"uc_trans.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There are 4787 classes of all unicode characters.  \nHowever, some characters might be useless in training images.  \nWe check useless unicodes which are in `uc_trans[\"Unicode\"]` and are not in all unicodes of `train_chars`."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_chars_ucs = set()\nfor train_chars_value in train_chars.values():\n    train_chars_ucs |= set(train_chars_value[\"Unicode\"])\nuc_trans[~uc_trans[\"Unicode\"].isin(train_chars_ucs)].info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There seems to be 575 useless unicodes in training images.  \nFinally, we shrink `uc_trans` from 4787 classes to **4212**(=4787-575)."},{"metadata":{"trusted":true},"cell_type":"code","source":"uc_trans = uc_trans[uc_trans[\"Unicode\"].isin(train_chars_ucs)]\nuc_trans.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We create a list of unicode `uc_list` whose index is used for training and test labels."},{"metadata":{"trusted":true},"cell_type":"code","source":"uc_list = uc_trans[\"Unicode\"].values.tolist()\nuc_list.index(\"U+306F\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Create Pytorch Dataset"},{"metadata":{},"cell_type":"markdown","source":"We define `KuzushijiCharDataset` class extended from `torch.utils.data.Dataset`.  \nAs it costs little time to get i-th training data, it creates as follows.\n1. Open PIL Image each `image_id`\n2. Crop as Character's PIL Image\n3. Resize Character's PIL Image\n4. Gray-Scale Character's PIL Image where the channel is 1\n5. Convert from Character's PIL Image to Tensor"},{"metadata":{"trusted":true},"cell_type":"code","source":"class KuzushijiCharDataset(torch.utils.data.Dataset):\n    def __init__(self,\n                 chars: dict,\n                 uc_list: list,\n                 train_imgs_path: Path,\n                 scale_resize: tuple):\n        self._x_in_list = []\n        self._y_list = []\n        for image_id, values in tqdm(chars.items()):\n            # Open PIL Image each image_id\n            img = Image.open(train_imgs_path/(image_id+\".jpg\"))\n            values_zip = zip(values[\"Unicode\"],\n                             values[\"X\"],\n                             values[\"Y\"],\n                             values[\"Width\"],\n                             values[\"Height\"])\n            for uc, x, y, w, h in values_zip:\n                # Crop as Character's PIL Image\n                img_char = img.crop((x, y, x+w, y+h))\n                # Resize Character's PIL Image\n                img_char = img_char.resize(scale_resize)\n                # Gray-Scale Character's PIL Image where the channel is 1\n                img_char = img_char.convert('L')\n                # Convert from Character's PIL Image to Tensor\n                img_char = torchvision.transforms.functional.to_tensor(img_char)\n                # Add Training Data\n                self._x_in_list.append(img_char)\n                # Add Training Label\n                uc_idx = uc_list.index(uc)\n                self._y_list.append(uc_idx)\n\n    def __len__(self):\n        return len(self._y_list)\n    \n    def __getitem__(self, idx: int):\n        x_in = self._x_in_list[idx]\n        y = self._y_list[idx]\n        return x_in, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndataset = KuzushijiCharDataset(train_chars,\n                               uc_list,\n                               train_imgs_path,\n                               (w_resize, h_resize))\nlen(dataset)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We split a dataset into training dataset(90%) and validation one(10%)."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_size = int(len(dataset) * 0.9)\nvalid_size = len(dataset) - train_size\ntrain_dataset, valid_dataset = torch.utils.data.random_split(dataset,\n                                                             [train_size, valid_size])\nargs = (len(dataset), len(train_dataset), len(valid_dataset))\nprint(\"Total:%d,Training:%d,Validation:%d\" % args)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Demonstration for Classifying Characters"},{"metadata":{},"cell_type":"markdown","source":"## Create Network"},{"metadata":{},"cell_type":"markdown","source":"We define an original model where\n* We define 1 input channel at the 1st layer `conv1` because of gray-scaled.\n* We define 4212(=Character Classes) input output features at the affine layer `fc`."},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"class DemoModel(torch.nn.Module):\n    def __init__(self):\n        super(DemoModel, self).__init__()\n        self.conv1 = torch.nn.Conv2d(in_channels=1,\n                                     out_channels=16,\n                                     kernel_size=7)\n        self.relu1 = torch.nn.ReLU(inplace=True)\n        self.maxpool1 = torch.nn.MaxPool2d(kernel_size=2)\n        self.conv2 = torch.nn.Conv2d(in_channels=16,\n                                     out_channels=128,\n                                     kernel_size=6)\n        self.relu2 = torch.nn.ReLU(inplace=True)\n        self.maxpool2 = torch.nn.MaxPool2d(kernel_size=2)\n        self.fc = torch.nn.Linear(in_features=128*8*8,\n                                  out_features=4212,\n                                  bias=True)\n        self.log_softmax = torch.nn.LogSoftmax(dim=-1)\n\n    def forward(self, x):\n        out = self.conv1(x) # (batch, 1, 48, 48) -> (batch, 16, 42, 42)\n        out = self.relu1(out)\n        out = self.maxpool1(out) # (batch, 16, 42, 42) -> (batch, 16, 21, 21)\n        out = self.conv2(out) # (batch, 16, 21, 21) -> (batch, 128, 16, 16)\n        out = self.relu2(out)\n        out = self.maxpool2(out) # (batch, 128, 16, 16) -> (batch, 128, 8, 8)\n        out = out.view(out.size(0), -1) # (batch, 128, 8, 8) -> (batch, 8192)\n        out = self.fc(out) # (batch, 8192) -> (batch, 4212)\n        out = self.log_softmax(out)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"network = DemoModel().to(device)\nnetwork","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Training Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_epochs = 10\nbatch_size = 1024\nlr = 0.005\noptimizer = torch.optim.Adam(network.parameters())\ncriterion = torch.nn.NLLLoss()\ntrain_dataLoader = torch.utils.data.DataLoader(train_dataset,\n                                               batch_size=batch_size,\n                                               shuffle=True)\nvalid_dataLoader = torch.utils.data.DataLoader(valid_dataset)\nargs = (len(train_dataLoader), len(valid_dataLoader))\nprint(\"Training:%d,Validation:%d\" % args)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nresult = {\"Epoch\" : [],\n          \"Type\" : [],\n          \"Average Loss\" : [],\n          \"Accuracy\" : []}\nfor epoch in range(1, max_epochs+1):\n    # Training\n    sum_loss = 0.0\n    correct = 0\n    for x_in, y in tqdm(train_dataLoader):\n        network.zero_grad()\n        x_out = network(x_in.to(device))\n        loss = criterion(x_out, y.to(device))\n        loss.backward()\n        optimizer.step()\n        sum_loss += loss.item() * x_in.shape[0]\n        correct += int(torch.sum(torch.argmax(x_out, 1) == y.to(device)))\n    ave_loss = sum_loss / len(train_dataset)\n    accuracy = 100.0 * correct / len(train_dataset)\n    result[\"Epoch\"].append(epoch)\n    result[\"Type\"].append(\"Training\")\n    result[\"Average Loss\"].append(ave_loss)\n    result[\"Accuracy\"].append(accuracy)\n    args = (datetime.now().isoformat(), epoch, max_epochs, ave_loss, accuracy)\n    print(\"Type:Training,Time:%s,Epoch:%d/%d,Average Loss:%.3f,Accuracy:%.3f%%\" % args)\n\n    # Validation\n    sum_loss = 0.0\n    correct = 0\n    for x_in, y in tqdm(valid_dataLoader):\n        x_out = network(x_in.to(device))\n        loss = criterion(x_out, y.to(device))\n        sum_loss += loss.item() * x_in.shape[0]\n        correct += int(torch.sum(torch.argmax(x_out, 1) == y.to(device)))\n    ave_loss = sum_loss / len(valid_dataset)\n    accuracy = 100.0 * correct / len(valid_dataset)\n    result[\"Epoch\"].append(epoch)\n    result[\"Type\"].append(\"Validation\")\n    result[\"Average Loss\"].append(ave_loss)\n    result[\"Accuracy\"].append(accuracy)\n    args = (datetime.now().isoformat(), epoch, max_epochs, ave_loss, accuracy)\n    print(\"Type:Validation,Time:%s,Epoch:%d/%d,Average Loss:%.3f,Accuracy:%.3f%%\" % args)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.relplot(x=\"Epoch\",\n            y=\"Average Loss\",\n            hue=\"Type\",\n            kind=\"line\",\n            data=pd.DataFrame(result))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.relplot(x=\"Epoch\",\n            y=\"Accuracy\",\n            hue=\"Type\",\n            kind=\"line\",\n            data=pd.DataFrame(result))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}