{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CSCI 6998 Master's Thesis\n## Chemical Element Transformations\n#### Narmin Jamalova, MSCSDA 2022\n","metadata":{}},{"cell_type":"markdown","source":"> This notebook implements a Chemical Element Identifier on originally provided dataset of chemical structure images, then applies various augmentations and text position correction techniques to generate a dataset of augmented images, where chemical elements are flipped/rotated in a correct manner. \n\n>> The results get saved as a stand-alone dataset of augmented images with proper labelling.","metadata":{}},{"cell_type":"markdown","source":"## Import Libraries","metadata":{}},{"cell_type":"code","source":"!pip install imutils","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-18T07:13:07.424144Z","iopub.execute_input":"2022-01-18T07:13:07.424465Z","iopub.status.idle":"2022-01-18T07:13:18.066159Z","shell.execute_reply.started":"2022-01-18T07:13:07.424389Z","shell.execute_reply":"2022-01-18T07:13:18.065168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport imutils\nimport numpy as np\nimport pandas as pd\nfrom imutils import contours\nimport matplotlib.pyplot as plt\nfrom collections import defaultdict","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:18.069267Z","iopub.execute_input":"2022-01-18T07:13:18.069537Z","iopub.status.idle":"2022-01-18T07:13:18.284332Z","shell.execute_reply.started":"2022-01-18T07:13:18.069505Z","shell.execute_reply":"2022-01-18T07:13:18.283656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Setup Dataset Directory\n\n> For this research, the first `N_IMAGES` chemical structure images are selected from Bristol-Myers chemical structure images database. In the cell below, project directory is set up and the labels of images are read-in. The labels are standard international chemical identifiers, called InChI.","metadata":{}},{"cell_type":"code","source":"START    = 13001\nN_IMAGES = 14000","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:18.287111Z","iopub.execute_input":"2022-01-18T07:13:18.287305Z","iopub.status.idle":"2022-01-18T07:13:18.292251Z","shell.execute_reply.started":"2022-01-18T07:13:18.28728Z","shell.execute_reply":"2022-01-18T07:13:18.291645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set the directory to project dataset\nproject_directory = '../input/bms-molecular-translation/'\n\n# read the labels in\nlabels  = pd.read_csv(project_directory + 'train_labels.csv')\n\n# restrict the labels to the first range\nlabels  = labels.iloc[START:N_IMAGES, :]\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:18.294373Z","iopub.execute_input":"2022-01-18T07:13:18.294896Z","iopub.status.idle":"2022-01-18T07:13:26.571105Z","shell.execute_reply.started":"2022-01-18T07:13:18.29486Z","shell.execute_reply":"2022-01-18T07:13:26.570431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# set the directory to train dataset only\ntrain_directory = project_directory + 'train/'","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:26.57248Z","iopub.execute_input":"2022-01-18T07:13:26.572953Z","iopub.status.idle":"2022-01-18T07:13:26.576414Z","shell.execute_reply.started":"2022-01-18T07:13:26.572915Z","shell.execute_reply":"2022-01-18T07:13:26.575773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read Image Data\n\n> The code below reads the image data by going through the depth of the given directory and looking for the provided image ID. The image is read in RGB mode.","metadata":{}},{"cell_type":"code","source":"def get_image(dir_path: str, image_id: str) -> np.array:\n    \"\"\"Get the image from the directory path + sub-folder pattern.\n    \n    :param dir_path: path to the directory in which the sub-folders are contained\n    :type dir_path:  str\n    :param image_id: image label\n    :type image_id:  str\n    :return:         read-in image\n    :rtype:          np.array\n    \"\"\"\n    img_path = (\n        dir_path \n        + image_id[0] \n        + '/' \n        + image_id[1] \n        + '/' \n        + image_id[2] \n        + '/' \n        + image_id \n        + '.png'\n    )\n    img      = cv2.imread(img_path, cv2.IMREAD_COLOR)\n    \n    return img","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:26.577744Z","iopub.execute_input":"2022-01-18T07:13:26.578189Z","iopub.status.idle":"2022-01-18T07:13:26.587264Z","shell.execute_reply.started":"2022-01-18T07:13:26.578153Z","shell.execute_reply":"2022-01-18T07:13:26.586576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalize Image Data\n\n> In the cell below, image pixel intensities are normalized using the MINMAX technique.","metadata":{}},{"cell_type":"code","source":"def get_normalized_image(img: np.array, dim: tuple = (300, 300)) -> np.array:\n    \"\"\"Normalize the input image pixel intensities.\n    \n    :param img: input image\n    :type img:  np.array\n    :return:    normalized image\n    :rtype:     np.array\n    \"\"\"    \n    # normalize the pixel intensities\n    norm_img    = cv2.normalize(img,  (300, 300), 0, 255, cv2.NORM_MINMAX)\n    \n    return norm_img","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:26.588649Z","iopub.execute_input":"2022-01-18T07:13:26.589098Z","iopub.status.idle":"2022-01-18T07:13:26.597158Z","shell.execute_reply.started":"2022-01-18T07:13:26.589063Z","shell.execute_reply":"2022-01-18T07:13:26.596364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Identify Chemical Element Characters on Image Data\n\n> In the cell below, an identifier of chemical elements is created. Its main purpose is to optically recognize textual elements on the image data and insert a red bounding box around the recognitions.","metadata":{}},{"cell_type":"code","source":"class ChemicalElementIdentifier():\n    \"\"\"Identifier of chemical elements on an image.\"\"\"\n    def __init__(self, img: np.array) -> None:\n        \"\"\"Initialize the class with input image.\n        \n        :param img: input image\n        :type img:  np.array\n        \"\"\"\n        self.img    = img\n\n    def _get_processed_image(self, threshold: int = 180, kernel_size: tuple = (1, 1)) -> np.array:\n        \"\"\"Process the input image.\n        \n        In particular, apply the following transformations:\n            1) gray-out the input image\n            2) apply basic black-and-white thresholding\n            3) dilate the image to join broken parts\n            4) detect image edges using Laplace second order derivative mask\n            \n        :param threshold:   threshold value for black-and-white (binary) thresholding\n        :type threshold:    int\n        :param kernel_size: kernel size for morphological structuring (dilation)\n        :type kernel_size:  tuple\n        :return:            processed image\n        :rtype:             np.array\n        \"\"\"\n        # get a grayscale version of the input image\n        gray_img    = cv2.cvtColor(self.img, cv2.COLOR_BGR2GRAY)\n        \n        # if pixel value is greater than threshold (default: 180), set it to white\n        ret, mask   = cv2.threshold(gray_img, threshold, 255, cv2.THRESH_BINARY)\n        \n        bitwise_img = cv2.bitwise_and(gray_img, gray_img, mask=mask)\n        \n        # if pixel value is greater than threshold (default: 180), set it to inverse\n        ret, mask   = cv2.threshold(bitwise_img, threshold, 255, cv2.THRESH_BINARY_INV)\n        \n        # apply a 1x1 cross-shaped filtered dilation to increase the size of foreground objects and join broken image parts\n        kernel      = cv2.getStructuringElement(cv2.MORPH_CROSS, kernel_size)\n        dilated_img = cv2.dilate(mask, kernel, iterations=9)\n        \n        # detect image edges using Laplace second order derivative mask\n        laplace_img = cv2.Laplacian(dilated_img, cv2.CV_8U)\n        \n        return laplace_img\n    \n    \n    def _get_contours(self, img: np.array) -> list:\n        \"\"\"Get image contours, i.e. curves that join continous parts on an image.\n        \n        :param img: grayscale input or processed image\n        :type img:  np.array\n        :return:    all contours of an image and their hierarchy\n        :rtype:     list\n        \"\"\"\n        # retrieve only the extreme outer contours and store all contour points\n        contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)\n\n        return contours, hierarchy\n    \n    \n    def _get_chemical_elements_mask(\n        self, \n        img:      np.array, \n        contours: np.array, \n        beta_1:   int = 20,\n        beta_2:   int = 650,\n        alpha:    int = 10,\n    ) -> np.array:\n        \"\"\"Get chemical-character-only mask of an image.\n        \n        :param img:      a given grayscale input or processed image\n        :type img:       np.array\n        :param contours: overall contours of an image\n        :type contours:  np.array\n        :param beta_1:   lower threshold for the enclosing circle area\n        :type beta_1:    int\n        :param beta_2:   upper threshold for the enclosing circle area\n        :type beta_2:    int\n        :param alpha:    threshold for box area\n        :type alpha:     int\n        :return:         image chemical characters mask\n        :rtype:          np.array\n        \"\"\"\n        # initialize a list with character-only contours\n        character_contours = []\n        \n        # function for box-side difference calculation\n        # reference: https://www.kaggle.com/thomaskonstantin/letter-retrieval-molecular-translation\n        box_side_diff      = lambda x: np.abs(np.linalg.norm(x[0] - x[1]) - np.linalg.norm(x[0] - x[3]))\n        \n        # for each contour, do\n        for contour in contours:\n            # find the circle covering that contour\n            (x, y), radius = cv2.minEnclosingCircle(contour)\n            \n            # find the minimum area rotated rectangle\n            rect           = cv2.minAreaRect(contour)\n            \n            # identify all four corner points of the rectangle, i.e. the box\n            box            = cv2.boxPoints(rect)\n            \n            # save the contour as a character-only contour if the rule is satisfied\n            # if bounding box area < alpha and circle area is between beta_1 and beta_2\n            if (beta_1 < np.pi * radius**2 < beta_2) and box_side_diff(box) < alpha:\n                character_contours.append(contour)\n        \n        # get the contoured image of the same shape as the input\n        contoured_img_mask = cv2.cvtColor(np.zeros_like(img),cv2.COLOR_GRAY2RGB)\n        \n        # draw the contours on the contoured image\n        contoured_img_mask = cv2.drawContours(contoured_img_mask, character_contours, -1, (255,255,255), 2)\n        \n        # gray-out the image\n        contoured_img_mask = cv2.cvtColor(contoured_img_mask,cv2.COLOR_RGB2GRAY)\n    \n        # get the subset of the input image defined by contoured mask\n        contoured_img      = cv2.bitwise_and(img, img, mask = contoured_img_mask)\n\n        return contoured_img\n    \n    \n    def _get_chemical_element_coordinates(self, chemical_elements_mask: np.array) -> list:\n        \"\"\"Get coordinates of chemical elements on an input or processed image.\n\n        :param chemical_elements_mask: chemical-elements-only mask\n        :type chemical_elements_mask:  np.array\n        :return:                       list of chemical element coordinates\n        :rtype:                        list\n        \"\"\"\n        # store chemical element coordinates as tuples in this list\n        coordinates         = []\n        \n        # get contours of the image containing chemical elements\n        contours, hierarchy = self._get_contours(img=chemical_elements_mask)\n        \n        # get a bounding rectangle and its point coordinates around the chemical element\n        for contour in contours:\n            # find coordinates of the bounding rectangle\n            x, y, w, h      = cv2.boundingRect(contour)\n            coordinates.append((x, y, w, h))\n            \n        return coordinates\n       \n    \n    def _draw_bounding_boxes_around_elements(self, img: np.array, coordinates: list) -> np.array:\n        \"\"\"Draw bounding boxes around the coordinates of chemical elements on a given input image.\n        \n        :param img:         input image\n        :type img:          np.array\n        :param coordinates: list of chemical element coordinates\n        :type coordinates:  list\n        :return:            image with bounding boxes around chemical elements\n        :rtype:             np.array\n        \"\"\"\n        img_with_boxes         = img.copy()\n        for coordinate in coordinates:\n            # unbundle coordinates\n            x, y, w, h         = coordinate\n\n            # draw a red bounding box around the given coordinates\n            img_with_boxes = cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 0), 1)\n            \n        return img_with_boxes\n    \n    def _get_chemical_elements(self, img: np.array, coordinates: list) -> np.array:\n        \"\"\"Get chemical elements from a given input image.\n        \n        :param img:         input image\n        :type img:          np.array\n        :param coordinates: list of chemical element coordinates\n        :type coordinates:  list\n        :return:            image with an identified chemical element\n        :rtype:             np.array\n        \"\"\"\n        chemical_elements         = []\n        for coordinate in coordinates:\n            # unbundle coordinates\n            x, y, w, h            = coordinate\n\n            # get a chemical element using the given coordinates\n            chemical_element      = img[y:y+h, x:x+w]\n            chemical_elements.append(chemical_element)\n\n        return chemical_elements","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:26.598159Z","iopub.execute_input":"2022-01-18T07:13:26.598359Z","iopub.status.idle":"2022-01-18T07:13:26.624001Z","shell.execute_reply.started":"2022-01-18T07:13:26.598334Z","shell.execute_reply":"2022-01-18T07:13:26.62307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read and normalize the images (using a subset of 500 images)\n\n# store the input images here\nsample_images = []\n\n# for each image in our subset\nfor i in range(START, N_IMAGES):\n    \n    # read the image in\n    img = get_image(dir_path=train_directory, image_id=labels.loc[i, 'image_id'])\n    \n    # normalize the image\n    img = get_normalized_image(img)\n    \n    # store the processed image\n    sample_images.append(img)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:26.625224Z","iopub.execute_input":"2022-01-18T07:13:26.625786Z","iopub.status.idle":"2022-01-18T07:13:34.955366Z","shell.execute_reply.started":"2022-01-18T07:13:26.625749Z","shell.execute_reply":"2022-01-18T07:13:34.954625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get input image with bounding boxes drawn around identified chemical characters\n\n# store individual chemical element images here\nall_chemical_elements          = []\n\n# store annotated images here (i.e. with red bounding boxes around chemical elements)\nfinal_annotated_images         = []\n\n# store un-annotated images here\nfinal_unannotated_images       = []\n\n# store coordinates of chemical elements here\nall_chemical_coordinates       = []\n\n# for each processed image \nfor img in sample_images:\n    \n    # instantiate the chemical element identifier\n    chem_identifier        = ChemicalElementIdentifier(img)\n    \n    # process the image\n    processed_img          = chem_identifier._get_processed_image()\n    \n    # get contours of the processed image\n    contours, hierarchy    = chem_identifier._get_contours(processed_img)\n    \n    # get the mask of only chemical elements\n    chemical_elements_mask = chem_identifier._get_chemical_elements_mask(\n        processed_img, \n        contours\n    )\n    \n    # get the coordinates of identified chemical elements\n    chemical_coordinates   = chem_identifier._get_chemical_element_coordinates(\n        chemical_elements_mask\n    )\n    \n    # get the images of chemical elements themselves\n    chemical_elements      = chem_identifier._get_chemical_elements(\n        img, \n        chemical_coordinates\n    )\n    \n    # store un-annotated images\n    final_unannotated_image  = img.copy()\n    \n    # draw bounding boxes around the chemical elements on the original image\n    final_annotated_image  = chem_identifier._draw_bounding_boxes_around_elements(\n        img, \n        chemical_coordinates\n    )\n    \n    \n    # store identified chemical elements\n    all_chemical_elements.append(chemical_elements)\n    \n    # store annotated original images (i.e. with bounding boxes)\n    final_annotated_images.append(final_annotated_image)\n    \n    # store un-annotated original images\n    final_unannotated_images.append(final_unannotated_image) \n    \n    # store the coordinates of chemical elements\n    all_chemical_coordinates.append(chemical_coordinates)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:34.958471Z","iopub.execute_input":"2022-01-18T07:13:34.958743Z","iopub.status.idle":"2022-01-18T07:13:39.250357Z","shell.execute_reply.started":"2022-01-18T07:13:34.958708Z","shell.execute_reply":"2022-01-18T07:13:39.24965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# view the results visually\nfor img in final_annotated_images[:3]:\n    plt.figure(figsize=(7, 7))\n    plt.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:39.251743Z","iopub.execute_input":"2022-01-18T07:13:39.252Z","iopub.status.idle":"2022-01-18T07:13:39.878833Z","shell.execute_reply.started":"2022-01-18T07:13:39.251965Z","shell.execute_reply":"2022-01-18T07:13:39.877956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Anomaly Detection\n\n> In the code below, an anomaly detection method is suggested by going through all image data, calculating the mean and standard deviation of the identified box areas and removing the outliers. This helps get rid of false positives that sometimes get recognized by the algorithm above.\n\n> This anomaly detection technique is based on a sample of 500 images used above for chemical element identification.","metadata":{}},{"cell_type":"code","source":"def compute_average_box_size(set_of_chemical_coordinates: list) -> tuple:\n    \"\"\"Computes the chemical element areas, their mean and standard deviation across the entire set of detected chemical elements.\n    \n    :param set_of_chemical_coordinates: list of all identified chemical coordinates\n    :type set_of_chemical_coordinates:  list\n    :return:                            computed box areas, the mean box size and standard deviation\n    :rtype:                             tuple\n    \"\"\"\n    # store all computed areas in the list below\n    areas              = []\n    \n    # for each set of chemical coordinates, do\n    for _set in set_of_chemical_coordinates:\n        \n        # for each coordinate in the set, do\n        for coordinate in _set:\n            # unravel the coordinate tuple\n            x, y, w, h = coordinate\n    \n            # compute area\n            area       = w*h\n            \n            # store the computed area in a container\n            areas.append(area)\n    \n    # get the mean area size and its standard deviation\n    average_box_size   = np.mean(areas)\n    box_size_stdev     = np.std(areas)\n    \n    # normalize the areas using the mean and stdev\n    areas              = [(float(idx) - average_box_size)/box_size_stdev for idx in areas]\n    \n    return areas, average_box_size, box_size_stdev","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:39.880491Z","iopub.execute_input":"2022-01-18T07:13:39.880802Z","iopub.status.idle":"2022-01-18T07:13:39.888102Z","shell.execute_reply.started":"2022-01-18T07:13:39.880762Z","shell.execute_reply":"2022-01-18T07:13:39.887072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# compute the areas of boxes, average box size and standard deviation\nareas, avg_box_size, stdev = compute_average_box_size(all_chemical_coordinates)\nprint('Average box size: ', avg_box_size, '\\n', 'Standard deviation: ', stdev)\n\n# plot the distribution of chemical element areas\nplt.hist(areas)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:39.889886Z","iopub.execute_input":"2022-01-18T07:13:39.890177Z","iopub.status.idle":"2022-01-18T07:13:40.166269Z","shell.execute_reply.started":"2022-01-18T07:13:39.890138Z","shell.execute_reply":"2022-01-18T07:13:40.165528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transform Chemical Elements (with Anomaly Removal)\n\n> Since some chemical element names change their direction upon rotations / flips (i.e. they are not symmetric), it's important to ensure that they are rotated correctly when the whole image is rotated. For example, the elements `O` and `H` will be correct regardless of the degree of rotation, but the same cannot be true for such an element as `OH`, which can transform into being `HO`, or `S` which will start looking into the opposite direction.\n\n> In the code below, we transform images by applying rotations and flips whilst preserving the textual orientation of chemical elements. Moreover, anomaly detection technique from the code above is applied to ensure only valid boxes are considered.\n\n> The resultant original and transformed images form a new dataset.","metadata":{}},{"cell_type":"markdown","source":"### Augment Image Data \n\n> The following augmentation techniques are applied: horizontal, vertical and two-way flips, 90 and -90 degree rotations.","metadata":{}},{"cell_type":"code","source":"def get_augmentations(img: np.array) -> list:\n    \"\"\"Get augmented images - rotations and flips.\n    \n    :param img:                  input image\n    :type img:                   np.array\n    :return:                     list of image augmentations\n    :rtype:                      list\n    \"\"\"\n    # generate flips along axes 0, 1 and both\n    flip_vertical   = cv2.flip(img, 0) # vertical\n    flip_horizontal = cv2.flip(img, 1) # horizontal\n    flip_both_axes  = cv2.flip(img, -1) # two-way\n    \n    # generate rotations by 90 and 270 degrees\n    rotate_90       = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)\n    rotate_270      = cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE)\n    \n    # return all transformations as elements of one list\n    augmentations   = [flip_vertical, flip_horizontal, flip_both_axes, rotate_90, rotate_270]\n    \n    return augmentations","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:40.16773Z","iopub.execute_input":"2022-01-18T07:13:40.168017Z","iopub.status.idle":"2022-01-18T07:13:40.174542Z","shell.execute_reply.started":"2022-01-18T07:13:40.16798Z","shell.execute_reply":"2022-01-18T07:13:40.173746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the N_IMAGES images (since we're restarting for the purpose of augmenting)\n\naugmented_images = {}\n\nfor i in range(START, N_IMAGES):\n    # get the original image and its label\n    img = get_image(dir_path=train_directory, image_id=labels.loc[i, 'image_id'])\n    lbl = labels.loc[i, 'image_id']\n    \n    # normalize the image\n    img = get_normalized_image(img)\n    \n    # get the image augmentations\n    aug = get_augmentations(img)\n    \n    # append augmentations to the label key\n    augmented_images[lbl] = aug","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:40.176264Z","iopub.execute_input":"2022-01-18T07:13:40.176877Z","iopub.status.idle":"2022-01-18T07:13:42.694554Z","shell.execute_reply.started":"2022-01-18T07:13:40.176833Z","shell.execute_reply":"2022-01-18T07:13:42.693729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Annotate Augmented Images (with Anomaly Detection)","metadata":{}},{"cell_type":"code","source":"# get input image with bounding boxes drawn around identified chemical characters\n\n# use these dictionaries to store key-value pairs (key: InChI label, values: augmented images, elements and coordinates)\naug_chemical_elements          = {}\naug_annotated_images           = {}\naug_chemical_coordinates       = {}\naug_unannotated_images         = {}\n\n# for each augmented image list, do\nfor lbl, augs in augmented_images.items():\n    \n    # initialization for storage dictionaries\n    aug_chemical_elements[lbl]    = [0, 0, 0, 0, 0]\n    aug_annotated_images[lbl]     = [0, 0, 0, 0, 0]\n    aug_unannotated_images[lbl]   = [0, 0, 0, 0, 0]\n    aug_chemical_coordinates[lbl] = [0, 0, 0, 0, 0]\n    idx                           = 0\n    \n    # for each augmentation, do\n    for aug in augs:\n        # instantiate the chemical element identifier\n        chem_identifier        = ChemicalElementIdentifier(aug)\n        # pre-process the image\n        processed_img          = chem_identifier._get_processed_image()\n        # get image contours\n        contours, hierarchy    = chem_identifier._get_contours(processed_img)\n        # get the mask of only chemical elements\n        chemical_elements_mask = chem_identifier._get_chemical_elements_mask(\n            processed_img, \n            contours\n        )\n        # get coordinates of chemical elements\n        chemical_coordinates   = chem_identifier._get_chemical_element_coordinates(\n            chemical_elements_mask\n        )\n        # store non-outlier coordinates only\n        updated_chem_coords    = []\n        # for each coordinate, do\n        for coordinate in chemical_coordinates:\n            # unbundle the coordinate\n            x, y, w, h         = coordinate\n            # compute the area of the box\n            area               = w*h\n            # if the normalized area is no more than 3 stdev away, keep the element\n            if (\n                np.mean(areas) - 3* np.std(areas) \n                < \n                ((area - avg_box_size) / stdev) \n                < \n                np.mean(areas) + 3* np.std(areas)\n            ):\n                updated_chem_coords.append(coordinate)\n        \n        # get chemical elements separately\n        chemical_elements      = chem_identifier._get_chemical_elements(\n            aug, \n            updated_chem_coords\n        )\n        \n        final_unannotated_image = aug.copy()\n        \n        # get final image with red bounding boxes around identified chemical elements\n        final_annotated_image  = chem_identifier._draw_bounding_boxes_around_elements(\n            aug, \n            updated_chem_coords\n        )\n        \n        # update key-value pairs with final results\n        aug_chemical_elements[lbl][idx]    = chemical_elements\n        aug_annotated_images[lbl][idx]     = final_annotated_image  \n        aug_unannotated_images[lbl][idx]   = final_unannotated_image\n        aug_chemical_coordinates[lbl][idx] = updated_chem_coords\n        \n        idx                                +=1","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:13:42.698125Z","iopub.execute_input":"2022-01-18T07:13:42.699769Z","iopub.status.idle":"2022-01-18T07:17:44.190166Z","shell.execute_reply.started":"2022-01-18T07:13:42.699718Z","shell.execute_reply":"2022-01-18T07:17:44.189436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correct Textual Orientations","metadata":{}},{"cell_type":"code","source":"# showcase results on an example -- plotting the original image\nim = get_image(dir_path=train_directory, image_id=lbl)\nplt.imshow(im)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:44.191591Z","iopub.execute_input":"2022-01-18T07:17:44.191859Z","iopub.status.idle":"2022-01-18T07:17:44.41951Z","shell.execute_reply.started":"2022-01-18T07:17:44.191825Z","shell.execute_reply":"2022-01-18T07:17:44.418889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Correct the Vertical Flip (flip = 0)","metadata":{}},{"cell_type":"code","source":"# show the augmented image before the correction\nplt.imshow(aug_unannotated_images[lbl][0])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:44.420635Z","iopub.execute_input":"2022-01-18T07:17:44.420881Z","iopub.status.idle":"2022-01-18T07:17:44.638079Z","shell.execute_reply.started":"2022-01-18T07:17:44.420847Z","shell.execute_reply":"2022-01-18T07:17:44.637426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vertical flip has index = 0 in the list of augmented images from above\n\n# for each vertically flipped image, do\nfor lbl, augs in aug_unannotated_images.items():\n    # for each chemical element coordinate on that image\n    for idx in range(len(aug_chemical_coordinates[lbl][0])):\n        # unbundle the coordinate\n        x, y, w, h            = aug_chemical_coordinates[lbl][0][idx]\n        # re-flip the box bounded by these coordinates\n        augs[0][y:y+h, x:x+w] = cv2.flip(augs[0][y:y+h, x:x+w], 0)\n        aug_unannotated_images[lbl][0] = augs[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:44.639316Z","iopub.execute_input":"2022-01-18T07:17:44.640763Z","iopub.status.idle":"2022-01-18T07:17:44.685331Z","shell.execute_reply.started":"2022-01-18T07:17:44.640725Z","shell.execute_reply":"2022-01-18T07:17:44.684646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display the example result\nplt.imshow(aug_unannotated_images[lbl][0])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:44.686582Z","iopub.execute_input":"2022-01-18T07:17:44.68685Z","iopub.status.idle":"2022-01-18T07:17:44.906165Z","shell.execute_reply.started":"2022-01-18T07:17:44.686815Z","shell.execute_reply":"2022-01-18T07:17:44.905428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Correct the Horizontal Flip (flip = 1)","metadata":{}},{"cell_type":"code","source":"# show the augmented image before the correction\nplt.imshow(aug_unannotated_images[lbl][1])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:44.90746Z","iopub.execute_input":"2022-01-18T07:17:44.907876Z","iopub.status.idle":"2022-01-18T07:17:45.127057Z","shell.execute_reply.started":"2022-01-18T07:17:44.907836Z","shell.execute_reply":"2022-01-18T07:17:45.126369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# horizontal flip has index = 1 in the list of augmented images from above\n\n# for each horizontally flipped image, do\nfor lbl, augs in aug_unannotated_images.items():\n    # for each chemical element coordinate on that image\n    for idx in range(len(aug_chemical_coordinates[lbl][1])):\n        # unbundle the coordinate\n        x, y, w, h            = aug_chemical_coordinates[lbl][1][idx]\n        # re-flip the box bounded by these coordinates\n        augs[1][y:y+h, x:x+w] = cv2.flip(augs[1][y:y+h, x:x+w], 1)\n        aug_unannotated_images[lbl][1] = augs[1]","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:45.12814Z","iopub.execute_input":"2022-01-18T07:17:45.128384Z","iopub.status.idle":"2022-01-18T07:17:45.178152Z","shell.execute_reply.started":"2022-01-18T07:17:45.128348Z","shell.execute_reply":"2022-01-18T07:17:45.177562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display the example result\nplt.imshow(aug_unannotated_images[lbl][1])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:45.179168Z","iopub.execute_input":"2022-01-18T07:17:45.17941Z","iopub.status.idle":"2022-01-18T07:17:45.398967Z","shell.execute_reply.started":"2022-01-18T07:17:45.179376Z","shell.execute_reply":"2022-01-18T07:17:45.398351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Correct Two-Way Flips (flip = -1)","metadata":{}},{"cell_type":"code","source":"# show the augmented image before the correction\nplt.imshow(aug_unannotated_images[lbl][2])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:45.400086Z","iopub.execute_input":"2022-01-18T07:17:45.400329Z","iopub.status.idle":"2022-01-18T07:17:45.630376Z","shell.execute_reply.started":"2022-01-18T07:17:45.400294Z","shell.execute_reply":"2022-01-18T07:17:45.62975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# two-way flip has index = 2 in the list of augmented images from above\n\n# for each image flipped two-ways, do\nfor lbl, augs in aug_unannotated_images.items():\n    # for each chemical element coordinate on that image\n    for idx in range(len(aug_chemical_coordinates[lbl][2])):\n        # unbundle the coordinate\n        x, y, w, h            = aug_chemical_coordinates[lbl][2][idx]\n        # re-flip the box bounded by these coordinates\n        augs[2][y:y+h, x:x+w] = cv2.flip(augs[2][y:y+h, x:x+w], -1)\n        aug_unannotated_images[lbl][2] = augs[2]","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:45.631465Z","iopub.execute_input":"2022-01-18T07:17:45.631717Z","iopub.status.idle":"2022-01-18T07:17:45.682518Z","shell.execute_reply.started":"2022-01-18T07:17:45.631682Z","shell.execute_reply":"2022-01-18T07:17:45.681924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display the example result\nplt.imshow(aug_unannotated_images[lbl][2])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:45.683702Z","iopub.execute_input":"2022-01-18T07:17:45.683951Z","iopub.status.idle":"2022-01-18T07:17:45.90215Z","shell.execute_reply.started":"2022-01-18T07:17:45.683918Z","shell.execute_reply":"2022-01-18T07:17:45.901437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Correct Rotations by +90 Degrees","metadata":{}},{"cell_type":"code","source":"# show the augmented image before the correction\nplt.imshow(aug_unannotated_images[lbl][3])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:45.903334Z","iopub.execute_input":"2022-01-18T07:17:45.903576Z","iopub.status.idle":"2022-01-18T07:17:46.100248Z","shell.execute_reply.started":"2022-01-18T07:17:45.903543Z","shell.execute_reply":"2022-01-18T07:17:46.099577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rotation by +90 has index = 3 in the list of augmented images from above\n\n# for each +90-rotated image, do\nfor lbl, augs in aug_unannotated_images.items():\n    # for each chemical element coordinate on that image\n    for idx in range(len(aug_chemical_coordinates[lbl][3])):\n        # unbundle the coordinatees\n        x, y, w, h                             = aug_chemical_coordinates[lbl][3][idx]\n        # re-rotate the chemical element area\n        chemical_element                       = cv2.rotate(augs[3][y:y+h, x:x+w], cv2.ROTATE_90_COUNTERCLOCKWISE)\n        # get chemical element area shape\n        res_width, res_height                  = chemical_element.shape[0], chemical_element.shape[1]\n        # whiten the original location\n        augs[3][y:y+h, x:x+w]                  = 255\n        # re-insert the rotated chemical element\n        try:\n            augs[3][y:y+res_width, x:x+res_height] = chemical_element\n        except:\n            pass\n        aug_unannotated_images[lbl][3] = augs[3]","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:46.103923Z","iopub.execute_input":"2022-01-18T07:17:46.104466Z","iopub.status.idle":"2022-01-18T07:17:46.171281Z","shell.execute_reply.started":"2022-01-18T07:17:46.104429Z","shell.execute_reply":"2022-01-18T07:17:46.170647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display the example result\nplt.imshow(aug_unannotated_images[lbl][3])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:46.172591Z","iopub.execute_input":"2022-01-18T07:17:46.172848Z","iopub.status.idle":"2022-01-18T07:17:46.407191Z","shell.execute_reply.started":"2022-01-18T07:17:46.172814Z","shell.execute_reply":"2022-01-18T07:17:46.406454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Correct Rotations by -90 Degrees","metadata":{}},{"cell_type":"code","source":"# show the augmented image before the correction\nplt.imshow(aug_unannotated_images[lbl][4])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:46.411997Z","iopub.execute_input":"2022-01-18T07:17:46.414858Z","iopub.status.idle":"2022-01-18T07:17:46.686632Z","shell.execute_reply.started":"2022-01-18T07:17:46.414815Z","shell.execute_reply":"2022-01-18T07:17:46.685983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rotation by -90 has index = 4 in the list of augmented images from above\n\n# for each -90-rotated image, do\nfor lbl, augs in aug_unannotated_images.items():\n    # for each chemical element coordinate on that image\n    for idx in range(len(aug_chemical_coordinates[lbl][4])):\n        # unbundle the coordinates\n        x, y, w, h                             = aug_chemical_coordinates[lbl][4][idx]\n        # re-rotate the chemical element area\n        chemical_element                       = cv2.rotate(augs[4][y:y+h, x:x+w], cv2.ROTATE_90_CLOCKWISE)\n        # get chemical element area shape\n        res_width, res_height                  = chemical_element.shape[0], chemical_element.shape[1]\n        # whiten the original location\n        augs[4][y:y+h, x:x+w]                  = 255\n        # re-insert the rotated chemical element\n        try:\n            augs[4][y:y+res_width, x:x+res_height] = chemical_element\n        except:\n            pass\n        aug_unannotated_images[lbl][4] = augs[4]","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:46.690326Z","iopub.execute_input":"2022-01-18T07:17:46.692347Z","iopub.status.idle":"2022-01-18T07:17:46.792399Z","shell.execute_reply.started":"2022-01-18T07:17:46.692306Z","shell.execute_reply":"2022-01-18T07:17:46.791741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display the example result\nplt.imshow(aug_unannotated_images[lbl][4])","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:46.796102Z","iopub.execute_input":"2022-01-18T07:17:46.79657Z","iopub.status.idle":"2022-01-18T07:17:47.041998Z","shell.execute_reply.started":"2022-01-18T07:17:46.796531Z","shell.execute_reply":"2022-01-18T07:17:47.041324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save the Results in a Dataset","metadata":{}},{"cell_type":"code","source":"# make a new directory for augmented outputs if doesn't exist\npath = 'augmentations/'\ntry:\n    os.mkdir(path)\nexcept:\n    pass\n\n# save each augmented image as PNG under that directory\nfor lbl, augs in aug_unannotated_images.items():\n    idx      = 0\n    for aug in augs:\n        cv2.imwrite(path+lbl+f'-aug{idx}.png', aug)\n        idx += 1","metadata":{"execution":{"iopub.status.busy":"2022-01-18T07:17:47.045682Z","iopub.execute_input":"2022-01-18T07:17:47.047636Z","iopub.status.idle":"2022-01-18T07:17:52.626305Z","shell.execute_reply.started":"2022-01-18T07:17:47.047582Z","shell.execute_reply":"2022-01-18T07:17:52.625576Z"},"trusted":true},"execution_count":null,"outputs":[]}]}