{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os,path\nfrom PIL import Image, ImageDraw\nfrom zipfile import ZipFile\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom concurrent.futures import ProcessPoolExecutor\nimport cv2 as cv\nnp.random.seed(1337)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"kaggle_input_folder = path.Path(\"../input\")\nkuzushiji_folder = kaggle_input_folder/\"kuzushiji-recognition\"\nunicode_translation = kuzushiji_folder/\"unicode_translation.csv\"\ntrain_images_zip = kuzushiji_folder/\"train_images.zip\"\ntrain_csv = kuzushiji_folder/\"train.csv\"\ntrain_images_folder = path.Path(\"../temp/train\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_labels = pd.read_csv(train_csv)\nmap_unicode_translation = {unicode_char:japanese_char for unicode_char,japanese_char in pd.read_csv(unicode_translation).iterrows()}\ntrain_zip_interface = ZipFile(train_images_zip)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_file_name = lambda x: \"{}.jpg\".format(x)\ndef get_label_array(x):\n    x = np.array(x.split(' ')).reshape(-1,5)\n    bboxes = x[:,1:].astype(int)\n    labels = x[:,0]\n    return pd.DataFrame.from_dict({'unicode_char':labels.tolist(), 'bboxes':bboxes.tolist()})\nget_japanese_char = lambda x: map_unicode_translation[x]\ndef get_image_as_np_array_from_file(file_handler,file_name):\n    with file_handler.open(file_name) as file:\n        data = Image.open(file)\n        data.load()\n    return np.asarray(data)\n\ndef get_sample_file(file_dir):\n    return np.random.choice(os.listdir(file_dir))\ndef convert_to_gray_scale(image):\n    return np.dot(image[...,:3], [0.299, 0.587, 0.114])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_zip_interface.extractall(train_images_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_file = train_images_folder/get_sample_file(train_images_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FileHandler:\n    def __init__(self,file_name,mode):\n        self.file_name = file_name\n        self.mode=mode\n        \n    def __enter__(self):\n        self.file_obj = open(self.file_name,self.mode)\n        return self.file_obj\n    \n    def __exit__(self,exc_type, exc_value, exc_traceback):\n        if exc_type is None:\n            self.file_obj.close()\n        else:\n            print(exc_value)\n        return True\n    \n    @classmethod\n    def open(cls,file_name,mode='rb'):\n        return cls(file_name,mode)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_arr = get_image_as_np_array_from_file(FileHandler,sample_file)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Get Image Sizes"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_image_size_dimension(image):\n    size = image.shape[:2]\n    if len(image.shape)>2:\n        dimension = image.shape[-1]\n    else:\n        dimension = 1\n    return size, dimension","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_images_from_dir(dir_folder):\n    file_names = os.listdir(dir_folder)\n    for i in file_names:\n        yield get_image_as_np_array_from_file(FileHandler,dir_folder/i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = get_images_from_dir(train_images_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with ProcessPoolExecutor() as executor:\n    future = executor.map(get_image_size_dimension,images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"future = list(future)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_df_from_shape_dim(array):\n    recs = []\n    for h_w, dim in array:\n        h = h_w[0]\n        w = h_w[1]\n        d = dim\n        recs.append([w,h,d])\n    return pd.DataFrame(recs,columns=[\"width\",\"height\",\"dimension\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_shape_dim = get_df_from_shape_dim(future)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_shape_dim[\"dimension\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### We have all the color Images as all the dimensions are of three"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.plotting.scatter_matrix(df_shape_dim[[\"width\",\"height\"]],figsize=(15,10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### It can be observed that Images have different Sizes. Also the Hist Plots are right skewed"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_shape_dim[[\"width\",\"height\"]].describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Looks like our images are centered at 2000,3000 with larger variation in height"},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_images(images,gray_scale=False):\n    images = list(images)\n    n_images = len(images)\n    cols = n_images//2\n    if n_images%2:\n        cols+=1\n    fig,axs = plt.subplots(2,cols,figsize=(10,20))\n    for i_image in range(n_images):\n        col = i_image//2\n        row = i_image%2\n        if gray_scale:\n            axs[row,col].imshow(convert_to_gray_scale(images[i_image]),cmap='gray')\n        else:\n            axs[row,col].imshow(images[i_image])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_sample_images_from_dir(file_dir,n_samples=6):\n    file_names = os.listdir(file_dir)\n    file_names = np.random.choice(file_names,n_samples)\n    for i in file_names:\n        yield get_image_as_np_array_from_file(FileHandler,file_dir/i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = get_sample_images_from_dir(train_images_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_images(images,gray_scale=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### As we would be focused on Characters Only Convert the Images to Grey Scale"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}