{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Bengali.AI Dataset EDA"},{"metadata":{},"cell_type":"markdown","source":"### List all the files"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load Packages"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport tensorflow.keras as keras\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport cv2\nprint(\"matplotlib version: {}\".format(matplotlib.__version__))\nprint(\"numpy version: {}\".format(np.__version__))\nprint(\"pandas version: {}\".format(pd.__version__))\nprint(\"matplotlib backend: {}\".format(matplotlib.get_backend()))\nprint(\"Tensorflow version: {}\".format(tf.__version__))\nprint(\"tf.keras version: {}\".format(tf.keras.__version__))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Exploration\n\n* Analyse train.csv and one of the image parquet files (train_image_data_0.parquet)"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv_df = pd.read_csv(\"/kaggle/input/bengaliai-cv19/train.csv\")\ntrain_csv_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_image_data_df_0 = pd.read_parquet(\"/kaggle/input/bengaliai-cv19/train_image_data_0.parquet\")\ntrain_image_data_df_0.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total number of training images(rows) in each training parquet file: {}\".format(train_image_data_df_0.shape[0]))\nprint(\"Total number of pixels in each image(colums) of the training dataset: {}\".format(train_image_data_df_0.shape[1]))\nprint(\"Total number of training images: {}\".format(train_csv_df['image_id'].count()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import cycle, islice\n\ntop_n_grapheme_root_df = train_csv_df['grapheme_root'].value_counts().nlargest(15)\ncolor_list = list(islice(cycle(['b', 'r', 'g', 'c', 'y']), None, len(top_n_grapheme_root_df)))\nax = top_n_grapheme_root_df.plot(kind = 'barh', color = color_list, rot = 0, figsize = (15, 10), fontsize = 15)\nax.set_xlabel(xlabel = 'Image Count', fontsize = 20)\nax.set_ylabel(ylabel = 'Grapheme Root', fontsize = 20)\nax.set_title(label = 'Grapheme Root Frequency Plot (TopN)', fontsize = 25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vowel_diacritic_df = train_csv_df['vowel_diacritic'].value_counts()\n\n\ncolor_list = list(islice(cycle(['b', 'r', 'g', 'c', 'y']), None, len(vowel_diacritic_df)))\nax = vowel_diacritic_df.plot(kind = 'barh', color = color_list, rot = 0, figsize = (15, 10), fontsize = 15)\nax.set_xlabel(xlabel = 'Image Count', fontsize = 20)\nax.set_ylabel(ylabel = 'Vowel Diacritic', fontsize = 20)\nax.set_title(label = 'Vowel Diacritic Frequency Plot', fontsize = 25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"consonant_diacritic_df = train_csv_df['consonant_diacritic'].value_counts()\n\n\ncolor_list = list(islice(cycle(['b', 'r', 'g', 'c', 'y']), None, len(consonant_diacritic_df)))\nax = consonant_diacritic_df.plot(kind = 'barh', color = color_list, rot = 0, figsize = (15, 10), fontsize = 15)\nax.set_xlabel(xlabel = 'Image Count', fontsize = 20)\nax.set_ylabel(ylabel = 'Consonant Diacritic', fontsize = 20)\nax.set_title(label = 'Consonant Diacritic Frequency Plot', fontsize = 25)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Analyse test.csv, sample_submission.csv and class_map.csv files"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv_df = pd.read_csv(\"/kaggle/input/bengaliai-cv19/test.csv\")\nsubmission_df = pd.read_csv(\"/kaggle/input/bengaliai-cv19/sample_submission.csv\")\nclass_map_df = pd.read_csv(\"/kaggle/input/bengaliai-cv19/class_map.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Total number of rows in test_csv dataframe: {}\".format(train_csv_df['image_id'].count()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_map_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_map_df.groupby(['component_type'])['label'].max()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display Grapheme Images\n\n* Create an ImageDataGenerator to process and display grapheme images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# one hot encoding of labels\n\none_hot_df = pd.concat([\n    train_csv_df[[\"image_id\"]],\n    pd.get_dummies(train_csv_df.grapheme_root, prefix=\"grapheme_root\"),\n    pd.get_dummies(train_csv_df.vowel_diacritic, prefix=\"vowel_diacritic\"),\n    pd.get_dummies(train_csv_df.consonant_diacritic, prefix=\"consonant_diacritic\"),\n], axis = 1)\n\none_hot_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data_df = train_image_data_df_0.set_index('image_id').join(one_hot_df.set_index('image_id'))\ntrain_data_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Label columns per attribute type\n_consonant_diacritic_cols_ = [col for col in train_data_df.columns if col.startswith(\"consonant_diacritic\")]\n_grapheme_root_cols_ = [col for col in train_data_df.columns if col.startswith(\"grapheme_root\")]\n_vowel_diacritic_cols_ = [col for col in train_data_df.columns if col.startswith(\"vowel_diacritic\")]\n\n\nclass BegaliImageDataGenerator(keras.utils.Sequence):\n\n  def __init__(self, df, augmentation = None, policy = None, new_shape = (128,128) , batch_size=32, shuffle=True):\n        self.df = df.iloc[:, 0:137*236]\n        self.label_df = df.iloc[:, 137*236: ]\n        self.batch_size=batch_size\n        self.shuffle = shuffle\n        self.augment = augmentation\n        self.policy = policy\n        self.on_epoch_end()\n\n  def __len__(self):\n        return int(np.floor(self.df.shape[0] / self.batch_size))\n\n  def __getitem__(self, index):\n        \"\"\"fetch batched images and targets\"\"\"\n        batch_slice = slice(index * self.batch_size, (index + 1) * self.batch_size)\n        items = self.df.iloc[batch_slice]\n        label_items = self.label_df.iloc[batch_slice]\n        if not self.augment:\n          image = np.stack([cv2.resize(np.reshape(item.values, (137, 236)).astype(np.float32), new_shape) for _, item in items.iterrows()])\n        else:\n          image = np.stack([cv2.resize(np.reshape(item.values, (137, 236)).astype(np.float32), new_shape) for _, item in items.iterrows()])\n\n        target = {\n            \"consonant_diacritic_output\": label_items[_consonant_diacritic_cols_].values,\n            \"grapheme_root_output\": label_items[_grapheme_root_cols_].values,\n            \"vowel_diacritic_output\": label_items[_vowel_diacritic_cols_].values,\n        }\n        \n        image = image.reshape(image.shape[0], image.shape[1], image.shape[2], 1)\n        return image, target\n\n  def on_epoch_end(self):\n        \"\"\"Updates indexes after each epoch\"\"\"\n        if self.shuffle == True:\n            self.df = self.df.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Show train images from ImageDataGenerator after resizing to 128x128 size\n\nnew_shape = (128,128)\ntrain_generator32 = BegaliImageDataGenerator(train_data_df, policy = None, batch_size=32, new_shape = new_shape, augmentation = ImageDataGenerator(\n        horizontal_flip=False,\n        vertical_flip=False,\n    ))\n\nx, y = next(iter(train_generator32))\nprint(x.shape)\n\nplt.figure(figsize=(32, 16))\nfor i, (img, label) in enumerate(zip(x, y['consonant_diacritic_output'])):\n    plt.subplot(4, 8, i+1)\n    plt.axis('off')\n    plt.imshow(img.reshape(img.shape[0],img.shape[1]), interpolation=\"nearest\", cmap = 'gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}