{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport PIL.Image\nimport time\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\n%matplotlib inline\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"for dirname,_, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_FOLDER = '/kaggle/input/bengaliai-cv19/'\ntrain_df = pd.read_csv(os.path.join(DATA_FOLDER,'train.csv'))\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(DATA_FOLDER,'test.csv'))\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_map_df = pd.read_csv(os.path.join(DATA_FOLDER,'class_map.csv'))\nclass_map_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_map_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission_df = pd.read_csv(os.path.join(DATA_FOLDER,'sample_submission.csv'))\nsample_submission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\ntrain_0_df = pd.read_parquet(os.path.join(DATA_FOLDER,'train_image_data_0.parquet'))\nprint(f\"'train_image_data_0' read in {round(time.time()-start_time,2)} sec.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\ntrain_1_df = pd.read_parquet(os.path.join(DATA_FOLDER,'train_image_data_1.parquet'))\nprint(f\"'train_image_data_1' read in {round(time.time()-start_time,2)} sec.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Each train_image_data_x(x=0,1,2,3,..) contains 50210 rows and 32333 columns - size of each image(137,230). Total there are 50210 x 4 = 200840 rows in training set","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\ntest_0_df = pd.read_parquet(os.path.join(DATA_FOLDER,'test_image_data_0.parquet'))\nprint(f\"'test_image_data_0' read in{round(time.time()-start_time,2)} sec.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_0_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_0_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now checking the distribution of graphene roots, vowel diacritics and consonant diacritics","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Train: unique graphene roots: {train_df.grapheme_root.nunique()}\")\nprint(f\"Train: unique vowel diacritics: {train_df.vowel_diacritic.nunique()}\")\nprint(f\"Train: unique consonant diacritics: {train_df.consonant_diacritic.nunique()}\")\nprint(f\"Train: total unique elements: {train_df.grapheme_root.nunique() + train_df.vowel_diacritic.nunique() + train_df.consonant_diacritic.nunique()}\")\nprint(f\"Class map: unique elements: \\n{class_map_df.component_type.value_counts()}\")\nprint(f\"Total combinations: {pd.DataFrame(train_df.groupby(['grapheme_root','vowel_diacritic','consonant_diacritic'])).shape[0]}\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cm_gr = class_map_df.loc[(class_map_df.component_type=='grapheme_root'),'component'].values\ncm_vd = class_map_df.loc[(class_map_df.component_type=='vowel_diacritic'),'component'].values\ncm_cd = class_map_df.loc[(class_map_df.component_type=='consonant_diacritic'),'component'].values\nprint(f\"grapheme root:\\n{15*'-'}\\n{cm_gr}\\n\\n vowel diacritic:\\n{18*'-'}\\n{cm_vd}\\n\\n consonant diacritic:\\n{20*'-'}\\n{cm_cd}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def most_frequent_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columsn = ['Total']\n    items=[]\n    vals = []\n    for col in data.columns:\n        itm = data[col].value_counts().index[0]\n        val = data[col].value_counts().values[0]\n        items.append(itm)\n        vals.append(val)\n    tt['Most frequent item']=items\n    tt['Frequency']=vals\n    tt['Percent from total'] = np.round(vals/total*100,3)\n    return(np.transpose(tt))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"most_frequent_values(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"most_frequent_values(test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Distribution of class values","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_count(feature, title, df, size=1):\n    f, ax = plt.subplots(1,1,figsize=(4*size,4))\n    total = float(len(df))\n    g = sns.countplot(df[feature],order=df[feature].value_counts().index[:20],palette='Set3')\n    g.set_title(\"Number and percentage of {}\".format(title))\n    if(size>2):\n        plt.xticks(rotation=90, size=8)\n    for p in ax.patches:\n        height = p.get_height()\n        ax.text(p.get_x()+p.get_width()/2.,\n               height + 3,\n               '{:1.2f}%'.format(100*height/total),\n                   ha=\"center\")\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_count('grapheme_root','grapheme_root(first most frequent 20 values - train)',train_df, size=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_count('vowel_diacritic', 'vowel_diacritic (train)',train_df, size=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_count('consonant_diacritic','consonant_diacritic (train)',train_df, size=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_count_heatmap(feature1, feature2, df, size=1):\n    tmp = train_df.groupby([feature1, feature2])['grapheme'].count()\n    df = tmp.reset_index()\n    df\n    df_m = df.pivot(feature1, feature2, \"grapheme\")\n    f,ax = plt.subplots(figsize=(9, size*4))\n    sns.heatmap(df_m, annot=True, fmt='3.0f',linewidths=.5, ax=ax)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_count_heatmap('vowel_diacritic','consonant_diacritic',train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_count_heatmap('grapheme_root','consonant_diacritic',train_df,size=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_image_from_data(data_df, size=5):\n    plt.figure()\n    fig, ax = plt.subplots(size, size, figsize=(12,12))\n    for i,index in enumerate(data_df.index):\n        image_id = data_df.iloc[i]['image_id']\n        flattened_image = data_df.iloc[i].drop('image_id').values.astype(np.uint8)\n        unpacked_image = PIL.Image.fromarray(flattened_image.reshape(137, 236))\n        ax[i//size, i%size].imshow(unpacked_image)\n        ax[i//size, i%size].set_title(image_id)\n        ax[i//size, i%size].axis('on')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image_from_data(train_0_df.sample(25))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_image_from_data(train_1_df.sample(16), size=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_writting_variety(data_df=train_0_df, grapheme_root=72, vowel_diacritic=0,\\\n                             consonant_diacritic=0, size=5):\n    \n    sample_train_df = train_df.loc[(train_df.grapheme_root == grapheme_root) & \\\n                                  (train_df.vowel_diacritic == vowel_diacritic) & \\\n                                  (train_df.consonant_diacritic == consonant_diacritic)]\n    print(f\"total: {sample_train_df.shape}\")\n    sample_df = data_df.merge(sample_train_df.image_id, how='inner')\n    print(f\"total: {sample_df.shape}\")\n    gr = sample_train_df.iloc[0]['grapheme']\n    cm_gr = class_map_df.loc[(class_map_df.component_type=='grapheme_root')& \\\n                             (class_map_df.label==grapheme_root), 'component'].values[0]\n    cm_vd = class_map_df.loc[(class_map_df.component_type=='vowel_diacritic')& \\\n                             (class_map_df.label==vowel_diacritic), 'component'].values[0]    \n    cm_cd = class_map_df.loc[(class_map_df.component_type=='consonant_diacritic')& \\\n                             (class_map_df.label==consonant_diacritic), 'component'].values[0]    \n    \n    print(f\"grapheme: {gr}, grapheme root: {cm_gr}, vowel discritic: {cm_vd}, consonant diacritic: {cm_cd}\")\n    sample_df = sample_df.sample(size * size)\n    display_image_from_data(sample_df, size=size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_writting_variety(train_0_df,72,1,1,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_writting_variety(train_0_df,64,1,2,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_writting_variety(train_1_df,13,0,0,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_writting_variety(train_1_df,23,3,2,4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}