{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":14897,"databundleVersionId":1020216},{"sourceType":"datasetVersion","sourceId":9530626,"datasetId":5791444,"databundleVersionId":9745320},{"sourceType":"datasetVersion","sourceId":852804,"datasetId":451010,"databundleVersionId":879257},{"sourceType":"datasetVersion","sourceId":187731,"datasetId":80814,"databundleVersionId":198687},{"sourceType":"kernelVersion","sourceId":199165791}],"dockerImageVersionId":30776,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport math\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.applications import DenseNet121\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import layers, optimizers\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom sklearn.model_selection import train_test_split\nfrom skimage.color import rgb2gray\nfrom skimage import measure\nimport seaborn as sns\nimport random\n\nimport cv2\nfrom tqdm import tqdm\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-02T10:04:15.945993Z","iopub.execute_input":"2024-10-02T10:04:15.946733Z","iopub.status.idle":"2024-10-02T10:04:34.797504Z","shell.execute_reply.started":"2024-10-02T10:04:15.946675Z","shell.execute_reply":"2024-10-02T10:04:34.796181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.random.set_seed(42)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:04:34.799641Z","iopub.execute_input":"2024-10-02T10:04:34.800328Z","iopub.status.idle":"2024-10-02T10:04:34.806111Z","shell.execute_reply.started":"2024-10-02T10:04:34.800280Z","shell.execute_reply":"2024-10-02T10:04:34.804746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load_dir = '/kaggle/input/bengaliai/256_train/256/'\n\n# train = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\n# train.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:04:34.807682Z","iopub.execute_input":"2024-10-02T10:04:34.808140Z","iopub.status.idle":"2024-10-02T10:04:34.921177Z","shell.execute_reply.started":"2024-10-02T10:04:34.808086Z","shell.execute_reply":"2024-10-02T10:04:34.919986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = train[30000:105000]\n# len(train)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:14.220288Z","iopub.execute_input":"2024-10-02T08:21:14.220706Z","iopub.status.idle":"2024-10-02T08:21:14.228685Z","shell.execute_reply.started":"2024-10-02T08:21:14.220661Z","shell.execute_reply":"2024-10-02T08:21:14.227379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def getPlot(target='grapheme_root', clr='blue'):\n#     # Get all unique graphemes and count their occurrences\n#     grapheme_counts = train[target].value_counts().reset_index()\n#     grapheme_counts.columns = [target, 'count']\n\n#     # Plot the grapheme frequency as a Seaborn bar chart\n#     plt.figure(figsize=(14, 4))\n#     sns.barplot(x=target, y='count', data=grapheme_counts, color=clr)\n\n#     # Customize the plot\n#     plt.title(f'Frequency of Unique {target} in the Dataset', fontsize=14)\n#     plt.xlabel(target, fontsize=12)\n#     plt.ylabel('Count', fontsize=12)\n#     #plt.xticks([])  # Hide x-axis labels to prevent clutter\n#     plt.xticks(rotation=90) \n#     plt.tight_layout()\n#     plt.savefig(f'{target}.png', dpi=300, bbox_inches='tight')\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:14.230330Z","iopub.execute_input":"2024-10-02T08:21:14.230839Z","iopub.status.idle":"2024-10-02T08:21:14.241589Z","shell.execute_reply.started":"2024-10-02T08:21:14.230784Z","shell.execute_reply":"2024-10-02T08:21:14.240103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getPlot()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:14.749509Z","iopub.execute_input":"2024-10-02T08:21:14.749987Z","iopub.status.idle":"2024-10-02T08:21:18.969827Z","shell.execute_reply.started":"2024-10-02T08:21:14.749945Z","shell.execute_reply":"2024-10-02T08:21:18.968597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getPlot('vowel_diacritic', 'red')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:18.971919Z","iopub.execute_input":"2024-10-02T08:21:18.972347Z","iopub.status.idle":"2024-10-02T08:21:20.125789Z","shell.execute_reply.started":"2024-10-02T08:21:18.972293Z","shell.execute_reply":"2024-10-02T08:21:20.123794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# getPlot('consonant_diacritic', 'green')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:20.128049Z","iopub.execute_input":"2024-10-02T08:21:20.128760Z","iopub.status.idle":"2024-10-02T08:21:21.127134Z","shell.execute_reply.started":"2024-10-02T08:21:20.128703Z","shell.execute_reply":"2024-10-02T08:21:21.125806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train['filename'] = train.image_id.apply(lambda filename: load_dir + filename + '.png')\n# train = train[:75000]\n# len(train)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:21.130264Z","iopub.execute_input":"2024-10-02T08:21:21.130830Z","iopub.status.idle":"2024-10-02T08:21:21.186098Z","shell.execute_reply.started":"2024-10-02T08:21:21.130772Z","shell.execute_reply":"2024-10-02T08:21:21.184410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:21.187646Z","iopub.execute_input":"2024-10-02T08:21:21.188047Z","iopub.status.idle":"2024-10-02T08:21:21.203140Z","shell.execute_reply.started":"2024-10-02T08:21:21.188006Z","shell.execute_reply":"2024-10-02T08:21:21.201741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_df = pd.read_csv('/kaggle/input/bengaliai-cv19/train.csv')\n# print(f'Size of training data: {train_df.shape}')\n# train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:21.204664Z","iopub.execute_input":"2024-10-02T08:21:21.205050Z","iopub.status.idle":"2024-10-02T08:21:21.481787Z","shell.execute_reply.started":"2024-10-02T08:21:21.205010Z","shell.execute_reply":"2024-10-02T08:21:21.480487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f'Number of unique graphemes: {train_df[\"grapheme_root\"].nunique()}')\n# print(f'Number of unique vowel diacritic: {train_df[\"vowel_diacritic\"].nunique()}')\n# print(f'Number of unique consonant diacritic: {train_df[\"consonant_diacritic\"].nunique()}')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:21.483244Z","iopub.execute_input":"2024-10-02T08:21:21.483704Z","iopub.status.idle":"2024-10-02T08:21:21.496644Z","shell.execute_reply.started":"2024-10-02T08:21:21.483660Z","shell.execute_reply":"2024-10-02T08:21:21.495471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pad_width(im, new_shape, is_rgb=True):\n    pad_diff = new_shape - np.array(im.shape[:2])\n    t, b = pad_diff[0] // 2, (pad_diff[0] + 1) // 2\n    l, r = pad_diff[1] // 2, (pad_diff[1] + 1) // 2\n    return ((t, b), (l, r), (0, 0)) if is_rgb else ((t, b), (l, r))\n\ndef crop_object(img, thresh=220, maxval=255, square=True):\n    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    _, thresh_gray = cv2.threshold(gray, thresh, maxval, cv2.THRESH_BINARY_INV)\n    contours, _ = cv2.findContours(thresh_gray, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n\n    x, y, w, h = max([cv2.boundingRect(c) for c in contours], key=lambda b: b[2]*b[3])\n    crop = img[y:y+h, x:x+w]\n\n    if square:\n        pad_width = get_pad_width(crop, max(crop.shape[:2]), is_rgb=(img.ndim == 3))\n        crop = np.pad(crop, pad_width=pad_width, mode='constant', constant_values=255)\n    \n    return crop","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:05:42.053625Z","iopub.execute_input":"2024-10-02T10:05:42.054095Z","iopub.status.idle":"2024-10-02T10:05:42.066352Z","shell.execute_reply.started":"2024-10-02T10:05:42.054052Z","shell.execute_reply":"2024-10-02T10:05:42.065015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datagen = ImageDataGenerator(\n#     rotation_range=5,\n#     width_shift_range=0.1,\n#     height_shift_range=0.1,\n#     shear_range=0.1,\n#     preprocessing_function= tf.keras.applications.xception.preprocess_input\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:05:42.808333Z","iopub.execute_input":"2024-10-02T10:05:42.808757Z","iopub.status.idle":"2024-10-02T10:05:42.814818Z","shell.execute_reply.started":"2024-10-02T10:05:42.808717Z","shell.execute_reply":"2024-10-02T10:05:42.813364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def data_generator(filenames, y, batch_size=128, shape=(64, 64, 1), random_state=42, preprocess=False, augmentation=False):\n#     y = y.copy()\n#     np.random.seed(random_state)\n#     indices = np.arange(len(filenames))\n\n#     while True:\n#         np.random.shuffle(indices)\n\n#         for i in range(0, len(indices), batch_size):\n#             batch_idx = indices[i:i + batch_size]\n#             size = len(batch_idx)\n\n#             batch_files = filenames[batch_idx]\n#             X_batch = np.zeros((size, *shape), dtype=np.float32)\n#             y_batch = y[batch_idx]\n\n#             for j, file in enumerate(batch_files):\n#                 #print(file)\n#                 img = cv2.imread(file)\n#                 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB\n\n#                 if preprocess:\n#                     img = crop_object(img, thresh=250)  # Ensure img is still RGB\n#                     img = cv2.resize(img, shape[:2])  # Resize to (64, 64)\n#                 else:\n#                     img = cv2.resize(img, shape[:2])  # Resize to (64, 64)\n\n#                 if augmentation:\n#                     img = datagen.random_transform(img)\n\n#                 # Convert the RGB image to grayscale\n#                 img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n\n#                 # Reshape to add a channel dimension\n#                 img = img.reshape(*shape)  # Now shape is (64, 64, 1)\n\n#                 X_batch[j] = img / 255.0  # Normalize pixel values to [0, 1]\n\n#             # Convert target labels from list of arrays to a tuple of arrays\n#             y_batch_tuple = tuple([y_batch[:, i] for i in range(y_batch.shape[1])])\n\n#             # Yield the batch and labels\n#             yield X_batch, y_batch_tuple","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:05:43.205827Z","iopub.execute_input":"2024-10-02T10:05:43.206295Z","iopub.status.idle":"2024-10-02T10:05:43.213492Z","shell.execute_reply.started":"2024-10-02T10:05:43.206225Z","shell.execute_reply":"2024-10-02T10:05:43.212105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_files, valid_files, y_train, y_valid = train_test_split(\n#     train.filename.values, \n#     train[['grapheme_root','vowel_diacritic', 'consonant_diacritic']].values, \n#     test_size=0.25, \n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:40.052079Z","iopub.execute_input":"2024-10-02T08:21:40.052544Z","iopub.status.idle":"2024-10-02T08:21:40.072663Z","shell.execute_reply.started":"2024-10-02T08:21:40.052499Z","shell.execute_reply":"2024-10-02T08:21:40.071360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image = cv2.cvtColor(cv2.imread('/kaggle/input/bengaliai/256_train/256/Train_30001.png'), cv2.COLOR_BGR2RGB)\n# plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:55.765704Z","iopub.execute_input":"2024-10-02T08:21:55.766348Z","iopub.status.idle":"2024-10-02T08:21:56.141799Z","shell.execute_reply.started":"2024-10-02T08:21:55.766181Z","shell.execute_reply":"2024-10-02T08:21:56.140368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #generator = data_generator(train_files, y_train, preprocess=False, augmentation=False)\n# generator = data_generator(train_files, y_train)\n# X_batch, _ = next(generator)\n\n# fig, axes = plt.subplots(5, 5, figsize=(5, 5), facecolor='lightgray')\n# axes = axes.flatten()\n\n# for img, ax in zip(X_batch, axes):\n#     ax.imshow(np.squeeze(img))  # Squeeze the image and use grayscale colormap\n#     ax.axis('off')  # Hide axis\n\n# plt.suptitle('Images from Training Set', fontsize=16)\n\n# # Adjust layout\n# plt.tight_layout(rect=[0, 0, 1, 1])  # Leave space for the title\n\n# # Save the figure\n# #plt.savefig('sample_images.png', bbox_inches='tight', dpi=400)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T08:21:40.881468Z","iopub.execute_input":"2024-10-02T08:21:40.882795Z","iopub.status.idle":"2024-10-02T08:21:43.751884Z","shell.execute_reply.started":"2024-10-02T08:21:40.882728Z","shell.execute_reply":"2024-10-02T08:21:43.750651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Usage of the generator\n# #generator = data_generator(train_files, y_train, preprocess=True, augmentation=False)\n# generator = data_generator(train_files, y_train, preprocess=True, augmentation=False)\n# X_batch, _ = next(generator)\n\n# fig, axes = plt.subplots(5, 5, figsize=(5, 5), facecolor='lightgray')\n# axes = axes.flatten()\n\n# for img, ax in zip(X_batch, axes):\n#     ax.imshow(img)  # Display the RGB image\n#     ax.axis('off')  # Hide axis\n\n# plt.suptitle('Preprocessed Images', fontsize=16)\n\n# # Adjust layout\n# plt.tight_layout(rect=[0, 0, 1, 1])  # Leave space for the title\n\n# # Save the figure\n# #plt.savefig('preprocessed.png', bbox_inches='tight', dpi=400)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T05:21:24.959769Z","iopub.execute_input":"2024-10-02T05:21:24.960816Z","iopub.status.idle":"2024-10-02T05:21:26.141041Z","shell.execute_reply.started":"2024-10-02T05:21:24.960725Z","shell.execute_reply":"2024-10-02T05:21:26.139799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Usage of the generator\n# #generator = data_generator(train_files, y_train, preprocess=True, augmentation=True)\n# generator = data_generator(train_files, y_train, preprocess=True, augmentation=True)\n# X_batch, _ = next(generator)\n\n# fig, axes = plt.subplots(5, 5, figsize=(5, 5), facecolor='lightgray')\n# axes = axes.flatten()\n\n# for img, ax in zip(X_batch, axes):\n#     ax.imshow(img)  # Display the RGB image\n#     ax.axis('off')  # Hide axis\n\n# plt.suptitle('Augmented + Preprocessed Images', fontsize=16)\n\n# # Adjust layout\n# plt.tight_layout(rect=[0, 0, 1, 1])  # Leave space for the title\n\n# # Save the figure\n# #plt.savefig('augmented.png', bbox_inches='tight', dpi=400)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T05:21:39.456897Z","iopub.execute_input":"2024-10-02T05:21:39.457348Z","iopub.status.idle":"2024-10-02T05:21:40.909534Z","shell.execute_reply.started":"2024-10-02T05:21:39.457305Z","shell.execute_reply":"2024-10-02T05:21:40.908263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# batch_size = 128\n\n# #train_gen = data_generator(train_files, y_train, preprocess=True, augmentation=True)\n# #valid_gen = data_generator(valid_files, y_valid, preprocess=True, augmentation=True)\n\n# train_gen = data_generator(train_files, y_train, preprocess=True, augmentation=True)\n# valid_gen = data_generator(valid_files, y_valid, preprocess=True, augmentation=True)\n                           \n# print((len(train_files) , len(valid_files)))\n\n# train_steps = len(train_files) // batch_size\n# valid_steps = len(valid_files) // batch_size","metadata":{"execution":{"iopub.status.busy":"2024-10-02T05:21:43.271707Z","iopub.execute_input":"2024-10-02T05:21:43.272189Z","iopub.status.idle":"2024-10-02T05:21:43.280351Z","shell.execute_reply.started":"2024-10-02T05:21:43.272145Z","shell.execute_reply":"2024-10-02T05:21:43.279010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from tensorflow.keras.layers import Input, Conv2D, SeparableConv2D, BatchNormalization, MaxPool2D, Dropout, Dense, Flatten, GlobalAveragePooling2D\n# from tensorflow.keras.models import Model\n\n# def bengali_ai():\n#     inputs = Input(shape=(64, 64, 1))\n\n#     x = SeparableConv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu')(inputs)\n#     x = SeparableConv2D(filters=32, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = BatchNormalization()(x)\n#     x = MaxPool2D(pool_size=(2, 2))(x)\n#     x = Dropout(0.15)(x)\n\n#     x = SeparableConv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = Conv2D(filters=64, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = BatchNormalization()(x)\n#     x = MaxPool2D(pool_size=(2, 2))(x)\n\n#     x = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = Conv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = MaxPool2D(pool_size=(2, 2))(x)\n    \n#     x = Dropout(0.2)(x)\n\n#     x = SeparableConv2D(filters=128, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = SeparableConv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = BatchNormalization()(x)\n#     x = MaxPool2D(pool_size=(2, 2))(x)\n    \n#     x = SeparableConv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = SeparableConv2D(filters=256, kernel_size=(3, 3), padding='SAME', activation='relu')(x)\n#     x = MaxPool2D(pool_size=(2, 2))(x)\n#     x = Dropout(0.15)(x)\n    \n#     x = GlobalAveragePooling2D()(x)\n#     x = BatchNormalization()(x)\n#     # Dense Layers\n    \n#     x = Dense(512, activation=\"relu\")(x)\n#     x = Dropout(0.2)(x)\n\n#     # Output Layers\n#     root_out = layers.Dense(168, activation='softmax', name='grapheme')(x)\n#     vowel_out = layers.Dense(11, activation='softmax', name='vowel')(x)\n#     consonant_out = layers.Dense(7, activation='softmax', name='consonant')(x)\n\n#     # Final Model\n#     model = Model(inputs=inputs, outputs=[root_out, vowel_out, consonant_out], name='rono27')\n\n#     return model\n\n# model = bengali_ai()\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:48.739685Z","iopub.execute_input":"2024-10-02T04:49:48.740181Z","iopub.status.idle":"2024-10-02T04:49:48.751491Z","shell.execute_reply.started":"2024-10-02T04:49:48.740122Z","shell.execute_reply":"2024-10-02T04:49:48.750197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model('/kaggle/input/bengali-ai-grapehem-classification-models/xception grapheme grayscale.h5')\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:07.352443Z","iopub.execute_input":"2024-10-02T10:06:07.352894Z","iopub.status.idle":"2024-10-02T10:06:10.150491Z","shell.execute_reply.started":"2024-10-02T10:06:07.352856Z","shell.execute_reply":"2024-10-02T10:06:10.149213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model.save_weights('bengali.ai custom.weights.h5')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.536153Z","iopub.execute_input":"2024-10-02T04:49:51.536519Z","iopub.status.idle":"2024-10-02T04:49:51.542088Z","shell.execute_reply.started":"2024-10-02T04:49:51.536480Z","shell.execute_reply":"2024-10-02T04:49:51.540663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(model.layers)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.543807Z","iopub.execute_input":"2024-10-02T04:49:51.545211Z","iopub.status.idle":"2024-10-02T04:49:51.558389Z","shell.execute_reply.started":"2024-10-02T04:49:51.545125Z","shell.execute_reply":"2024-10-02T04:49:51.556909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tf.keras.utils.plot_model(model, to_file='model.png', show_shapes=True, show_layer_activations=True) # , show_trainable=True","metadata":{"execution":{"iopub.status.busy":"2024-10-02T05:21:58.818788Z","iopub.execute_input":"2024-10-02T05:21:58.819175Z","iopub.status.idle":"2024-10-02T05:22:00.261283Z","shell.execute_reply.started":"2024-10-02T05:21:58.819135Z","shell.execute_reply":"2024-10-02T05:22:00.260118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.compile(\n#     optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001),\n#     loss={\n#         'grapheme': 'sparse_categorical_crossentropy',\n#         'vowel': 'sparse_categorical_crossentropy',\n#         'consonant': 'sparse_categorical_crossentropy'\n#     },\n#     metrics={\n#         'grapheme': 'accuracy',\n#         'vowel': 'accuracy',\n#         'consonant': 'accuracy'\n#     }\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-02T05:22:06.846521Z","iopub.execute_input":"2024-10-02T05:22:06.846982Z","iopub.status.idle":"2024-10-02T05:22:06.859937Z","shell.execute_reply.started":"2024-10-02T05:22:06.846940Z","shell.execute_reply":"2024-10-02T05:22:06.858680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n#     filepath='xception.weights.h5',\n#     monitor='val_loss',\n#     mode='min',\n#     save_freq = 'epoch',\n#     save_weights_only=True,\n#     save_best_only=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T05:22:24.078586Z","iopub.execute_input":"2024-10-02T05:22:24.079021Z","iopub.status.idle":"2024-10-02T05:22:24.085193Z","shell.execute_reply.started":"2024-10-02T05:22:24.078981Z","shell.execute_reply":"2024-10-02T05:22:24.083818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# early_stopping_callback = tf.keras.callbacks.EarlyStopping(\n#     monitor='val_loss',  # Monitor validation accuracy\n#     min_delta=0.001,         # Minimum change in monitored value to qualify as improvement\n#     patience=5,             # Stop after 10 epochs of no improvement\n#     mode='min',              # Maximize the validation accuracy\n#     restore_best_weights=True,  # Restore model weights from the best epoch\n#     verbose=1\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.601033Z","iopub.execute_input":"2024-10-02T04:49:51.601443Z","iopub.status.idle":"2024-10-02T04:49:51.610423Z","shell.execute_reply.started":"2024-10-02T04:49:51.601402Z","shell.execute_reply":"2024-10-02T04:49:51.609082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reduce_lr = ReduceLROnPlateau(\n#     monitor='val_loss',       # Monitor validation loss\n#     factor=0.9,               # Reduce learning rate by half\n#     patience=2,               # Wait 3 epochs before reducing learning rate\n#     verbose=1,                # Print learning rate reduction messages\n#     min_lr=1e-10               # Do not reduce learning rate below this threshold\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.612185Z","iopub.execute_input":"2024-10-02T04:49:51.612586Z","iopub.status.idle":"2024-10-02T04:49:51.623394Z","shell.execute_reply.started":"2024-10-02T04:49:51.612541Z","shell.execute_reply":"2024-10-02T04:49:51.622098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_history = model.fit(\n#     train_gen,\n#     steps_per_epoch=train_steps,\n#     epochs=50,\n#     validation_data=valid_gen,\n#     validation_steps=valid_steps,\n#     callbacks=[early_stopping_callback, reduce_lr]\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.625141Z","iopub.execute_input":"2024-10-02T04:49:51.625612Z","iopub.status.idle":"2024-10-02T04:49:51.636021Z","shell.execute_reply.started":"2024-10-02T04:49:51.625556Z","shell.execute_reply":"2024-10-02T04:49:51.634637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_history.history.keys()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.637764Z","iopub.execute_input":"2024-10-02T04:49:51.638323Z","iopub.status.idle":"2024-10-02T04:49:51.651196Z","shell.execute_reply.started":"2024-10-02T04:49:51.638270Z","shell.execute_reply":"2024-10-02T04:49:51.649703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.save('xception grapheme grayscale.h5')","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.653181Z","iopub.execute_input":"2024-10-02T04:49:51.653624Z","iopub.status.idle":"2024-10-02T04:49:51.662995Z","shell.execute_reply.started":"2024-10-02T04:49:51.653572Z","shell.execute_reply":"2024-10-02T04:49:51.661681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.style.use('ggplot')\n# plt.figure(figsize = (10, 5))\n# plt.plot(train_history.history['loss'], '--o', label='train loss')\n# plt.plot(train_history.history['val_loss'], '--o', label='val loss')\n# plt.legend()\n# plt.title('training loss & val loss')\n# plt.savefig('fig_total_loss.png', format='png', dpi=400)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.664943Z","iopub.execute_input":"2024-10-02T04:49:51.665777Z","iopub.status.idle":"2024-10-02T04:49:51.676120Z","shell.execute_reply.started":"2024-10-02T04:49:51.665711Z","shell.execute_reply":"2024-10-02T04:49:51.674516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.style.use('ggplot')\n# plt.figure(figsize = (10, 5))\n# plt.plot(train_history.history['grapheme_accuracy'], '--o', label='grapheme accuracy')\n# plt.plot(train_history.history['val_grapheme_accuracy'], '--o', label='val grapheme accuracy')\n# plt.legend()\n# plt.title('training grapheme acc & val grapheme acc')\n# plt.savefig('fig_grapheme_acc.png', format='png', dpi=400)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.678460Z","iopub.execute_input":"2024-10-02T04:49:51.678939Z","iopub.status.idle":"2024-10-02T04:49:51.694663Z","shell.execute_reply.started":"2024-10-02T04:49:51.678892Z","shell.execute_reply":"2024-10-02T04:49:51.693218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.style.use('ggplot')\n# plt.figure(figsize = (10, 5))\n# plt.plot(train_history.history['vowel_accuracy'], '--o', label='vowel accuracy')\n# plt.plot(train_history.history['val_vowel_accuracy'], '--o', label='val vowel accuracy')\n# plt.legend()\n# plt.title('training vowel acc & val vowel acc')\n# plt.savefig('fig_vowel_acc.png', format='png', dpi=400)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.696440Z","iopub.execute_input":"2024-10-02T04:49:51.697333Z","iopub.status.idle":"2024-10-02T04:49:51.707100Z","shell.execute_reply.started":"2024-10-02T04:49:51.697265Z","shell.execute_reply":"2024-10-02T04:49:51.705404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.style.use('ggplot')\n# plt.figure(figsize = (10, 5))\n# plt.plot(train_history.history['consonant_accuracy'], '--o', label='consonant accuracy')\n# plt.plot(train_history.history['val_consonant_accuracy'], '--o', label='val consonant accuracy')\n# plt.legend()\n# plt.title('training consonant acc & val consonant acc')\n# plt.savefig('fig_consonant_acc.png', format='png', dpi=400)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.709188Z","iopub.execute_input":"2024-10-02T04:49:51.710067Z","iopub.status.idle":"2024-10-02T04:49:51.720457Z","shell.execute_reply.started":"2024-10-02T04:49:51.709988Z","shell.execute_reply":"2024-10-02T04:49:51.719156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.DataFrame(train_history.history).to_csv('history.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.722441Z","iopub.execute_input":"2024-10-02T04:49:51.722916Z","iopub.status.idle":"2024-10-02T04:49:51.733903Z","shell.execute_reply.started":"2024-10-02T04:49:51.722873Z","shell.execute_reply":"2024-10-02T04:49:51.732255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = pd.read_csv('/kaggle/working/history.csv')\n# print(df.shape) \n# df.head(df.shape[0])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T04:49:51.736045Z","iopub.execute_input":"2024-10-02T04:49:51.736558Z","iopub.status.idle":"2024-10-02T04:49:51.750229Z","shell.execute_reply.started":"2024-10-02T04:49:51.736511Z","shell.execute_reply":"2024-10-02T04:49:51.748561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preds","metadata":{}},{"cell_type":"code","source":"class_map = pd.read_csv('/kaggle/input/bengaliai-cv19/class_map.csv')\nclass_map.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:34.897520Z","iopub.execute_input":"2024-10-02T10:06:34.898005Z","iopub.status.idle":"2024-10-02T10:06:34.935504Z","shell.execute_reply.started":"2024-10-02T10:06:34.897963Z","shell.execute_reply":"2024-10-02T10:06:34.934331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grouped = class_map.groupby('component_type')\n\n# Create sub-DataFrames\ngrapheme_root_df = grouped.get_group('grapheme_root')\nvowel_diacritic_df = grouped.get_group('vowel_diacritic')\nconsonant_diacritic_df = grouped.get_group('consonant_diacritic')\n\n# Display the sub-DataFrames\nprint(\"Grapheme Root DataFrame:\")\nprint(grapheme_root_df)\n\nprint(\"\\nConsonant Diacritic DataFrame:\")\nprint(consonant_diacritic_df)\n\nprint(\"\\nvowel Diacritic DataFrame:\")\nprint(vowel_diacritic_df)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:35.125286Z","iopub.execute_input":"2024-10-02T10:06:35.125761Z","iopub.status.idle":"2024-10-02T10:06:35.154277Z","shell.execute_reply.started":"2024-10-02T10:06:35.125713Z","shell.execute_reply":"2024-10-02T10:06:35.153055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grapheme_root_df.drop('component_type', axis=1, inplace=True)\nvowel_diacritic_df.drop('component_type', axis=1, inplace=True)\nconsonant_diacritic_df.drop('component_type', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:35.659543Z","iopub.execute_input":"2024-10-02T10:06:35.660067Z","iopub.status.idle":"2024-10-02T10:06:35.676046Z","shell.execute_reply.started":"2024-10-02T10:06:35.660020Z","shell.execute_reply":"2024-10-02T10:06:35.674442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grapheme, vowel, consonant = grapheme_root_df.to_numpy(), vowel_diacritic_df.to_numpy(), consonant_diacritic_df.to_numpy()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:36.330995Z","iopub.execute_input":"2024-10-02T10:06:36.331490Z","iopub.status.idle":"2024-10-02T10:06:36.338787Z","shell.execute_reply.started":"2024-10-02T10:06:36.331444Z","shell.execute_reply":"2024-10-02T10:06:36.337007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(grapheme[:10])\nprint(vowel[:10])\nprint(consonant[:10])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:37.772733Z","iopub.execute_input":"2024-10-02T10:06:37.773182Z","iopub.status.idle":"2024-10-02T10:06:37.780401Z","shell.execute_reply.started":"2024-10-02T10:06:37.773141Z","shell.execute_reply":"2024-10-02T10:06:37.779091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pad_width(im, new_shape, is_rgb=True):\n    pad_diff = new_shape - np.array(im.shape[:2])\n    t, b = pad_diff[0] // 2, (pad_diff[0] + 1) // 2\n    l, r = pad_diff[1] // 2, (pad_diff[1] + 1) // 2\n    return ((t, b), (l, r), (0, 0)) if is_rgb else ((t, b), (l, r))\n\ndef crop_object(img, thresh=220, maxval=255, square=True):\n    # Check if the image is grayscale and convert to RGB\n    if img.ndim == 2:  # Grayscale image\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)\n\n    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n    _, thresh_gray = cv2.threshold(gray, thresh, maxval, cv2.THRESH_BINARY_INV)\n    contours, _ = cv2.findContours(thresh_gray, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n\n    if contours:  # Check if contours are found\n        x, y, w, h = max([cv2.boundingRect(c) for c in contours], key=lambda b: b[2]*b[3])\n        crop = img[y:y+h, x:x+w]\n\n        if square:\n            pad_width = get_pad_width(crop, max(crop.shape[:2]), is_rgb=True)\n            crop = np.pad(crop, pad_width=pad_width, mode='constant', constant_values=255)\n\n        return crop\n    else:\n        return img  # Return original image if no contours are found","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:38.340695Z","iopub.execute_input":"2024-10-02T10:06:38.341149Z","iopub.status.idle":"2024-10-02T10:06:38.354819Z","shell.execute_reply.started":"2024-10-02T10:06:38.341101Z","shell.execute_reply":"2024-10-02T10:06:38.353636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # List to hold temporary DataFrames\n# dataframes = []\n\n# for i in tqdm(range(4)):\n#     temp = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/train_image_data_{i}.parquet')\n#     dataframes.append(temp)  # Add the temporary DataFrame to the list\n#     del temp  # Delete the temporary variable to free memory\n#     gc.collect()  # Force garbage collection\n\n# # Concatenate all DataFrames at once\n# test_df = pd.concat(dataframes, ignore_index=True)\n\n# # Clean up the dataframes list\n# del dataframes\n# gc.collect()  # Optional: force garbage collection again\n\n# # Display the head of the concatenated DataFrame\n# test_df.head(test_df.shape[0])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:40.301054Z","iopub.execute_input":"2024-10-02T10:06:40.302084Z","iopub.status.idle":"2024-10-02T10:06:40.307767Z","shell.execute_reply.started":"2024-10-02T10:06:40.302031Z","shell.execute_reply":"2024-10-02T10:06:40.306322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test = test_df.drop('image_id', axis=1)\n# test.head(test.shape[0])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:40.771202Z","iopub.execute_input":"2024-10-02T10:06:40.772341Z","iopub.status.idle":"2024-10-02T10:06:40.779196Z","shell.execute_reply.started":"2024-10-02T10:06:40.772275Z","shell.execute_reply":"2024-10-02T10:06:40.777499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# first_row = test.iloc[4]\n# first_row = first_row.to_numpy()\n# first_row = np.array(first_row)\n# first_row.resize((137, 236))\n# resized_image = crop_object(first_row, 250)\n# resized_image = cv2.resize(resized_image, (64, 64))\n# resized_image = cv2.cvtColor(resized_image, cv2.COLOR_RGB2GRAY)\n# resized_image = resized_image / 255.\n# plt.imshow(resized_image)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:41.445089Z","iopub.execute_input":"2024-10-02T10:06:41.445811Z","iopub.status.idle":"2024-10-02T10:06:41.452833Z","shell.execute_reply.started":"2024-10-02T10:06:41.445763Z","shell.execute_reply":"2024-10-02T10:06:41.451521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# first_row = test.iloc[0]\n\n# first_row_array = first_row.to_numpy() \n\n# first_row_image = first_row_array.reshape(137, 236) \n# plt.imshow(first_row_image, cmap='gray')  # Use 'gray' for grayscale images\n# plt.axis('off')  # Hide axis\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:42.645025Z","iopub.execute_input":"2024-10-02T10:06:42.645507Z","iopub.status.idle":"2024-10-02T10:06:42.652130Z","shell.execute_reply.started":"2024-10-02T10:06:42.645464Z","shell.execute_reply":"2024-10-02T10:06:42.650419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getImg(df):\n  images = []\n  for i in range(df.shape[0]):\n    first_row = df.iloc[i]\n    first_row = first_row.to_numpy()\n    first_row = np.array(first_row)\n    first_row.resize((137, 236))\n    resized_image = crop_object(first_row, 250)\n    resized_image = cv2.resize(resized_image, (64, 64))\n    resized_image = cv2.cvtColor(resized_image, cv2.COLOR_RGB2GRAY)\n    resized_image = resized_image / 255.\n    plt.imshow(resized_image)\n    images.append(resized_image)\n  return images","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:43.057059Z","iopub.execute_input":"2024-10-02T10:06:43.057682Z","iopub.status.idle":"2024-10-02T10:06:43.066461Z","shell.execute_reply.started":"2024-10-02T10:06:43.057618Z","shell.execute_reply":"2024-10-02T10:06:43.064683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# images = getImg(test)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:43.557030Z","iopub.execute_input":"2024-10-02T10:06:43.557669Z","iopub.status.idle":"2024-10-02T10:06:43.564427Z","shell.execute_reply.started":"2024-10-02T10:06:43.557580Z","shell.execute_reply":"2024-10-02T10:06:43.562652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getPred(img, getIndice=False):\n  pred = model.predict(np.expand_dims(img, axis=0), verbose=0)\n  grapheme_pred = pred[0]  # Predictions for grapheme\n  vowel_pred = pred[1]      # Predictions for vowel\n  consonant_pred = pred[2]  # Predictions for consonant\n\n  grapheme_label = np.argmax(grapheme_pred, axis=-1)\n  vowel_label = np.argmax(vowel_pred, axis=-1)\n  consonant_label = np.argmax(consonant_pred, axis=-1)\n\n  if not getIndice:\n    print(\"Grapheme Prediction:\", grapheme[grapheme_label][0][1])\n    print(\"Vowel Prediction:\", vowel[vowel_label][0][1])\n    print(\"Consonant Prediction:\", consonant[consonant_label][0][1])\n  else:\n    return consonant_label[0], grapheme_label[0], vowel_label[0],","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:43.892945Z","iopub.execute_input":"2024-10-02T10:06:43.893400Z","iopub.status.idle":"2024-10-02T10:06:43.904289Z","shell.execute_reply.started":"2024-10-02T10:06:43.893357Z","shell.execute_reply":"2024-10-02T10:06:43.902736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# i = random.randint(0, len(images)-1)\n# img = images[i]\n# plt.imshow(img)\n# getPred(img)","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:44.699922Z","iopub.execute_input":"2024-10-02T10:06:44.700400Z","iopub.status.idle":"2024-10-02T10:06:44.706425Z","shell.execute_reply.started":"2024-10-02T10:06:44.700353Z","shell.execute_reply":"2024-10-02T10:06:44.704311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_test_img = pd.read_parquet('/kaggle/input/bengaliai-cv19/test_image_data_3.parquet') \n# df_test_img.head(df_test_img.shape[0])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:06:44.955878Z","iopub.execute_input":"2024-10-02T10:06:44.956343Z","iopub.status.idle":"2024-10-02T10:06:44.961805Z","shell.execute_reply.started":"2024-10-02T10:06:44.956299Z","shell.execute_reply":"2024-10-02T10:06:44.960298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Initialize lists for row_id and target\n# row_id = []\n# target = []\n# batch_size = 100  # Adjust based on your memory limits\n\n# # Use tqdm to iterate over the range of batches\n# for start in tqdm(range(0, test_df.shape[0], batch_size), desc=\"Processing batches\"):\n#     end = min(start + batch_size, test_df.shape[0])\n#     batch_df = test_df.iloc[start:end]\n\n#     # Use tqdm to iterate over rows in the current batch\n#     for idx, row in tqdm(batch_df.iterrows(), total=batch_df.shape[0], desc=\"Predicting images\", leave=False):\n#         image_data = row[1:].to_numpy()\n#         image_data = np.array(image_data, dtype=np.uint8)\n#         image_data.resize((137, 236))\n#         resized_image = crop_object(image_data, 250)\n#         resized_image = cv2.resize(resized_image, (64, 64))\n#         resized_image = cv2.cvtColor(resized_image, cv2.COLOR_RGB2GRAY)\n#         resized_image = resized_image / 255.\n\n#         grapheme_pred, vowel_pred, consonant_pred = getPred(resized_image, getIndice=True)\n        \n#         row_id.append(f'{row[\"image_id\"]}_consonant_diacritic')\n#         target.append(consonant_pred)\n        \n#         row_id.append(f'{row[\"image_id\"]}_grapheme_root')\n#         target.append(grapheme_pred)\n\n#         row_id.append(f'{row[\"image_id\"]}_vowel_diacritic')\n#         target.append(vowel_pred)\n\n#         # Clean up to free memory\n#         del image_data, resized_image, grapheme_pred, vowel_pred, consonant_pred\n#         gc.collect()\n\n# # Create and save submission DataFrame\n# submission_df = pd.DataFrame({'row_id': row_id, 'target': target})\n# submission_df.to_csv('submission.csv', index=False)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-10-02T10:06:44.973546Z","iopub.execute_input":"2024-10-02T10:06:44.973991Z","iopub.status.idle":"2024-10-02T10:06:44.980975Z","shell.execute_reply.started":"2024-10-02T10:06:44.973944Z","shell.execute_reply":"2024-10-02T10:06:44.979625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"components = ['consonant_diacritic', 'grapheme_root', 'vowel_diacritic']\nrow_id = []  # row_id placeholder\ntarget = []  # model predictions placeholder\nbatch_size = 32  # Batch size for prediction\nIMG_SIZE = 64  # Resized image dimensions\nN_CHANNELS = 1  # Number of channels (grayscale)\n\n# Iterate over all test image parquet files (assuming there are 4)\nfor i in tqdm(range(4)):\n    # Read parquet file\n    df_test_img = pd.read_parquet(f'/kaggle/input/bengaliai-cv19/test_image_data_{i}.parquet')\n    df_test_img.set_index('image_id', inplace=True)\n    \n    # Initialize dictionary to store predictions for each component\n    preds_dict = {comp: [] for comp in components}\n    \n    # Preprocess and resize the images\n    test_images = []\n    for idx, (image_id, row) in enumerate(tqdm(df_test_img.iterrows(), desc=f\"Processing rows for test_image_data_{i}\")):\n        image_data = row.to_numpy(dtype=np.uint8).reshape(137, 236)\n        #resized_image = tf.keras.applications.xception.preprocess_input(resized_image)\n        resized_image = crop_object(image_data, 250)\n        resized_image = cv2.resize(resized_image, (IMG_SIZE, IMG_SIZE))\n        resized_image = cv2.cvtColor(resized_image, cv2.COLOR_RGB2GRAY) / 255.0\n        test_images.append(resized_image)\n\n        # Print the current row being processed\n        if (idx + 1) % 100 == 0 or idx == 0:  # Every 100 rows or the first row\n            print(f\"Processing row {idx + 1} of {len(df_test_img)} in test_image_data_{i}\")\n    \n    # Convert list to numpy array and reshape for model input\n    test_images = np.array(test_images).reshape(-1, IMG_SIZE, IMG_SIZE, N_CHANNELS)\n    \n    # Process in batches for prediction\n    for start in tqdm(range(0, len(test_images), batch_size), desc=f\"Processing batches for test_image_data_{i}\"):\n        end = min(start + batch_size, len(test_images))\n        batch_images = test_images[start:end]\n        \n        # Predict batch\n        preds = model.predict(batch_images, verbose=0)\n        \n        # Assign predictions for each component correctly\n        consonant_preds = np.argmax(preds[2], axis=1)  # Predictions for consonant_diacritic\n        grapheme_preds = np.argmax(preds[0], axis=1)  # Predictions for grapheme_root\n        vowel_preds = np.argmax(preds[1], axis=1)  # Predictions for vowel_diacritic\n        \n        # Store predictions in preds_dict\n        preds_dict['consonant_diacritic'].extend(consonant_preds)\n        preds_dict['grapheme_root'].extend(grapheme_preds)\n        preds_dict['vowel_diacritic'].extend(vowel_preds)\n    \n    # Populate row_id and target lists based on predictions\n    for k, image_id in enumerate(df_test_img.index.values):\n        for comp in components:\n            row_id.append(f'{image_id}_{comp}')\n            target.append(preds_dict[comp][k])\n    \n    # Clean up memory after each parquet file\n    del df_test_img, test_images, preds_dict\n    gc.collect()\n\n# Create submission DataFrame\ndf_sample = pd.DataFrame({\n    'row_id': row_id,\n    'target': target\n})\n\n# Save to CSV\ndf_sample.to_csv('submission.csv', index=False)\ndf_sample.head()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-10-02T10:28:20.583620Z","iopub.execute_input":"2024-10-02T10:28:20.584111Z","iopub.status.idle":"2024-10-02T10:28:29.841982Z","shell.execute_reply.started":"2024-10-02T10:28:20.584065Z","shell.execute_reply":"2024-10-02T10:28:29.840722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission = pd.read_csv('submission.csv')\n# submission.head(submission.shape[0])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:28:29.843834Z","iopub.execute_input":"2024-10-02T10:28:29.844229Z","iopub.status.idle":"2024-10-02T10:28:29.849931Z","shell.execute_reply.started":"2024-10-02T10:28:29.844186Z","shell.execute_reply":"2024-10-02T10:28:29.848556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample.head(df_sample.shape[0])","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:28:29.851709Z","iopub.execute_input":"2024-10-02T10:28:29.852139Z","iopub.status.idle":"2024-10-02T10:28:29.873477Z","shell.execute_reply.started":"2024-10-02T10:28:29.852097Z","shell.execute_reply":"2024-10-02T10:28:29.872143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}