{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":6799,"databundleVersionId":4225553,"sourceType":"competition"},{"sourceId":998277,"sourceType":"datasetVersion","datasetId":547506},{"sourceId":8196408,"sourceType":"datasetVersion","datasetId":4854960}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Input, Dense, Flatten, Reshape, Conv2D, MaxPool2D, Conv1D, Lambda, Concatenate, ZeroPadding1D, Activation\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.models import Model, Sequential\nimport cv2\nimport matplotlib.pyplot as plt\nimport os \nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom skimage.metrics import structural_similarity as ssim\nfrom skimage.metrics import peak_signal_noise_ratio as psnr","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:34.541516Z","iopub.execute_input":"2024-04-23T21:12:34.541817Z","iopub.status.idle":"2024-04-23T21:12:38.608036Z","shell.execute_reply.started":"2024-04-23T21:12:34.54179Z","shell.execute_reply":"2024-04-23T21:12:38.606966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Function to create text to image","metadata":{}},{"cell_type":"code","source":"# Custom Activation Function\ndef custom_activation(x):\n    return tf.math.round(tf.math.sigmoid(x) * 255)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:38.60949Z","iopub.execute_input":"2024-04-23T21:12:38.610285Z","iopub.status.idle":"2024-04-23T21:12:38.615712Z","shell.execute_reply.started":"2024-04-23T21:12:38.610244Z","shell.execute_reply":"2024-04-23T21:12:38.614712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to convert text to binary\ndef text_to_image(text):\n    image = []\n    case_flags = []  # Array to store case information\n    \n    for char in text:\n        if char.isupper():\n            image.append((ord(char) - ord(\"A\")) * 255 / 24)\n            case_flags.append(0)  # 0 indicates uppercase\n        elif char.islower():\n            image.append((ord(char) - ord(\"a\")) * 255 / 24)\n            case_flags.append(1)  # 1 indicates lowercase\n        else:\n            image.append((ord(char) - ord(\"0\")) * 255 / 24)\n            case_flags.append(2)\n    return np.array(image), np.array(case_flags)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:38.616804Z","iopub.execute_input":"2024-04-23T21:12:38.617064Z","iopub.status.idle":"2024-04-23T21:12:38.636571Z","shell.execute_reply.started":"2024-04-23T21:12:38.617041Z","shell.execute_reply":"2024-04-23T21:12:38.635689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to convert binary to text\ndef image_to_text(image, case_flags):\n    text = \"\"\n    for value, flag in zip(image, case_flags):\n        if flag == 0:  # Uppercase\n            char_value = int((value * 24 / 255) + ord(\"A\"))\n        elif flag == 1:  # Lowercase\n            char_value = int((value * 24 / 255) + ord(\"a\"))\n        \n        else:\n            char_value = int((value * 24 / 255) + ord(\"0\"))\n        text += chr(char_value)\n    return text","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:38.639215Z","iopub.execute_input":"2024-04-23T21:12:38.639476Z","iopub.status.idle":"2024-04-23T21:12:38.649348Z","shell.execute_reply.started":"2024-04-23T21:12:38.639452Z","shell.execute_reply":"2024-04-23T21:12:38.648604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data paths\ntrain_paths = ['/kaggle/input/imagenetmini-1000/imagenet-mini/train/n01440764','/kaggle/input/imagenetmini-1000/imagenet-mini/train/n01443537']\n\nval_paths = ['/kaggle/input/imagenetmini-1000/imagenet-mini/val/n01440764','/kaggle/input/imagenetmini-1000/imagenet-mini/val/n01443537']","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:38.650532Z","iopub.execute_input":"2024-04-23T21:12:38.651122Z","iopub.status.idle":"2024-04-23T21:12:38.660098Z","shell.execute_reply.started":"2024-04-23T21:12:38.651098Z","shell.execute_reply":"2024-04-23T21:12:38.659251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to load and preprocess image\ndef load_image(img_path):\n    img = cv2.imread(img_path)\n    img = cv2.resize(img, (256, 256))\n    img = img / 255.0\n    return img\n\n# Load training data\ntrain_images = []","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:38.66124Z","iopub.execute_input":"2024-04-23T21:12:38.661527Z","iopub.status.idle":"2024-04-23T21:12:38.669544Z","shell.execute_reply.started":"2024-04-23T21:12:38.661504Z","shell.execute_reply":"2024-04-23T21:12:38.66879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_message = \"I have hidden this data 1\"\nbinary_text,flags = text_to_image(text_message)\nprint(image_to_text(binary_text,flags))\nbin_img = np.resize(binary_text,(256,256,1))\nprint(image_to_text(np.resize(binary_text,(256,)),flags))\nplt.imshow(bin_img)\nplt.show()\nbin_img = bin_img/255\nplt.imshow(bin_img)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:38.670469Z","iopub.execute_input":"2024-04-23T21:12:38.670915Z","iopub.status.idle":"2024-04-23T21:12:39.208882Z","shell.execute_reply.started":"2024-04-23T21:12:38.670891Z","shell.execute_reply":"2024-04-23T21:12:39.207923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for train_path in train_paths:\n    for root, dirs, files in os.walk(train_path):\n    #     for dirs2 in dirs:\n    #         for root3,dirs3,files3 in os.walk(os.path.join(root,dirs2)):\n                for file in files:\n                    if file.endswith('.JPEG'):\n                        img_path = os.path.join(root, file)\n                        train_images.append(load_image(img_path))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:39.210126Z","iopub.execute_input":"2024-04-23T21:12:39.210494Z","iopub.status.idle":"2024-04-23T21:12:39.714426Z","shell.execute_reply.started":"2024-04-23T21:12:39.210457Z","shell.execute_reply":"2024-04-23T21:12:39.71358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text = [bin_img]*len(train_images)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:39.715525Z","iopub.execute_input":"2024-04-23T21:12:39.716122Z","iopub.status.idle":"2024-04-23T21:12:39.720329Z","shell.execute_reply.started":"2024-04-23T21:12:39.716092Z","shell.execute_reply":"2024-04-23T21:12:39.719163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load validation data\nval_images = []\nval_texts = []\nfor val_path in val_paths:\n    for root, dirs, files in os.walk(val_path):\n    #     for dirs2 in dirs:\n    #         for root3,dirs3,files3 in os.walk(os.path.join(root,dirs2)):\n                for file in files:\n                    if file.endswith('.JPEG'):\n                        img_path = os.path.join(root, file)\n                        val_images.append(load_image(img_path))\n\n    val_texts = [bin_img]*len(val_images)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:39.721679Z","iopub.execute_input":"2024-04-23T21:12:39.721972Z","iopub.status.idle":"2024-04-23T21:12:39.781469Z","shell.execute_reply.started":"2024-04-23T21:12:39.72195Z","shell.execute_reply":"2024-04-23T21:12:39.780814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create data generator\ntrain_datagen = ImageDataGenerator()\ntrain_generator = train_datagen.flow((np.array(train_images), np.array(train_text)), np.array(train_images), batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:39.782462Z","iopub.execute_input":"2024-04-23T21:12:39.782729Z","iopub.status.idle":"2024-04-23T21:12:39.888525Z","shell.execute_reply.started":"2024-04-23T21:12:39.782707Z","shell.execute_reply":"2024-04-23T21:12:39.887483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create validation data generator\nval_datagen = ImageDataGenerator()\nval_generator = val_datagen.flow((np.array(val_images), np.array(val_texts)), np.array(val_images), batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:39.890007Z","iopub.execute_input":"2024-04-23T21:12:39.890391Z","iopub.status.idle":"2024-04-23T21:12:39.909167Z","shell.execute_reply.started":"2024-04-23T21:12:39.890356Z","shell.execute_reply":"2024-04-23T21:12:39.908383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Image Encoder\ninput_img = Input(shape=(256, 256, 3))\n# conv1 = Conv2D(8, (3, 3), padding=\"same\", activation='relu')(input_img)\nconv2 = Conv2D(16, (3, 3), padding=\"same\", activation='relu')(input_img)\nconv3 = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(conv2)\n\n# Text Encoder\ninput_text = Input(shape=(256, 256, 1))  # Adjust input shape for text\n# text_conv1 = Conv2D(8, (3, 3), padding=\"same\", activation='relu')(input_text)\ntext_conv2 = Conv2D(16, (3, 3), padding=\"same\", activation='relu')(input_text)\ntext_conv3 = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(text_conv2)\n\n# Merge Encoded Features\nmerge = Concatenate()([conv3, text_conv3])\n\n# Embedding Network\n# em_conv1 = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(merge)\n# em_conv2 = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(em_conv1)\n# em_conv3 = Conv2D(128, (3, 3), padding=\"same\", activation='relu')(em_conv2)\nem_conv4 = Conv2D(64, (3, 3), padding=\"same\", activation='relu')(merge)\nem_conv5 = Conv2D(32, (3, 3), padding=\"same\", activation='relu')(em_conv4)\nem_conv6 = Conv2D(16, (3, 3), padding=\"same\", activation='relu')(em_conv5)\nem_conv7 = Conv2D(8, (3, 3), padding=\"same\", activation='relu')(em_conv6)\nem_conv8 = Conv2D(3, (3, 3), padding=\"same\", activation='sigmoid')(em_conv7)\n\n# Combined Model\nautoencoder = Model(inputs=[input_img, input_text], outputs=em_conv8)\nautoencoder.compile(optimizer='adam', loss='mean_squared_error', metrics = ['mae'])\n\n# Print Model Summary\nautoencoder.summary()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:39.91498Z","iopub.execute_input":"2024-04-23T21:12:39.915336Z","iopub.status.idle":"2024-04-23T21:12:40.677567Z","shell.execute_reply.started":"2024-04-23T21:12:39.915311Z","shell.execute_reply":"2024-04-23T21:12:40.676685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert train_images and train_text to numpy arrays\ntrain_images = np.array(train_images)\ntrain_text = np.array(train_text)\n\n# Reshape train_images and train_text to match the input shape of the model\ntrain_images = train_images.reshape(-1, 256, 256, 3)\ntrain_text = train_text.reshape(-1, 256, 256, 1)\n\n# Similarly, convert val_images and val_texts to numpy arrays and reshape them\nval_images = np.array(val_images)\nval_texts = np.array(val_texts)\n\nval_images = val_images.reshape(-1, 256, 256, 3)\nval_texts = val_texts.reshape(-1, 256, 256, 1)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:40.678877Z","iopub.execute_input":"2024-04-23T21:12:40.679231Z","iopub.status.idle":"2024-04-23T21:12:40.730684Z","shell.execute_reply.started":"2024-04-23T21:12:40.679198Z","shell.execute_reply":"2024-04-23T21:12:40.729669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_images.shape)\nprint(train_text.shape)\nprint(val_images.shape)\nprint(val_texts.shape)\n\nprint(train_images.dtype)\nprint(train_text.dtype)\nprint(val_images.dtype)\nprint(val_texts.dtype)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:40.732184Z","iopub.execute_input":"2024-04-23T21:12:40.732538Z","iopub.status.idle":"2024-04-23T21:12:40.739251Z","shell.execute_reply.started":"2024-04-23T21:12:40.732508Z","shell.execute_reply":"2024-04-23T21:12:40.738175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checkpoint to save the best model\ncheckpoint = ModelCheckpoint('autoencoder_model.keras', save_best_only=True)\n\n# Train the model\nautoencoder.fit([train_images, train_text], train_images, epochs=180, validation_data=([val_images, val_texts], val_images), callbacks=[checkpoint])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:12:40.740724Z","iopub.execute_input":"2024-04-23T21:12:40.74105Z","iopub.status.idle":"2024-04-23T21:16:31.350613Z","shell.execute_reply.started":"2024-04-23T21:12:40.741021Z","shell.execute_reply":"2024-04-23T21:16:31.349698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extract the loss and validation loss from the history object\nloss = autoencoder.history.history['loss']\nval_loss = autoencoder.history.history['val_loss']\n\n# Plotting the training and validation loss\nplt.figure(figsize=(10, 6))\nplt.plot(loss, label='Training Loss', color='blue')\nplt.plot(val_loss, label='Validation Loss', color='red')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:31.351886Z","iopub.execute_input":"2024-04-23T21:16:31.352183Z","iopub.status.idle":"2024-04-23T21:16:31.694608Z","shell.execute_reply.started":"2024-04-23T21:16:31.352157Z","shell.execute_reply":"2024-04-23T21:16:31.693677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecting a sample pair of images and text from the training data\nsample_img = train_images[0]\nsample_text = train_text[0]\n\n# Reshaping the input for prediction\nsample_img = sample_img.reshape(1, 256, 256, 3)\nsample_text = sample_text.reshape(1, 256, 256, 1)\n\n# Predicting the output using the autoencoder\npredicted_output = autoencoder.predict([sample_img, sample_text])\n\n# Reshaping the predicted output to the original shape\npredicted_output = predicted_output.reshape(256, 256, 3)\n\n# Plotting the original and predicted images side by side\nplt.figure(figsize=(10, 5))\n\n# Original Image\nplt.subplot(1, 2, 1)\nplt.imshow(sample_img.reshape(256, 256, 3))\nplt.title('Original Image')\nplt.axis('off')\n\n# Predicted Image\nplt.subplot(1, 2, 2)\nplt.imshow(predicted_output)\nplt.title('Predicted Image')\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:31.695843Z","iopub.execute_input":"2024-04-23T21:16:31.696208Z","iopub.status.idle":"2024-04-23T21:16:33.895571Z","shell.execute_reply.started":"2024-04-23T21:16:31.696171Z","shell.execute_reply":"2024-04-23T21:16:33.894684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecting a sample pair of images and text from the training data\nsample_img = train_images[1]\nsample_text = train_text[1]\n\n# Reshaping the input for prediction\nsample_img = sample_img.reshape(1, 256, 256, 3)\nsample_text = sample_text.reshape(1, 256, 256, 1)\n\n# Predicting the output using the autoencoder\npredicted_output = autoencoder.predict([sample_img, sample_text])\n\n# Reshaping the predicted output to the original shape\npredicted_output = predicted_output.reshape(256, 256, 3)\n\n# Plotting the original and predicted images side by side\nplt.figure(figsize=(10, 5))\n\n# Original Image\nplt.subplot(1, 2, 1)\nplt.imshow(sample_img.reshape(256, 256, 3))\nplt.title('Original Image')\nplt.axis('off')\n\n# Predicted Image\nplt.subplot(1, 2, 2)\nplt.imshow(predicted_output)\nplt.title('Predicted Image')\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:33.896842Z","iopub.execute_input":"2024-04-23T21:16:33.897172Z","iopub.status.idle":"2024-04-23T21:16:34.393042Z","shell.execute_reply.started":"2024-04-23T21:16:33.897127Z","shell.execute_reply":"2024-04-23T21:16:34.392181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_img = load_image(\"/kaggle/input/myimage/photograph.jpg\")\nsample_text = train_text[0]\n\n# Reshaping the input for prediction\nsample_img = sample_img.reshape(1, 256, 256, 3)\nsample_text = sample_text.reshape(1, 256, 256, 1)\n\n# Predicting the output using the autoencoder\npredicted_output = autoencoder.predict([sample_img, sample_text])\n\n# Reshaping the predicted output to the original shape\npredicted_output = predicted_output.reshape(256, 256, 3)\n\n# Plotting the original and predicted images side by side\nplt.figure(figsize=(10, 5))\n\n# Original Image\nplt.subplot(1, 2, 1)\nplt.imshow(sample_img.reshape(256, 256, 3))\nplt.title('Original Image')\nplt.axis('off')\n\n# Predicted Image\nplt.subplot(1, 2, 2)\nplt.imshow(predicted_output)\nplt.title('Predicted Image')\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:34.394328Z","iopub.execute_input":"2024-04-23T21:16:34.394712Z","iopub.status.idle":"2024-04-23T21:16:35.134064Z","shell.execute_reply.started":"2024-04-23T21:16:34.394677Z","shell.execute_reply":"2024-04-23T21:16:35.133109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecting a sample pair of images and text from the training data\nsample_img = train_images[2]\nsample_text = train_text[2]\n\n# Reshaping the input for prediction\nsample_img = sample_img.reshape(1, 256, 256, 3)\nsample_text = sample_text.reshape(1, 256, 256, 1)\n\n# Predicting the output using the autoencoder\npredicted_output = autoencoder.predict([sample_img, sample_text])\n\n# Reshaping the predicted output to the original shape\npredicted_output = predicted_output.reshape(256, 256, 3)\n\n# Plotting the original and predicted images side by side\nplt.figure(figsize=(10, 5))\n\n# Original Image\nplt.subplot(1, 2, 1)\nplt.imshow(sample_img.reshape(256, 256, 3))\nplt.title('Original Image')\nplt.axis('off')\n\n# Predicted Image\nplt.subplot(1, 2, 2)\nplt.imshow(predicted_output)\nplt.title('Predicted Image')\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:35.135264Z","iopub.execute_input":"2024-04-23T21:16:35.135571Z","iopub.status.idle":"2024-04-23T21:16:35.563297Z","shell.execute_reply.started":"2024-04-23T21:16:35.135543Z","shell.execute_reply":"2024-04-23T21:16:35.562316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecting a sample pair of images and text from the training data\nsample_img = train_images[10]\nsample_text = train_text[10]\n\n# Reshaping the input for prediction\nsample_img = sample_img.reshape(1, 256, 256, 3)\nsample_text = sample_text.reshape(1, 256, 256, 1)\n\n# Predicting the output using the autoencoder\npredicted_output = autoencoder.predict([sample_img, sample_text])\n\n# Reshaping the predicted output to the original shape\npredicted_output = predicted_output.reshape(256, 256, 3)\n\n# Plotting the original and predicted images side by side\nplt.figure(figsize=(10, 5))\n\n# Original Image\nplt.subplot(1, 2, 1)\nplt.imshow(sample_img.reshape(256, 256, 3))\nplt.title('Original Image')\nplt.axis('off')\n\n# Predicted Image\nplt.subplot(1, 2, 2)\nplt.imshow(predicted_output)\nplt.title('Predicted Image')\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:35.564559Z","iopub.execute_input":"2024-04-23T21:16:35.564894Z","iopub.status.idle":"2024-04-23T21:16:36.073188Z","shell.execute_reply.started":"2024-04-23T21:16:35.564866Z","shell.execute_reply":"2024-04-23T21:16:36.072192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# extraction model","metadata":{}},{"cell_type":"code","source":"pred = []\nsample_text = train_text[0]\nfor i in train_images:\n    # Reshaping the input for prediction\n    i = i.reshape(1, 256, 256, 3)\n    sample_text = sample_text.reshape(1, 256, 256, 1)\n\n    # Predicting the output using the autoencoder\n    pred.append(autoencoder.predict([i, sample_text]).reshape(256, 256, 3))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:36.074419Z","iopub.execute_input":"2024-04-23T21:16:36.074764Z","iopub.status.idle":"2024-04-23T21:16:40.72819Z","shell.execute_reply.started":"2024-04-23T21:16:36.074726Z","shell.execute_reply":"2024-04-23T21:16:40.727368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred = []\nsample_text = train_text[0]\nfor i in val_images:\n    # Reshaping the input for prediction\n    i = i.reshape(1, 256, 256, 3)\n    sample_text = sample_text.reshape(1, 256, 256, 1)\n\n    # Predicting the output using the autoencoder\n    val_pred.append(autoencoder.predict([i, sample_text]).reshape(256, 256, 3))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:40.729372Z","iopub.execute_input":"2024-04-23T21:16:40.729706Z","iopub.status.idle":"2024-04-23T21:16:41.136797Z","shell.execute_reply.started":"2024-04-23T21:16:40.72967Z","shell.execute_reply":"2024-04-23T21:16:41.13583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert train_images and train_text to numpy arrays\npred = np.array(pred)\nval_pred = np.array(val_pred)\n\n# Reshape train_images and train_text to match the input shape of the model\npred = pred.reshape(-1, 256, 256, 3)\nval_pred = val_pred.reshape(-1, 256, 256, 3)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:41.138307Z","iopub.execute_input":"2024-04-23T21:16:41.139113Z","iopub.status.idle":"2024-04-23T21:16:41.16431Z","shell.execute_reply.started":"2024-04-23T21:16:41.139074Z","shell.execute_reply":"2024-04-23T21:16:41.163546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify the shapes and data types of the input data\nprint(\"Shape of pred:\", pred.shape)  # Should be (num_samples, 256, 256, 3)\nprint(\"Shape of train_text:\", train_text.shape)  # Should be (num_samples, 256, 256, 3)\nprint(\"Shape of val_pred:\", val_pred.shape)  # Should be (num_samples, 256, 256, 3)\nprint(\"Shape of val_texts:\", val_texts.shape)  # Should be (num_samples, 256, 256, 3)\n\nprint(\"Data type of pred:\", pred.dtype)  # Should be float32\nprint(\"Data type of train_text:\", train_text.dtype)  # Should be float32\nprint(\"Data type of val_pred:\", val_pred.dtype)  # Should be float32\nprint(\"Data type of val_texts:\", val_texts.dtype)  # Should be float32","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:41.165354Z","iopub.execute_input":"2024-04-23T21:16:41.165628Z","iopub.status.idle":"2024-04-23T21:16:41.172243Z","shell.execute_reply.started":"2024-04-23T21:16:41.165604Z","shell.execute_reply":"2024-04-23T21:16:41.171386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#embedding network\nex_input_img = Input(shape=(256, 256, 3))\nex_conv1 = Conv2D(8,(3,3),padding=\"same\", activation='relu')(ex_input_img)\nex_conv2 = Conv2D(16,(3,3),padding=\"same\", activation='relu')(ex_conv1)\nex_conv3 = Conv2D(32,(3,3),padding=\"same\", activation='relu')(ex_conv2)\n# ex_conv4 = Conv2D(64,(3,3),padding=\"same\", activation='relu')(ex_conv3)\n# # ex_conv5 = Conv2D(128,(3,3),padding=\"same\", activation='relu')(ex_conv4)\n# # ex_conv6 = Conv2D(128,(3,3),padding=\"same\", activation='relu')(ex_conv5)\n# ex_conv7 = Conv2D(64,(3,3),padding=\"same\", activation='relu')(ex_conv3)\nex_conv8 = Conv2D(32,(3,3),padding=\"same\", activation='relu')(ex_conv3)\nex_conv9 = Conv2D(16,(3,3),padding=\"same\", activation='relu')(ex_conv8)\nex_conv10 = Conv2D(8,(3,3),padding=\"same\", activation='relu')(ex_conv9)\nex_conv11 = Conv2D(1,(3,3),padding=\"same\", activation='sigmoid')(ex_conv10)\n# # Flatten and Dense Layers for Classification\n# flatten = Flatten()(ex_conv11)\n# dense1 = Dense(2048, activation='sigmoid')(flatten)\n# dense2 = Dense(1024, activation='sigmoid')(dense1)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:26:15.382995Z","iopub.execute_input":"2024-04-23T21:26:15.38374Z","iopub.status.idle":"2024-04-23T21:26:15.446094Z","shell.execute_reply.started":"2024-04-23T21:26:15.383703Z","shell.execute_reply":"2024-04-23T21:26:15.445278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Combined model\nautodecoder = Model(inputs=ex_input_img, outputs=ex_conv11)\nautodecoder.compile(optimizer='adam', loss='mean_squared_error')\nautodecoder.summary()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:26:15.447479Z","iopub.execute_input":"2024-04-23T21:26:15.447795Z","iopub.status.idle":"2024-04-23T21:26:15.481422Z","shell.execute_reply.started":"2024-04-23T21:26:15.447768Z","shell.execute_reply":"2024-04-23T21:26:15.480412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# #embedding network\n# ex_input_img = Input(shape=(256, 256, 3))\n# ex_conv1 = Conv2D(8,(3,3),padding=\"same\", activation='relu')(ex_input_img)\n# ex_conv2 = Conv2D(16,(3,3),padding=\"same\", activation='relu')(ex_conv1)\n# ex_conv3 = Conv2D(32,(3,3),padding=\"same\", activation='relu')(ex_conv2)\n# # ex_conv4 = Conv2D(64,(3,3),padding=\"same\", activation='relu')(ex_conv3)\n# # # ex_conv5 = Conv2D(128,(3,3),padding=\"same\", activation='relu')(ex_conv4)\n# # # ex_conv6 = Conv2D(128,(3,3),padding=\"same\", activation='relu')(ex_conv5)\n# # ex_conv7 = Conv2D(64,(3,3),padding=\"same\", activation='relu')(ex_conv4)\n# ex_conv8 = Conv2D(32,(3,3),padding=\"same\", activation='relu')(ex_conv3)\n# ex_conv9 = Conv2D(16,(3,3),padding=\"same\", activation='relu')(ex_conv8)\n# ex_conv10 = Conv2D(8,(3,3),padding=\"same\", activation='relu')(ex_conv9)\n# ex_conv11 = Conv2D(1,(3,3),padding=\"same\", activation='sigmoid')(ex_conv10)\n# # ex_output = Activation(custom_activation)(ex_conv11)\n\n\n# # Combined model\n# autodecoder = Model(inputs=ex_input_img, outputs=ex_conv11)\n# autodecoder.compile(optimizer='adam', loss='mean_squared_error')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:41.290318Z","iopub.execute_input":"2024-04-23T21:16:41.29057Z","iopub.status.idle":"2024-04-23T21:16:41.29491Z","shell.execute_reply.started":"2024-04-23T21:16:41.290548Z","shell.execute_reply":"2024-04-23T21:16:41.294057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Evaluation Function\n# def evaluate_model(model, data):\n#     reconstructed_images = model.predict(data)\n#     ssim_scores = []\n#     psnr_scores = []\n    \n#     for i in range(len(data[0])):\n#         ssim_score = ssim(data[0][i], reconstructed_images[i], multichannel=True)\n#         psnr_score = psnr(data[0][i], reconstructed_images[i], data_range=255)\n#         ssim_scores.append(ssim_score)\n#         psnr_scores.append(psnr_score)\n    \n#     avg_ssim = np.mean(ssim_scores)\n#     avg_psnr = np.mean(psnr_scores)\n    \n#     print(f\"Average SSIM: {avg_ssim}\")\n#     print(f\"Average PSNR: {avg_psnr}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:41.296153Z","iopub.execute_input":"2024-04-23T21:16:41.296412Z","iopub.status.idle":"2024-04-23T21:16:41.309165Z","shell.execute_reply.started":"2024-04-23T21:16:41.29639Z","shell.execute_reply":"2024-04-23T21:16:41.308286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_img.size","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:16:41.310257Z","iopub.execute_input":"2024-04-23T21:16:41.310519Z","iopub.status.idle":"2024-04-23T21:16:41.32073Z","shell.execute_reply.started":"2024-04-23T21:16:41.310496Z","shell.execute_reply":"2024-04-23T21:16:41.319728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checkpoint to save the best model\ndecheckpoint = ModelCheckpoint('autodecoder_model.keras', save_best_only=True)\n\n# Train the model\nautodecoder.fit(pred, np.array([bin_img]*len(pred)), epochs=4, validation_data=(val_pred, np.array([bin_img]*len(val_pred))), callbacks=[decheckpoint])\n# autodecoder.fit(pred, np.array([np.resize(bin_img,(256,))]*len(pred)), epochs=6, validation_data=(val_pred, np.array([np.resize(bin_img,(256,))]*len(val_pred))), callbacks=[decheckpoint])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:20.663473Z","iopub.execute_input":"2024-04-23T21:27:20.66426Z","iopub.status.idle":"2024-04-23T21:27:22.746184Z","shell.execute_reply.started":"2024-04-23T21:27:20.664224Z","shell.execute_reply":"2024-04-23T21:27:22.745224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Evaluate the combined model\n# evaluate_model(autoencoder, [train_images[0],train_text[0]])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:17:59.943817Z","iopub.execute_input":"2024-04-23T21:17:59.944088Z","iopub.status.idle":"2024-04-23T21:17:59.948215Z","shell.execute_reply.started":"2024-04-23T21:17:59.944064Z","shell.execute_reply":"2024-04-23T21:17:59.947196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Extract the loss and validation loss from the history object\nloss = autodecoder.history.history['loss']\nval_loss = autodecoder.history.history['val_loss']\n\n# Plotting the training and validation loss\nplt.figure(figsize=(10, 6))\nplt.plot(loss, label='Training Loss', color='blue')\nplt.plot(val_loss, label='Validation Loss', color='red')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:26.032089Z","iopub.execute_input":"2024-04-23T21:27:26.032773Z","iopub.status.idle":"2024-04-23T21:27:26.29047Z","shell.execute_reply.started":"2024-04-23T21:27:26.032739Z","shell.execute_reply":"2024-04-23T21:27:26.289503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecting a sample pair of images and text from the training data\nsample_img = pred[1]\nsample_text = bin_img\n\n# Reshaping the input for prediction\nsample_img = sample_img.reshape(1, 256, 256, 3)\n# sample_text = sample_text.reshape(1, 256,256,1)\n\n# Predicting the output using the autoencoder\npredicted_output = autodecoder.predict(sample_img)\n\n# # Reshaping the predicted output to the original shape\npredicted_output = predicted_output.reshape(256, 256, 1)\n\n# Plotting the original and predicted images side by side\nplt.figure(figsize=(10, 5))\n\n# Original Image\nplt.subplot(1, 2, 1)\nplt.imshow(sample_text)\nplt.title('Original Image')\nplt.axis('off')\n\n# Predicted Image\nplt.subplot(1, 2, 2)\nplt.imshow(predicted_output)\nplt.title('Predicted Image')\nplt.axis('off')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:26.78723Z","iopub.execute_input":"2024-04-23T21:27:26.787595Z","iopub.status.idle":"2024-04-23T21:27:27.051354Z","shell.execute_reply.started":"2024-04-23T21:27:26.787565Z","shell.execute_reply":"2024-04-23T21:27:27.050359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_text[0]*255","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:29.917603Z","iopub.execute_input":"2024-04-23T21:27:29.918242Z","iopub.status.idle":"2024-04-23T21:27:29.930136Z","shell.execute_reply.started":"2024-04-23T21:27:29.918199Z","shell.execute_reply":"2024-04-23T21:27:29.929093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_output[0]*255","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:30.804637Z","iopub.execute_input":"2024-04-23T21:27:30.805277Z","iopub.status.idle":"2024-04-23T21:27:30.816692Z","shell.execute_reply.started":"2024-04-23T21:27:30.805236Z","shell.execute_reply":"2024-04-23T21:27:30.815562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = np.resize(predicted_output*255,(256,))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:35.503225Z","iopub.execute_input":"2024-04-23T21:27:35.504Z","iopub.status.idle":"2024-04-23T21:27:35.509143Z","shell.execute_reply.started":"2024-04-23T21:27:35.503968Z","shell.execute_reply":"2024-04-23T21:27:35.50789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_data = test_data.astype(\"int64\")\nimage_to_text(test_data,flags)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:37.003281Z","iopub.execute_input":"2024-04-23T21:27:37.004022Z","iopub.status.idle":"2024-04-23T21:27:37.009865Z","shell.execute_reply.started":"2024-04-23T21:27:37.003985Z","shell.execute_reply":"2024-04-23T21:27:37.008855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bin_pred = [ 1 if i<1 and i>0.4 else 0 for i in test_data*255]\nimage_to_text(bin_pred,flags)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T21:27:00.430563Z","iopub.execute_input":"2024-04-23T21:27:00.430971Z","iopub.status.idle":"2024-04-23T21:27:00.438591Z","shell.execute_reply.started":"2024-04-23T21:27:00.430941Z","shell.execute_reply":"2024-04-23T21:27:00.437651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}