{"cells":[{"metadata":{},"cell_type":"markdown","source":"### This notebook covers following:\n* Basic image denoising using opencv\n* Extraction of features using Unet for a random sample of 50000 images\n\nRef: https://www.kaggle.com/paulorzp/denoise-images"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Import libraries\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport collections\nimport random\n\nimport cv2\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.utils import shuffle\nfrom PIL import Image\nfrom numpy import asarray\nimport pickle\nfrom tqdm import tqdm_notebook\n\n\nfrom tensorflow.keras.backend import int_shape\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Conv2D, Conv3D, MaxPooling2D, MaxPooling3D, UpSampling2D, UpSampling3D, Add, BatchNormalization, Input, Activation, Lambda, Concatenate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Read the train data labels into a dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_LABELS_PATH = \"../input/bms-molecular-translation/train_labels.csv\"\n# setting the index to the image_id column\ndf_train_labels = pd.read_csv(TRAIN_LABELS_PATH, index_col=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize train images"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ref: https://www.kaggle.com/ihelon/molecular-translation-exploratory-data-analysis \ndef convert_image_id_2_path(image_id: str) -> str:\n    return \"../input/bms-molecular-translation/train/{}/{}/{}/{}.png\".format(\n        image_id[0], image_id[1], image_id[2], image_id \n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ref: https://www.kaggle.com/ihelon/molecular-translation-exploratory-data-analysis\ndef visualize_train_batch(image_ids, labels):\n    plt.figure(figsize=(16, 12))\n    \n    for ind, (image_id, label) in enumerate(zip(image_ids, labels)):\n        plt.subplot(3, 3, ind + 1)\n        image = cv2.imread(convert_image_id_2_path(image_id))\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n\n        plt.imshow(image)\n#         print(f\"{ind}: {label}\")\n        plt.title(f\"{label[:30]}...\", fontsize=10)\n        plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#ref: https://www.kaggle.com/ihelon/molecular-translation-exploratory-data-analysis\ndef visualize_train_image(image_id, label):\n    plt.figure(figsize=(10, 8))\n    \n    image = cv2.imread(convert_image_id_2_path(image_id))\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    plt.imshow(image)\n    plt.title(f\"{label}\", fontsize=14)\n    plt.axis(\"off\")\n    \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_image_denoise(image_id):\n    plt.figure(figsize=(10, 8))  \n    image = cv2.imread(convert_image_id_2_path(image_id), cv2.IMREAD_GRAYSCALE)\n    _, blackAndWhite = cv2.threshold(image, 127, 255, cv2.THRESH_BINARY_INV)\n    nlabels, labels, stats, centroids = cv2.connectedComponentsWithStats(blackAndWhite, None, None, None, 8, cv2.CV_32S)\n    sizes = stats[1:, -1] #get CC_STAT_AREA component\n    img2 = np.zeros((labels.shape), np.uint8)\n    for i in range(0, nlabels - 1):\n        if sizes[i] >= 2:   #filter small dotted regions\n            img2[labels == i + 1] = 255\n    image = cv2.bitwise_not(img2)\n    plt.imshow(image)\n    plt.title(f\"{image_id}\", fontsize=14)\n    plt.axis(\"off\")\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Image visualization\n\nsample_row = df_train_labels.sample(5)\nfor i in range(5):\n    visualize_train_image(\n        sample_row.index[i], sample_row[\"InChI\"][i]\n    )\n    visualize_image_denoise(\n        sample_row.index[i]\n    )\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Some of the statistics from train data:\n*As we can see each of the chemical identifier is unique*"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Length of training-data:',len(df_train_labels))\nprint('Number of unique chemical identifier:',len(df_train_labels['InChI'].value_counts().index))\nprint('Max count of any chemical identifier in training data:',max(df_train_labels['InChI'].value_counts().values))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Extract image_path and caption to store as key-value pair in a dictionary"},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path_to_caption = collections.defaultdict(list)\nfor idx,path in enumerate(df_train_labels.index):\n    caption = df_train_labels['InChI'].iloc[idx]\n    image_path = convert_image_id_2_path(path)\n    image_path_to_caption[image_path].append(caption)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Sample images from the complete dataset  "},{"metadata":{"trusted":true},"cell_type":"code","source":"image_paths = list(image_path_to_caption.keys())\nrandom.shuffle(image_paths)\n# Let us take just first 6000 images for training now \ntrain_image_paths = image_paths[:50000]\nprint(len(train_image_paths))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a list of image paths and corresponding captions\ntrain_captions = []\nimg_name_vector = []\n\nfor image_path in train_image_paths:\n  caption_list = image_path_to_caption[image_path]\n  train_captions.extend(caption_list)\n  img_name_vector.extend([image_path] * len(caption_list))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Basic EDA"},{"metadata":{},"cell_type":"markdown","source":"*Extract width and height pixels distribution*"},{"metadata":{"trusted":true},"cell_type":"code","source":"# h_shape=[]\n# w_shape=[]\n# aspect_ratio=[]\n# for image_path in train_image_paths:\n#     image = cv2.imread(image_path)\n#     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#     h_shape.append(image.shape[0])\n#     w_shape.append(image.shape[1])\n#     aspect_ratio.append(1.0 * (image.shape[1] / image.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(\"Mean of height, width for a random sample of {} images is: ({}, {}) \".format(len(train_image_paths), sum(h_shape) / len(h_shape), sum(w_shape) / len(w_shape)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Importing additional libraries for training and feature extraction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set some parameters\nim_width = 224\nim_height = 224\nborder = 5\nchannels = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Keras custom data generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"def generator(samples, batch_size=32,shuffle_data=True,resize=224):\n    \"\"\"\n    Yields the next training batch.\n    Suppose `samples` is an array [[image1_filename,label1], [image2_filename,label2],...].\n    \"\"\"\n    num_samples = len(samples)\n    while True: # Loop forever so the generator never terminates\n        samples = shuffle(samples)\n\n        # Get index to start each batch: [0, batch_size, 2*batch_size, ..., max multiple of batch_size <= num_samples]\n        for offset in range(0, num_samples, batch_size):\n            # Get the samples you'll use in this batch\n            batch_samples = samples[offset:offset+batch_size]\n\n            # Initialise X_train and y_train arrays for this batch\n#             X_train = []\n#             y_train = []\n            X_train = np.zeros((len(batch_samples), im_height, im_width, channels), dtype=np.float32)\n            y_train = np.zeros((len(batch_samples), im_height, im_width, channels), dtype=np.float32)\n\n            # For each batch\n            for n, batch_sample in enumerate(batch_samples):\n                \n                # Denoise, resize and normalize images \n                img = cv2.imread(batch_sample, cv2.IMREAD_GRAYSCALE)\n                _, blackAndWhite = cv2.threshold(img, 127, 255, cv2.THRESH_BINARY_INV)\n                nlabels, labels, stats, centroids = cv2.connectedComponentsWithStats(blackAndWhite, None, None, None, 8, cv2.CV_32S)\n                sizes = stats[1:, -1] #get CC_STAT_AREA component\n                img2 = np.zeros((labels.shape), np.uint8)\n                for i in range(0, nlabels - 1):\n                    if sizes[i] >= 2:   #filter small dotted regions\n                        img2[labels == i + 1] = 255\n                image = cv2.bitwise_not(img2)\n                \n                img = cv2.resize(image,(resize,resize))\n                img = np.expand_dims(img, axis=-1)\n                img = img/255.0\n                \n                # Add example to numpy arrays\n                X_train[n] = img\n                y_train[n] = img\n\n            # The generator-y part: yield the next training batch            \n            yield X_train, y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# this will create a generator object\nencode_train = sorted(set(img_name_vector))\ntrain_datagen = generator(encode_train,batch_size=8)\n\nx,y = next(train_datagen)\nprint(x.shape, y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split train and valid\nX_train, X_valid, y_train, y_valid = train_test_split(encode_train, encode_train, test_size=0.1, random_state=42)\nlen(X_valid), len(y_valid), len(X_train), len(y_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Unet model structure"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv2d_block(input_tensor, n_filters, kernel_size = 3, batchnorm = True):\n    \"\"\"Function to add 2 convolutional layers with the parameters passed to it\"\"\"\n    # first layer\n    x = tf.keras.layers.Conv2D(filters = n_filters, kernel_size = (kernel_size, kernel_size),\\\n              kernel_initializer = 'he_normal', padding = 'same')(input_tensor)\n    if batchnorm:\n        x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    \n    # second layer\n    x = tf.keras.layers.Conv2D(filters = n_filters, kernel_size = (kernel_size, kernel_size),\\\n              kernel_initializer = 'he_normal', padding = 'same')(input_tensor)\n    if batchnorm:\n        x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Activation('relu')(x)\n    \n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_unet(input_img, n_filters = 16, dropout = 0.1, batchnorm = True):\n    \"\"\"Function to define the UNET Model\"\"\"\n    # Contracting Path\n    c1 = conv2d_block(input_img, n_filters * 1, kernel_size = 3, batchnorm = batchnorm)\n    p1 = tf.keras.layers.MaxPooling2D((2, 2))(c1)\n    p1 = tf.keras.layers.Dropout(dropout)(p1)\n    \n    c2 = conv2d_block(p1, n_filters * 2, kernel_size = 3, batchnorm = batchnorm)\n    p2 = tf.keras.layers.MaxPooling2D((2, 2))(c2)\n    p2 = tf.keras.layers.Dropout(dropout)(p2)\n    \n    c3 = conv2d_block(p2, n_filters * 4, kernel_size = 3, batchnorm = batchnorm)\n    p3 = tf.keras.layers.MaxPooling2D((2, 2))(c3)\n    p3 = tf.keras.layers.Dropout(dropout)(p3)\n    \n    c4 = conv2d_block(p3, n_filters * 8, kernel_size = 3, batchnorm = batchnorm)\n    p4 = tf.keras.layers.MaxPooling2D((2, 2))(c4)\n    p4 = tf.keras.layers.Dropout(dropout)(p4)\n    \n    c5 = conv2d_block(p4, n_filters = n_filters * 16, kernel_size = 3, batchnorm = batchnorm)\n    \n    # Expansive Path\n    u6 = tf.keras.layers.Conv2DTranspose(n_filters * 8, (3, 3), strides = (2, 2), padding = 'same')(c5)\n    u6 = tf.keras.layers.concatenate([u6, c4])\n    u6 = tf.keras.layers.Dropout(dropout)(u6)\n    c6 = conv2d_block(u6, n_filters * 8, kernel_size = 3, batchnorm = batchnorm)\n    \n    u7 = tf.keras.layers.Conv2DTranspose(n_filters * 4, (3, 3), strides = (2, 2), padding = 'same')(c6)\n    u7 = tf.keras.layers.concatenate([u7, c3])\n    u7 = tf.keras.layers.Dropout(dropout)(u7)\n    c7 = conv2d_block(u7, n_filters * 4, kernel_size = 3, batchnorm = batchnorm)\n    \n    u8 = tf.keras.layers.Conv2DTranspose(n_filters * 2, (3, 3), strides = (2, 2), padding = 'same')(c7)\n    u8 = tf.keras.layers.concatenate([u8, c2])\n    u8 = tf.keras.layers.Dropout(dropout)(u8)\n    c8 = conv2d_block(u8, n_filters * 2, kernel_size = 3, batchnorm = batchnorm)\n    \n    u9 = tf.keras.layers.Conv2DTranspose(n_filters * 1, (3, 3), strides = (2, 2), padding = 'same')(c8)\n    u9 = tf.keras.layers.concatenate([u9, c1])\n    u9 = tf.keras.layers.Dropout(dropout)(u9)\n    c9 = conv2d_block(u9, n_filters * 1, kernel_size = 3, batchnorm = batchnorm)\n    \n    outputs = tf.keras.layers.Conv2D(1, (1, 1), activation='sigmoid')(c9)\n    model = tf.keras.Model(inputs=[input_img], outputs=[outputs])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_img = tf.keras.Input((im_height, im_width, 1), name='img')\nmodel = get_unet(input_img, n_filters=16, dropout=0.05, batchnorm=True)\nmodel.compile(optimizer=tf.keras.optimizers.Adam(), loss=\"binary_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.EarlyStopping(patience=5, verbose=1),\n    tf.keras.callbacks.ReduceLROnPlateau(factor=0.1, patience=5, min_lr=0.00001, verbose=1),\n    tf.keras.callbacks.ModelCheckpoint('model-unet.h5', verbose=1, save_best_only=True, save_weights_only=True)\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_generator = generator(X_train, batch_size=32)\nvalid_generator = generator(X_valid, batch_size=32)\nbatch_size=32","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train the unet model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_generator,\n            steps_per_epoch=len(X_train) // batch_size,\n            epochs=10,\n            validation_data=valid_generator,\n            validation_steps=len(X_valid) // batch_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Download the features and model structure for reuse"},{"metadata":{"trusted":true},"cell_type":"code","source":"# serialize model to JSON\nmodel_json = model.to_json()\nwith open(\"model.json\", \"w\") as json_file:\n    json_file.write(model_json)\n# serialize weights to HDF5\nmodel.save_weights(\"model.h5\")\nprint(\"Saved model to disk\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}