{"metadata": {"kernelspec": {"name": "python3", "language": "python", "display_name": "Python 3"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "name": "python", "version": "3.6.1", "pygments_lexer": "ipython3", "file_extension": ".py"}}, "nbformat": 4, "cells": [{"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_uuid": "3d0c30df9d08ce0547f26f75777b5217510b8551", "_cell_guid": "7dc67788-ff70-4010-a7e8-0af5c1f32162"}, "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "fba014be-48c5-4ece-a712-0c56357a88ec", "_uuid": "42bf855973d8734b70a925fb839863b3a4457b38"}, "source": ["# partial code for unet architecture is taken from https://github.com/kkweon/UNet-in-Tensorflow/blob/master/train.py\n", "\n", "import tensorflow as tf\n", "import glob\n", "import os\n", "import cv2\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "from subprocess import check_output\n", "\n", "WIDTH = 1840\n", "HEIGHT = 1200"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "63d90366-1e5d-4e52-a900-2810a7da3761", "_uuid": "80d7c36b1a97190750e74db3a3944c741c8ed838"}, "source": ["def prepare_queue(input_dir, mask_dir):\n", "    files = glob.glob(input_dir + \"/*.jpg\")\n", "    base_files = []\n", "    for file in files:\n", "        base_files.append(os.path.basename(file).split(\".\")[0])\n", "\n", "    base_tensor = tf.convert_to_tensor(base_files)\n", "    \n", "    input_queue = tf.train.string_input_producer(input_dir + base_tensor + \".jpg\", shuffle=True, seed=123)\n", "    mask_queue = tf.train.string_input_producer(mask_dir + base_tensor + \"_mask.gif\", shuffle=True, seed=123)\n", "    \n", "    return input_queue, mask_queue"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "7f784a2e-a11b-4a42-8fa3-bfaf1b070100", "_uuid": "8aa79e2206642a9cec6b716509c2db31b770d0d6"}, "source": ["def read_input_image(input_queue):\n", "    reader = tf.WholeFileReader()\n", "    key, value = reader.read(input_queue)\n", "    \n", "    input_image = tf.image.decode_jpeg(value)\n", "    \n", "    input_image = tf.image.resize_images(input_image, (HEIGHT,WIDTH))\n", "    input_image = tf.reshape(input_image, (HEIGHT,WIDTH,3))\n", "#     input_image = tf.cast(input_image, dtype=tf.uint8)\n", "    \n", "    return key, input_image\n", "    \n", "\n", "def read_mask_image(mask_queue):\n", "    reader = tf.WholeFileReader()\n", "    key, value = reader.read(mask_queue)\n", "    \n", "    mask_image = tf.image.decode_gif(value)\n", "    mask_image = tf.image.rgb_to_grayscale(mask_image)\n", "    mask_image = tf.image.resize_images(mask_image, (HEIGHT,WIDTH))\n", "    mask_image = tf.reshape(mask_image, (HEIGHT,WIDTH))\n", "    return key, mask_image"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "6fda0ee4-ad0b-4b30-bc30-8c9686e78a64", "_uuid": "36d04ebe1fc5d6750d093a2b50f644e15eb1d08d"}, "source": ["def get_input_mask_batch(input_queue, mask_queue):\n", "    input_name, input_image = read_input_image(input_queue)\n", "    mask_name, mask_image = read_mask_image(mask_queue)\n", "    \n", "    input_image_batch, mask_image_batch, input_name_batch, mask_name_batch = \\\n", "                tf.train.shuffle_batch([input_image, mask_image, input_name, mask_name],\\\n", "                batch_size=1, capacity=4, min_after_dequeue=2)\n", "    \n", "    \n", "    \n", "    return input_image_batch, mask_image_batch, input_name_batch, mask_name_batch"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "042ec8be-3daf-48ed-b599-e63c693e7655", "_uuid": "5247987e723dcae3f59f8408ceb210ccf9784891"}, "source": ["def conv_conv_pool(input_, n_filters, training, name, pool=True, activation=tf.nn.relu):\n", "    \"\"\"{Conv -> BN -> RELU}x2 -> {Pool, optional}\n", "    Args:\n", "        input_ (4-D Tensor): (batch_size, H, W, C)\n", "        n_filters (list): number of filters [int, int]\n", "        training (1-D Tensor): Boolean Tensor\n", "        name (str): name postfix\n", "        pool (bool): If True, MaxPool2D\n", "        activation: Activaion functions\n", "    Returns:\n", "        net: output of the Convolution operations\n", "        pool (optional): output of the max pooling operations\n", "    \"\"\"\n", "    net = input_\n", "\n", "    with tf.variable_scope(\"layer{}\".format(name)):\n", "        for i, F in enumerate(n_filters):\n", "            net = tf.layers.conv2d(net, F, (3, 3), activation=None, padding='same', name=\"conv_{}\".format(i + 1))\n", "            net = tf.layers.batch_normalization(net, training=training, name=\"bn_{}\".format(i + 1))\n", "            net = activation(net, name=\"relu{}_{}\".format(name, i + 1))\n", "\n", "        if pool is False:\n", "            return net\n", "\n", "        pool = tf.layers.max_pooling2d(net, (2, 2), strides=(2, 2), name=\"pool_{}\".format(name))\n", "\n", "        return net, pool"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "59bdc708-c727-4e61-9d11-c5589cf70bcd", "_uuid": "c86e34dcd1ac8213e60f79dbfb719a60858ac90f"}, "source": ["def upsample_concat(inputA, input_B, name):\n", "    \"\"\"Upsample `inputA` and concat with `input_B`\n", "    Args:\n", "        input_A (4-D Tensor): (N, H, W, C)\n", "        input_B (4-D Tensor): (N, 2*H, 2*H, C2)\n", "        name (str): name of the concat operation\n", "    Returns:\n", "        output (4-D Tensor): (N, 2*H, 2*W, C + C2)\n", "    \"\"\"\n", "    upsample = upsampling_2D(inputA, size=(2, 2), name=name)\n", "\n", "    return tf.concat([upsample, input_B], axis=-1, name=\"concat_{}\".format(name))\n"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "931a2bd6-389d-48ce-811c-423d4f2b90e3", "_uuid": "df16fa2c4daaf176106452a021b1e1f0a88ad038"}, "source": ["def upsampling_2D(tensor, name, size=(2, 2)):\n", "    \"\"\"Upsample/Rescale `tensor` by size\n", "    Args:\n", "        tensor (4-D Tensor): (N, H, W, C)\n", "        name (str): name of upsampling operations\n", "        size (tuple, optional): (height_multiplier, width_multiplier)\n", "            (default: (2, 2))\n", "    Returns:\n", "        output (4-D Tensor): (N, h_multiplier * H, w_multiplier * W, C)\n", "    \"\"\"\n", "    H, W, _ = tensor.get_shape().as_list()[1:]\n", "\n", "    H_multi, W_multi = size\n", "    target_H = H * H_multi\n", "    target_W = W * W_multi\n", "\n", "    return tf.image.resize_nearest_neighbor(tensor, (target_H, target_W), name=\"upsample_{}\".format(name))"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "aacc750e-ed5c-4a9c-b9bb-ca55d56f255c", "_uuid": "b46a8789bf0fdf65baf898c5de068cac257cb5d9"}, "source": ["def make_unet(X, training):\n", "    \"\"\"Build a U-Net architecture\n", "    Args:\n", "        X (4-D Tensor): (N, H, W, C)\n", "        training (1-D Tensor): Boolean Tensor is required for batchnormalization layers\n", "    Returns:\n", "        output (4-D Tensor): (N, H, W, C)\n", "            Same shape as the `input` tensor\n", "    Notes:\n", "        U-Net: Convolutional Networks for Biomedical Image Segmentation\n", "        https://arxiv.org/abs/1505.04597\n", "    \"\"\"\n", "#     net = X / 127.5 - 1\n", "    net = X\n", "    net = tf.layers.conv2d(net, 3, (1, 1), name=\"color_space_adjust\")\n", "    conv1, pool1 = conv_conv_pool(net, [8, 8], training, name=1)\n", "    conv2, pool2 = conv_conv_pool(pool1, [16, 16], training, name=2)\n", "    conv3, pool3 = conv_conv_pool(pool2, [32, 32], training, name=3)\n", "    conv4, pool4 = conv_conv_pool(pool3, [64, 64], training, name=4)\n", "    conv5 = conv_conv_pool(pool4, [128, 128], training, name=5, pool=False)\n", "\n", "    up6 = upsample_concat(conv5, conv4, name=6)\n", "    conv6 = conv_conv_pool(up6, [64, 64], training, name=6, pool=False)\n", "\n", "    up7 = upsample_concat(conv6, conv3, name=7)\n", "    conv7 = conv_conv_pool(up7, [32, 32], training, name=7, pool=False)\n", "\n", "    up8 = upsample_concat(conv7, conv2, name=8)\n", "    conv8 = conv_conv_pool(up8, [16, 16], training, name=8, pool=False)\n", "\n", "    up9 = upsample_concat(conv8, conv1, name=9)\n", "    conv9 = conv_conv_pool(up9, [8, 8], training, name=9, pool=False)\n", "    \n", "    final = tf.layers.conv2d(conv9, 1, (1, 1), name='final', activation=tf.nn.sigmoid, padding='same')\n", "    \n", "    return final"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "fa5ec87f-6b8d-439e-ba99-f02f03540f64", "_uuid": "92d3a256a76321248d1570727af3e823eb715bf7"}, "source": ["def IOU_(y_pred, y_true):\n", "    \n", "    \"\"\"Returns a (approx) IOU score\n", "    intesection = y_pred.flatten() * y_true.flatten()\n", "    Then, IOU = 2 * intersection / (y_pred.sum() + y_true.sum() + 1e-7) + 1e-7\n", "    Args:\n", "        y_pred (4-D array): (N, H, W, 1)\n", "        y_true (4-D array): (N, H, W, 1)\n", "    Returns:\n", "        float: IOU score\n", "    \"\"\"\n", "    H, W, _ = y_pred.get_shape().as_list()[1:]\n", "\n", "    pred_flat = tf.reshape(y_pred, [-1, H * W])\n", "    true_flat = tf.reshape(y_true, [-1, H * W])\n", "\n", "    intersection = 2 * tf.reduce_sum(pred_flat * true_flat, axis=1) + 1e-7\n", "    denominator = tf.reduce_sum(pred_flat, axis=1) + tf.reduce_sum(true_flat, axis=1) + 1e-7\n", "\n", "    return tf.reduce_mean(intersection / denominator)"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "c3627018-9e26-427c-bb6b-000837efe762", "_uuid": "f60b80dede1582a159acb17eb48a377c8ff5618b"}, "source": ["def make_train_op(y_pred, y_true, n_iteration):\n", "    \"\"\"Returns a training operation\n", "    Loss function = - IOU(y_pred, y_true)\n", "    IOU is\n", "        (the area of intersection)\n", "        --------------------------\n", "        (the area of two boxes)\n", "    Args:\n", "        y_pred (4-D Tensor): (N, H, W, 1)\n", "        y_true (4-D Tensor): (N, H, W, 1)\n", "    Returns:\n", "        train_op: minimize operation\n", "    \"\"\"\n", "    starter_learning_rate = 0.001\n", "    loss = -IOU_(y_pred, y_true)\n", "\n", "    global_step = tf.train.get_or_create_global_step()\n", "    \n", "    learning_rate = tf.train.exponential_decay(starter_learning_rate, \\\n", "                        global_step,n_iteration, decay_rate=0.5, staircase=False)\n", "    tf.summary.scalar(\"learning_rate\", learning_rate)\n", "    \n", "    optim = tf.train.AdamOptimizer(learning_rate=learning_rate)\n", "    train_op = optim.minimize(loss, global_step=global_step, name=\"train_op\")\n", "    \n", "    return train_op"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "cdb873e4-d297-4019-aa86-4cc52a135019", "_uuid": "dbc9cf2441f13eefe5ba8b78b06f3f6b631a929d"}, "source": ["def train():\n", "#     os.chdir(\"/home/tejas/Documents/selfstudy/carvana/\")\n", "#     checkpoint_dir = \"../checkpoints/\"\n", "#     summary_dir = \"../summary/\"\n", "\n", "    n_iteration = 1000\n", "    tf.reset_default_graph()\n", "\n", "    # prepare the queue for reading input and mask images\n", "    input_queue, mask_queue = prepare_queue(input_dir = \"../input/train/\", mask_dir = \"../input/train_masks/\")\n", "    input_image_batch, mask_image_batch, _, _ = get_input_mask_batch(input_queue, mask_queue)\n", "\n", "    x = input_image_batch\n", "    y = mask_image_batch\n", "\n", "    pred = make_unet(x, training=True)\n", "    \n", "    tf.add_to_collection(\"inputs\", x)\n", "    tf.add_to_collection(\"outputs\", pred)\n", "    \n", "    tf.summary.histogram(\"predicted_mask\", pred)\n", "    tf.summary.image(\"predicted_mask\", pred)\n", "    \n", "    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n", "\n", "    with tf.control_dependencies(update_ops):\n", "        train_op = make_train_op(pred, y, n_iteration)\n", "\n", "    IOU_op = IOU_(pred, y)\n", "    IOU_op = tf.Print(IOU_op, [IOU_op])\n", "    tf.summary.scalar(\"IOU\", IOU_op)\n", "    \n", "    summary_op = tf.summary.merge_all()\n", "    \n", "    with tf.Session() as sess:\n", "        sess.run(tf.local_variables_initializer())\n", "        sess.run(tf.global_variables_initializer())\n", "\n", "#         saver = tf.train.Saver(max_to_keep=10, keep_checkpoint_every_n_hours=2)\n", "#         summary_writer = tf.summary.FileWriter(summary_dir, sess.graph)\n", "\n", "#         if os.path.exists(checkpoint_dir) and tf.train.checkpoint_exists(checkpoint_dir):\n", "#             latest_check_point = tf.train.latest_checkpoint(checkpoint_dir)\n", "#             saver.restore(sess, latest_check_point)\n", "#         else:\n", "#             try:\n", "#                 os.rmdir(checkpoint_dir)\n", "#             except FileNotFoundError:\n", "#                 pass\n", "#             os.mkdir(checkpoint_dir)\n", "\n", "#         if not os.path.exists(summary_dir):\n", "#             os.mkdir(checkpoint_dir)\n", "        \n", "        global_step = tf.train.get_global_step(sess.graph)\n", "\n", "        coord = tf.train.Coordinator()\n", "        threads = tf.train.start_queue_runners(coord=coord)\n", "\n", "        for epoch in range(n_iteration):\n", "            if epoch%2 == 0:\n", "#                 iou_val, summary_val, global_step_val = sess.run([IOU_op, summary_op, global_step])\n", "#                 saver.save(sess, checkpoints_dir, global_step=i)\n", "#                 summary_writer.add_summary(summary_val, epoch)\n", "                iou_val, global_step_val = sess.run([IOU_op, global_step]) #comment this line\n", "                print(iou_val, global_step_val)\n", "    \n", "            train_op, _ = sess.run([train_op, global_step])\n", "        \n", "#         saver.save(sess, checkpoints_dir, global_step=i)\n", "        \n", "        coord.request_stop()\n", "        coord.join(threads)"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"_uuid": "948e3baef05d8000cc4c7f69eb3587e26996d0e8", "_cell_guid": "91a301ce-db1f-4b9a-8a47-68e97360b5f3"}, "source": ["if __name__ == '__main__':\n", "    train()"]}, {"outputs": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "bed2cf8a-399b-46c7-af5b-093248059e22", "_uuid": "5c1b5e6018004f9d1425fced3c9bdf59c5c3764a"}, "source": []}], "nbformat_minor": 1}