{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Purpose\n\nThe purpose of this notebook is to detect target(whale or dolphine) in images, crop the target and place them at the center of images. This preprocessing consists of following steps:\n\n* step1: **Detecting whale or dolphine** in an image using **pre-trained model**\n* step2: cropping the object from the image\n* step3: centering (locate the object at the center of the image)","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nimport cv2\nimport time\nimport tensorflow as tf\nimport tensorflow_hub as hub\n\nimport matplotlib\nmatplotlib.rcParams['image.cmap'] = 'gray'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-08T07:36:56.857600Z","iopub.execute_input":"2023-05-08T07:36:56.858220Z","iopub.status.idle":"2023-05-08T07:37:08.949363Z","shell.execute_reply.started":"2023-05-08T07:36:56.858167Z","shell.execute_reply":"2023-05-08T07:37:08.948055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"train_dir = \"/kaggle/input/happy-whale-and-dolphin/train_images/\"\ntest_dir = \"/kaggle/input/happy-whale-and-dolphin/test_images/\"\n\ntrain_file = \"/kaggle/input/happy-whale-and-dolphin/train.csv\"\nsample_file = \"/kaggle/input/happy-whale-and-dolphin/sample_submission.csv\"\n\nsample_df = pd.read_csv(sample_file)\ntrain_df = pd.read_csv(train_file)\n\ntrain_files = train_df[\"image\"].to_numpy()\ntest_files = sample_df[\"image\"].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:08.951525Z","iopub.execute_input":"2023-05-08T07:37:08.952333Z","iopub.status.idle":"2023-05-08T07:37:09.173588Z","shell.execute_reply.started":"2023-05-08T07:37:08.952294Z","shell.execute_reply":"2023-05-08T07:37:09.172648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_train = train_df.shape[0]\nn_test = sample_df.shape[0]\nprint(\"train n\", n_train)\nprint(\"test n\", n_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:09.175135Z","iopub.execute_input":"2023-05-08T07:37:09.176367Z","iopub.status.idle":"2023-05-08T07:37:09.184383Z","shell.execute_reply.started":"2023-05-08T07:37:09.176314Z","shell.execute_reply":"2023-05-08T07:37:09.182920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Object Detection","metadata":{}},{"cell_type":"markdown","source":"## Import pre-trained detector\n\n`mobilenet_v2` is selected as detector. `inception_resnet_v2` has better accuracy but it takes much longer (not used).  ","metadata":{}},{"cell_type":"code","source":"model_path = \"https://tfhub.dev/google/openimages_v4/ssd/mobilenet_v2/1\"\ndetector_mobilenet = hub.load(model_path).signatures[\"default\"]","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:09.188677Z","iopub.execute_input":"2023-05-08T07:37:09.189127Z","iopub.status.idle":"2023-05-08T07:37:29.389198Z","shell.execute_reply.started":"2023-05-08T07:37:09.189084Z","shell.execute_reply":"2023-05-08T07:37:29.387913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#module_handle = \"https://tfhub.dev/google/faster_rcnn/openimages_v4/inception_resnet_v2/1\"\n#detector_inception = hub.load(module_handle).signatures[\"default\"]","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:29.390941Z","iopub.execute_input":"2023-05-08T07:37:29.391284Z","iopub.status.idle":"2023-05-08T07:37:29.396497Z","shell.execute_reply.started":"2023-05-08T07:37:29.391249Z","shell.execute_reply":"2023-05-08T07:37:29.395188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def obj_detector(img, draw_box=False):\n    #input: image\n    #output: box (yxyx) of object\n    \n    #STEP1------------------------\n    #Object Detection by Mobile Net\n\n    img_tf = tf.image.convert_image_dtype(img, tf.float32)[tf.newaxis,...]\n    IY, IX, _ = img.shape\n    \n    result = detector_mobilenet(img_tf)\n    \n    check = (result[\"detection_class_entities\"] == b'Dolphin') | \\\n        (result[\"detection_class_entities\"] == b'Whale') | \\\n        (result[\"detection_class_entities\"] == b'Marine mammal') \n    \n    # if mobile net cannot detect\n    if np.sum(check) == 0: \n        \n        check = (result[\"detection_class_entities\"] == b'Animal') | \\\n        (result[\"detection_class_entities\"] == b'Fish') | \\\n        (result[\"detection_class_entities\"] == b'Shark') \n    \n        \n        # Object Detection by Inception (NOT USED)\n        #result = detector_inception(img_tf)\n\n        #check = (result[\"detection_class_entities\"] == b'Dolphin') | \\\n        #    (result[\"detection_class_entities\"] == b'Whale') | \\\n        #    (result[\"detection_class_entities\"] == b'Marine mammal')\n        \n        # if Inception also cannot detect\n        if np.sum(check) == 0:        \n            #return original size\n            return [0,0,IY, IX]\n    \n    detect_idx = np.where(check)[0][0]\n    ymin, xmin, ymax, xmax = np.array(result[\"detection_boxes\"][detect_idx])\n    \n    margin = 0.03\n    ymin = max(ymin - margin, 0)\n    xmin = max(xmin - margin, 0)\n    ymax = min(ymax + margin, 1)\n    xmax = min(xmax + margin, 1)\n    \n    IXmin = int(xmin*IX)\n    IYmin = int(ymin*IY)\n    IXmax = int(xmax*IX) \n    IYmax = int(ymax*IY)\n    \n    if draw_box:\n        T = max(int(((IX + IY)/2)/100), 2)\n        cv2.rectangle(img, (IXmin, IYmin ), (IXmax, IYmax), (255,0,0), thickness  = T)\n    \n    return [IYmin, IXmin, IYmax, IXmax]\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:29.398254Z","iopub.execute_input":"2023-05-08T07:37:29.398682Z","iopub.status.idle":"2023-05-08T07:37:29.414156Z","shell.execute_reply.started":"2023-05-08T07:37:29.398579Z","shell.execute_reply":"2023-05-08T07:37:29.412995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_centering(img):\n    \n    L1, L2, _  = img.shape\n    \n    Background = 150\n    \n    if L2 > L1:\n        L = L2\n        M = L1\n\n        img_sq = np.zeros((L,L, 3), dtype = 'uint8')\n        img_sq[:,:,:] = Background\n\n        center = int(L/2)\n        start = center - int(M/2)\n        end = start + M\n        \n        img_sq[start:end,:,:] = img[:,:,:]\n        \n    else:\n        L = L1\n        M = L2\n        \n        img_sq = np.zeros((L,L, 3), dtype = 'uint8')\n        img_sq[:,:,:] = Background\n        \n        center = int(L/2)\n        start = center - int(M/2)\n        end = start + M\n        \n        img_sq[:,start:end,:] = img[:,:,:]\n        \n    return img_sq","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:29.416015Z","iopub.execute_input":"2023-05-08T07:37:29.416396Z","iopub.status.idle":"2023-05-08T07:37:29.432397Z","shell.execute_reply.started":"2023-05-08T07:37:29.416357Z","shell.execute_reply":"2023-05-08T07:37:29.431365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_preprocess(img, draw_box = False ):\n    \n    #STEP1: Object Detection\n    obj_box = obj_detector(img, draw_box)\n    y1, x1, y2, x2 = obj_box\n    \n    if draw_box:\n        return None\n    \n    #STEP2: Cropping the object\n    img_cripped = img[y1:y2, x1:x2,:]\n    \n    #STEP3: Centering\n    img_sq = image_centering(img_cripped)\n    \n    #STEP4: Resize\n    s = 128\n    img_sq_resized = cv2.resize(img_sq, (s, s))\n    \n    return img_sq_resized","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:29.434573Z","iopub.execute_input":"2023-05-08T07:37:29.434923Z","iopub.status.idle":"2023-05-08T07:37:29.449637Z","shell.execute_reply.started":"2023-05-08T07:37:29.434889Z","shell.execute_reply":"2023-05-08T07:37:29.448232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Selecting train data index\n\nTo run this notebook within few hours, number of train data should be limited to 10,000 for each run. `m` shall be changed every time. ","metadata":{}},{"cell_type":"code","source":"m = 5\nn = 10000\ntest_clean = True","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:51:55.436683Z","iopub.execute_input":"2023-05-08T07:51:55.437938Z","iopub.status.idle":"2023-05-08T07:51:55.444295Z","shell.execute_reply.started":"2023-05-08T07:51:55.437851Z","shell.execute_reply":"2023-05-08T07:51:55.442400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if test_clean:\n    X_mat = np.zeros((n_test, 128, 128, 3), dtype = \"uint8\")\n    idx_small = np.arange(n_test)\n    \nelse:\n    train_idx_all = np.arange(n_train)\n\n    idx_small = train_idx_all[n*m:n*(m+1)]\n    n2 = idx_small.shape[0]\n    X_mat = np.zeros((n2, 128, 128, 3), dtype = \"uint8\")\n","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:29.464563Z","iopub.execute_input":"2023-05-08T07:37:29.465543Z","iopub.status.idle":"2023-05-08T07:37:29.481190Z","shell.execute_reply.started":"2023-05-08T07:37:29.465503Z","shell.execute_reply":"2023-05-08T07:37:29.479926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"step = 500","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:29.484421Z","iopub.execute_input":"2023-05-08T07:37:29.484840Z","iopub.status.idle":"2023-05-08T07:37:29.489562Z","shell.execute_reply.started":"2023-05-08T07:37:29.484800Z","shell.execute_reply":"2023-05-08T07:37:29.488649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time1 = time.time()\ncount = 0\nfor ID in idx_small:\n    \n    if test_clean:\n        img = cv2.imread(test_dir+test_files[ID])\n    else:\n        img = cv2.imread(train_dir+train_files[ID])\n        \n        \n    img_sq = image_preprocess(img, False)\n    \n    X_mat[count, :,:,:] = img_sq[:,:,:]\n    count += 1\n    \n    if count % step == 0:\n        time2 = time.time()\n        time_took = np.round(time2 - time1)\n        print(time_took,\"sec, count\", count)\n    \ntime2 = time.time()\ntime_took = np.round(time2 - time1)\nprint(time_took,\"sec, finished\")","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:37:29.491033Z","iopub.execute_input":"2023-05-08T07:37:29.491601Z","iopub.status.idle":"2023-05-08T07:43:38.497298Z","shell.execute_reply.started":"2023-05-08T07:37:29.491538Z","shell.execute_reply":"2023-05-08T07:43:38.495981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save file","metadata":{}},{"cell_type":"code","source":"if test_clean:\n    fname = \"X_test\"\nelse:\n    fname = \"X_train_\" + str(m)\nprint(fname)\nnp.save(fname, X_mat)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T07:52:02.342950Z","iopub.execute_input":"2023-05-08T07:52:02.343347Z","iopub.status.idle":"2023-05-08T07:52:02.349909Z","shell.execute_reply.started":"2023-05-08T07:52:02.343312Z","shell.execute_reply":"2023-05-08T07:52:02.348680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#mat = np.zeros((100, 100, 3), dtype = 'uint8')\n#mat[:,:,:] = 150\n#plt.imshow(mat)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T05:31:34.858874Z","iopub.execute_input":"2023-05-08T05:31:34.859862Z","iopub.status.idle":"2023-05-08T05:31:34.864591Z","shell.execute_reply.started":"2023-05-08T05:31:34.859802Z","shell.execute_reply":"2023-05-08T05:31:34.863235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Examples","metadata":{}},{"cell_type":"code","source":"def plot_images(img, img_sq):\n    fig, ax = plt.subplots(1, 2, figsize = (9, 4))\n    ax[0].imshow(img)\n    ax[1].imshow(img_sq)\n    \n    ax[0].set_title(\"Original Image & Object Detection\")\n    ax[1].set_title(\"Output Image\")\n    \n    plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-08T05:31:34.866878Z","iopub.execute_input":"2023-05-08T05:31:34.867380Z","iopub.status.idle":"2023-05-08T05:31:34.883863Z","shell.execute_reply.started":"2023-05-08T05:31:34.867329Z","shell.execute_reply":"2023-05-08T05:31:34.882379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(3)\nrandom_choice = np.random.choice(n_train, 40)\nrandom_choice","metadata":{"execution":{"iopub.status.busy":"2023-05-08T05:31:34.886324Z","iopub.execute_input":"2023-05-08T05:31:34.887640Z","iopub.status.idle":"2023-05-08T05:31:34.910525Z","shell.execute_reply.started":"2023-05-08T05:31:34.887579Z","shell.execute_reply":"2023-05-08T05:31:34.909119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ID in random_choice:\n    img = cv2.imread(train_dir+train_files[ID])\n    img_sq = image_preprocess(img, False)\n    \n    img = cv2.imread(train_dir+train_files[ID])\n    _ = image_preprocess(img, True)\n    \n    plot_images(img, img_sq)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T05:31:34.912319Z","iopub.execute_input":"2023-05-08T05:31:34.913567Z","iopub.status.idle":"2023-05-08T05:32:43.330841Z","shell.execute_reply.started":"2023-05-08T05:31:34.913479Z","shell.execute_reply":"2023-05-08T05:32:43.328949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reference\n\nhttps://tfhub.dev/google/openimages_v4/ssd/mobilenet_v2/1\n\nhttps://tfhub.dev/google/faster_rcnn/openimages_v4/inception_resnet_v2/1","metadata":{}}]}