{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# This is a fork of:\nhttps://www.kaggle.com/hengck23/augmentation-using-image-blending\nand uses cots-masks from \nhttps://www.kaggle.com/alexandrecc/cots-masks\n\nSome changes are needed like style transfer to get best results. ","metadata":{}},{"cell_type":"markdown","source":"# Augmentation using image blending\n\n- The images used for illustration purposes only. I just downloaded some creative commons license images from google image search. I haven't checked their licenses from the source website yet ... you are advised to **use your own images ** after checking their license !!!!\n\n- Alternatively, you can just use kaggle train images and transfer COTS objects from one frame to another frame\n","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd  \nimport cv2\nimport random\n\n%matplotlib inline\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-01T02:32:04.768838Z","iopub.execute_input":"2022-02-01T02:32:04.769471Z","iopub.status.idle":"2022-02-01T02:32:05.001442Z","shell.execute_reply.started":"2022-02-01T02:32:04.769429Z","shell.execute_reply":"2022-02-01T02:32:05.000687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#blending algorithms\n\n\n# 1. simple color transfer by rgb normalisation\n#https://github.com/chia56028/Color-Transfer-between-Images/blob/master/color_transfer.py\n\ndef norm_color_transfer(src, dst):\n\n    def get_mean_and_std(x):\n        x_mean, x_std = cv2.meanStdDev(x)\n        x_mean = np.hstack(np.around(x_mean,2)).reshape(1,1,3)\n        x_std = np.hstack(np.around(x_std,2)).reshape(1,1,3)\n        return x_mean, x_std\n\n    s = cv2.cvtColor(src,cv2.COLOR_BGR2LAB)\n    t = cv2.cvtColor(dst,cv2.COLOR_BGR2LAB)\n    s_mean, s_std = get_mean_and_std(s)\n    t_mean, t_std = get_mean_and_std(t)\n    \n    # print(s_mean, s_std, t_mean, t_std)\n\n    m = (s-s_mean)*(t_std/s_std)+t_mean\n    m = np.round(m)\n    m = np.clip(m,0,255).astype(np.uint8)\n\n    m = cv2.cvtColor(m,cv2.COLOR_LAB2BGR)\n    return m\n\n\n\n\n# 2. deep blending (in progress)\n# https://github.com/owenzlz/DeepImageBlending\n\n\n\n\n# 3. piosson editing  \n# https://github.com/PPPW/poisson-image-editing\nimport scipy.sparse\nfrom scipy.sparse.linalg import spsolve\n\n\ndef laplacian_matrix(n, m):\n    \"\"\"Generate the Poisson matrix.\n    Refer to:\n    https://en.wikipedia.org/wiki/Discrete_Poisson_equation\n    Note: it's the transpose of the wiki's matrix\n    \"\"\"\n    mat_D = scipy.sparse.lil_matrix((m, m))\n    mat_D.setdiag(-1, -1)\n    mat_D.setdiag(4)\n    mat_D.setdiag(-1, 1)\n\n    mat_A = scipy.sparse.block_diag([mat_D] * n).tolil()\n\n    mat_A.setdiag(-1, 1*m)\n    mat_A.setdiag(-1, -1*m)\n\n    return mat_A\n\n\ndef poisson_edit(source, target, mask, offset=(0,0)):\n    \"\"\"The poisson blending function.\n    Refer to:\n    Perez et. al., \"Poisson Image Editing\", 2003.\n    \"\"\"\n\n    # Assume:\n    # target is not smaller than source.\n    # shape of mask is same as shape of target.\n    y_max, x_max = target.shape[:-1]\n    y_min, x_min = 0, 0\n\n    x_range = x_max - x_min\n    y_range = y_max - y_min\n\n    M = np.float32([[1,0,offset[0]],[0,1,offset[1]]])\n    source = cv2.warpAffine(source,M,(x_range,y_range))\n\n    mask = mask[y_min:y_max, x_min:x_max]\n    mask[mask != 0] = 1\n    #mask = cv2.threshold(mask, 127, 1, cv2.THRESH_BINARY)\n\n    mat_A = laplacian_matrix(y_range, x_range)\n\n    # for \\Delta g\n    laplacian = mat_A.tocsc()\n\n    # set the region outside the mask to identity\n    for y in range(1, y_range - 1):\n        for x in range(1, x_range - 1):\n            if mask[y, x] == 0:\n                k = x + y * x_range\n                mat_A[k, k] = 1\n                mat_A[k, k + 1] = 0\n                mat_A[k, k - 1] = 0\n                mat_A[k, k + x_range] = 0\n                mat_A[k, k - x_range] = 0\n\n    # corners\n    # mask[0, 0]\n    # mask[0, y_range-1]\n    # mask[x_range-1, 0]\n    # mask[x_range-1, y_range-1]\n\n    mat_A = mat_A.tocsc()\n\n    mask_flat = mask.flatten()\n    for channel in range(source.shape[2]):\n        source_flat = source[y_min:y_max, x_min:x_max, channel].flatten()\n        target_flat = target[y_min:y_max, x_min:x_max, channel].flatten()\n\n        #concat = source_flat*mask_flat + target_flat*(1-mask_flat)\n\n        # inside the mask:\n        # \\Delta f = div v = \\Delta g\n        alpha = 1\n        mat_b = laplacian.dot(source_flat)*alpha\n\n        # outside the mask:\n        # f = t\n        mat_b[mask_flat==0] = target_flat[mask_flat==0]\n\n        x = spsolve(mat_A, mat_b)\n        #print(x.shape)\n        x = x.reshape((y_range, x_range))\n        #print(x.shape)\n        x[x > 255] = 255\n        x[x < 0] = 0\n        x = x.astype('uint8')\n        #x = cv2.normalize(x, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)\n        #print(x.shape)\n\n        target[y_min:y_max, x_min:x_max, channel] = x\n    return target","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:32:05.003076Z","iopub.execute_input":"2022-02-01T02:32:05.003461Z","iopub.status.idle":"2022-02-01T02:32:05.405292Z","shell.execute_reply.started":"2022-02-01T02:32:05.00343Z","shell.execute_reply":"2022-02-01T02:32:05.404642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#helper\ndef make_blend_mask(size, object_box):\n    x,y,w,h = object_box\n    x0=x\n    x1=x+w\n    y0=y\n    y1=y+h\n\n\n    w,h = size\n    mask = np.ones((h,w,3),np.float32)\n\n    for i in range(0,y0):\n        mask[i]=i/(y0)\n    for i in range(y1,h):\n        mask[i]=(h-i)/(h-y1+1)\n    for i in range(0,x0):\n        mask[:,i]=np.minimum(mask[:,i],i/(x0))\n    for i in range(x1,w):\n        mask[:,i]=np.minimum(mask[:,i],(w-i)/(w-x1+1))\n\n    return mask\n\n\ndef insert_object(mix, box, crop, mask):\n    x,y,w,h = box\n    crop = cv2.resize(crop, dsize=(w,h), interpolation=cv2.INTER_AREA)\n    mask = cv2.resize(mask, dsize=(w,h), interpolation=cv2.INTER_AREA)\n\n    mix_crop = mix[y:y+h,x:x+w]\n    crop = norm_color_transfer(crop, mix_crop)\n    mix[y:y+h,x:x+w] = mask*crop +(1-mask)*mix_crop\n    return mix","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:32:05.40652Z","iopub.execute_input":"2022-02-01T02:32:05.406886Z","iopub.status.idle":"2022-02-01T02:32:05.41866Z","shell.execute_reply.started":"2022-02-01T02:32:05.406856Z","shell.execute_reply":"2022-02-01T02:32:05.417433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\n\nfns = os.listdir('/kaggle/input/cots-masks/images')\n\ntotal_objects = []\n\nfor fn in tqdm(fns):\n    image = cv2.imread('../input/cots-masks/images/' + fn, cv2.IMREAD_COLOR)\n    mask =  cv2.imread('../input/cots-masks/masks/' + fn[:-3] + 'png', cv2.IMREAD_COLOR)\n\n    total_objects.append([image, mask])","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:32:05.420471Z","iopub.execute_input":"2022-02-01T02:32:05.420903Z","iopub.status.idle":"2022-02-01T02:32:08.614957Z","shell.execute_reply.started":"2022-02-01T02:32:05.420863Z","shell.execute_reply":"2022-02-01T02:32:08.613777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/tensorflow-great-barrier-reef/train.csv')\ndf = df[df['annotations'] == '[]']\n\ndef get_rand_background():\n    global df\n    \n    row = df.sample()\n    \n    image = cv2.imread(f'../input/tensorflow-great-barrier-reef/train_images/video_{row.video_id.values[0]}/{row.video_frame.values[0]}.jpg', cv2.IMREAD_COLOR)\n#     print(f'../input/tensorflow-great-barrier-reef/train_images/video_{row.video_id.values[0]}/{row.video_frame.values[0]}.jpg')\n\n    if random.random() > .5:\n        image = cv2.flip(image, 0)\n        \n    if random.random() > .5:\n        image = cv2.flip(image, 1)\n    \n    return image\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:32:08.617211Z","iopub.execute_input":"2022-02-01T02:32:08.617556Z","iopub.status.idle":"2022-02-01T02:32:08.694515Z","shell.execute_reply.started":"2022-02-01T02:32:08.61752Z","shell.execute_reply":"2022-02-01T02:32:08.693812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def image_resize(image, width = None, height = None, inter = cv2.INTER_AREA):\n    # initialize the dimensions of the image to be resized and\n    # grab the image size\n    dim = None\n    (h, w) = image.shape[:2]\n\n    # if both the width and height are None, then return the\n    # original image\n    if width is None and height is None:\n        return image\n\n    # check to see if the width is None\n    if width is None:\n        # calculate the ratio of the height and construct the\n        # dimensions\n        r = height / float(h)\n        dim = (int(w * r), height)\n\n    # otherwise, the height is None\n    else:\n        # calculate the ratio of the width and construct the\n        # dimensions\n        r = width / float(w)\n        dim = (width, int(h * r))\n\n    # resize the image\n    resized = cv2.resize(image, dim, interpolation = inter)\n\n    # return the resized image\n    return resized","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:32:08.695704Z","iopub.execute_input":"2022-02-01T02:32:08.696451Z","iopub.status.idle":"2022-02-01T02:32:08.704077Z","shell.execute_reply.started":"2022-02-01T02:32:08.696415Z","shell.execute_reply":"2022-02-01T02:32:08.70333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf labels\n!rm -rf images\n\n!mkdir images\n!mkdir labels","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:38:44.447565Z","iopub.execute_input":"2022-02-01T02:38:44.448252Z","iopub.status.idle":"2022-02-01T02:38:47.475758Z","shell.execute_reply.started":"2022-02-01T02:38:44.448212Z","shell.execute_reply":"2022-02-01T02:38:47.474758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kernel = np.ones((5,5),np.float32)/25\n\n# cv2.rectangle(mix1, (x-50,y-50), (x+w+50,y+h+50), (255,255,255), 2)\n\n# mix = background.copy()\n\n\nfor im_id in tqdm(range(10000)):\n    mix = get_rand_background()\n    bboxes = []\n    for i in range(5):\n        crop, mask = total_objects[i]\n\n        x, y = random.randint(int(1280 * .1), int(1280 * .9)), random.randint(int(720 * .1), int(720 * .9))\n\n        # random resize\n        hi = random.randint(15, 50)\n        crop = image_resize(crop, height=hi)\n        mask = image_resize(mask, height=hi)\n\n\n        # w = random.randint(20, 100)\n        # h = max(20, w - random.randint(20, 30))\n\n    #     crop = cv2.resize(crop, dsize=(h,w), interpolation=cv2.INTER_AREA)\n    #     mask = cv2.resize(mask, dsize=(h,w), interpolation=cv2.INTER_AREA)\n\n\n        mask = cv2.filter2D(mask,-1,kernel)\n\n        # augmentations\n        if random.random() > .5:\n            crop = cv2.flip(crop, 0)\n            mask = cv2.flip(mask, 0)\n\n        if random.random() > .5:\n            crop = cv2.flip(crop, 1)\n            mask = cv2.flip(mask, 1)\n\n        if random.random() > .5:\n            crop = cv2.rotate(crop, cv2.ROTATE_90_COUNTERCLOCKWISE)\n            mask = cv2.rotate(mask, cv2.ROTATE_90_COUNTERCLOCKWISE)\n\n        if random.random() > .5:\n            crop = cv2.rotate(crop, cv2.ROTATE_180)\n            mask = cv2.rotate(mask, cv2.ROTATE_180)\n\n        if random.random() > .5:\n            crop = cv2.rotate(crop, cv2.ROTATE_90_CLOCKWISE)\n            mask = cv2.rotate(mask, cv2.ROTATE_90_CLOCKWISE)\n\n        h, w, _ = mask.shape\n\n    #     mask = np.float32(mask)\n        mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)\n\n        nmask = np.zeros((h, w, 3))\n        nmask[:, :, 0] = mask\n        nmask[:, :, 1] = mask\n        nmask[:, :, 2] = mask\n\n    #     print(h,w)\n\n        nmask = (nmask/255) * random.uniform(.7, .95)\n        # nmask =nmask\n\n        mix_crop = mix[y:y+h,x:x+w]\n        crop = norm_color_transfer(crop, mix_crop) \n    #     crop = poisson_edit(crop, mix_crop, (mask).astype(np.float32), offset=(0,0))\n\n        mix[y:y+h,x:x+w] = nmask*crop +(1-nmask)*mix_crop\n        \n        # blur top of it \n        # mix[y-int(h * .10):y+int(h * .10),x:x+w] = cv2.filter2D(mix[y-int(h * .10):y+int(h * .10),x:x+w],-1,kernel)\n\n        bboxes.append([x, y, w, h])\n\n        # cv2.rectangle(mix, (x-20,y-20), (x+w+20,y+h+20), (255,255,255), 2)\n    # Save image\n    cv2.imwrite(f'./images/{im_id}.jpg', mix)\n    \n    with open(f'./labels/{im_id}.txt', 'w') as f:\n        for x, y, w, h in bboxes:\n            f.write(f'0 {x / 1280} {y / 720} {w / 1280} {h / 720}\\n')\n\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:39:15.637881Z","iopub.execute_input":"2022-02-01T02:39:15.639221Z","iopub.status.idle":"2022-02-01T02:40:19.091992Z","shell.execute_reply.started":"2022-02-01T02:39:15.639144Z","shell.execute_reply":"2022-02-01T02:40:19.090637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nshutil.make_archive('labels', 'zip', 'labels')\n\nimport shutil\nshutil.make_archive('images', 'zip', 'images')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf images\n!rm -rf labels","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(mask[...,::-1])\n\nplt.figure(figsize=(25,25))\nplt.imshow(mix[...,::-1])","metadata":{"execution":{"iopub.status.busy":"2022-02-01T02:32:10.567661Z","iopub.status.idle":"2022-02-01T02:32:10.568362Z","shell.execute_reply.started":"2022-02-01T02:32:10.568073Z","shell.execute_reply":"2022-02-01T02:32:10.568114Z"},"trusted":true},"execution_count":null,"outputs":[]}]}