{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd  \nimport cv2\n\n%matplotlib inline\nfrom matplotlib import pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-22T11:25:30.330955Z","iopub.execute_input":"2022-01-22T11:25:30.331229Z","iopub.status.idle":"2022-01-22T11:25:30.337359Z","shell.execute_reply.started":"2022-01-22T11:25:30.331202Z","shell.execute_reply":"2022-01-22T11:25:30.336411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#blending algorithms\n\n\n# 1. simple color transfer by rgb normalisation\n#https://github.com/chia56028/Color-Transfer-between-Images/blob/master/color_transfer.py\n\ndef norm_color_transfer(src, dst):\n\n    def get_mean_and_std(x):\n        x_mean, x_std = cv2.meanStdDev(x)\n        x_mean = np.hstack(np.around(x_mean,2)).reshape(1,1,3)\n        x_std = np.hstack(np.around(x_std,2)).reshape(1,1,3)\n        return x_mean, x_std\n\n    s = cv2.cvtColor(src,cv2.COLOR_BGR2LAB)\n    t = cv2.cvtColor(dst,cv2.COLOR_BGR2LAB)\n    s_mean, s_std = get_mean_and_std(s)\n    t_mean, t_std = get_mean_and_std(t)\n\n    m = (s-s_mean)*(t_std/s_std)+t_mean\n    m = np.round(m)\n    m = np.clip(m,0,255).astype(np.uint8)\n\n    m = cv2.cvtColor(m,cv2.COLOR_LAB2BGR)\n    return m\n\n\n\n\n# 2. deep blending (in progress)\n# https://github.com/owenzlz/DeepImageBlending\n\n\n\n\n# 3. piosson editing  \n# https://github.com/PPPW/poisson-image-editing\nimport scipy.sparse\nfrom scipy.sparse.linalg import spsolve\n\n\ndef laplacian_matrix(n, m):\n    \"\"\"Generate the Poisson matrix.\n    Refer to:\n    https://en.wikipedia.org/wiki/Discrete_Poisson_equation\n    Note: it's the transpose of the wiki's matrix\n    \"\"\"\n    mat_D = scipy.sparse.lil_matrix((m, m))\n    mat_D.setdiag(-1, -1)\n    mat_D.setdiag(4)\n    mat_D.setdiag(-1, 1)\n\n    mat_A = scipy.sparse.block_diag([mat_D] * n).tolil()\n\n    mat_A.setdiag(-1, 1*m)\n    mat_A.setdiag(-1, -1*m)\n\n    return mat_A\n\n\ndef poisson_edit(source, target, mask, offset=(0,0)):\n    \"\"\"The poisson blending function.\n    Refer to:\n    Perez et. al., \"Poisson Image Editing\", 2003.\n    \"\"\"\n\n    # Assume:\n    # target is not smaller than source.\n    # shape of mask is same as shape of target.\n    y_max, x_max = target.shape[:-1]\n    y_min, x_min = 0, 0\n\n    x_range = x_max - x_min\n    y_range = y_max - y_min\n\n    M = np.float32([[1,0,offset[0]],[0,1,offset[1]]])\n    source = cv2.warpAffine(source,M,(x_range,y_range))\n\n    mask = mask[y_min:y_max, x_min:x_max]\n    mask[mask != 0] = 1\n    #mask = cv2.threshold(mask, 127, 1, cv2.THRESH_BINARY)\n\n    mat_A = laplacian_matrix(y_range, x_range)\n\n    # for \\Delta g\n    laplacian = mat_A.tocsc()\n\n    # set the region outside the mask to identity\n    for y in range(1, y_range - 1):\n        for x in range(1, x_range - 1):\n            if mask[y, x] == 0:\n                k = x + y * x_range\n                mat_A[k, k] = 1\n                mat_A[k, k + 1] = 0\n                mat_A[k, k - 1] = 0\n                mat_A[k, k + x_range] = 0\n                mat_A[k, k - x_range] = 0\n\n    # corners\n    # mask[0, 0]\n    # mask[0, y_range-1]\n    # mask[x_range-1, 0]\n    # mask[x_range-1, y_range-1]\n\n    mat_A = mat_A.tocsc()\n\n    mask_flat = mask.flatten()\n    for channel in range(source.shape[2]):\n        source_flat = source[y_min:y_max, x_min:x_max, channel].flatten()\n        target_flat = target[y_min:y_max, x_min:x_max, channel].flatten()\n\n        #concat = source_flat*mask_flat + target_flat*(1-mask_flat)\n\n        # inside the mask:\n        # \\Delta f = div v = \\Delta g\n        alpha = 1\n        mat_b = laplacian.dot(source_flat)*alpha\n\n        # outside the mask:\n        # f = t\n        mat_b[mask_flat==0] = target_flat[mask_flat==0]\n\n        x = spsolve(mat_A, mat_b)\n        #print(x.shape)\n        x = x.reshape((y_range, x_range))\n        #print(x.shape)\n        x[x > 255] = 255\n        x[x < 0] = 0\n        x = x.astype('uint8')\n        #x = cv2.normalize(x, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)\n        #print(x.shape)\n\n        target[y_min:y_max, x_min:x_max, channel] = x\n    return target","metadata":{"execution":{"iopub.status.busy":"2022-01-22T11:25:30.338862Z","iopub.execute_input":"2022-01-22T11:25:30.339118Z","iopub.status.idle":"2022-01-22T11:25:30.357792Z","shell.execute_reply.started":"2022-01-22T11:25:30.33907Z","shell.execute_reply":"2022-01-22T11:25:30.356983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#helper\ndef make_blend_mask(size, object_box):\n    x,y,w,h = object_box\n    x0=x\n    x1=x+w\n    y0=y\n    y1=y+h\n\n\n    w,h = size\n    mask = np.ones((h,w,3),np.float32)\n\n    for i in range(0,y0):\n        mask[i]=i/(y0)\n    for i in range(y1,h):\n        mask[i]=(h-i)/(h-y1+1)\n    for i in range(0,x0):\n        mask[:,i]=np.minimum(mask[:,i],i/(x0))\n    for i in range(x1,w):\n        mask[:,i]=np.minimum(mask[:,i],(w-i)/(w-x1+1))\n\n    return mask\n\n\ndef insert_object(mix, box, crop, mask):\n    x,y,w,h = box\n    crop = cv2.resize(crop, dsize=(w,h), interpolation=cv2.INTER_AREA)\n    mask = cv2.resize(mask, dsize=(w,h), interpolation=cv2.INTER_AREA)\n\n    mix_crop = mix[y:y+h,x:x+w]\n    crop = norm_color_transfer(crop, mix_crop)\n    mix[y:y+h,x:x+w] = mask*crop +(1-mask)*mix_crop\n    return mix","metadata":{"execution":{"iopub.status.busy":"2022-01-22T11:25:30.359029Z","iopub.execute_input":"2022-01-22T11:25:30.359371Z","iopub.status.idle":"2022-01-22T11:25:30.375172Z","shell.execute_reply.started":"2022-01-22T11:25:30.359344Z","shell.execute_reply":"2022-01-22T11:25:30.374344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#load dummy object\nobject_infor = [\n    ['00012.jpg',[0,0,1024,848],[230,129,667,623],], # image_file, context_box, object_box\n    ['00001.jpg',[0,0,767,1023],[14,75,717,897],],\n   # ['00014.jpg',[0,0,1000,567],[168,87,702,419],],\n    ['00021.jpg',[680,1144,1244,884],[760,1256,992,724],],\n]\n\ndef load_dummy_object():\n\n    object = []\n    for image_file, context_box, object_box in object_infor:\n\n        image_file = '../input/cots-non-verified-license-yet/' + image_file #download-0.jpeg'\n        image = cv2.imread(image_file, cv2.IMREAD_COLOR)\n\n        x,y,w,h = context_box\n        crop = image[y:y+h,x:x+w]\n        object_box = np.array(object_box)-[x,y,0,0]\n        mask = make_blend_mask((w,h),object_box)\n\n        #image_show('crop',crop, resize=0.5)\n        #image_show('mask',mask, resize=0.5)\n        #cv2.waitKey(0)\n        object.append([crop, mask])\n    return object\n\ndef load_dummy_background():\n    video_id = 1\n    video_frame = 9187 #9287 #9187\n    image_file = 'video_%d/%d.jpg' % (video_id, video_frame)\n    image = cv2.imread('../input/tensorflow-great-barrier-reef/train_images/' + image_file, cv2.IMREAD_COLOR)\n    return image\n\n# demo here!\n\nobject = load_dummy_object()\nbackground = load_dummy_background()\n\nmix = background.copy()\nmix1 = None\nfor i, box in enumerate([\n    [330,158,70,61],\n    [208,598,45,52],\n    [930,498,120,100],\n]):\n    crop,mask = object[i]\n    \n    #mix = insert_object (mix, box, crop, mask*0.5) ------\n    x,y,w,h = box\n    crop = cv2.resize(crop, dsize=(w,h), interpolation=cv2.INTER_AREA)\n    mask = cv2.resize(mask, dsize=(w,h), interpolation=cv2.INTER_AREA)\n\n    mix_crop = mix[y:y+h,x:x+w]\n    crop = norm_color_transfer(crop, mix_crop) \n    #crop = poisson_edit(crop, mix_crop, (mask[:,:,0]>0.5).astype(np.float32), offset=(0,0))\n\n    mask = mask*0.8  #mixup ratio\n    mix[y:y+h,x:x+w] = mask*crop +(1-mask)*mix_crop\n    \n    #-------------------------------------------------------\n    #     image_show('background',background, resize=1)\n    #     image_show('mix',mix, resize=1)\n    #     image_show('crop',crop, resize=4)\n    #     image_show('mix_crop',mix_crop, resize=4)\n    #     image_show('mask1',mask1, resize=4)\n    #     image_show('mask',mask, resize=4)\n    #     cv2.waitKey(0)\n    \n    \n#show object location for debug\nmix1= mix.copy()\nfor i, box in enumerate([\n    [330,158,70,61],\n    [208,598,45,52],\n    [930,498,120,100],\n]):\n    x,y,w,h = box\n    cv2.rectangle(mix1, (x-50,y-50), (x+w+50,y+h+50), (255,255,255), 2)\n\n\n    \nplt.figure(figsize=(15,20))\nplt.title('original background')\nplt.imshow(background[...,::-1])\n\nplt.figure(figsize=(15,20))\nplt.title('augmeted image')\nplt.imshow(mix[...,::-1])\n\nplt.figure(figsize=(15,20))\nplt.title('same augmeted image with marking')\nplt.imshow(mix1[...,::-1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-22T11:25:30.514317Z","iopub.execute_input":"2022-01-22T11:25:30.514607Z","iopub.status.idle":"2022-01-22T11:25:32.529645Z","shell.execute_reply.started":"2022-01-22T11:25:30.514573Z","shell.execute_reply":"2022-01-22T11:25:32.528747Z"},"trusted":true},"execution_count":null,"outputs":[]}]}