{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Preprocess BMS data \n## refferrence\n1. frog dudes masterpieces https://www.kaggle.com/c/bms-molecular-translation/discussion/231190\n2. notebook fail because of output too many file take more than one hour to upload https://www.kaggle.com/product-feedback/233383\n\n## Warnning \n\nthis notebook is under development, just show how to proccess data parallelization and make all file into a achive file, there maybe some mistake in code \n\n\n\n## version \n- v1. setup from hengk's code , [drawback] output file will overcome 20 GB in about 160k pickle files\n- v2 - v4.\n    \n    try to split into multiproccess without change output order\n    \n    try to split dataframe into two part to avoid overcome 20GB\n- v10 \n    \n    try to split to 8 part to tar in case out of disk\n    \n- v14\n\nfinally I came up with two problem during make patchs\n\n```\nProblems: \n\n1. patchs pickle file will exceed 20GB\n\n2. patch files will take more than one hour to upload , that will cause upload fail\n```\n\n```\nSolutions: \n\n1. compress pickle file to tar.gz then kaggle will auto uncompress it after we make datasets https://www.kaggle.com/docs/datasets\n\n2. in case of exceed 20GB during make tar file, I divide proccess into 24 parts and delete pickle files per parts,\n```","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# there are too many output \ninputDir = '/kaggle/input'\n\ndef _walkdir(dirPath): \n    for dirname, _, filenames in os.walk(dirPath):\n        for filename in filenames:\n            print(os.path.join(dirname, filename))\n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nimport random\n# use for speedup reading images\nimport io\n\nimport cv2\nimport torch\nimport torch.nn.functional as F\n\nimport pickle\nimport matplotlib.pyplot as plt\n\nimport tarfile\n\nimport os.path\nimport shutil","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debug = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from joblib import Parallel, delayed\nimport multiprocessing\nimport subprocess\nMAX_THREAD = multiprocessing.cpu_count()\nMAX_THREAD","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration","metadata":{}},{"cell_type":"code","source":"# this mate come from henks preprocess to calculate oritation scale etc ...\ndata_dir = '../input/bms-molecular-translation'\ncsv_data_dir = '../input/bmd-mate-csv/csv'\n\npatch_size   = 16\npixel_pad    = 3\npixel_scale  = 0.8  #1.0  #0.62=36/58 #1.0\n\n_walkdir(csv_data_dir)\n\ntar_file_name = 'tar-bms-moleular-translation.tar.gz'\ntar = tarfile.open(tar_file_name, \"w:gz\")\ndef make_tarfile(output_filename, source_dir):\n    tar.add(source_dir, arcname=os.path.basename(source_dir))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils Function","metadata":{}},{"cell_type":"code","source":"def write_pickle_to_file(pickle_file, x):\n    with open(pickle_file, 'wb') as f:\n        pickle.dump(x, f, pickle.HIGHEST_PROTOCOL)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# draw -----------------------------------\ndef image_show(name, image, resize=1):\n    H,W = image.shape[0:2]\n    \"\"\"\n    cv2.namedWindow(name, cv2.WINDOW_GUI_NORMAL)  #WINDOW_NORMAL\n    #cv2.namedWindow(name, cv2.WINDOW_GUI_EXPANDED)  #WINDOW_GUI_EXPANDED\n    cv2.imshow(name, image.astype(np.uint8))\n    cv2.resizeWindow(name, round(resize*W), round(resize*H))\n    \"\"\"\n    plt.figure(figsize = (round(resize*W), round(resize*H)))\n    plt.imshow(image, cmap = plt.cm.gray)\n    #plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create patch dataset","metadata":{}},{"cell_type":"code","source":"# tools to make patches ...\n# see https://www.kaggle.com/yasufuminakama/inchi-resnet-lstm-with-attention-inference/data\ndef remove_rotate(image, orientation):\n    l = orientation\n    if l == 1:\n        image = np.rot90(image,-1)\n    if l == 2:\n        image = np.rot90(image, 1)\n    if l == 3:\n        image = np.rot90(image, 2)\n    return image\n\n\ndef resize_image(image, scale=1):\n    if scale==1 :\n        f = pixel_scale * 58/36  #1.2414 #0.80555\n        b = int(round(36*0.5))\n\n    if scale==2 :\n        f = pixel_scale * 1\n        b = int(round(58*0.5))\n\n    image = image[b:-b,b:-b] #remove border\n    if not np.isclose(1,f, rtol=1e-02, atol=1e-02):\n        h, w = image.shape\n        fw = int(round(f*w))\n        fh = int(round(f*h))\n        image = cv2.resize(image, dsize=(fw, fh), interpolation=cv2.INTER_AREA)\n    return image\n\ndef repad_image(image, multiplier=16):\n    h, w = image.shape\n    fh = int(np.ceil(h/multiplier))*multiplier\n    fw = int(np.ceil(w/multiplier))*multiplier\n    m  = np.full((fh, fw), 255, np.uint8)\n    m[0:h, 0:w] = image\n    return m\n\ndef image_to_patch(image, patch_size, pixel_pad, threshold=0):\n    p = pixel_pad\n    h, w = image.shape\n\n    x, y = np.meshgrid(np.arange(w // patch_size), np.arange(h // patch_size))\n    yx = np.stack([y, x], 2).reshape(-1, 2)\n\n    s = patch_size + 2*p\n    m = torch.from_numpy(image).reshape(1, 1, h, w).float()\n    k = F.unfold(m, kernel_size=s, stride=patch_size, padding=p)\n    k = k.permute(0, 2, 1).reshape(-1, s * s)\n    k = k.data.cpu().numpy().reshape(-1, s, s)\n    #print(k.shape)\n\n    sum = (1 - k[:, p:-p, p:-p]/255).reshape(len(k), -1).sum(-1)\n    i = np.where(sum > threshold)\n    #print(sum)\n    patch = k[i]\n    coord = yx[i]\n    return  patch, coord\n\ndef patch_to_image(patch, coord, width, height):\n    image = np.full((height,width), 255, np.uint8)\n    p = pixel_pad\n    patch = patch[:, p:-p, p:-p]\n    num_patch = len(patch)\n\n    for i in range(num_patch):\n        y,x = coord[i]\n        x = x * patch_size\n        y = y * patch_size\n        image[y:y+patch_size,x:x+patch_size] = patch[i]\n        cv2.rectangle(image, (x, y), (x + patch_size, y + patch_size), 128, 1)\n    return image\n\n#<todo>\n# np compression is very slow!!!\n# https://stackoverflow.com/questions/39035983/compress-zip-numpy-arrays-in-memory\ndef compress_array(k):\n    compressed_k = io.BytesIO()\n    np.savez_compressed(compressed_k, k)\n    return compressed_k\n\ndef uncompress_array(compressed_k):\n    compressed_k.seek(0)\n    k  = np.load(compressed_k,allow_pickle=True)['arr_0']\n    return k\n\n#<todo> add token to mark 4 corner of image\n#############################################################################################\ndef make_chessbord_image(w, h, patch_size=16):\n    m = np.zeros((h, w), np.float32)\n    s = patch_size\n    u = 1\n    for y in range(0, h, s):\n        v = u\n        for x in range(0, w, s):\n            m[y:y + s, x:x + s] = v\n            v *= -1\n        u *= -1\n    m = ((0.5 * m + 0.5) * 255).astype(np.uint8)\n    return m\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Divide data into two part","metadata":{}},{"cell_type":"code","source":"mode = 'train'\n# outout will be too large so we have to divide into two part\nPARTNUM = 24\n#_PART_ = 0\n#dump_dir = './bms-moleular-translation-part' + str(_PART_)\ndump_dir = './bms-moleular-translation'\n\ndump_dir_mutaion = ['0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f']\n\n\nif mode == 'train':\n    df = pd.read_csv(csv_data_dir+'/df_train_image_meta.csv')\n    folder = 'train_patch16_s%0.3f'%(pixel_scale)\n\nif mode == 'test':\n    df = pd.read_csv(csv_data_dir+'/df_test_image_meta.csv')\n    folder = 'test_patch16_s%0.3f'%(pixel_scale)\n\n#df = df.head(4000)\n# calculate the chunk size as an integer\npart_size = round(df.shape[0]/PARTNUM)\n# split dateframe \npart_chunks = [df.iloc[df.index[i:i + part_size]] for i in range(0, df.shape[0], part_size)]\n\n# just for part1\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# multiprocess data iterrows https://stackoverflow.com/questions/40357434/pandas-df-iterrows-parallelization\n        \ndef _parafunc(df):\n    num_patch = []\n    for i, d in df.iterrows():\n        image_id = d.image_id\n        if i%1000==0: print(i, image_id)\n        scale = d.scale\n        orientation = d.orientation\n\n\n        image_file = data_dir + '/%s/%s/%s/%s/%s.png' % (mode, image_id[0], image_id[1], image_id[2], image_id)\n\n        image = cv2.imread(image_file, cv2.IMREAD_GRAYSCALE)\n        if mode=='test':\n            image = remove_rotate(image, orientation)\n\n        image = resize_image(image, scale)\n        image = repad_image(image, patch_size)  # remove border and repad        \n\n        #k, yx = image_to_patch(image, patch_size, pixel_pad, threshold=pixel_scale*4)\n        k, yx = image_to_patch(image, patch_size, pixel_pad, threshold=4)\n\n        \"\"\"\n            if debug and i == 0:  #debug\n            for y, x in yx:\n                x = x * patch_size\n                y = y * patch_size\n                cv2.rectangle(image, (x, y), (x + patch_size, y + patch_size), 128, 1)\n\n            image_show('image', image, resize=1)\n            #cv2.waitKey(0)\n        \"\"\"\n        #-------------------------------------------\n        h,w = image.shape\n        yx  = yx.astype(np.int32)\n        k   = compress_array(k.astype(np.uint8))\n        write_pickle_to_file(dump_dir + '/%s/%s/%s/%s/%s.pickle' % (\n            folder, image_id[0], image_id[1], image_id[2], image_id),\n            {'patch':k, 'coord': yx, 'width': w,'height': h}\n        )\n        num_patch.append(len(yx))\n    return num_patch\n    \ndef run_make_patch_data():\n    \n    num_patch = []\n        \n    print(mode, \"total data length\" , len(df))\n    \n    for _PART_ in range(PARTNUM):\n        \n        _PARTITION_DF_ = part_chunks[_PART_]\n        print(mode, \"part\", _PART_ , \"data length\" , len(_PARTITION_DF_))\n        \n        for f in dump_dir_mutaion:\n            for g in dump_dir_mutaion:\n                for h in dump_dir_mutaion:\n                    os.makedirs(dump_dir+'/%s/%s/%s/%s'%(folder,f,g,h),exist_ok=True)\n\n\n        # divide data frame by multiprocessing thread\n        chunk_size = int(_PARTITION_DF_.shape[0]/MAX_THREAD)\n        _shape = _PARTITION_DF_.shape[0]\n        chunks = [_PARTITION_DF_.iloc[_PARTITION_DF_.index[i:i + chunk_size] - part_size *_PART_ ] for i in range(0, _shape, chunk_size)]\n\n        # create our pool with `num_processes` processes\n        pool = multiprocessing.Pool(processes=MAX_THREAD)\n\n        # apply our function to each chunk in the list        \n        result = pool.map(_parafunc, chunks)\n        num_patch += [item for sublist in result for item in sublist]\n        pool.close()\n        pool.join()\n\n        \"\"\"\n\n        num_patch2 = []\n        for i,d in _PARTITION_DF_.iterrows():\n            if i%1000==0: print(i, d.image_id)\n            image_id = d.image_id\n            scale = d.scale\n            orientation = d.orientation\n\n            image_file = data_dir + '/%s/%s/%s/%s/%s.png' % (mode, image_id[0], image_id[1], image_id[2], image_id)\n            image = cv2.imread(image_file, cv2.IMREAD_GRAYSCALE)\n\n            if mode=='test':\n                image = remove_rotate(image, orientation)\n\n            image = resize_image(image, scale)\n            image = repad_image(image, patch_size)  # remove border and repad        \n\n            #k, yx = image_to_patch(image, patch_size, pixel_pad, threshold=pixel_scale*4)\n            k, yx = image_to_patch(image, patch_size, pixel_pad, threshold=4)\n\n            h,w = image.shape\n            yx  = yx.astype(np.int32)\n            k   = compress_array(k.astype(np.uint8))\n\n            write_pickle_to_file(dump_dir + '/%s/%s/%s/%s/%s.pickle' % (\n                folder, d.image_id[0], d.image_id[1], d.image_id[2], d.image_id),\n                {'patch':k, 'coord': yx, 'width': w,'height': h}\n            )\n            num_patch2.append(len(yx))\n        \"\"\"\n        \n        make_tarfile(tar_file_name, dump_dir)\n        shutil.rmtree(dump_dir)\n        \n    #print(num_patch == num_patch2)\n    \n    df_patch = pd.DataFrame({\n        'image_id': df.image_id.values,\n        'num_patch': num_patch\n    })\n    _cvsfile_name = './df_train_patch_s%0.3f.csv'%(pixel_scale)\n    df_patch.to_csv(_cvsfile_name, index=False)\n    \n    make_tarfile(tar_file_name, _cvsfile_name)\n    os.remove(_cvsfile_name)\n    \n    tar.close()\n    exit(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run_make_patch_data()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}