{"cells":[{"metadata":{"_uuid":"4b2c59b5e07245d2ec2a341a2ae03ef049b73bde"},"cell_type":"markdown","source":"\nref:\n- [Siamese (pretrained) 0.822](https://www.kaggle.com/seesee/siamese-pretrained-0-822)\n- [Whale Recognition Model with score 0.78563](https://www.kaggle.com/martinpiotte/whale-recognition-model-with-score-0-78563)\n- [pretrain model: piotte](https://www.kaggle.com/seesee/piotte)\n- [tensorflow-triplet-loss](https://github.com/omoindrot/tensorflow-triplet-loss)"},{"metadata":{"_uuid":"f3e6128b6af8416f77f35cf7b48f2768c0f75f10"},"cell_type":"markdown","source":"# 1.install packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# install \n!pip -q install aiohttp faiss-prebuilt pyxtools pymltools\n!apt -qq install -y libopenblas-base libomp-dev\n    \nimport tensorflow\ntensorflow.__version__","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# 2. image preprocess"},{"metadata":{"trusted":true,"_uuid":"e5ed8e83470532c81a21f0bf33a4daf07641b393"},"cell_type":"code","source":"import csv\nimport glob\nimport logging\nimport pickle\nimport subprocess\nimport time\n\nimport numpy as np\nimport os\nimport pandas as pd\nimport random\nimport shutil\nimport tensorflow as tf\nfrom tensorflow.python.keras import Input, regularizers\nfrom tensorflow.python.keras import backend as K\nfrom tensorflow.python.keras.layers import Activation, Add, BatchNormalization, Conv2D, GlobalMaxPooling2D, \\\n    MaxPooling2D\nfrom tensorflow.python.keras.models import Model\nfrom tensorflow.python.ops import control_flow_ops\n\nfrom pymltools.tf_utils import Params, DatasetUtils, parse_bounding_boxes_list, tf_image_crop, \\\n    show_embedding, keras_convert_model_to_estimator_ckpt, InitFromPretrainedCheckpointHook, \\\n    AbstractEstimator, estimator_iter_process, colab_save_file_func, OptimizerType, tf_model_fn, \\\n    get_wsl_path, map_per_set, LossStepHookForTrain, ProcessMode, load_data_from_h5file, \\\n    store_data_in_h5file, get_triplet_pair_np, map_per_image, show_distance_dense_plot, init_logger\nfrom pyxtools import byte_to_string, create_fake_random_string, iter_list_with_size, download_big_file, list_files, \\\n    FileCache, random_choice, NormType, get_pretty_float, image_utils\n\ntry:\n    from pyxtools.faiss_tools import ImageIndexUtils\nexcept ImportError:\n    from pyxtools.pyxtools.faiss_tools import ImageIndexUtils\n\n\ndef apply_with_random_selector(x, func, num_cases):\n    \"\"\"Computes func(x, sel), with sel sampled from [0...num_cases-1].\n\n    Args:\n      x: input Tensor.\n      func: Python function to apply.\n      num_cases: Python int32, number of cases to sample sel from.\n\n    Returns:\n      The result of func(x, sel), where func receives the value of the\n      selector as a python integer, but sel is sampled dynamically.\n    \"\"\"\n    sel = tf.random_uniform([], maxval=num_cases, dtype=tf.int32)\n    # Pass the real x only to one of the func calls.\n    return control_flow_ops.merge([\n        func(control_flow_ops.switch(x, tf.equal(sel, case))[1], case)\n        for case in range(num_cases)])[0]\n\n\ndef distorted_bounding_box_crop(image,\n                                bbox,\n                                min_object_covered=0.1,\n                                aspect_ratio_range=(0.75, 1.33),\n                                area_range=(0.05, 1.0),\n                                max_attempts=100,\n                                scope=None):\n    \"\"\"Generates cropped_image using a one of the bboxes randomly distorted.\n\n    See `tf.image.sample_distorted_bounding_box` for more documentation.\n\n    Args:\n      image: 3-D Tensor of image (it will be converted to floats in [0, 1]).\n      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]\n        where each coordinate is [0, 1) and the coordinates are arranged\n        as [ymin, xmin, ymax, xmax]. If num_boxes is 0 then it would use the whole\n        image.\n      min_object_covered: An optional `float`. Defaults to `0.1`. The cropped\n        area of the image must contain at least this fraction of any bounding box\n        supplied.\n      aspect_ratio_range: An optional list of `floats`. The cropped area of the\n        image must have an aspect ratio = width / height within this range.\n      area_range: An optional list of `floats`. The cropped area of the image\n        must contain a fraction of the supplied image within in this range.\n      max_attempts: An optional `int`. Number of attempts at generating a cropped\n        region of the image of the specified constraints. After `max_attempts`\n        failures, return the entire image.\n      scope: Optional scope for name_scope.\n    Returns:\n      A tuple, a 3-D Tensor cropped_image and the distorted bbox\n    \"\"\"\n    with tf.name_scope(scope, 'distorted_bounding_box_crop', [image, bbox]):\n        # Each bounding box has shape [1, num_boxes, box coords] and\n        # the coordinates are ordered [ymin, xmin, ymax, xmax].\n\n        # A large fraction of image datasets contain a human-annotated bounding\n        # box delineating the region of the image containing the object of interest.\n        # We choose to create a new bounding box for the object which is a randomly\n        # distorted version of the human-annotated bounding box that obeys an\n        # allowed range of aspect ratios, sizes and overlap with the human-annotated\n        # bounding box. If no box is supplied, then we assume the bounding box is\n        # the entire image.\n        sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(\n            tf.shape(image),\n            bounding_boxes=bbox,\n            min_object_covered=min_object_covered,\n            aspect_ratio_range=aspect_ratio_range,\n            area_range=area_range,\n            max_attempts=max_attempts,\n            use_image_if_no_bounding_boxes=True)\n        bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box\n\n        # Crop the image to the specified bounding box.\n        cropped_image = tf.slice(image, bbox_begin, bbox_size)\n        return cropped_image, distort_bbox\n\n\ndef _whale_gray_preprocess_for_train(image, height, width, bbox,\n                                     fast_mode=True,\n                                     scope=None,\n                                     add_image_summaries=True, mean_tf_func=None):\n    \"\"\"Distort one image for training a network.\n\n    Distorting images provides a useful technique for augmenting the data\n    set during training in order to make the network invariant to aspects\n    of the image that do not effect the label.\n\n    Additionally it would create image_summaries to display the different\n    transformations applied to the image.\n\n    Args:\n      mean_tf_func (func):\n      image: 3-D Tensor of image. If dtype is tf.float32 then the range should be\n        [0, 1], otherwise it would converted to tf.float32 assuming that the range\n        is [0, MAX], where MAX is largest positive representable number for\n        int(8/16/32) data type (see `tf.image.convert_image_dtype` for details).\n      height: integer\n      width: integer\n      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]\n        where each coordinate is [0, 1) and the coordinates are arranged\n        as [ymin, xmin, ymax, xmax].\n      fast_mode: Optional boolean, if True avoids slower transformations (i.e.\n        bi-cubic resizing, random_hue or random_contrast).\n      scope: Optional scope for name_scope.\n      add_image_summaries: Enable image summaries.\n    Returns:\n      3-D float Tensor of distorted image used for training with range [-1, 1].\n    \"\"\"\n    with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]):\n        if bbox is None:\n            bbox = tf.constant([0.0, 0.0, 1.0, 1.0],\n                               dtype=tf.float32,\n                               shape=[1, 1, 4])\n        if image.shape[-1] == 3:\n            image = tf.image.rgb_to_grayscale(image)\n        if image.dtype != tf.float32:\n            image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n        # Each bounding box has shape [1, num_boxes, box coords] and\n        # the coordinates are ordered [ymin, xmin, ymax, xmax].\n        image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), bbox)\n        if add_image_summaries:\n            tf.summary.image('image_with_bounding_boxes', image_with_box)\n\n        # 1.random crop\n        distorted_image, distorted_bbox = distorted_bounding_box_crop(\n            image,\n            bbox,\n            min_object_covered=0.90,\n            aspect_ratio_range=(0.75, 1.33),\n            area_range=(0.90, 1.0), )\n\n        # Restore the shape since the dynamic slice based upon the bbox_size loses\n        # the third dimension.\n        distorted_image.set_shape([None, None, 1])\n        image_with_distorted_box = tf.image.draw_bounding_boxes(\n            tf.expand_dims(image, 0), distorted_bbox)\n        if add_image_summaries:\n            tf.summary.image('images_with_distorted_bounding_box',\n                             image_with_distorted_box)\n\n        # This resizing operation may distort the images because the aspect\n        # ratio is not respected. We select a resize method in a round robin\n        # fashion based on the thread number.\n        # Note that ResizeMethod contains 4 enumerated resizing methods.\n\n        # We select only 1 case for fast_mode bilinear.\n        # todo siamese use nearest\n        num_resize_cases = 1 if fast_mode else 4\n        distorted_image = apply_with_random_selector(\n            distorted_image,\n            lambda x, method: tf.image.resize_images(x, [height, width], method),\n            num_cases=num_resize_cases)\n\n        if add_image_summaries:\n            tf.summary.image('cropped_resized_image',\n                             tf.expand_dims(distorted_image, 0))\n\n        # 2.Randomly flip the image horizontally.\n        distorted_image = tf.image.random_flip_left_right(distorted_image)\n\n        # 3.旋转\n        # distorted_image = tf.contrib.image.rotate(\n        #     distorted_image,\n        #     angles=tf.random.uniform(shape=(1,), minval=-0.18, maxval=0.18, )[0]\n        # )\n\n        # 4.brightness\n        distorted_image = tf.image.random_brightness(distorted_image, max_delta=30)\n\n        # 5.mean\n        if mean_tf_func is None:\n            mean_tf_func = tf_image_mean_inception\n\n        distorted_image = mean_tf_func(distorted_image)\n\n        # todo 1.rotate\n        # todo 2.shift\n\n        if add_image_summaries:\n            tf.summary.image('final_distorted_image',\n                             tf.expand_dims(distorted_image, 0))\n        return distorted_image\n\n\ndef _whale_gray_preprocess_for_eval(image, height, width, bbox=None, scope=None, mean_tf_func=None):\n    \"\"\"Prepare one image for evaluation.\n\n    If height and width are specified it would output an image with that size by\n    applying resize_bilinear.\n\n    If central_fraction is specified it would crop the central fraction of the\n    input image.\n\n    Args:\n      mean_tf_func (func):\n      image: 3-D Tensor of image. If dtype is tf.float32 then the range should be\n        [0, 1], otherwise it would converted to tf.float32 assuming that the range\n        is [0, MAX], where MAX is largest positive representable number for\n        int(8/16/32) data type (see `tf.image.convert_image_dtype` for details).\n      height: integer\n      width: integer\n      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]\n        where each coordinate is [0, 1) and the coordinates are arranged\n        as [ymin, xmin, ymax, xmax].\n      scope: Optional scope for name_scope.\n    Returns:\n      3-D float Tensor of prepared image.\n    \"\"\"\n    with tf.name_scope(scope, 'eval_image', [image, height, width]):\n        if image.shape[-1] == 3:\n            image = tf.image.rgb_to_grayscale(image)\n        if image.dtype != tf.float32:\n            image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n\n        if bbox is not None:\n            image, distorted_bbox = distorted_bounding_box_crop(\n                image, bbox,\n                min_object_covered=1.0,\n                aspect_ratio_range=(0.75, 1.33),\n                area_range=(0.99, 1.0),\n            )\n            image.set_shape([None, None, 1])\n\n        if height and width:\n            # Resize the image to the specified height and width.\n            image = tf.expand_dims(image, 0)\n            image = tf.image.resize_bilinear(image, [height, width],\n                                             align_corners=False)  # todo siamese use nearest\n            image = tf.squeeze(image, [0])\n\n        # 5.mean\n        if mean_tf_func is None:\n            mean_tf_func = tf_image_mean_inception\n\n        image = mean_tf_func(image)\n\n        return image\n\n\ndef _whale_rgb_preprocess_for_train(image, height, width, bbox, fast_mode=True, easy_train: bool = False,\n                                    scope=None, add_image_summaries=True, mean_tf_func=None):\n    \"\"\"Distort one image for training a network.\n\n    Distorting images provides a useful technique for augmenting the data\n    set during training in order to make the network invariant to aspects\n    of the image that do not effect the label.\n\n    Additionally it would create image_summaries to display the different\n    transformations applied to the image.\n\n    Args:\n      mean_tf_func (func):\n      image: 3-D Tensor of image. If dtype is tf.float32 then the range should be\n        [0, 1], otherwise it would converted to tf.float32 assuming that the range\n        is [0, MAX], where MAX is largest positive representable number for\n        int(8/16/32) data type (see `tf.image.convert_image_dtype` for details).\n      height: integer\n      width: integer\n      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]\n        where each coordinate is [0, 1) and the coordinates are arranged\n        as [ymin, xmin, ymax, xmax].\n      fast_mode: Optional boolean, if True avoids slower transformations (i.e.\n        bi-cubic resizing, random_hue or random_contrast).\n      scope: Optional scope for name_scope.\n      add_image_summaries: Enable image summaries.\n    Returns:\n      3-D float Tensor of distorted image used for training with range [-1, 1].\n    \"\"\"\n\n    with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]):\n        if bbox is None:\n            bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])\n        if image.dtype != tf.float32:\n            image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n        # Each bounding box has shape [1, num_boxes, box coords] and\n        # the coordinates are ordered [ymin, xmin, ymax, xmax].\n        image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), bbox)\n        if add_image_summaries:\n            tf.summary.image('image_with_bounding_boxes', image_with_box)\n\n        if easy_train:\n            rotate_angle, min_cover = 0.01, 0.90\n        else:\n            rotate_angle, min_cover = 0.18, 0.30\n\n        # 1. random rotate\n        image = tf.contrib.image.rotate(\n            image,\n            angles=tf.random.uniform(shape=(1,), minval=-rotate_angle, maxval=rotate_angle, )[0],  # -10 ~ 10\n            interpolation=\"BILINEAR\",\n        )\n\n        # 2. random crop\n        distorted_image, distorted_bbox = distorted_bounding_box_crop(\n            image,\n            bbox,\n            min_object_covered=min_cover,\n            aspect_ratio_range=(0.75, 1.33),\n            area_range=(min_cover, 1.0), )\n\n        # Restore the shape since the dynamic slice based upon the bbox_size loses\n        # the third dimension.\n        distorted_image.set_shape([None, None, 3])\n        image_with_distorted_box = tf.image.draw_bounding_boxes(\n            tf.expand_dims(image, 0), distorted_bbox)\n        if add_image_summaries:\n            tf.summary.image('images_with_distorted_bounding_box', image_with_distorted_box)\n\n        # This resizing operation may distort the images because the aspect\n        # ratio is not respected. We select a resize method in a round robin\n        # fashion based on the thread number.\n        # Note that ResizeMethod contains 4 enumerated resizing methods.\n\n        # We select only 1 case for fast_mode bilinear.\n        # todo siamese use nearest\n        num_resize_cases = 1 if fast_mode else 4\n        distorted_image = apply_with_random_selector(\n            distorted_image,\n            lambda x, method: tf.image.resize_images(x, [height, width], method),\n            num_cases=num_resize_cases)\n\n        if add_image_summaries:\n            tf.summary.image('cropped_resized_image', tf.expand_dims(distorted_image, 0))\n\n        # 3.Randomly flip the image horizontally.\n        distorted_image = tf.image.random_flip_left_right(distorted_image)\n\n        # 4.brightness\n        distorted_image = tf.image.random_brightness(distorted_image, max_delta=30)\n\n        # 5.mean\n        if mean_tf_func is None:\n            mean_tf_func = tf_image_mean_inception\n\n        distorted_image = mean_tf_func(distorted_image)\n\n        if add_image_summaries:\n            tf.summary.image('final_distorted_image', tf.expand_dims(distorted_image, 0))\n\n        return distorted_image\n\n\ndef _whale_rgb_preprocess_for_eval(image, height, width, bbox=None, scope=None, mean_tf_func=None):\n    \"\"\"Prepare one image for evaluation.\n\n    If height and width are specified it would output an image with that size by\n    applying resize_bilinear.\n\n    If central_fraction is specified it would crop the central fraction of the\n    input image.\n\n    Args:\n      mean_tf_func (func):\n      image: 3-D Tensor of image. If dtype is tf.float32 then the range should be\n        [0, 1], otherwise it would converted to tf.float32 assuming that the range\n        is [0, MAX], where MAX is largest positive representable number for\n        int(8/16/32) data type (see `tf.image.convert_image_dtype` for details).\n      height: integer\n      width: integer\n      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]\n        where each coordinate is [0, 1) and the coordinates are arranged\n        as [ymin, xmin, ymax, xmax].\n      scope: Optional scope for name_scope.\n    Returns:\n      3-D float Tensor of prepared image.\n    \"\"\"\n\n    with tf.name_scope(scope, 'eval_image', [image, height, width]):\n        if image.dtype != tf.float32:\n            image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n\n        if bbox is not None:\n            image, distorted_bbox = distorted_bounding_box_crop(\n                image, bbox,\n                min_object_covered=1.0,\n                aspect_ratio_range=(0.75, 1.33),\n                area_range=(0.99, 1.0),\n            )\n            image.set_shape([None, None, 3])\n\n        if height and width:\n            # Resize the image to the specified height and width.\n            image = tf.expand_dims(image, 0)\n            image = tf.image.resize_bilinear(image, [height, width], align_corners=False)  # todo siamese use nearest\n            image = tf.squeeze(image, [0])\n\n        # 5.mean\n        if mean_tf_func is None:\n            mean_tf_func = tf_image_mean_inception\n\n        image = mean_tf_func(image)\n\n        return image\n\n\ndef tf_image_mean_inception(image, ):\n    \"\"\"\n        Test Pass, equal to `whale_siamese_image_mean_np`\n    Args:\n        image:\n\n    Returns:\n\n    \"\"\"\n    # rescale the image to [-1,1] from [0,1]\n    image = tf.subtract(image, 0.5)\n    image = tf.multiply(image, 2.0)\n    return image\n\n\ndef whale_siamese_image_mean_tf(image, ):\n    \"\"\"\n        Test Pass, equal to `whale_siamese_image_mean_np`\n    Args:\n        image:\n\n    Returns:\n\n    \"\"\"\n    # image = tf.image.per_image_standardization(image)\n    image = tf.subtract(image, tf.reduce_mean(image, keepdims=True))\n\n    dev_squared = tf.square(image - tf.reduce_mean(image, keepdims=True))\n    std_image = tf.sqrt(tf.reduce_mean(dev_squared, keepdims=True))\n\n    return tf.divide(image, std_image + K.epsilon())\n\n\ndef whale_gray_preprocess_image(image, height, width,\n                                is_training=False,\n                                bbox=None,\n                                fast_mode=True,\n                                add_image_summaries=True, mean_tf_func=None):\n    \"\"\"Pre-process one image for training or evaluation.\n\n    Args:\n      mean_tf_func:\n      image: 3-D Tensor [height, width, channels] with the image. If dtype is\n        tf.float32 then the range should be [0, 1], otherwise it would converted\n        to tf.float32 assuming that the range is [0, MAX], where MAX is largest\n        positive representable number for int(8/16/32) data type (see\n        `tf.image.convert_image_dtype` for details).\n      height: integer, image expected height.\n      width: integer, image expected width.\n      is_training: Boolean. If true it would transform an image for train,\n        otherwise it would transform it for evaluation.\n      bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords]\n        where each coordinate is [0, 1) and the coordinates are arranged as\n        [ymin, xmin, ymax, xmax].\n      fast_mode: Optional boolean, if True avoids slower transformations.\n      add_image_summaries: Enable image summaries.\n\n    Returns:\n      3-D float Tensor containing an appropriately scaled image\n\n    Raises:\n      ValueError: if user does not provide bounding box\n    \"\"\"\n    if is_training:\n        return _whale_gray_preprocess_for_train(\n            image, height, width, bbox, fast_mode,\n            add_image_summaries=add_image_summaries, mean_tf_func=mean_tf_func)\n    else:\n        return _whale_gray_preprocess_for_eval(image, height, width, bbox, mean_tf_func=mean_tf_func)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8e7b42fb45044079bf7d931715c35ff275b61cb"},"cell_type":"markdown","source":"# 3.utils for project"},{"metadata":{"trusted":true,"_uuid":"5465a8dbc724784bbe01524abecbf306d286895e"},"cell_type":"code","source":"\ndef combine_csv(file_weight: dict, out_file: str):\n    sub_files = []\n    sub_weight = []\n    for csv_file, weight in file_weight.items():\n        sub_files.append(csv_file)\n        sub_weight.append(weight)\n\n    place_weights = {}\n    for i in range(5):\n        place_weights[i] = 10 - i * 2\n\n    h_label = 'Image'\n    h_target = 'Id'\n\n    sub = [None] * len(sub_files)\n    for i, file in enumerate(sub_files):\n        print(\"Reading {}: w={} - {}\".format(i, sub_weight[i], file))\n        reader = csv.DictReader(open(file, \"r\"))\n        sub[i] = sorted(reader, key=lambda d: d[h_label])\n\n    out = open(out_file, \"w\", newline='')\n    writer = csv.writer(out)\n    writer.writerow([h_label, h_target])\n    p = 0\n    for row in sub[0]:\n        target_weight = {}\n        for s in range(len(sub_files)):\n            row1 = sub[s][p]\n            for ind, trgt in enumerate(row1[h_target].split(' ')):\n                target_weight[trgt] = target_weight.get(trgt, 0) + (place_weights[ind] * sub_weight[s])\n        tops_trgt = sorted(target_weight, key=target_weight.get, reverse=True)[:5]\n        writer.writerow([row1[h_label], \" \".join(tops_trgt)])\n        p += 1\n    out.close()\n\n\ndef expend_bounding_box(boxes, pix: int = 5):\n    left, upper, right, lower = boxes\n    return max(0, left - pix), max(0, upper - pix), right + pix, lower + pix\n\n\nclass PathManager(object):\n    def __init__(self, mode: str = \"default\"):\n        self._mode = mode\n        self.init_all_path()\n        self.init_bounding_boxes()\n        self._submission_csv = None\n\n    @property\n    def data_set_root_path(self) -> str:\n        if self._mode == \"default\":\n            return get_wsl_path(\"E:/frkhit/Download/AI/data-set/kaggle/whale\")\n        elif self._mode == \"fake\":\n            return get_wsl_path(\"E:/frkhit/Download/AI/data-set/kaggle/whale/fake\")\n        elif self._mode == \"colab\":\n            return \"/content\"\n        elif self._mode == \"kaggle\":\n            return \"../input/humpback-whale-identification\"\n\n        return \"./\"\n\n    @property\n    def working_path(self):\n        if self._mode == \"kaggle\":\n            return \"/kaggle/working/\"\n\n        return self.data_set_root_path\n\n    @property\n    def output_path(self, ):\n        if self._mode == \"default\" or self._mode == \"fake\":\n            return \"/mnt/e/s\"\n\n        return self.working_path\n\n    @property\n    def data_set_train_path(self):\n        return os.path.join(self.data_set_root_path, \"train\")\n\n    @property\n    def data_set_test_path(self, ):\n        return os.path.join(self.data_set_root_path, \"test\")\n\n    @property\n    def train_csv(self, ):\n        return os.path.join(self.data_set_root_path, \"train.csv\")\n\n    @property\n    def submission_csv(self, ):\n        if self._submission_csv:\n            return self._submission_csv\n        return os.path.join(self.output_path, \"whale.submission.csv\")\n\n    @submission_csv.setter\n    def submission_csv(self, submission_csv):\n        self._submission_csv = submission_csv\n\n    @property\n    def bounding_boxes_csv(self, ):\n        return os.path.join(self.working_path, \"bounding_boxes.csv\")\n\n    @property\n    def ckpt_pretrained_rv250(self, ):\n        if self._mode == \"default\" or self._mode == \"fake\":\n            return get_wsl_path(\n                \"E:/frkhit/Download/AI/pre-trained-model/resnet_v2_50.ckpt\"\n            )\n        return os.path.join(self.working_path, \"resnet_v2_50.ckpt\")\n\n    def init_bounding_boxes(self):\n        if not os.path.exists(self.bounding_boxes_csv):\n            download_big_file(\"https://raw.githubusercontent.com/frkhit/file_servers/master/bounding_boxes.csv\",\n                              self.bounding_boxes_csv)\n            if not os.path.exists(self.bounding_boxes_csv):\n                raise ValueError(\"fail to download bounding_boxes_csv!\")\n\n    def init_image_net_model(self):\n        working_dir = os.path.dirname(self.ckpt_pretrained_mv2)\n        if not os.path.exists(working_dir):\n            os.mkdir(working_dir)\n\n        if not os.path.exists(self.ckpt_pretrained_mv2 + \".index\"):\n            tar_file = \"mobilenet_v2_1.0_224.tgz\"\n            download_big_file(\"https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_224.tgz\",\n                              os.path.join(working_dir, tar_file))\n            if not os.path.exists(tar_file):\n                raise ValueError(\"fail to download pretrained model!\")\n\n            raw_dir = os.getcwd()\n            try:\n                os.chdir(working_dir)\n                cmd = subprocess.Popen([\"tar\", \"-xvf\", tar_file])\n                cmd.wait()\n                if not os.path.exists(self.ckpt_pretrained_mv2 + \".index\"):\n                    raise ValueError(\"fail to download pretrained model!\")\n            finally:\n                os.chdir(raw_dir)\n\n    def download_resnet_v2_50(self):\n        working_dir = os.path.dirname(self.ckpt_pretrained_rv250)\n        if not os.path.exists(working_dir):\n            os.mkdir(working_dir)\n\n        if not os.path.exists(self.ckpt_pretrained_rv250):\n            tar_file = \"resnet_v2_50_2017_04_14.tar.gz\"\n            download_big_file(\"http://download.tensorflow.org/models/resnet_v2_50_2017_04_14.tar.gz\",\n                              os.path.join(working_dir, tar_file))\n            if not os.path.exists(tar_file):\n                raise ValueError(\"fail to download pretrained model!\")\n\n            raw_dir = os.getcwd()\n            try:\n                os.chdir(working_dir)\n                cmd = subprocess.Popen([\"tar\", \"-xzvf\", tar_file])\n                cmd.wait()\n                if not os.path.exists(self.ckpt_pretrained_rv250):\n                    raise ValueError(\"fail to download pretrained model!\")\n            finally:\n                os.chdir(raw_dir)\n\n    def init_all_path(self):\n        members = [attr for attr in dir(self) if not attr.startswith(\"__\")]\n        for member in members:\n            if (member.endswith(\"path\") or member.endswith(\"dir\")) and not member.endswith(\"init_all_path\"):\n                tmp_dir = getattr(self, member)\n                if callable(tmp_dir):\n                    tmp_dir = tmp_dir()\n\n                if not os.path.exists(tmp_dir):\n                    try:\n                        os.mkdir(tmp_dir)\n                    except Exception as e:\n                        logging.error(e)\n\n    @property\n    def is_kaggle(self):\n        return bool(self._mode == \"kaggle\")\n\n    def create_fake_data(self, x: int = 1):\n        raw_path_manager = PathManager()\n        whales = pd.read_csv(raw_path_manager.train_csv)\n        train_info = {}\n        for index, image_name in enumerate(whales.Image):\n            train_info[image_name] = whales.Id[index]\n\n        shutil.rmtree(self.data_set_root_path)\n        os.mkdir(self.data_set_root_path)\n\n        # test path\n        os.mkdir(self.data_set_test_path)\n        test_image_list = list_files(raw_path_manager.data_set_test_path)\n        for image_file in random.choices(test_image_list, k=10 * x):\n            shutil.copy(image_file, os.path.join(self.data_set_test_path, os.path.basename(image_file)))\n\n        # train path\n        os.mkdir(self.data_set_train_path)\n        raw_image_list = list_files(raw_path_manager.data_set_train_path)\n        image_id_vs_image_file = {\n            os.path.basename(image_file): image_file for image_file in raw_image_list}\n\n        class_id_vs_image_list = {}\n\n        for image_id, class_id in train_info.items():\n            class_id_vs_image_list.setdefault(class_id, []).append(image_id_vs_image_file[image_id])\n\n        anchor_class_id_list = [class_id for class_id, _image_list in class_id_vs_image_list.items() if\n                                len(_image_list) > 1]\n        all_anchor_class_id_list = list(train_info.values())\n        try:\n            anchor_class_id_list.remove(WhaleDataUtils.blank_class_id)\n        except Exception:\n            pass\n\n        to_move_list = []\n        for class_id in random.choices(anchor_class_id_list, k=2 * x):\n            to_move_list.extend(class_id_vs_image_list[class_id][:3 * x])\n        for class_id in random.choices(all_anchor_class_id_list, k=3 * x):\n            to_move_list.extend(class_id_vs_image_list[class_id][:3 * x])\n        to_move_list.extend(class_id_vs_image_list[WhaleDataUtils.blank_class_id][:4 * x])\n        for image_file in to_move_list:\n            shutil.copy(image_file, os.path.join(self.data_set_train_path, os.path.basename(image_file)))\n        with open(self.train_csv, \"w\") as f:\n            f.write(\"Image,Id\\n\")\n            for image_file in to_move_list:\n                image_id = os.path.basename(image_file)\n                class_id = train_info[image_id]\n                f.write(\"{},{}\\n\".format(image_id, class_id))\n\n        # bounding box\n        self.init_bounding_boxes()\n\n\nclass WhaleDataUtils(object):\n    blank_class_id = \"new_whale\"\n    not_gray_str = \".ng\"\n\n    def __init__(self, path_manager: PathManager, loss_hook: LossStepHookForTrain = None,\n                 gen_data_setting: dict = None):\n        self.logger = logging.getLogger(self.__class__.__name__)\n        self._file = \"./{}.pkl\".format(self.__class__.__name__)\n        self.path_manager = path_manager\n        self.loss_hook = loss_hook\n        self.file_cache = FileCache(pickle_file=self._file)\n        self.dimension = 512\n        self.margin = 1.0\n        self.gen_data_setting = {\n            \"x_train_num\": 1,\n            \"ignore_blank_prob\": 0.95,\n            \"ignore_single_prob\": 0.2,\n            \"stop_calc_feature\": False,\n            \"gen_data_by_random_prob\": 0,\n            \"use_norm_when_calc_apn\": True,\n        }\n        if gen_data_setting:\n            self.gen_data_setting.update(gen_data_setting)\n\n        _cache = None\n        if os.path.exists(self._file):\n            _cache = self.file_cache.get(\"cache\")\n\n        if not _cache:\n            self._cache = ([], [], {}, {}, {}, [])\n        else:\n            self._cache = _cache\n\n        self._baseline = None\n        self._to_use_random = False\n\n    def _update_cache(self):\n        self.file_cache.set(\"cache\", self._cache)\n\n    def clear_cache(self):\n        self.file_cache.cache.clear()\n        self.file_cache.set(\"__key__\", \"value\", ttl=10)  # flush\n\n        _pkl_file = self._file + \".feature.h5\"\n        if os.path.exists(_pkl_file):\n            os.remove(_pkl_file)\n\n    def load_train_info(self) -> dict:\n        if self._cache and self._cache[3]:\n            return self._cache[3]\n\n        whales = pd.read_csv(self.path_manager.train_csv)\n        info = self._cache[3]\n        for index, image_name in enumerate(whales.Image):\n            info[image_name] = whales.Id[index]\n        self.logger.info(\"there are {} record in train csv!\".format(len(info)))\n        self._update_cache()\n        return info\n\n    def load_train_label_info(self) -> dict:\n        if self._cache and self._cache[4]:\n            return self._cache[4]\n\n        class_id_list = [class_id for class_id in self.load_train_info().values()]\n        class_id_list = list(set(class_id_list))\n        class_id_list.sort()\n\n        info = self._cache[4]\n        for index, class_id in enumerate(class_id_list):\n            info[class_id] = index\n        self.logger.info(\"there are {} classes in train set!\".format(len(info)))\n        self._update_cache()\n        return info\n\n    def load_boxes_info(self):\n        if self._cache and self._cache[2]:\n            return self._cache[2]\n\n        boxes = pd.read_csv(self.path_manager.bounding_boxes_csv)\n        info = self._cache[2]\n        for index, image_id in enumerate(boxes.Image):\n            info[image_id] = (boxes.x0[index], boxes.y0[index], boxes.x1[index], boxes.y1[index])\n        self.logger.info(\"there are {} record in boxes csv!\".format(len(info)))\n\n        # check\n        file_list = self.list_train_image_file().copy()  # fix bug\n        file_list.extend(self.list_test_image_file())\n        for image_file in file_list:\n            base_name = os.path.basename(image_file)\n            if base_name not in info:\n                raise ValueError(\"{} not in boxes info!\".format(image_file))\n\n        self._update_cache()\n        return self._cache[2]\n\n    def get_boxes(self, image_file) -> (int, int, int, int):\n        base_name = os.path.basename(image_file)\n        return self.load_boxes_info()[base_name]\n\n    def list_train_image_file(self, ignore_blank: bool = False) -> list:\n        if not self._cache[0]:\n            image_list = list_files(self.path_manager.data_set_train_path)\n            result_list = [file_name for file_name in image_list if file_name.endswith(\".jpg\")]\n            assert len(result_list) == len(image_list)\n            self.logger.info(\"there are {} jpg in train path!\".format(len(result_list)))\n            self._cache[0].clear()\n            self._cache[0].extend(result_list)\n\n            self._update_cache()\n\n        # ignore blank image_file\n        if not self._cache[5]:\n            for _image_file in self._cache[0]:\n                if self.get_class_id(_image_file) != self.blank_class_id:\n                    self._cache[5].append(_image_file)\n\n            self._update_cache()\n\n        if ignore_blank is False:\n            return self._cache[0]\n\n        return self._cache[5]\n\n    def list_test_image_file(self) -> list:\n        if self._cache and self._cache[1]:\n            return self._cache[1]\n\n        image_list = list_files(self.path_manager.data_set_test_path)\n        result_list = [file_name for file_name in image_list if file_name.endswith(\".jpg\")]\n        assert len(result_list) == len(image_list)\n        self.logger.info(\"there are {} jpg in test path!\".format(len(result_list)))\n        self._cache[1].clear()\n        self._cache[1].extend(result_list)\n        self._update_cache()\n        return self._cache[1]\n\n    def get_class_id(self, image_file: str) -> str:\n        base_name = os.path.basename(image_file)\n        if base_name.endswith(self.not_gray_str):\n            base_name = base_name[:-len(self.not_gray_str)]\n        return self.load_train_info()[base_name]\n\n    def list_class_id_with_multi_instance(self) -> dict:\n        info = {}\n        for class_id in self.load_train_info().values():\n            info.setdefault(class_id, 0)\n            info[class_id] += 1\n\n        key_to_remove_list = []\n        for key, count in info.items():\n            if count <= 1:\n                key_to_remove_list.append(key)\n\n        for key in key_to_remove_list:\n            info.pop(key)\n\n        return info\n\n    def get_label(self, image_file: str) -> int:\n        class_id = self.get_class_id(image_file)\n        return self.load_train_label_info()[class_id]\n\n    def submit_test_result(self, test_image_list: list, result_list: list, feature_list: list = None):\n        if len(test_image_list) != len(result_list):\n            self.logger.error(\"len(test_image_list) != len(result_list)!\")\n        if feature_list and len(feature_list) != len(test_image_list):\n            self.logger.error(\"len(test_image_list) != len(feature_list)!\")\n\n        with open(self.path_manager.submission_csv, \"w\") as f:\n            f.write(\"Image,Id\\n\")\n            for index, image_file in enumerate(test_image_list):\n                f.write(\"{},{}\\n\".format(os.path.basename(image_file),\n                                         \" \".join([class_id for class_id in result_list[index]])))\n            self.logger.info(\"success to save result in {}\".format(self.path_manager.submission_csv))\n\n        if feature_list and feature_list[0]:\n            self.logger.info(\"saving feature list ...\")\n            with open(self.path_manager.submission_csv + \".feature\", \"w\") as f:\n                for index, image_file in enumerate(test_image_list):\n                    f.write(\"{},{}\\n\".format(os.path.basename(image_file), feature_list[index]))\n\n                self.logger.info(\"success to save feature result in {}\".format(\n                    self.path_manager.submission_csv + \".feature\"))\n\n    def get_random_baseline(self) -> (float, float):\n        \"\"\"\n            0.384, 0.381\n        Returns:\n            (float, float):\n        \"\"\"\n        if self._baseline is None:\n            class_count_dict = self.list_class_id_with_multi_instance()\n            class_list = sorted(class_count_dict.items(), key=lambda x: x[1], reverse=True)\n            class_id_list = [_class_id for (_class_id, _count) in class_list]\n            random_class_id_list = [self.blank_class_id]\n            for _class_id in class_id_list:\n                if _class_id not in random_class_id_list:\n                    random_class_id_list.append(_class_id)\n                    if len(random_class_id_list) >= 5:\n                        break\n\n            labels = [self.get_class_id(image_file) for image_file in self.list_train_image_file()]\n            class_id_result_list = [random_class_id_list] * len(labels)\n            total_map_5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)\n            total_map_1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)\n            self._baseline = (total_map_5, total_map_1)\n\n        return self._baseline\n\n    def list_debug_image_list(self, count: int = 128) -> list:\n        all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n            self.get_basic_info(self.list_train_image_file())\n        if count >= len(all_image_list):\n            return all_image_list\n\n        # info\n        single_blank_class_files = []\n        anchor_class_id_set = set(anchor_class_id_list)\n        for class_id, file_list in class_id_vs_image_list.items():\n            if class_id not in anchor_class_id_set:\n                single_blank_class_files.extend(file_list)\n\n        image_list = []\n\n        # anchor_file_count\n        anchor_file_count = int(count * (1 - len(single_blank_class_files) / len(all_image_list)))\n        if (anchor_file_count + len(single_blank_class_files)) > len(all_image_list):\n            anchor_file_count = len(all_image_list) - len(single_blank_class_files)\n\n        # multi class\n        for class_id in anchor_class_id_list:\n            image_list.extend(class_id_vs_image_list[class_id])\n            if len(image_list) >= anchor_file_count:\n                break\n\n        while len(image_list) > anchor_file_count:\n            image_list.pop(-1)\n\n        # add single class and blank class\n        image_list.extend(random_choice(single_blank_class_files, k=count - anchor_file_count, unique=True))\n\n        assert len(image_list) == count\n\n        return image_list\n\n    def get_basic_info(self, all_image_list: list = None) -> (list, dict, list, list, list):\n        \"\"\"\n            list train info\n        Args:\n\n        Returns:\n\n        \"\"\"\n        if all_image_list is None:\n            all_image_list = list(self.list_train_image_file(ignore_blank=False))\n\n        image_id_vs_image_file = {os.path.basename(image_file): image_file for image_file in all_image_list}\n\n        class_id_vs_image_list = {}\n\n        for image_id, class_id in self.load_train_info().items():\n            if image_id in image_id_vs_image_file:\n                class_id_vs_image_list.setdefault(class_id, []).append(image_id_vs_image_file[image_id])\n\n        anchor_class_id_list = [class_id for class_id, _image_list in class_id_vs_image_list.items() if\n                                len(_image_list) > 1]\n        all_class_id_list = list(class_id_vs_image_list.keys())\n        single_class_id_list = list(set(all_class_id_list) - set(anchor_class_id_list))\n        try:\n            anchor_class_id_list.remove(self.blank_class_id)\n        except Exception:\n            pass\n        try:\n            single_class_id_list.remove(self.blank_class_id)\n        except Exception:\n            pass\n\n        return all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list\n\n    def get_file_list(self, is_training: bool = False, shuffle: bool = False, num_epochs: int = 1,\n                      batch_size: int = None, online_batch_count: int = 0) -> list:\n\n        def get_random_batch_list(batch_count: int, add_blank_class: bool, class_id_vs_image_list: dict,\n                                  anchor_class_id_list: list, all_image_list: list, single_class_id_list: list,\n                                  anchor_unique: bool, single_unique: bool, ) -> list:\n            batch_list = []\n            # add one image from new blank\n            if add_blank_class:\n                batch_list.append(random.choice(class_id_vs_image_list[self.blank_class_id]))\n\n            # add anchor(has more than one image)\n            chosen_class_id_list = random_choice(anchor_class_id_list, k=batch_count, unique=anchor_unique)\n            for class_id in chosen_class_id_list:\n                if len(class_id_vs_image_list[class_id]) >= 2 * online_batch_count:\n                    batch_list.extend(\n                        random_choice(class_id_vs_image_list[class_id], k=2 * online_batch_count, unique=True))\n                else:\n                    batch_list.extend(class_id_vs_image_list[class_id])\n\n                if (batch_count - len(batch_list)) < online_batch_count:\n                    break\n\n            # add single class\n            if len(batch_list) < batch_count:\n                if single_class_id_list:\n                    batch_list.extend([\n                        class_id_vs_image_list[class_id][0] for class_id in\n                        random_choice(single_class_id_list, k=batch_count - len(batch_list),\n                                      unique=single_unique)])\n                else:\n                    batch_list.extend(\n                        random_choice(all_image_list, k=batch_count - len(batch_list), unique=anchor_unique))\n            else:\n                while len(batch_list) > batch_count:\n                    batch_list.pop(-1)\n\n            return batch_list\n\n        def create_file_list_by_random() -> list:\n            all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n                self.get_basic_info()\n            total_count = int(num_epochs * len(all_image_list) * self.gen_data_setting[\"x_train_num\"])\n            batch_count = total_count // batch_size\n            multi_batch_list = []\n\n            _unique = True if len(anchor_class_id_list) >= batch_size else False\n            _unique_single = True if len(single_class_id_list) >= batch_size else False\n            _has_blank_class = True if class_id_vs_image_list.get(self.blank_class_id) else False\n\n            for i in range(batch_count):\n                batch_list = get_random_batch_list(\n                    batch_count=batch_size, add_blank_class=_has_blank_class,\n                    class_id_vs_image_list=class_id_vs_image_list, anchor_class_id_list=anchor_class_id_list,\n                    all_image_list=all_image_list, single_class_id_list=single_class_id_list,\n                    anchor_unique=_unique, single_unique=_unique_single,\n                )\n\n                random.shuffle(batch_list)\n                multi_batch_list.append(batch_list)\n\n            return multi_batch_list\n\n        def create_file_list_by_distance() -> list:\n            # calc distance\n            feature_arr, file_arr = self._load_feature(create_if_not_exist=False)\n            if feature_arr is None:\n                self.logger.warning(\n                    \"no feature in h5, use create_file_list_by_random instead of create_file_list_by_distance!\")\n                return create_file_list_by_random()\n\n            if self.gen_data_setting.get(\"use_norm_when_calc_apn\", False):\n                feature_arr, norm_float = NormType.all.normalize_and_return_norm(feature_arr)\n                self.logger.info(\"norm of feature is {}<type: {}>\".format(norm_float, type(norm_float)))\n                assert isinstance(norm_float, float)\n                margin = self.margin / norm_float\n            else:\n                margin = self.margin\n\n            _file_list = []\n            for i in range(file_arr.shape[0]):\n                _file_list.append(file_arr[i].decode(\"utf-8\"))\n\n            all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n                self.get_basic_info(_file_list)\n            total_count = int(num_epochs * len(all_image_list) * self.gen_data_setting[\"x_train_num\"])\n            batch_count = total_count // batch_size\n            multi_batch_list = []\n\n            single_class_id_set = set(single_class_id_list)\n            anchor_index_end = -1\n            _anchor_search_end = False\n            labels = np.zeros(shape=file_arr.shape, dtype=np.int)\n            for i in range(file_arr.shape[0]):\n                _file_name = file_arr[i].decode(\"utf-8\")\n                labels[i] = self.get_label(_file_name)\n                if not _anchor_search_end:\n                    class_id = self.get_class_id(_file_name)\n                    if class_id in single_class_id_set:\n                        _anchor_search_end = True\n                    else:\n                        anchor_index_end = i\n\n            anchor_feature = feature_arr[:anchor_index_end + 1]\n            _apn_np_start = time.time()\n            apn_list = get_triplet_pair_np(anchor_feature, all_feature=feature_arr, all_label=labels,\n                                           margin=margin, logger=self.logger)\n            _apn_set = set()\n            for (a, p, n) in apn_list:\n                _apn_set.add(a)\n                _apn_set.add(p)\n                _apn_set.add(n)\n\n            self.logger.info(\n                \"get {} hardest-apn-pairs in {} files, time cost {}s. Unique file in apn pairs is {}.\".format(\n                    len(apn_list), feature_arr.shape[0], time.time() - _apn_np_start, len(_apn_set)))\n\n            _unique = True if len(anchor_class_id_list) >= batch_size else False\n            _unique_single = True if len(single_class_id_list) >= batch_size else False\n            _has_blank_class = True if class_id_vs_image_list.get(self.blank_class_id) else False\n            _blank_file_id_set = set()\n            if _has_blank_class:\n                _blank_file_label = self.get_label(class_id_vs_image_list[self.blank_class_id][0])\n                for i in range(labels.shape[0]):  # todo == len(labels)\n                    if labels[i] == _blank_file_label:\n                        _blank_file_id_set.add(i)\n\n            assert len(_blank_file_id_set) == len(class_id_vs_image_list.get(self.blank_class_id, []))\n\n            if len(apn_list) <= batch_size or (len(_apn_set - _blank_file_id_set) + 1) <= batch_size:\n                self.logger.warning(\"apn list is too small, use create_file_list_by_random instead!\")\n                return create_file_list_by_random()\n\n            tmp_apn_list = []\n            for i in range(batch_count):\n                batch_file_id_id = set()\n                _blank_exists = False\n                while True:\n                    if len(tmp_apn_list) == 0:\n                        tmp_apn_list.extend(list(apn_list))\n                        random.shuffle(tmp_apn_list)\n\n                    (a, p, n) = tmp_apn_list.pop(-1)\n                    for _file_id in [a, p, n]:\n                        if _file_id in _blank_file_id_set:\n                            if _blank_exists is False:\n                                batch_file_id_id.add(_file_id)\n                                _blank_exists = True\n                            else:\n                                continue\n                        else:\n                            batch_file_id_id.add(_file_id)\n\n                    if len(batch_file_id_id) >= batch_size:\n                        break\n\n                batch_list = [file_arr[file_id].decode(\"utf-8\") for file_id in batch_file_id_id]\n\n                assert len(batch_list) >= batch_size\n\n                batch_list = batch_list[:batch_size]\n\n                random.shuffle(batch_list)\n                multi_batch_list.append(batch_list)\n\n            return multi_batch_list\n\n        if is_training:\n            if online_batch_count > 0:\n                self.logger.info(\"using create_file_list_by_distance...\")\n\n                if self._to_use_random:\n                    _multi_batch_list = create_file_list_by_random()\n                    self._to_use_random = False\n                else:\n                    _multi_batch_list = create_file_list_by_distance()\n                # shuffle\n                if shuffle is True:\n                    random.shuffle(_multi_batch_list)\n\n                image_file_list = []\n                for _batch_list in _multi_batch_list:\n                    image_file_list.extend(_batch_list)\n\n                return image_file_list\n            else:\n                return self.list_train_image_file()\n\n        return self.list_test_image_file()\n\n    def _list_file_to_calc_feature(self) -> list:\n        if random.random() < self.gen_data_setting[\"ignore_blank_prob\"]:\n            all_image_list = list(self.list_train_image_file(ignore_blank=True))\n\n            if random.random() < self.gen_data_setting[\"ignore_single_prob\"]:\n                _, class_id_vs_image_list, _, anchor_class_id_list, _ = \\\n                    self.get_basic_info(all_image_list)\n\n                all_file_list = []\n                anchor_class_id_set = set(anchor_class_id_list)\n                for class_id, _file_list in class_id_vs_image_list.items():\n                    if class_id in anchor_class_id_set:\n                        all_file_list.extend(_file_list)\n\n                self.logger.info(\"list train image file [ignore_single, ignore_blank] to calc feature!\")\n                return all_file_list\n            else:\n                self.logger.info(\"list train image file [ignore_blank] to calc feature!\")\n                return all_image_list\n\n        self.logger.info(\"list all train image file to calc feature!\")\n        return list(self.list_train_image_file())\n\n    def calc_feature(self, callback, epoch_num: int = None):\n        \"\"\"\n            调用callback计算feature\n        Args:\n            epoch_num (int): total count\n            callback:\n        \"\"\"\n        if self.gen_data_setting.get(\"stop_calc_feature\", False):\n            self.logger.info(\"no need to calc feature: stop_calc_feature is True!\")\n            return\n\n        if epoch_num is not None and epoch_num == 0 and os.path.exists(self._file + \".feature.h5\"):\n            self.logger.info(\"no need to calc feature: feature file exist and epoch_num == 0!\")\n            return\n\n        if random.random() < self.gen_data_setting.get(\"gen_data_by_random_prob\", 0):\n            self.logger.info(\"no need to calc feature: going to gen data by random!\")\n            self._to_use_random = True\n            return\n\n        file_list = self._list_file_to_calc_feature()\n        if not file_list:\n            self.logger.info(\"no need to calc feature!\")\n            return\n\n        self.logger.info(\"calculating feature of {} file...\".format(len(file_list)))\n        feature_list, feature_file_list = callback(file_list, mode=ProcessMode.train)\n        self.logger.info(\"success to calculate feature of {} file!\".format(len(feature_file_list)))\n\n        self.update_feature(feature_list, feature_file_list, is_training=True)\n\n    def update_feature(self, feature_list, file_list, is_training: bool):\n        \"\"\"\n\n        Returns:\n            object:\n        \"\"\"\n        if not is_training:\n            self.logger.info(\"give up update-feature for no train mode!\")\n            return\n\n        if not feature_list:\n            return\n\n        # 保存 feature\n        self._save_feature(feature_list, file_list)\n\n    def _save_feature(self, feature_list: list, file_list: list):\n        # clear exist feature\n        _pkl_file = self._file + \".feature.h5\"\n        if os.path.exists(_pkl_file):\n            os.remove(_pkl_file)\n\n        # feature save\n        all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n            self.get_basic_info(all_image_list=file_list)\n\n        all_file_list = []\n        for class_id in anchor_class_id_list:\n            all_file_list.extend(class_id_vs_image_list[class_id])\n\n        for class_id in single_class_id_list:\n            all_file_list.extend(class_id_vs_image_list[class_id])\n\n        all_file_list.extend(class_id_vs_image_list.get(self.blank_class_id, []))\n        assert len(all_image_list) == len(all_file_list)\n\n        # sort file index\n        sort_index_list = []\n        _dict = {}\n        for i, file_name in enumerate(file_list):\n            _dict[file_name] = i\n        for file_name in all_file_list:\n            sort_index_list.append(_dict[file_name])\n\n        # feature\n        file_arr = np.asarray([f.encode(\"utf-8\") for f in all_file_list], dtype=np.string_)\n        feature_arr = np.zeros(shape=(len(all_file_list), self.dimension), dtype=np.float32)\n        # update feature\n        for index, key_index in enumerate(sort_index_list):\n            feature_arr[index][:] = feature_list[key_index].reshape((self.dimension,))\n\n        _pkl_file = self._file + \".feature.h5\"\n        store_data_in_h5file(_pkl_file, [feature_arr, file_arr], key_list=[\"feature\", \"file\"])\n\n    def _load_feature(self, create_if_not_exist: bool = True) -> (np.ndarray, np.ndarray):\n        \"\"\"\n\n        Returns:\n            feature, file, is_created\n        \"\"\"\n        _pkl_file = self._file + \".feature.h5\"\n        if os.path.exists(_pkl_file):\n            feature, file_arr = None, None\n            try:\n                feature, file_arr = load_data_from_h5file(_pkl_file, key_list=[\"feature\", \"file\"])\n            except Exception as e:\n                os.remove(_pkl_file)\n                self.logger.error(e)\n\n            if feature is not None:\n                self.logger.info(\"feature shape is {}, file_arr shape is {}\".format(feature.shape, file_arr.shape))\n                assert feature.shape[1] == self.dimension\n                assert file_arr.shape[0] == feature.shape[0]\n                return feature, file_arr\n\n        if not create_if_not_exist:\n            return None, None\n\n        all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n            self.get_basic_info()\n        all_file_list = []\n        for class_id in anchor_class_id_list:\n            all_file_list.extend(class_id_vs_image_list[class_id])\n\n        for class_id in single_class_id_list:\n            all_file_list.extend(class_id_vs_image_list[class_id])\n\n        all_file_list.extend(class_id_vs_image_list[self.blank_class_id])\n        assert len(all_image_list) == len(all_file_list)\n\n        file_arr = np.asarray([f.encode(\"utf-8\") for f in all_file_list], dtype=np.string_)\n        feature = np.zeros(shape=(len(all_file_list), self.dimension))\n\n        return feature, file_arr\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f3417e0c9b291d0c846f71aea65731229215dddd"},"cell_type":"markdown","source":"# 4.predict tools "},{"metadata":{"trusted":true,"_uuid":"10ae6a4b20404d42c89abc6781a5568803914380"},"cell_type":"code","source":"\nclass WhaleRankingUtils(object):\n    def __init__(self, data_utils: WhaleDataUtils, top_k: int = 5):\n        self.logger = logging.getLogger(self.__class__.__name__)\n        self.top_k = top_k\n        self.data_utils = data_utils\n\n    def simple_rank(self, result_list: list, distance_cutoff: float = None, only_distance_fit: bool = False) -> list:\n        \"\"\"\n\n        Args:\n            result_list: list, consist of (distance, class_id, image_file) elements\n            distance_cutoff: float\n            only_distance_fit: bool\n\n        Returns:\n            rank_list: list, like, [class_id_1, class_id_2, ... ]\n        \"\"\"\n        # deal with image_result_list\n        _class_id_list = []\n        _class_id_set = set()\n\n        if distance_cutoff is None:\n            for result in result_list:\n                class_id = result[1]\n                if class_id not in _class_id_set:\n                    _class_id_list.append(class_id)\n                    _class_id_set.add(class_id)\n\n            if len(_class_id_list) < self.top_k:\n                if self.data_utils.blank_class_id not in _class_id_set:\n                    _class_id_list.append(self.data_utils.blank_class_id)\n                    _class_id_set.add(self.data_utils.blank_class_id)\n\n        else:\n            if result_list[0][0] > distance_cutoff:\n                _class_id_list.append(self.data_utils.blank_class_id)\n                if only_distance_fit:\n                    return _class_id_list\n            else:\n                _class_id_list.append(result_list[0][1])\n            _class_id_set.add(_class_id_list[0])\n\n            for result in result_list[1:]:\n                if result[0] <= distance_cutoff:\n                    class_id = result[1]\n                    if class_id not in _class_id_set:\n                        _class_id_list.append(class_id)\n                        _class_id_set.add(class_id)\n                else:\n                    if only_distance_fit:\n                        return _class_id_list\n                    else:\n                        break\n\n            if len(_class_id_list) < self.top_k:\n                if self.data_utils.blank_class_id not in _class_id_set:\n                    _class_id_list.append(self.data_utils.blank_class_id)\n                    _class_id_set.add(self.data_utils.blank_class_id)\n\n            if len(_class_id_list) < self.top_k:\n                for result in result_list[1:]:\n                    if result[0] > distance_cutoff:\n                        class_id = result[1]\n                        if class_id not in _class_id_set:\n                            _class_id_list.append(class_id)\n                            _class_id_set.add(class_id)\n\n        return _class_id_list\n\n    def recreate_class_id_list(self, cutoff: float, class_id_list: list, feature_list: list) -> list:\n        new_class_id_list = []\n        count = 0\n        for i, class_list in enumerate(class_id_list):\n            _new_class_list = list(class_list)\n            if feature_list[i][0][0] <= cutoff or _new_class_list[0] == self.data_utils.blank_class_id:\n                new_class_id_list.append(_new_class_list)\n                continue\n\n            try:\n                found_index = _new_class_list.index(self.data_utils.blank_class_id)\n                if found_index > -1:\n                    _new_class_list.pop(found_index)\n            except ValueError:\n                pass\n\n            _new_class_list.insert(0, self.data_utils.blank_class_id)\n            new_class_id_list.append(_new_class_list)\n            count += 1\n\n        self.logger.info(\"recreate_class_id_list change {}/{} record\".format(count, len(class_id_list)))\n        return new_class_id_list\n\n    def get_similar_cutoff(self, validating_labels: list, validating_feature_list: list) -> float:\n        # calc similar distance cutoff\n        _distance_list = []\n        _, class_id_vs_image_list, _, anchor_class_id_list, _ = \\\n            self.data_utils.get_basic_info(self.data_utils.list_train_image_file())\n\n        anchor_class_set = set(anchor_class_id_list)\n        for i, class_id in enumerate(validating_labels):\n            if class_id not in anchor_class_set:\n                continue\n\n            for (_distance, _class_id, _file_name) in validating_feature_list[i]:\n                if class_id == _class_id:\n                    _distance_list.append(_distance)\n\n        self.logger.info(\"distance list: len is {}, first 5 is {}\".format(len(_distance_list), _distance_list[:5]))\n        distance_mean, distance_std = show_distance_dense_plot(\n            np.asarray(_distance_list), self.data_utils.path_manager.submission_csv + \".distance.jpg\")\n\n        distance_similar_cutoff = distance_mean + 3 * distance_std\n        self.logger.info(\"distance_similar_cutoff is {}\".format(distance_similar_cutoff))\n\n        return distance_similar_cutoff\n\n    @staticmethod\n    def get_feature_list(feature_file: str) -> (list, list):\n        feature_list = []\n        file_id_list = []\n        with open(feature_file, \"r\", encoding=\"utf-8\") as f:\n            for line in f:\n                if len(line) < 2:\n                    continue\n\n                line = line.rstrip()\n                file_id_list.append(line.split(\",\")[0])\n                feature_list.append(eval(line[line.find(\",\") + 1:]))\n\n        return file_id_list, feature_list\n\n    def analyze_validating_result(self, feature_file: str):\n        # read feature list\n        file_id_list, feature_list = self.get_feature_list(feature_file)\n\n        class_id_result_list = [self.simple_rank(feature)[:self.top_k] for feature in feature_list]\n\n        # calc map@5\n        labels = self.list_class_id(file_id_list)\n        total_map_5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)\n        total_map_1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)\n        self.logger.info(\"validating data result is: map@5 is {},  map@1 is {}!\".format(total_map_5, total_map_1))\n\n        # calc similar distance cutoff\n        distance_similar_cutoff = self.get_similar_cutoff(labels, feature_list)\n\n        # find error\n        for i, image_id in enumerate(file_id_list):\n            map_5 = map_per_image(label=labels[i], predictions=class_id_result_list[i], k=5)\n            if map_5 < 1.0:\n                # error\n                self.logger.info(\"map@5 is {}\\nTrue: {}, Predict with cutoff is {}\\n{}\\n\".format(\n                    map_5, labels[i],\n                    self.data_utils.blank_class_id if feature_list[i][0][0] > distance_similar_cutoff else\n                    feature_list[i][0][1],\n                    feature_list[i])\n                )\n\n        # create new class id result\n        new_class_id_result = self.recreate_class_id_list(\n            cutoff=distance_similar_cutoff, class_id_list=class_id_result_list, feature_list=feature_list)\n\n        total_map_5 = map_per_set(labels=labels, predictions=new_class_id_result, k=5)\n        total_map_1 = map_per_set(labels=labels, predictions=new_class_id_result, k=1)\n        self.logger.info(\n            \"validating data result with cutoff is: map@5 is {},  map@1 is {}!\".format(total_map_5, total_map_1))\n\n    def compare_submission_file(self, file_1, file_2):\n        def _compare_same_count(_result_1, _result_2, info):\n            assert set(_result_1.keys()) == set(_result_2.keys())\n\n            all_same_count = 0\n            for image_id, result_list in _result_1.items():\n                if \",\".join(result_list[:self.top_k]) == \",\".join(_result_2[image_id][:self.top_k]):\n                    all_same_count += 1\n\n            self.logger.info(\"[{}] same count is {}/{}\".format(info, all_same_count, len(_result_1)))\n\n        def _compare_top1_count(_result_1, _result_2, info):\n            assert set(_result_1.keys()) == set(_result_2.keys())\n\n            all_same_count = 0\n            for image_id, result_list in _result_1.items():\n                if result_list[0] == _result_2[image_id][0]:\n                    all_same_count += 1\n\n            self.logger.info(\"[{}] same top1 count is {}/{}\".format(info, all_same_count, len(_result_1)))\n\n        def _compare_map(_result_1, _result_2, info):\n            assert set(_result_1.keys()) == set(_result_2.keys())\n\n            _file_id_list = list(_result_1.keys())\n            _file_id_list.sort()\n            labels = self.list_class_id(_file_id_list)\n\n            # 1\n            _class_id_result_list_1 = [_result_1[_image_id] for _image_id in _file_id_list]\n            map5_1 = map_per_set(labels=labels, predictions=_class_id_result_list_1, k=5)\n            map1_1 = map_per_set(labels=labels, predictions=_class_id_result_list_1, k=1)\n\n            # 2\n            _class_id_result_list_2 = [_result_2[_image_id] for _image_id in _file_id_list]\n            map5_2 = map_per_set(labels=labels, predictions=_class_id_result_list_2, k=5)\n            map1_2 = map_per_set(labels=labels, predictions=_class_id_result_list_2, k=1)\n\n            self.logger.info(\"[{}] map@5 is {}/{}, map@1 is {}/{}\".format(\n                info,\n                get_pretty_float(map5_1, count=3),\n                get_pretty_float(map5_2, count=3),\n                get_pretty_float(map1_1, count=3),\n                get_pretty_float(map1_2, count=3),\n            ))\n\n        # true same count\n        result_1, result_2 = self.read_submission_file(file_1), self.read_submission_file(file_2)\n        result_1_x, result_2_x = self._get_result_from_feature_file(file_1), self._get_result_from_feature_file(file_2)\n        result_1_true, result_2_true = self._get_result_by_cutoff(file_1), self._get_result_by_cutoff(file_2)\n        self.logger.info(\"Start to compare:\\n\\n\")\n\n        # map\n        self.logger.info(\"compare map5, map1\")\n        _compare_map(result_1, result_2, \"map, raw, resenet vs siamese\")\n        _compare_map(result_1, result_1_x, \"map, raw, resenet, csv vs feature\")\n        _compare_map(result_2, result_2_x, \"map, raw, siamese, csv vs feature\")\n        _compare_map(result_1_true, result_2_true, \"map, true, resenet vs siamese\")\n        _compare_map(result_1, result_1_true, \"map, resent, raw vs true\")\n        _compare_map(result_2, result_2_true, \"map, siamese, raw vs true\")\n        self.logger.info(\"End\\n\\n\")\n\n        # top5 same\n        self.logger.info(\"compare top5\")\n        _compare_same_count(result_1, result_2, \"top5, raw, resenet vs siamese\")\n        _compare_same_count(result_1_true, result_2_true, \"top5, true, resenet vs siamese\")\n        _compare_same_count(result_1, result_1_true, \"top5, resent, raw vs true\")\n        _compare_same_count(result_2, result_2_true, \"top5, siamese, raw vs true\")\n        self.logger.info(\"End\\n\\n\")\n\n        # top1 same\n        self.logger.info(\"compare top1\")\n        _compare_top1_count(result_1, result_2, \"top1, raw, resenet vs siamese\")\n        _compare_top1_count(result_1_true, result_2_true, \"top1, true, resenet vs siamese\")\n        _compare_top1_count(result_1, result_1_true, \"top1, resent, raw vs true\")\n        _compare_top1_count(result_2, result_2_true, \"top1, siamese, raw vs true\")\n        self.logger.info(\"End\\n\\n\")\n\n    @staticmethod\n    def read_submission_file(csv_file: str) -> dict:\n        submission = pd.read_csv(csv_file)\n        _result = {}\n        for index, _image_id in enumerate(submission.Image):\n            _result[_image_id] = submission.Id[index].rstrip().split(\" \")\n\n        return _result\n\n    def _get_result_from_feature_file(self, csv_file: str) -> dict:\n        feature_file = csv_file + \".feature\"\n        assert os.path.exists(feature_file)\n        cutoff, _, file_id_list, feature_list = self._read_feature_file(feature_file)\n\n        _new_result = {}\n        for _index, file_id in enumerate(file_id_list):\n            _new_result[file_id] = self.simple_rank(\n                feature_list[_index], distance_cutoff=cutoff, only_distance_fit=False)\n\n        return _new_result\n\n    def _get_result_by_cutoff(self, csv_file: str) -> dict:\n        feature_file = csv_file + \".feature\"\n        assert os.path.exists(feature_file)\n        cutoff, labels, file_id_list, feature_list = self._read_feature_file(feature_file)\n        self.logger.info(\"cutoff is {}\".format(get_pretty_float(cutoff, count=3)))\n\n        _new_result = {}\n        for _index, file_id in enumerate(file_id_list):\n            _new_result[file_id] = self.simple_rank(\n                feature_list[_index], distance_cutoff=cutoff, only_distance_fit=True)\n\n        return _new_result\n\n    def _read_feature_file(self, feature_file):\n        assert os.path.exists(feature_file)\n        file_id_list, feature_list = self.get_feature_list(feature_file)\n        labels = self.list_class_id(file_id_list)\n        cutoff = self.get_similar_cutoff(labels, feature_list)\n        return cutoff, labels, file_id_list, feature_list\n\n    @staticmethod\n    def _calc_weight_score(feature_list, file_id_list, cutoff) -> dict:\n        # todo simple score\n        keep_count = 10\n        cutoff_score = 5.0\n\n        _new_result = {}\n        for _index, file_id in enumerate(file_id_list):\n            _new_result[file_id] = {}\n\n            _sort_num = -1\n            for (distance, class_id, file_name) in feature_list[_index]:\n                if class_id not in _new_result[file_id]:\n                    _sort_num += 1\n                    _new_result[file_id][class_id] = keep_count - _sort_num\n                    if distance < cutoff:\n                        _new_result[file_id][class_id] += cutoff_score\n\n                    if _sort_num >= keep_count:\n                        break\n\n        return _new_result\n\n    @staticmethod\n    def _get_result_by_weight_score(score_dict: dict, top_k: int = 5) -> dict:\n        def _sort_by_value(info_dict: dict) -> list:\n            _tuple = sorted(info_dict.items(), key=lambda x: x[1], reverse=True)\n            return [_class_id for (_class_id, _weight) in _tuple]\n\n        result_dict = {}\n        for image_id, score in score_dict.items():\n            result_dict[image_id] = _sort_by_value(score)[:top_k]\n\n        return result_dict\n\n    def combine_submission_file(self, validating_file_list: list, submission_file_list: list, output_file: str):\n        def _create_final_result(_result_dict: dict):\n            _file_list = []\n            _result_list = []\n            for _image_id, _class_list in _result_dict.items():\n                _file_list.append(_image_id)\n                _result_list.append(_class_list)\n            return _file_list, _result_list\n\n        def _update_weight_score(all_dict: dict, one_dict: dict, model_weight: float):\n            for _image_id, _score_dict in one_dict.items():\n                if _image_id not in all_dict:\n                    all_dict[_image_id] = {}\n\n                for _class_id, _score in _score_dict.items():\n                    all_dict[_image_id].setdefault(_class_id, 0.0)\n                    all_dict[_image_id][_class_id] += _score * model_weight\n\n            return all_dict\n\n        def eval_result(_result_dict: dict):\n            _file_list, _result_list = _create_final_result(_result_dict)\n            _labels = self.list_class_id(_file_list)\n            _map5 = map_per_set(labels=_labels, predictions=_result_list, k=5)\n            _map1 = map_per_set(labels=_labels, predictions=_result_list, k=1)\n            return _map5, _map1\n\n        # check input\n        assert len(validating_file_list) == len(submission_file_list)\n        for i, validating_submission_csv in enumerate(validating_file_list):\n            predict_submission_csv = submission_file_list[i]\n            assert \".\".join(validating_submission_csv.split(\".\")[:3]) == \".\".join(predict_submission_csv.split(\".\")[:3])\n            assert os.path.exists(validating_submission_csv + \".feature\")\n            assert os.path.exists(predict_submission_csv + \".feature\")\n\n        total_predict_score_dict = {}\n        total_validate_score_dict = {}\n        total_eval_result_list = []\n\n        for i, validating_submission_csv in enumerate(validating_file_list):\n            predict_submission_csv = submission_file_list[i]\n\n            # validate\n            cutoff, _, v_file_id_list, v_feature_list = self._read_feature_file(validating_submission_csv + \".feature\")\n            v_model_weight = float(\"0.{}\".format(validating_submission_csv.split(\".\")[-3]))\n\n            validating_score_dict = self._calc_weight_score(\n                feature_list=v_feature_list, file_id_list=v_file_id_list, cutoff=cutoff)\n            total_validate_score_dict = _update_weight_score(\n                total_validate_score_dict, validating_score_dict, model_weight=v_model_weight)\n\n            # predict\n            p_file_id_list, p_feature_list = self.get_feature_list(predict_submission_csv + \".feature\")\n            predict_score_dict = self._calc_weight_score(\n                feature_list=p_feature_list, file_id_list=p_file_id_list, cutoff=cutoff)\n            total_predict_score_dict = _update_weight_score(\n                total_predict_score_dict, predict_score_dict, model_weight=v_model_weight)\n\n            # eval validate\n            _validate_result_dict = self.read_submission_file(validating_submission_csv)\n            total_eval_result_list.append(eval_result(_validate_result_dict))\n\n        # evaluate validating result\n        _validate_result_dict = self._get_result_by_weight_score(total_validate_score_dict, top_k=self.top_k)\n        total_eval_result_list.append(eval_result(_validate_result_dict))\n        self.logger.info(\"after combine {} models: \\nvalidate map@5 is {}/{}, \\nvalidate map@1 is {}/{}\\n\\n\".format(\n            len(validating_file_list),\n            total_eval_result_list[-1][0], [x[0] for x in total_eval_result_list[:-1]],\n            total_eval_result_list[-1][1], [x[1] for x in total_eval_result_list[:-1]],\n        ))\n\n        # evaluate and save prediction result\n        raw_submission = self.data_utils.path_manager.submission_csv\n        try:\n            predict_result_dict = self._get_result_by_weight_score(total_predict_score_dict, top_k=self.top_k)\n            predict_file_list, predict_result_list = _create_final_result(predict_result_dict)\n            self.data_utils.path_manager.submission_csv = output_file\n            self.data_utils.submit_test_result(test_image_list=predict_file_list, result_list=predict_result_list)\n            self.logger.info(\"save final submission result in {}\".format(self.data_utils.path_manager.submission_csv))\n        finally:\n            self.data_utils.path_manager.submission_csv = raw_submission\n\n    def debug_map_csv(self, csv_file):\n        self.logger.info(\"debug map set from submission file...\\n\\n\")\n        result_dict = self.read_submission_file(csv_file)\n\n        class_id_result_list = []\n        file_id_list = []\n        for _image_id, _result_list in result_dict.items():\n            file_id_list.append(_image_id)\n            class_id_result_list.append(_result_list)\n\n        labels = self.list_class_id(file_id_list)\n\n        self.logger.info(\"map5 is {}\".format(map_per_set(labels=labels, predictions=class_id_result_list, k=5)))\n        self.logger.info(\"map1 is {}\".format(map_per_set(labels=labels, predictions=class_id_result_list, k=1)))\n\n    def list_class_id(self, file_id_list) -> list:\n        _, class_id_vs_image_list, _, _, _ = self.data_utils.get_basic_info(self.data_utils.list_train_image_file())\n        class_id_vs_image_count = {\n            class_id: len(_image_list) for class_id, _image_list in class_id_vs_image_list.items()}\n        return [self.get_class_id(class_id_vs_image_count, _image_id) for _image_id in file_id_list]\n\n    def get_class_id(self, class_id_vs_image_count, image_file_or_id) -> str:\n        # ignore same image\n        _class_id = self.data_utils.get_class_id(image_file_or_id)\n        if class_id_vs_image_count[_class_id] > 1:\n            return _class_id\n        else:\n            return self.data_utils.blank_class_id\n\n\nclass SimpleSearch(object):\n    model = None\n\n    def __init__(self, data_utils: WhaleDataUtils, shape: tuple, dimension: int, top_k: int = 5,\n                 norm_type: NormType = NormType.none, more_search_top_k: int = None):\n        self.logger = logging.getLogger(self.__class__.__name__)\n        self.data_utils = data_utils\n        self.faiss_index_path = os.path.join(self.data_utils.path_manager.working_path, self.__class__.__name__)\n        self.shape = shape\n        self.dimension = dimension\n        self.top_k = top_k\n        self._distance_cutoff = None\n        self.manager = ImageIndexUtils(\n            index_dir=self.faiss_index_path,\n            dimension=self.dimension)\n        self._norm_type = norm_type\n\n        if more_search_top_k is None or more_search_top_k < 0:\n            more_search_top_k = max(20, self.top_k)\n        self._search_top_k = more_search_top_k + self.top_k\n\n    def show_embedding(self, mode: ProcessMode):\n        key_name = \"train\" if mode == ProcessMode.train else \"test\"\n        list_image_file = self.data_utils.list_train_image_file if mode == ProcessMode.train \\\n            else self.data_utils.list_test_image_file\n        log_dir = os.path.join(self.faiss_index_path, \"logs_{}\".format(key_name))\n\n        self.logger.info(\"trying to show {} embedding...\".format(key_name))\n\n        feature_list, image_list = self.calc_or_load_feature(mode)\n        self.logger.info(\"got {}-image feature: {}/{}\".format(\n            key_name, len(image_list), len(list_image_file()))\n        )\n\n        labels = []\n        if mode == ProcessMode.train:\n            for image_file in image_list:\n                labels.append(self.data_utils.get_class_id(image_file))\n        else:\n            for image_file in image_list:\n                labels.append(os.path.basename(image_file))\n\n        show_embedding(feature_list=feature_list, labels=labels, log_dir=log_dir)\n        self.logger.info(\"success to show {} embedding!\".format(key_name))\n\n    def list_features(self, mode: ProcessMode) -> (list, list):\n        \"\"\"\n            input: ?X224X224X3\n            feature: list of ?X7X7X320, files: list of str\n        \"\"\"\n\n        raise NotImplementedError\n\n    def calc_or_load_feature(self, mode: ProcessMode) -> (list, list):\n        key_name = \"train\" if mode == ProcessMode.train else \"test\"\n\n        # calc feature or load image feature\n        feature_npy = os.path.join(self.faiss_index_path, \"{}.npy.pkl\".format(key_name))\n\n        self.logger.info(\"{}ing image...\".format(key_name))\n\n        if not os.path.exists(feature_npy):\n            self.logger.info(\"calculating feature of {} image...\".format(key_name))\n            feature_list, image_list = self.list_features(mode)\n            self.logger.info(\"success to calculate feature of {} image!\".format(key_name))\n            with open(feature_npy, \"wb\") as f:\n                pickle.dump((feature_list, image_list), f)\n        else:\n            self.logger.info(\"loading feature of {} image...\".format(key_name))\n            with open(feature_npy, \"rb\") as f:\n                feature_list, image_list = pickle.load(f)\n            self.logger.info(\"success to load feature of {} image!\".format(key_name))\n\n        return feature_list, image_list\n\n    def train(self):\n        if os.path.exists(self.manager.manager.faiss_index_file):\n            self.logger.info(\"train before, no need to train again!\")\n            return\n\n        # train: calc feature or load image feature\n        feature_list, image_list = self.calc_or_load_feature(ProcessMode.train)\n        self.logger.info(\"got train-image feature: {}/{}\".format(\n            len(image_list), len(self.data_utils.list_train_image_file())))\n\n        image_info_list = []\n        for index, image_file in enumerate(image_list):\n            image_info_list.append({\n                \"index\": index,\n                \"file\": image_file,\n                \"class_id\": self.data_utils.get_class_id(image_file)}\n            )\n        self.manager.add_images(feature_list, image_info_list=image_info_list)\n        self.logger.info(\"success to train image!\")\n\n    def test(self):\n        # calc feature\n        feature_list, image_list = self.calc_or_load_feature(ProcessMode.test)\n        if len(image_list) != len(self.data_utils.list_test_image_file()):\n            self.logger.warning(\"got test-image file: {}/{}\".format(\n                len(image_list), len(self.data_utils.list_test_image_file())))\n            assert len(image_list) % len(self.data_utils.list_test_image_file()) == 0\n\n        self.logger.info(\"got test-image feature: {}/{}\".format(\n            len(image_list), len(self.data_utils.list_test_image_file())))\n\n        # image search\n        num_feature = len(image_list) // len(self.data_utils.list_test_image_file())\n        self.logger.info(\"testing image...\")\n        test_image_list = [image_list[index * num_feature] for index in\n                           range(len(self.data_utils.list_test_image_file()))]\n        class_id_result_list, test_feature_list = self.search(\n            image_list=test_image_list, feature_list=feature_list, return_feature_list=True)\n        self.logger.info(\"success to test image!\")\n\n        # submit result\n        assert len(class_id_result_list) == len(self.data_utils.list_test_image_file())\n        self.data_utils.submit_test_result(test_image_list=test_image_list,\n                                           result_list=class_id_result_list,\n                                           feature_list=test_feature_list)\n        if self._distance_cutoff is None:\n            self.logger.info(\"submit predict result[without distance cutoff] in {}\".format(\n                self.data_utils.path_manager.submission_csv))\n        else:\n            self.logger.info(\"submit predict result[with distance cutoff {}] in {}\".format(\n                get_pretty_float(self._distance_cutoff, count=3), self.data_utils.path_manager.submission_csv))\n\n    def search(self, image_list: list, feature_list: list, cache_file: str = None, ignore_same: bool = False,\n               return_feature_list: bool = True) -> (list, list):\n        if cache_file is None:\n            cache_file = os.path.join(self.faiss_index_path, \"cache.pkl\")\n\n        cache = FileCache(cache_file)\n\n        # one_file -> multi feature\n        assert len(feature_list) % len(image_list) == 0\n        num_feature = len(feature_list) // len(image_list)\n        if num_feature > 1:\n            self.logger.info(\"1 image vs {} feature when searching!\".format(num_feature))\n\n        # cache\n        src_list = []\n        for index in range(len(image_list)):\n            image_file = image_list[index]\n            if not cache.get(image_file):\n                src_list.append((image_file, feature_list[index * num_feature:(index + 1) * num_feature]))\n\n        # image search\n        whale_rank_utils = WhaleRankingUtils(data_utils=self.data_utils, top_k=self.top_k)\n        extend = False\n        _show_log_per_steps = 100 if len(image_list) < 1000 else 1000\n        for part_src_list in iter_list_with_size(src_list, size=10240 // num_feature):\n            feature_list = []\n            for (_, _feature_list) in part_src_list:\n                feature_list.extend(_feature_list)\n\n            raw_result_iterator = self.manager.image_search_iterator(\n                feature_list=feature_list,\n                top_k=self._search_top_k,\n                extend=extend\n            )\n\n            index = -1\n            cache_list = []\n            for raw_image_result_list in raw_result_iterator:\n                index += 1\n                if index % num_feature == 0:\n                    # the first result for image file\n                    _image_file = part_src_list[index // num_feature][0] if ignore_same else None\n                    image_result_list = []\n\n                for (faiss_image_index, faiss_image_extend_list, faiss_distance) in raw_image_result_list:\n                    image_result = self.manager.get_image_info(faiss_image_index)\n                    if _image_file and image_result.get(\"file\") and image_result.get(\"file\").find(_image_file) == 0:\n                        continue\n\n                    image_result_list.append((faiss_distance, image_result[\"class_id\"], image_result[\"file\"]))\n\n                if (index + 1) % num_feature == 0:\n                    # the last result for image file\n                    image_result_list = sorted(image_result_list, key=lambda x: x[0], reverse=False)\n                    _class_id_list = whale_rank_utils.simple_rank(image_result_list,\n                                                                  distance_cutoff=self._distance_cutoff)\n                    if len(_class_id_list) < self.top_k:\n                        self.logger.debug(\"length of class_id_list < {}!\".format(self.top_k))\n\n                    # save in cache\n                    if return_feature_list:\n                        cache_list.append((_class_id_list[:self.top_k], list(image_result_list)))\n                    else:\n                        cache_list.append((_class_id_list[:self.top_k], None))\n\n                # logging\n                if index % _show_log_per_steps == 0:\n                    self.logger.info(\"got class_id_list of {} images\".format(index))\n\n            # bulk save in cache\n            for j, _class_id_list_feature_list in enumerate(cache_list):\n                cache.unsafe_set(part_src_list[j][0], _class_id_list_feature_list)\n            cache.set(part_src_list[0][0], cache_list[0])\n\n        # class_id_result_list\n        class_id_result_list = []\n        search_feature_list = []\n        for image_file in image_list:\n            _class_id_list, _search_feature_list = cache.get(image_file)\n            class_id_result_list.append(_class_id_list)\n            search_feature_list.append(_search_feature_list)\n\n        self.logger.info(\"success to parse search result for test images!\")\n\n        return class_id_result_list, search_feature_list\n\n    def _calc_map(self, info, data_percent, labels, class_id_result_list,\n                  image_list, multi_index_list) -> (float, float):\n        fake_map5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)\n        fake_map1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)\n\n        multi_map_5 = map_per_set(\n            labels=[labels[index] for index in multi_index_list],\n            predictions=[class_id_result_list[index] for index in multi_index_list],\n            k=5)\n        multi_map_1 = map_per_set(\n            labels=[labels[index] for index in multi_index_list],\n            predictions=[class_id_result_list[index] for index in multi_index_list],\n            k=1)\n\n        # new whale\n        new_index_list = []\n        for _index, _image_file in enumerate(image_list):\n            if self.data_utils.get_class_id(_image_file) == self.data_utils.blank_class_id:\n                new_index_list.append(_index)\n\n        new_map_5 = map_per_set(\n            labels=[labels[index] for index in new_index_list],\n            predictions=[class_id_result_list[index] for index in new_index_list],\n            k=5)\n        new_map_1 = map_per_set(\n            labels=[labels[index] for index in new_index_list],\n            predictions=[class_id_result_list[index] for index in new_index_list],\n            k=1)\n\n        # logging\n        self.logger.info(\n            \"map@5 for validating data[{} of train data, {}] is: standard {}, new-whale {}, all {}!\".format(\n                data_percent, info, multi_map_5, new_map_5, fake_map5)\n        )\n        self.logger.info(\n            \"map@1 for validating data[{} of train data, {}] is: standard {}, new-whale {}, all {}!\".format(\n                data_percent, info, multi_map_1, new_map_1, fake_map1)\n        )\n\n        return multi_map_5, multi_map_1\n\n    def validate(self, data_percent: float = 1.0) -> (float, float):\n        \"\"\"\n            validate with training data\n\n        Returns:\n            (float, float): map@5, map@1\n        \"\"\"\n        # calc feature\n        train_feature_list, train_image_list = self.calc_or_load_feature(ProcessMode.train)\n        num_feature = len(train_image_list) // len(self.data_utils.list_train_image_file())\n        if len(train_image_list) != len(self.data_utils.list_train_image_file()):\n            self.logger.warning(\"got validating-image file: {}/{}\".format(\n                len(train_image_list), len(self.data_utils.list_train_image_file())))\n            assert len(train_image_list) % len(self.data_utils.list_train_image_file()) == 0\n\n        # chosen data\n        chosen_index_list = random_choice([index for index in range(len(self.data_utils.list_train_image_file()))],\n                                          k=int(len(self.data_utils.list_train_image_file()) * data_percent),\n                                          unique=True)\n        if num_feature == 1:\n            image_list = [train_image_list[index] for index in chosen_index_list]\n            feature_list = [train_feature_list[index] for index in chosen_index_list]\n        else:\n            image_list = [train_image_list[index * num_feature] for index in chosen_index_list]\n            feature_list = []\n            for index in chosen_index_list:\n                feature_list.extend(train_feature_list[index * num_feature:(index + 1) * num_feature])\n\n        # search\n        self.logger.info(\"validating image...\")\n        _raw_class_id_result_list, validating_feature_list = self.search(\n            image_list=image_list, feature_list=feature_list, ignore_same=True, return_feature_list=True)\n        self.logger.info(\"success to validate image!\")\n\n        # calc map@5\n        image_id_vs_image_file = {os.path.basename(image_file): image_file for image_file in image_list}\n        class_id_vs_image_list = {}\n        for image_id, class_id in self.data_utils.load_train_info().items():\n            class_id_vs_image_list.setdefault(class_id, []).append(image_id_vs_image_file[image_id])\n        class_id_vs_image_count = {\n            class_id: len(_image_list) for class_id, _image_list in class_id_vs_image_list.items()}\n\n        # rand utils\n        rank_utils = WhaleRankingUtils(data_utils=self.data_utils, top_k=self.top_k)\n\n        # all data\n        labels = [rank_utils.get_class_id(class_id_vs_image_count, image_file) for image_file in image_list]\n\n        # multi instance\n        multi_index_list = []\n        for _index, _image_file in enumerate(image_list):\n            if class_id_vs_image_count[self.data_utils.get_class_id(_image_file)] > 1:\n                multi_index_list.append(_index)\n\n        distance_cutoff = rank_utils.get_similar_cutoff(\n            validating_labels=labels, validating_feature_list=validating_feature_list)\n        self.logger.info(\"got distance_cutoff {} and use it!\".format(get_pretty_float(distance_cutoff, count=3)))\n        self._distance_cutoff = distance_cutoff\n\n        class_id_result_list = rank_utils.recreate_class_id_list(\n            cutoff=self._distance_cutoff, class_id_list=_raw_class_id_result_list, feature_list=validating_feature_list)\n\n        total_map_5, total_map_1 = self._calc_map(\n            \"with distance cutoff {}\".format(get_pretty_float(self._distance_cutoff, count=3)),\n            data_percent, labels, class_id_result_list, image_list, multi_index_list\n        )\n\n        # baseline\n        random_map5, random_map1 = self.data_utils.get_random_baseline()\n        self.logger.info(\"random baseline for validating data[{} of train data]: map@5 is {}, map@1 is {}!\".format(\n            data_percent, random_map5, random_map1))\n\n        # submit result\n        raw_submission_csv = self.data_utils.path_manager.submission_csv\n        try:\n            self.data_utils.path_manager.submission_csv = \\\n                raw_submission_csv + \".{}.validate.csv\".format(str((total_map_5 + total_map_1) / 2)[2:])\n            self.data_utils.submit_test_result(\n                test_image_list=image_list, result_list=class_id_result_list, feature_list=validating_feature_list)\n            self.logger.info(\"submit validating result[{} of train data, with distance cutoff {}] in {}\".format(\n                data_percent, get_pretty_float(distance_cutoff, count=3), self.data_utils.path_manager.submission_csv))\n        finally:\n            self.data_utils.path_manager.submission_csv = raw_submission_csv\n\n        return total_map_5, total_map_1\n\n\nclass TripletSearch(SimpleSearch):\n    \"\"\"\n        Name: TripletSearch\n    \"\"\"\n    model = None\n\n    def __init__(self, instance, data_utils: WhaleDataUtils, clear_cache_after_exists: bool,\n                 norm_type: NormType = NormType.l2):\n        self.instance = instance\n        super(TripletSearch, self).__init__(\n            data_utils, shape=(None,), dimension=instance.params.dimension, top_k=5)\n        self.smart_iterator = None\n        self.feature_record = {}\n        self._clear_cache_after_exists = clear_cache_after_exists\n        self._norm_type = norm_type\n\n    def list_features(self, mode: ProcessMode) -> (list, list):\n        \"\"\"\n            input: ?X224X224X3\n            feature: list of ?X5005, files: list of str\n        \"\"\"\n        if \"train\" not in self.feature_record and \"test\" not in self.feature_record:\n            train_feature_list, train_feature_file_list = self.instance.list_features(mode=ProcessMode.train)\n            test_feature_list, test_feature_file_list = self.instance.list_features(mode=ProcessMode.test)\n\n            # l2 norm\n            feature = np.vstack(train_feature_list + test_feature_list). \\\n                reshape((len(train_feature_list) + len(test_feature_list), self.dimension))\n            if self._norm_type is None:\n                normalized_feature = feature\n            else:\n                self.logger.info(\"using {} to normalize feature\".format(self._norm_type))\n                normalized_feature = self._norm_type.normalize(feature)\n\n            self.feature_record[\"train\"] = train_feature_file_list\n            self.feature_record[\"test\"] = test_feature_file_list\n            self.feature_record[\"feature\"] = normalized_feature\n\n        if mode == ProcessMode.train:\n            feature_file_list = self.feature_record[\"train\"]\n            feature_list = [f.reshape(1, self.dimension) for f in\n                            self.feature_record[\"feature\"][:len(feature_file_list)]]\n        else:\n            feature_file_list = self.feature_record[\"test\"]\n            feature_list = [f.reshape(1, self.dimension) for f in\n                            self.feature_record[\"feature\"][len(self.feature_record[\"train\"]):]]\n\n        assert len(feature_list) == len(feature_file_list)\n        assert feature_list[0].shape == (1, self.dimension)\n\n        return feature_list, feature_file_list\n\n    def clear_cache(self, ):\n        if os.path.exists(self.faiss_index_path):\n            try:\n                if not self.data_utils.path_manager.is_kaggle:\n                    tmp_dir = os.path.join(os.path.dirname(self.faiss_index_path), self.instance.__class__.__name__)\n                    if os.path.exists(tmp_dir):\n                        shutil.rmtree(tmp_dir)\n                    shutil.copytree(self.faiss_index_path, tmp_dir)\n                    self.logger.info(\"backup Search Cache in {}\".format(os.path.abspath(tmp_dir)))\n                shutil.rmtree(self.faiss_index_path)\n            except Exception as e:\n                self.logger.error(e, exc_info=True)\n\n    def __enter__(self):\n        return self\n\n    def __exit__(self, exc_type, exc_val, exc_tb):\n        if self._clear_cache_after_exists:\n            self.clear_cache()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77384eb4c8bbb4b1a1d1b0d2e5c85b5eca8b3c35"},"cell_type":"markdown","source":"# 5.Triplet loss net from Siamese net "},{"metadata":{"trusted":true,"_uuid":"c7bd8fdc3285166b54f3bd6bec578faa82cf6176"},"cell_type":"code","source":"class TripletLossModelCNN(AbstractEstimator):\n    def __init__(self, train_ckpt_dir, data_utils: WhaleDataUtils, timeout: int = int(3600 * 5),\n                 pretrained_ckpt_file: str = None):\n        super(TripletLossModelCNN, self).__init__(\n            model_name=\"TripletLoss\",\n            train_ckpt_dir=train_ckpt_dir,\n            pretrained_ckpt_file=pretrained_ckpt_file\n        )\n        self.data_utils = data_utils\n        self.timeout = timeout + time.time()\n        self.batch_size = 8\n        self._features_images_key = \"images\"\n        self._features_filename_key = \"filenames\"\n        self._features_embedding_key = \"embeddings\"\n        self.params = Params({\n            \"num_channels\": 1,\n            \"margin\": 1.0,\n            \"dimension\": 512,\n            \"image_size\": 384,\n            \"online_batch_count\": 4,\n        })\n        self.learning_rate = 64e-5\n        self.image_tmp_size = 400\n        assert self.image_tmp_size >= self.params.image_size\n\n        self.train_loss_hook = LossStepHookForTrain(log_after_run=True, log_after_end=False,\n                                                    show_log_per_steps=10, run_after_run_per_steps=5)\n        self.data_utils.loss_hook = self.train_loss_hook\n        self.using_training_status_of_train_data = None\n        self.optimizer_type = OptimizerType.adam\n\n    def show_embedding(self, mode: ProcessMode, count: int = -1, remove_log_dir_if_exists: bool = False,\n                       norm_type: NormType = NormType.all):\n        key_name = \"train\" if mode == ProcessMode.train else \"test\"\n        log_dir = os.path.join(self.TRAIN_CKPT_DIR, \"logs_{}\".format(key_name))\n        if os.path.exists(log_dir) and remove_log_dir_if_exists:\n            shutil.rmtree(log_dir)\n\n        self.logger.info(\"trying to show {} embedding...\".format(key_name))\n\n        if count > 1:\n            if mode == ProcessMode.train:\n                image_list = self.data_utils.list_debug_image_list(count=count)\n            else:\n                image_list = self.data_utils.get_file_list(\n                    online_batch_count=0, is_training=bool(mode == ProcessMode.train))\n                image_list = image_list[:count]\n            if len(image_list) != count:\n                self.logger.warning(\"only have {} files\".format(len(image_list)))\n        else:\n            image_list = self.data_utils.get_file_list(\n                online_batch_count=0, is_training=bool(mode == ProcessMode.train))\n\n        if mode == ProcessMode.train:\n            label_list = [self.data_utils.get_class_id(image_file) for image_file in image_list]\n        else:\n            label_list = [os.path.basename(image_file) for image_file in image_list]\n\n        feature_list, _ = self.list_features(file_list=image_list)\n\n        # normalize\n        if norm_type is not None:\n            feature = np.vstack(feature_list).reshape((len(feature_list), self.params.dimension))\n            self.logger.info(\"using {} to normalize feature\".format(norm_type))\n            normalized_feature = norm_type.normalize(feature)\n            feature_list = [f.reshape(1, self.params.dimension) for f in normalized_feature]\n\n        self.logger.info(\"got {}-image feature: {}/{}\".format(\n            key_name, len(feature_list), len(image_list))\n        )\n\n        show_embedding(feature_list=feature_list, labels=label_list, log_dir=log_dir)\n        self.logger.info(\"success to show {} embedding!\".format(key_name))\n\n    def show_predict_result(self, count: int = 10, top_k: int = 5):\n        tmp_feature_list = glob.glob(\n            os.path.join(self.data_utils.path_manager.output_path, \"whale.sub_.*.submission.csv.feature\"))\n        tmp_feature_list = sorted(tmp_feature_list, key=lambda x: os.path.getmtime(x), reverse=True)\n        file_id_list, feature_list = WhaleRankingUtils.get_feature_list(feature_file=tmp_feature_list[0])\n\n        # get index list\n        if count < len(file_id_list):\n            index_list = random_choice(src_list=[i for i in range(len(file_id_list))], k=count, unique=True)\n        else:\n            index_list = [i for i in range(len(file_id_list))]\n\n        # image list\n        image_file_list = []\n        for index in index_list:\n            _image_file_list = [os.path.join(self.data_utils.path_manager.data_set_test_path, file_id_list[index])]\n            _count = 0\n            for (_distance, _class_id, _file_name) in feature_list[index]:\n                _image_file_list.append(\n                    os.path.join(self.data_utils.path_manager.data_set_train_path, os.path.basename(_file_name))\n                )\n                _count += 1\n\n                if _count >= top_k:\n                    break\n\n            image_file_list.append(_image_file_list)\n\n        # show image\n        image_utils.show_images_file(image_list=image_file_list, image_save_file=None, image_size=(200, 200), dpi=100)\n\n    def model_fun(self, ):\n        \"\"\" 返回func \"\"\"\n\n        raise NotImplementedError\n\n    def list_features(self, file_list: list = None, mode: ProcessMode = None) -> (list, list):\n        if file_list is None and mode is not None:\n            if mode == ProcessMode.train:\n                input_list = self.data_utils.get_file_list(is_training=True)\n            else:\n                input_list = self.data_utils.get_file_list(is_training=False)\n        else:\n            input_list = file_list\n\n        input_fn = self.get_dataset_func(\n            split_name=DatasetUtils.SPLIT_PREDICT, input_list=input_list,\n            num_epochs=1, shuffle=False, batch_size=self.batch_size, num_parallel_calls=2, prefetch_size=2,\n        )\n\n        # predict\n        self.logger.info(\"trying to list feature for {} file...\".format(len(input_list)))\n        classifier = self.get_classifier()\n        result_list = classifier.predict(input_fn=input_fn)\n\n        # parse feature\n        feature_list = []\n        feature_file_list = []\n        count = 0\n        _steps_ = 100 if len(input_list) <= 1000 else 1000\n        for score in result_list:\n            feature_file_list.append(byte_to_string(score[self._features_filename_key]))\n            feature_list.append(score[self._features_embedding_key].reshape(1, self.params.dimension))\n            count += 1\n            if count % _steps_ == 0:\n                self.logger.info(\"calculated {} image...\".format(count))\n\n        if count % _steps_ != 0:\n            self.logger.info(\"calculated {} image...\".format(count))\n\n        self.logger.info(\"success to list feature for {} file!\".format(len(input_list)))\n\n        return feature_list, feature_file_list\n\n    def get_dataset_func(self, split_name, num_epochs=1, shuffle=True, batch_size=64, num_parallel_calls=2,\n                         prefetch_size=2, shuffle_size=4, input_list=None):\n        if self.train_loss_hook is None:\n            raise ValueError(\"self.train_loss_hook cannot be null in this class!\")\n\n        def tf_decode_with_crop(file_name, label, offset_height, offset_width, target_height, target_width):\n            image_str = tf.read_file(file_name)\n            image = tf.image.decode_jpeg(image_str, channels=3)\n            image = tf_image_crop(image, offset_height, offset_width, target_height, target_width)\n\n            image = tf.image.resize_images(tf.image.rgb_to_grayscale(image),\n                                           size=(self.image_tmp_size, self.image_tmp_size))\n\n            processed_images = whale_gray_preprocess_image(\n                image, self.params.image_size, self.params.image_size,\n                is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),\n                mean_tf_func=whale_siamese_image_mean_tf)\n            return {self._features_images_key: processed_images, self._features_filename_key: file_name}, label\n\n        def input_fn():\n            if input_list:\n                file_list = input_list\n            else:\n                file_list = self.data_utils.get_file_list(\n                    is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),\n                    shuffle=shuffle, num_epochs=num_epochs, batch_size=batch_size,\n                    online_batch_count=self.params.online_batch_count)\n\n            self.logger.info(\"info of file_list: len is {}.\".format(len(file_list)))\n            if split_name == DatasetUtils.SPLIT_TRAIN:\n                labels = [self.data_utils.get_label(image_file) for image_file in file_list]\n            else:\n                labels = [0] * len(file_list)\n\n            bounding_boxes = [expend_bounding_box(self.data_utils.get_boxes(image_file)) for\n                              image_file in file_list]\n            offset_height_list, offset_width_list, target_height_list, target_width_list = \\\n                parse_bounding_boxes_list(bounding_boxes)\n            dataset = tf.data.Dataset.from_tensor_slices(\n                (file_list, labels, offset_height_list, offset_width_list, target_height_list, target_width_list))\n            dataset = dataset.map(tf_decode_with_crop, num_parallel_calls=num_parallel_calls)\n            dataset = dataset.prefetch(buffer_size=prefetch_size * batch_size)\n            dataset = dataset.batch(batch_size)\n            iterator = dataset.make_one_shot_iterator()\n            features, labels = iterator.get_next()\n            return features, labels\n\n        return input_fn\n\n    def map_search(self, only_validate: bool = False, clear_cache_after_exists: bool = True,\n                   norm_type: NormType = NormType.all):\n        # norm_type == NormType.all, 相当于所有元素同时除以一个数值. 与模型一致; 不改变距离顺序\n        with TripletSearch(instance=self, data_utils=self.data_utils, norm_type=norm_type,\n                           clear_cache_after_exists=clear_cache_after_exists) as search:\n            self.logger.info(\"trying to search with {}\".format(self.__class__.__name__))\n            raw_submission_csv = self.data_utils.path_manager.submission_csv\n            try:\n\n                _time_start = time.time()\n                search.train()\n\n                base_name = \"whale.sub_.{}\".format(create_fake_random_string(length=8))\n                self.data_utils.path_manager.submission_csv = os.path.join(self.data_utils.path_manager.output_path,\n                                                                           base_name)\n\n                map_5, map_1 = search.validate(data_percent=1.0)\n                if only_validate:\n                    self.logger.info(\"success to validate with {}, time cost {} seconds!\".format(\n                        self.__class__.__name__, round(time.time() - _time_start, 2)))\n                    return\n\n                # predict\n                self.data_utils.path_manager.submission_csv = \\\n                    self.data_utils.path_manager.submission_csv + \\\n                    \".{}.submission.csv\".format(str((map_5 + map_1) / 2)[2:])\n                search.test()\n                self.logger.info(\"success to search with {}, time cost {} seconds, submission save in {}\".format(\n                    self.__class__.__name__, round(time.time() - _time_start, 2),\n                    self.data_utils.path_manager.submission_csv))\n            finally:\n                self.data_utils.path_manager.submission_csv = raw_submission_csv\n\n    def combine_submission_csv(self):\n        # combine result\n        tmp_submission_csv_list = glob.glob(\n            os.path.join(self.data_utils.path_manager.output_path, \"whale.sub_.*.submission.csv\"))\n\n        if not tmp_submission_csv_list:\n            self.logger.warning(\"no submission file found!\")\n            return\n        self.logger.info(\"found tmp submission files: {}\".format(tmp_submission_csv_list))\n\n        result_weight = {}\n        for csv_file in tmp_submission_csv_list:\n            try:\n                result_weight[csv_file] = float(\"0.{}\".format(csv_file.split(\".\")[-3]))\n            except Exception as e:\n                self.logger.error(e)\n\n        combine_csv(result_weight, out_file=os.path.join(self.data_utils.path_manager.output_path, \"sub_ens.csv\"))\n        self.logger.info(\"success to merge all submission file into sub_ens.csv\")\n\n        if self.data_utils.path_manager.is_kaggle:\n            for csv_file in tmp_submission_csv_list:\n                os.remove(csv_file)\n\n    def train_with_predict(self, safe_max_batch_size=32, calc_every_epoch=1, shuffle_data_every_epoch=1,\n                           predict_every_epoch=100, max_epoch=100, ignore_error_in_train: bool = True):\n        assert max_epoch >= predict_every_epoch >= shuffle_data_every_epoch\n        assert max_epoch >= calc_every_epoch >= shuffle_data_every_epoch\n\n        self.batch_size = safe_max_batch_size\n\n        def end_process_func():\n            self.map_search(only_validate=False, clear_cache_after_exists=True)\n\n        def loop_process(total_epoch, num_epoch):\n            if total_epoch % calc_every_epoch == 0:\n                if total_epoch == 0:\n                    self.train(batch_size=safe_max_batch_size, num_epochs=1, shuffle=True, steps=1)\n\n                self.data_utils.calc_feature(self.list_features, epoch_num=total_epoch)\n\n                if total_epoch > 0 and not self.data_utils.path_manager.is_kaggle and \\\n                        os.path.abspath(self.TRAIN_CKPT_DIR).find(\"/content/\") > -1:\n                    colab_save_file_func(train_dir=self.TRAIN_CKPT_DIR, logger=self.logger, daemon=False,\n                                         only_save_latest_checkpoint=True)\n\n            self.train(batch_size=safe_max_batch_size, num_epochs=num_epoch, shuffle=True, steps=None)\n            self.set_epoch_num(count=total_epoch + num_epoch)\n\n            if total_epoch > 0 and total_epoch % predict_every_epoch == 0:\n                end_process_func()\n\n        # run\n        estimator_iter_process(loop_process, iter_stop_time=self.timeout,\n                               loop_process_min_epoch=shuffle_data_every_epoch,\n                               loop_process_start_epoch=self.get_epoch_num(),\n                               end_process_func=end_process_func, loop_process_max_epoch=max_epoch,\n                               ignore_error_in_loop_process=ignore_error_in_train, logger=self.logger)\n\n        # combine result\n        self.combine_submission_csv()\n\n\nclass TripletLossModelSiamese(TripletLossModelCNN):\n    def __init__(self, train_ckpt_dir, data_utils: WhaleDataUtils, timeout: int = int(3600 * 5),\n                 pretrained_ckpt_file: str = None, keras_model: str = None):\n        super(TripletLossModelSiamese, self).__init__(\n            train_ckpt_dir=train_ckpt_dir,\n            data_utils=data_utils,\n            timeout=timeout,\n            pretrained_ckpt_file=None if keras_model else pretrained_ckpt_file,\n        )\n\n        # restore hook\n        self.keras_model_file = keras_model\n        if not self._checkpoint_exist and keras_model:\n            ckpt_file = keras_convert_model_to_estimator_ckpt(\n                keras_model_path=keras_model, log_dir=self.TRAIN_CKPT_DIR, logger=self.logger)\n\n            if not ckpt_file:\n                self.logger.warning(\"fail to convert keras model to estimator!\")\n            else:\n                self.train_restore_hook = InitFromPretrainedCheckpointHook(\n                    ckpt_file, exclusion_list=[\"global_step\"], index_add=1, root_name=self.model_name)\n\n    def get_dataset_func(self, split_name, num_epochs=1, shuffle=True, batch_size=64, num_parallel_calls=2,\n                         prefetch_size=2, shuffle_size=4, input_list=None):\n        if self.train_loss_hook is None:\n            raise ValueError(\"self.train_loss_hook cannot be null in this class!\")\n\n        def tf_decode_with_crop(file_name, label, offset_height, offset_width, target_height, target_width):\n            image_str = tf.read_file(file_name)\n            image = tf.image.decode_jpeg(image_str, channels=3)\n            image = tf_image_crop(image, offset_height, offset_width, target_height, target_width)\n\n            image = tf.image.resize_images(tf.image.rgb_to_grayscale(image),\n                                           size=(self.image_tmp_size, self.image_tmp_size))\n\n            processed_images = whale_gray_preprocess_image(\n                image, self.params.image_size, self.params.image_size,\n                is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),\n                mean_tf_func=whale_siamese_image_mean_tf)\n            return {self._features_images_key: processed_images, self._features_filename_key: file_name}, label\n\n        def input_fn():\n            if input_list:\n                file_list = input_list\n            else:\n                file_list = self.data_utils.get_file_list(\n                    is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),\n                    shuffle=shuffle, num_epochs=num_epochs, batch_size=batch_size,\n                    online_batch_count=self.params.online_batch_count)\n\n            self.logger.info(\"info of file_list: len is {}.\".format(len(file_list)))\n            if split_name == DatasetUtils.SPLIT_TRAIN:\n                labels = [self.data_utils.get_label(image_file) for image_file in file_list]\n            else:\n                labels = [0] * len(file_list)\n\n            bounding_boxes = [expend_bounding_box(self.data_utils.get_boxes(image_file)) for\n                              image_file in file_list]\n            offset_height_list, offset_width_list, target_height_list, target_width_list = \\\n                parse_bounding_boxes_list(bounding_boxes)\n            dataset = tf.data.Dataset.from_tensor_slices(\n                (file_list, labels, offset_height_list, offset_width_list, target_height_list, target_width_list))\n            dataset = dataset.map(tf_decode_with_crop, num_parallel_calls=num_parallel_calls)\n            dataset = dataset.prefetch(buffer_size=prefetch_size * batch_size)\n            dataset = dataset.batch(batch_size)\n            iterator = dataset.make_one_shot_iterator()\n            features, labels = iterator.get_next()\n            return features, labels\n\n        return input_fn\n\n    def get_bone_net(self, ):\n        \"\"\" 返回func \"\"\"\n\n        def sub_block(x, n_filter, **kwargs):\n            x = BatchNormalization()(x)\n            y = x\n            y = Conv2D(n_filter, (1, 1), activation='relu', **kwargs)(y)  # Reduce the number of features to 'filter'\n            y = BatchNormalization()(y)\n            y = Conv2D(n_filter, (3, 3), activation='relu', **kwargs)(y)  # Extend the feature field\n            y = BatchNormalization()(y)\n            y = Conv2D(K.int_shape(x)[-1], (1, 1), **kwargs)(\n                y)  # no activation # Restore the number of original features\n            y = Add()([x, y])  # Add the bypass connection\n            y = Activation('relu')(y)\n            return y\n\n        def get_model_net(is_training: bool, img_shape: tuple, ):\n            if is_training:\n                regul = regularizers.l2(0.0002)\n            else:\n                regul = regularizers.l2(0)\n            kwargs = {'padding': 'same', 'kernel_regularizer': regul}\n\n            inp = Input(shape=img_shape)  # 384x384x1\n            x = Conv2D(64, (9, 9), strides=2, activation='relu', **kwargs)(inp)\n\n            x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 96x96x64\n            for _ in range(2):\n                x = BatchNormalization()(x)\n                x = Conv2D(64, (3, 3), activation='relu', **kwargs)(x)\n\n            x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 48x48x64\n            x = BatchNormalization()(x)\n            x = Conv2D(128, (1, 1), activation='relu', **kwargs)(x)  # 48x48x128\n            for _ in range(4):\n                x = sub_block(x, 64, **kwargs)\n\n            x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 24x24x128\n            x = BatchNormalization()(x)\n            x = Conv2D(256, (1, 1), activation='relu', **kwargs)(x)  # 24x24x256\n            for _ in range(4):\n                x = sub_block(x, 64, **kwargs)\n\n            x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 12x12x256\n            x = BatchNormalization()(x)\n            x = Conv2D(384, (1, 1), activation='relu', **kwargs)(x)  # 12x12x384\n            for _ in range(4):\n                x = sub_block(x, 96, **kwargs)\n\n            x = MaxPooling2D((2, 2), strides=(2, 2))(x)  # 6x6x384\n            x = BatchNormalization()(x)\n            x = Conv2D(512, (1, 1), activation='relu', **kwargs)(x)  # 6x6x512\n            for _ in range(4):\n                x = sub_block(x, 128, **kwargs)\n\n            x = GlobalMaxPooling2D()(x)  # 512\n            return Model(inputs=inp, outputs=x)\n\n        return get_model_net\n\n    def model_fun(self, ):\n        \"\"\" 返回func \"\"\"\n\n        def network(scope_name: str, features: dict, params: Params, is_training: bool = True, labels=None):\n            assert params.num_channels == 1\n            img_shape = [params.image_size, params.image_size, params.num_channels]\n            input_shape = [-1] + img_shape\n\n            inputs = tf.reshape(features[self._features_images_key], input_shape)\n\n            with tf.variable_scope(scope_name, 'triplet_loss', [inputs]):\n                branch_model = self.get_bone_net()(is_training=is_training, img_shape=img_shape)\n                embeddings = branch_model(inputs)\n\n                return embeddings, {}\n\n        return tf_model_fn.tf_triplet_loss_model_fn(\n            network=network,\n            scope_name=self.model_name,\n            features_embedding_key=self._features_embedding_key,\n            features_filename_key=self._features_filename_key,\n            get_learning_rate_func=self.get_learning_rate,\n            optimizer_type=self.optimizer_type,\n            logger=self.logger,\n            use_l2_normalize=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3382b9fa20db2bab6ef24da722756daaf53e21a"},"cell_type":"markdown","source":"# 6.training model"},{"metadata":{"trusted":true,"_uuid":"ce7cdd83a72ecdf2ea155f558c912554786f3e01"},"cell_type":"code","source":"!mkdir -p ./keras\n!cp ../input/piotte/mpiotte-standard.model ./keras/\n!cp ../input/whale-triplet-pretrained-model/tripletk/tripletK triplet -R\n\n# logger\ninit_logger()\n\npath_manager = PathManager(\"kaggle\")\ndata_utils = WhaleDataUtils(path_manager=path_manager, gen_data_setting={\n        \"x_train_num\": 4,\n        \"ignore_blank_prob\": 0.9,\n        \"ignore_single_prob\": 0.5,\n        \"stop_calc_feature\": False,\n        \"gen_data_by_random_prob\": 0,\n        \"use_norm_when_calc_apn\": True,\n    })\n    \n# # init model with siamese weight\n# shutil.rmtree(\"./triplet\")\n# estimator = TripletLossModelSiamese(\n#         train_ckpt_dir=\"./triplet\",\n#         data_utils=data_utils,\n#         timeout=int(5 * 3600),\n#         pretrained_ckpt_file=None,\n#         keras_model=\"./keras/mpiotte-standard.model\",\n#     )\n    \n# init model with pretrained triplet loss model (finetuning from Sianmese net)\nestimator = TripletLossModelSiamese(\n        train_ckpt_dir=\"./triplet\",\n        data_utils=data_utils,\n        timeout=int(5 * 3600),\n        pretrained_ckpt_file=None,\n        keras_model=None,\n    )\n    \n# predict\nestimator.map_search(only_validate=False, clear_cache_after_exists=True)\nestimator.combine_submission_csv() # create submission csv file: sub_ens.csv\n    \n# show embeddings (you need to download logdata and show it with tensorboard)\nestimator.show_embedding(\n        mode=ProcessMode.train,\n        count=1000,\n        remove_log_dir_if_exists=True,\n        norm_type=NormType.all,\n    )\n    \n# # train and predict\n# try:\n#     estimator.train_with_predict(safe_max_batch_size=64, calc_every_epoch=1, shuffle_data_every_epoch=1,\n#                                      predict_every_epoch=100, max_epoch=10000, ignore_error_in_train=True)\n# finally:\n#     if data_utils.path_manager.is_kaggle:\n#         if os.path.exists(path_manager.bounding_boxes_csv):\n#             os.remove(path_manager.bounding_boxes_csv)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9e88cecf34612b39de0ef17774c6ae0ea0e12c4"},"cell_type":"code","source":"# show top5\nestimator.show_predict_result(count=10, top_k=5)\n\n# show top3\nestimator.show_predict_result(count=20, top_k=3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25df05d8e8db5365c78902ce0333e4709f13cdf9"},"cell_type":"code","source":"# exit\n!rm *.pkl\n!rm ./keras/ -R\n\n!pip uninstall aiohttp faiss-prebuilt pyxtools pymltools -y\n!apt remove -y libopenblas-base libomp-dev\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"016708b17db3022a17d59e54b5ac6594ef7c0564"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}