{"cells":[{"metadata":{"_uuid":"4b2c59b5e07245d2ec2a341a2ae03ef049b73bde"},"cell_type":"markdown","source":"\nref:\n- [tensorflow-triplet-loss](https://github.com/omoindrot/tensorflow-triplet-loss)"},{"metadata":{"_uuid":"f3e6128b6af8416f77f35cf7b48f2768c0f75f10"},"cell_type":"markdown","source":"# 1.install packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# install \n!pip -q install aiohttp faiss-prebuilt pyxtools pymltools\n!apt -qq install -y libopenblas-base libomp-dev\n \nimport tensorflow\ntensorflow.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8e7b42fb45044079bf7d931715c35ff275b61cb"},"cell_type":"markdown","source":"# 2.utils for project"},{"metadata":{"trusted":true,"_uuid":"5465a8dbc724784bbe01524abecbf306d286895e"},"cell_type":"code","source":"import csv\nimport glob\nimport logging\nimport pickle\nimport subprocess\nimport time\n\nimport numpy as np\nimport os\nimport pandas as pd\nimport random\nimport shutil\nimport tensorflow as tf\nimport tensorflow.contrib.slim as slim\nfrom tensorflow.contrib.slim.nets import resnet_utils, resnet_v2\n\nfrom pymltools.tf_utils import Params, DatasetUtils, parse_bounding_boxes_list, tf_image_crop, \\\n show_embedding, AbstractEstimator, estimator_iter_process, colab_save_file_func, OptimizerType, get_wsl_path, \\\n map_per_set, LossStepHookForTrain, ProcessMode, load_data_from_h5file, tf_model_fn, balance_class_dict, \\\n store_data_in_h5file, get_triplet_pair_np, map_per_image, show_distance_dense_plot, init_logger, pre_process_utils\nfrom pyxtools import byte_to_string, create_fake_random_string, iter_list_with_size, download_big_file, list_files, \\\n FileCache, random_choice, NormType, get_pretty_float, expend_bounding_box, image_utils\n\n\ntry:\n from pyxtools.faiss_tools import ImageIndexUtils\nexcept ImportError:\n from pyxtools.pyxtools.faiss_tools import ImageIndexUtils\n\nresnet_arg_scope = resnet_utils.resnet_arg_scope\nresnet_v2_50 = resnet_v2.resnet_v2_50\n\n\ndef combine_csv(file_weight: dict, out_file: str):\n sub_files = []\n sub_weight = []\n for csv_file, weight in file_weight.items():\n sub_files.append(csv_file)\n sub_weight.append(weight)\n\n place_weights = {}\n for i in range(5):\n place_weights[i] = 10 - i * 2\n\n h_label = 'Image'\n h_target = 'Id'\n\n sub = [None] * len(sub_files)\n for i, file in enumerate(sub_files):\n print(\"Reading {}: w={} - {}\".format(i, sub_weight[i], file))\n reader = csv.DictReader(open(file, \"r\"))\n sub[i] = sorted(reader, key=lambda d: d[h_label])\n\n out = open(out_file, \"w\", newline='')\n writer = csv.writer(out)\n writer.writerow([h_label, h_target])\n p = 0\n for row in sub[0]:\n target_weight = {}\n for s in range(len(sub_files)):\n row1 = sub[s][p]\n for ind, trgt in enumerate(row1[h_target].split(' ')):\n target_weight[trgt] = target_weight.get(trgt, 0) + (place_weights[ind] * sub_weight[s])\n tops_trgt = sorted(target_weight, key=target_weight.get, reverse=True)[:5]\n writer.writerow([row1[h_label], \" \".join(tops_trgt)])\n p += 1\n out.close()\n\n\nclass PathManager(object):\n def __init__(self, mode: str = \"default\"):\n self._mode = mode\n self.init_all_path()\n self.init_bounding_boxes()\n self._submission_csv = None\n\n @property\n def data_set_root_path(self) -> str:\n if self._mode == \"default\":\n return get_wsl_path(\"E:/frkhit/Download/AI/data-set/kaggle/whale\")\n elif self._mode == \"fake\":\n return get_wsl_path(\"E:/frkhit/Download/AI/data-set/kaggle/whale/fake\")\n elif self._mode == \"colab\":\n return \"/content\"\n elif self._mode == \"kaggle\":\n return \"../input/humpback-whale-identification\"\n\n return \"./\"\n\n @property\n def working_path(self):\n if self._mode == \"kaggle\":\n return \"/kaggle/working/\"\n\n return self.data_set_root_path\n\n @property\n def output_path(self, ):\n if self._mode == \"default\" or self._mode == \"fake\":\n return \"/mnt/e/s\"\n\n return self.working_path\n\n @property\n def data_set_train_path(self):\n return os.path.join(self.data_set_root_path, \"train\")\n\n @property\n def data_set_test_path(self, ):\n return os.path.join(self.data_set_root_path, \"test\")\n\n @property\n def train_csv(self, ):\n return os.path.join(self.data_set_root_path, \"train.csv\")\n\n @property\n def submission_csv(self, ):\n if self._submission_csv:\n return self._submission_csv\n return os.path.join(self.output_path, \"whale.submission.csv\")\n\n @submission_csv.setter\n def submission_csv(self, submission_csv):\n self._submission_csv = submission_csv\n\n @property\n def bounding_boxes_csv(self, ):\n return os.path.join(self.working_path, \"bounding_boxes.csv\")\n\n @property\n def ckpt_pretrained_rv250(self, ):\n if self._mode == \"default\" or self._mode == \"fake\":\n return get_wsl_path(\n \"E:/frkhit/Download/AI/pre-trained-model/resnet_v2_50.ckpt\"\n )\n return os.path.join(self.working_path, \"resnet_v2_50.ckpt\")\n\n def init_bounding_boxes(self):\n if not os.path.exists(self.bounding_boxes_csv):\n download_big_file(\"https://raw.githubusercontent.com/frkhit/file_servers/master/bounding_boxes.csv\",\n self.bounding_boxes_csv)\n if not os.path.exists(self.bounding_boxes_csv):\n raise ValueError(\"fail to download bounding_boxes_csv!\")\n\n def init_image_net_model(self):\n working_dir = os.path.dirname(self.ckpt_pretrained_mv2)\n if not os.path.exists(working_dir):\n os.mkdir(working_dir)\n\n if not os.path.exists(self.ckpt_pretrained_mv2 + \".index\"):\n tar_file = \"mobilenet_v2_1.0_224.tgz\"\n download_big_file(\"https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_224.tgz\",\n os.path.join(working_dir, tar_file))\n if not os.path.exists(tar_file):\n raise ValueError(\"fail to download pretrained model!\")\n\n raw_dir = os.getcwd()\n try:\n os.chdir(working_dir)\n cmd = subprocess.Popen([\"tar\", \"-xvf\", tar_file])\n cmd.wait()\n if not os.path.exists(self.ckpt_pretrained_mv2 + \".index\"):\n raise ValueError(\"fail to download pretrained model!\")\n finally:\n os.chdir(raw_dir)\n\n def download_resnet_v2_50(self):\n working_dir = os.path.dirname(self.ckpt_pretrained_rv250)\n if not os.path.exists(working_dir):\n os.mkdir(working_dir)\n\n if not os.path.exists(self.ckpt_pretrained_rv250):\n tar_file = \"resnet_v2_50_2017_04_14.tar.gz\"\n download_big_file(\"http://download.tensorflow.org/models/resnet_v2_50_2017_04_14.tar.gz\",\n os.path.join(working_dir, tar_file))\n if not os.path.exists(tar_file):\n raise ValueError(\"fail to download pretrained model!\")\n\n raw_dir = os.getcwd()\n try:\n os.chdir(working_dir)\n cmd = subprocess.Popen([\"tar\", \"-xzvf\", tar_file])\n cmd.wait()\n if not os.path.exists(self.ckpt_pretrained_rv250):\n raise ValueError(\"fail to download pretrained model!\")\n finally:\n os.chdir(raw_dir)\n\n def init_all_path(self):\n members = [attr for attr in dir(self) if not attr.startswith(\"__\")]\n for member in members:\n if (member.endswith(\"path\") or member.endswith(\"dir\")) and not member.endswith(\"init_all_path\"):\n tmp_dir = getattr(self, member)\n if callable(tmp_dir):\n tmp_dir = tmp_dir()\n\n if not os.path.exists(tmp_dir):\n try:\n os.mkdir(tmp_dir)\n except Exception as e:\n logging.error(e)\n\n @property\n def is_kaggle(self):\n return bool(self._mode == \"kaggle\")\n\n def create_fake_data(self, x: int = 1):\n raw_path_manager = PathManager()\n whales = pd.read_csv(raw_path_manager.train_csv)\n train_info = {}\n for index, image_name in enumerate(whales.Image):\n train_info[image_name] = whales.Id[index]\n\n shutil.rmtree(self.data_set_root_path)\n os.mkdir(self.data_set_root_path)\n\n # test path\n os.mkdir(self.data_set_test_path)\n test_image_list = list_files(raw_path_manager.data_set_test_path)\n for image_file in random.choices(test_image_list, k=10 * x):\n shutil.copy(image_file, os.path.join(self.data_set_test_path, os.path.basename(image_file)))\n\n # train path\n os.mkdir(self.data_set_train_path)\n raw_image_list = list_files(raw_path_manager.data_set_train_path)\n image_id_vs_image_file = {\n os.path.basename(image_file): image_file for image_file in raw_image_list}\n\n class_id_vs_image_list = {}\n\n for image_id, class_id in train_info.items():\n class_id_vs_image_list.setdefault(class_id, []).append(image_id_vs_image_file[image_id])\n\n anchor_class_id_list = [class_id for class_id, _image_list in class_id_vs_image_list.items() if\n len(_image_list) > 1]\n all_anchor_class_id_list = list(train_info.values())\n try:\n anchor_class_id_list.remove(WhaleDataUtils.blank_class_id)\n except Exception:\n pass\n\n to_move_list = []\n for class_id in random.choices(anchor_class_id_list, k=2 * x):\n to_move_list.extend(class_id_vs_image_list[class_id][:3 * x])\n for class_id in random.choices(all_anchor_class_id_list, k=3 * x):\n to_move_list.extend(class_id_vs_image_list[class_id][:3 * x])\n to_move_list.extend(class_id_vs_image_list[WhaleDataUtils.blank_class_id][:4 * x])\n for image_file in to_move_list:\n shutil.copy(image_file, os.path.join(self.data_set_train_path, os.path.basename(image_file)))\n with open(self.train_csv, \"w\") as f:\n f.write(\"Image,Id\\n\")\n for image_file in to_move_list:\n image_id = os.path.basename(image_file)\n class_id = train_info[image_id]\n f.write(\"{},{}\\n\".format(image_id, class_id))\n\n # bounding box\n self.init_bounding_boxes()\n\n\nclass WhaleDataUtils(object):\n blank_class_id = \"new_whale\"\n not_gray_str = \".ng\"\n\n def __init__(self, path_manager: PathManager, loss_hook: LossStepHookForTrain = None,\n gen_data_setting: dict = None):\n self.logger = logging.getLogger(self.__class__.__name__)\n self._file = \"./{}.pkl\".format(self.__class__.__name__)\n self.path_manager = path_manager\n self.loss_hook = loss_hook\n self.file_cache = FileCache(pickle_file=self._file)\n self.dimension = 512\n self.margin = 1.0\n self.gen_data_setting = {\n \"x_train_num\": 1,\n \"ignore_blank_prob\": 0.95,\n \"ignore_single_prob\": 0.2,\n \"stop_calc_feature\": False,\n \"gen_data_by_random_prob\": 0,\n \"use_norm_when_calc_apn\": True,\n }\n if gen_data_setting:\n self.gen_data_setting.update(gen_data_setting)\n\n _cache = None\n if os.path.exists(self._file):\n _cache = self.file_cache.get(\"cache\")\n\n if not _cache:\n self._cache = ([], [], {}, {}, {}, [])\n else:\n self._cache = _cache\n\n self._baseline = None\n self._to_use_random = False\n\n def _update_cache(self):\n self.file_cache.set(\"cache\", self._cache)\n\n def clear_cache(self):\n self.file_cache.cache.clear()\n self.file_cache.set(\"__key__\", \"value\", ttl=10) # flush\n\n _pkl_file = self._file + \".feature.h5\"\n if os.path.exists(_pkl_file):\n os.remove(_pkl_file)\n\n def load_train_info(self) -> dict:\n if self._cache and self._cache[3]:\n return self._cache[3]\n\n whales = pd.read_csv(self.path_manager.train_csv)\n info = self._cache[3]\n for index, image_name in enumerate(whales.Image):\n info[image_name] = whales.Id[index]\n self.logger.info(\"there are {} record in train csv!\".format(len(info)))\n self._update_cache()\n return info\n\n def load_train_label_info(self) -> dict:\n if self._cache and self._cache[4]:\n return self._cache[4]\n\n class_id_list = [class_id for class_id in self.load_train_info().values()]\n class_id_list = list(set(class_id_list))\n class_id_list.sort()\n\n info = self._cache[4]\n for index, class_id in enumerate(class_id_list):\n info[class_id] = index\n self.logger.info(\"there are {} classes in train set!\".format(len(info)))\n self._update_cache()\n return info\n\n def load_boxes_info(self):\n if self._cache and self._cache[2]:\n return self._cache[2]\n\n boxes = pd.read_csv(self.path_manager.bounding_boxes_csv)\n info = self._cache[2]\n for index, image_id in enumerate(boxes.Image):\n info[image_id] = (boxes.x0[index], boxes.y0[index], boxes.x1[index], boxes.y1[index])\n self.logger.info(\"there are {} record in boxes csv!\".format(len(info)))\n\n # check\n file_list = self.list_train_image_file().copy() # fix bug\n file_list.extend(self.list_test_image_file())\n for image_file in file_list:\n base_name = os.path.basename(image_file)\n if base_name not in info:\n raise ValueError(\"{} not in boxes info!\".format(image_file))\n\n self._update_cache()\n return self._cache[2]\n\n def get_boxes(self, image_file) -> (int, int, int, int):\n base_name = os.path.basename(image_file)\n return self.load_boxes_info()[base_name]\n\n def list_train_image_file(self, ignore_blank: bool = False) -> list:\n if not self._cache[0]:\n image_list = list_files(self.path_manager.data_set_train_path)\n result_list = [file_name for file_name in image_list if file_name.endswith(\".jpg\")]\n assert len(result_list) == len(image_list)\n self.logger.info(\"there are {} jpg in train path!\".format(len(result_list)))\n self._cache[0].clear()\n self._cache[0].extend(result_list)\n\n self._update_cache()\n\n # ignore blank image_file\n if not self._cache[5]:\n for _image_file in self._cache[0]:\n if self.get_class_id(_image_file) != self.blank_class_id:\n self._cache[5].append(_image_file)\n\n self._update_cache()\n\n if ignore_blank is False:\n return self._cache[0]\n\n return self._cache[5]\n\n def list_test_image_file(self) -> list:\n if self._cache and self._cache[1]:\n return self._cache[1]\n\n image_list = list_files(self.path_manager.data_set_test_path)\n result_list = [file_name for file_name in image_list if file_name.endswith(\".jpg\")]\n assert len(result_list) == len(image_list)\n self.logger.info(\"there are {} jpg in test path!\".format(len(result_list)))\n self._cache[1].clear()\n self._cache[1].extend(result_list)\n self._update_cache()\n return self._cache[1]\n\n def get_class_id(self, image_file: str) -> str:\n base_name = os.path.basename(image_file)\n if base_name.endswith(self.not_gray_str):\n base_name = base_name[:-len(self.not_gray_str)]\n return self.load_train_info()[base_name]\n\n def list_class_id_with_multi_instance(self) -> dict:\n info = {}\n for class_id in self.load_train_info().values():\n info.setdefault(class_id, 0)\n info[class_id] += 1\n\n key_to_remove_list = []\n for key, count in info.items():\n if count <= 1:\n key_to_remove_list.append(key)\n\n for key in key_to_remove_list:\n info.pop(key)\n\n return info\n\n def get_label(self, image_file: str) -> int:\n class_id = self.get_class_id(image_file)\n return self.load_train_label_info()[class_id]\n\n def submit_test_result(self, test_image_list: list, result_list: list, feature_list: list = None):\n if len(test_image_list) != len(result_list):\n self.logger.error(\"len(test_image_list) != len(result_list)!\")\n if feature_list and len(feature_list) != len(test_image_list):\n self.logger.error(\"len(test_image_list) != len(feature_list)!\")\n\n with open(self.path_manager.submission_csv, \"w\") as f:\n f.write(\"Image,Id\\n\")\n for index, image_file in enumerate(test_image_list):\n f.write(\"{},{}\\n\".format(os.path.basename(image_file),\n \" \".join([class_id for class_id in result_list[index]])))\n self.logger.info(\"success to save result in {}\".format(self.path_manager.submission_csv))\n\n if feature_list and feature_list[0]:\n self.logger.info(\"saving feature list ...\")\n with open(self.path_manager.submission_csv + \".feature\", \"w\") as f:\n for index, image_file in enumerate(test_image_list):\n f.write(\"{},{}\\n\".format(os.path.basename(image_file), feature_list[index]))\n\n self.logger.info(\"success to save feature result in {}\".format(\n self.path_manager.submission_csv + \".feature\"))\n\n def get_random_baseline(self) -> (float, float):\n \"\"\"\n 0.384, 0.381\n Returns:\n (float, float):\n \"\"\"\n if self._baseline is None:\n class_count_dict = self.list_class_id_with_multi_instance()\n class_list = sorted(class_count_dict.items(), key=lambda x: x[1], reverse=True)\n class_id_list = [_class_id for (_class_id, _count) in class_list]\n random_class_id_list = [self.blank_class_id]\n for _class_id in class_id_list:\n if _class_id not in random_class_id_list:\n random_class_id_list.append(_class_id)\n if len(random_class_id_list) >= 5:\n break\n\n labels = [self.get_class_id(image_file) for image_file in self.list_train_image_file()]\n class_id_result_list = [random_class_id_list] * len(labels)\n total_map_5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)\n total_map_1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)\n self._baseline = (total_map_5, total_map_1)\n\n return self._baseline\n\n def list_debug_image_list(self, count: int = 128) -> list:\n all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n self.get_basic_info(self.list_train_image_file())\n if count >= len(all_image_list):\n return all_image_list\n\n # info\n single_blank_class_files = []\n anchor_class_id_set = set(anchor_class_id_list)\n for class_id, file_list in class_id_vs_image_list.items():\n if class_id not in anchor_class_id_set:\n single_blank_class_files.extend(file_list)\n\n image_list = []\n\n # anchor_file_count\n anchor_file_count = int(count * (1 - len(single_blank_class_files) / len(all_image_list)))\n if (anchor_file_count + len(single_blank_class_files)) > len(all_image_list):\n anchor_file_count = len(all_image_list) - len(single_blank_class_files)\n\n # multi class\n for class_id in anchor_class_id_list:\n image_list.extend(class_id_vs_image_list[class_id])\n if len(image_list) >= anchor_file_count:\n break\n\n while len(image_list) > anchor_file_count:\n image_list.pop(-1)\n\n # add single class and blank class\n image_list.extend(random_choice(single_blank_class_files, k=count - anchor_file_count, unique=True))\n\n assert len(image_list) == count\n\n return image_list\n\n def get_basic_info(self, all_image_list: list = None) -> (list, dict, list, list, list):\n \"\"\"\n list train info\n Args:\n\n Returns:\n\n \"\"\"\n if all_image_list is None:\n all_image_list = list(self.list_train_image_file(ignore_blank=False))\n\n image_id_vs_image_file = {os.path.basename(image_file): image_file for image_file in all_image_list}\n\n class_id_vs_image_list = {}\n\n for image_id, class_id in self.load_train_info().items():\n if image_id in image_id_vs_image_file:\n class_id_vs_image_list.setdefault(class_id, []).append(image_id_vs_image_file[image_id])\n\n anchor_class_id_list = [class_id for class_id, _image_list in class_id_vs_image_list.items() if\n len(_image_list) > 1]\n all_class_id_list = list(class_id_vs_image_list.keys())\n single_class_id_list = list(set(all_class_id_list) - set(anchor_class_id_list))\n try:\n anchor_class_id_list.remove(self.blank_class_id)\n except Exception:\n pass\n try:\n single_class_id_list.remove(self.blank_class_id)\n except Exception:\n pass\n\n return all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list\n\n def get_file_list(self, is_training: bool = False, shuffle: bool = False, num_epochs: int = 1,\n batch_size: int = None, online_batch_count: int = 0) -> list:\n\n def get_random_batch_list(batch_count: int, add_blank_class: bool, class_id_vs_image_list: dict,\n anchor_class_id_list: list, all_image_list: list, single_class_id_list: list,\n anchor_unique: bool, single_unique: bool, ) -> list:\n batch_list = []\n # add one image from new blank\n if add_blank_class:\n batch_list.append(random.choice(class_id_vs_image_list[self.blank_class_id]))\n\n # add anchor(has more than one image)\n chosen_class_id_list = random_choice(anchor_class_id_list, k=batch_count, unique=anchor_unique)\n for class_id in chosen_class_id_list:\n if len(class_id_vs_image_list[class_id]) >= 2 * online_batch_count:\n batch_list.extend(\n random_choice(class_id_vs_image_list[class_id], k=2 * online_batch_count, unique=True))\n else:\n batch_list.extend(class_id_vs_image_list[class_id])\n\n if (batch_count - len(batch_list)) < online_batch_count:\n break\n\n # add single class\n if len(batch_list) < batch_count:\n if single_class_id_list:\n batch_list.extend([\n class_id_vs_image_list[class_id][0] for class_id in\n random_choice(single_class_id_list, k=batch_count - len(batch_list),\n unique=single_unique)])\n else:\n batch_list.extend(\n random_choice(all_image_list, k=batch_count - len(batch_list), unique=anchor_unique))\n else:\n while len(batch_list) > batch_count:\n batch_list.pop(-1)\n\n return batch_list\n\n def create_file_list_by_random() -> list:\n all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n self.get_basic_info()\n total_count = int(num_epochs * len(all_image_list) * self.gen_data_setting[\"x_train_num\"])\n batch_count = total_count // batch_size\n multi_batch_list = []\n\n _unique = True if len(anchor_class_id_list) >= batch_size else False\n _unique_single = True if len(single_class_id_list) >= batch_size else False\n _has_blank_class = True if class_id_vs_image_list.get(self.blank_class_id) else False\n\n for i in range(batch_count):\n batch_list = get_random_batch_list(\n batch_count=batch_size, add_blank_class=_has_blank_class,\n class_id_vs_image_list=class_id_vs_image_list, anchor_class_id_list=anchor_class_id_list,\n all_image_list=all_image_list, single_class_id_list=single_class_id_list,\n anchor_unique=_unique, single_unique=_unique_single,\n )\n\n random.shuffle(batch_list)\n multi_batch_list.append(batch_list)\n\n return multi_batch_list\n\n def create_file_list_by_distance() -> list:\n # calc distance\n feature_arr, file_arr = self._load_feature(create_if_not_exist=False)\n if feature_arr is None:\n self.logger.warning(\n \"no feature in h5, use create_file_list_by_random instead of create_file_list_by_distance!\")\n return create_file_list_by_random()\n\n if self.gen_data_setting.get(\"use_norm_when_calc_apn\", False):\n feature_arr, norm_float = NormType.all.normalize_and_return_norm(feature_arr)\n self.logger.info(\"norm of feature is {}\".format(norm_float, type(norm_float)))\n assert isinstance(norm_float, float)\n margin = self.margin / norm_float\n else:\n margin = self.margin\n\n _file_list = []\n for i in range(file_arr.shape[0]):\n _file_list.append(file_arr[i].decode(\"utf-8\"))\n\n all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n self.get_basic_info(_file_list)\n total_count = int(num_epochs * len(all_image_list) * self.gen_data_setting[\"x_train_num\"])\n batch_count = total_count // batch_size\n multi_batch_list = []\n\n single_class_id_set = set(single_class_id_list)\n anchor_index_end = -1\n _anchor_search_end = False\n labels = np.zeros(shape=file_arr.shape, dtype=np.int)\n for i in range(file_arr.shape[0]):\n _file_name = file_arr[i].decode(\"utf-8\")\n labels[i] = self.get_label(_file_name)\n if not _anchor_search_end:\n class_id = self.get_class_id(_file_name)\n if class_id in single_class_id_set:\n _anchor_search_end = True\n else:\n anchor_index_end = i\n\n anchor_feature = feature_arr[:anchor_index_end + 1]\n _apn_np_start = time.time()\n apn_list = get_triplet_pair_np(anchor_feature, all_feature=feature_arr, all_label=labels,\n margin=margin, logger=self.logger)\n _apn_set = set()\n for (a, p, n) in apn_list:\n _apn_set.add(a)\n _apn_set.add(p)\n _apn_set.add(n)\n\n self.logger.info(\n \"get {} hardest-apn-pairs in {} files, time cost {}s. Unique file in apn pairs is {}.\".format(\n len(apn_list), feature_arr.shape[0], time.time() - _apn_np_start, len(_apn_set)))\n\n _unique = True if len(anchor_class_id_list) >= batch_size else False\n _unique_single = True if len(single_class_id_list) >= batch_size else False\n _has_blank_class = True if class_id_vs_image_list.get(self.blank_class_id) else False\n _blank_file_id_set = set()\n if _has_blank_class:\n _blank_file_label = self.get_label(class_id_vs_image_list[self.blank_class_id][0])\n for i in range(labels.shape[0]): # todo == len(labels)\n if labels[i] == _blank_file_label:\n _blank_file_id_set.add(i)\n\n assert len(_blank_file_id_set) == len(class_id_vs_image_list.get(self.blank_class_id, []))\n\n if len(apn_list) <= batch_size or (len(_apn_set - _blank_file_id_set) + 1) <= batch_size:\n self.logger.warning(\"apn list is too small, use create_file_list_by_random instead!\")\n return create_file_list_by_random()\n\n tmp_apn_list = []\n for i in range(batch_count):\n batch_file_id_id = set()\n _blank_exists = False\n while True:\n if len(tmp_apn_list) == 0:\n tmp_apn_list.extend(list(apn_list))\n random.shuffle(tmp_apn_list)\n\n (a, p, n) = tmp_apn_list.pop(-1)\n for _file_id in [a, p, n]:\n if _file_id in _blank_file_id_set:\n if _blank_exists is False:\n batch_file_id_id.add(_file_id)\n _blank_exists = True\n else:\n continue\n else:\n batch_file_id_id.add(_file_id)\n\n if len(batch_file_id_id) >= batch_size:\n break\n\n batch_list = [file_arr[file_id].decode(\"utf-8\") for file_id in batch_file_id_id]\n\n assert len(batch_list) >= batch_size\n\n batch_list = batch_list[:batch_size]\n\n random.shuffle(batch_list)\n multi_batch_list.append(batch_list)\n\n return multi_batch_list\n\n if is_training:\n if online_batch_count > 0:\n self.logger.info(\"using create_file_list_by_distance...\")\n\n if self._to_use_random:\n _multi_batch_list = create_file_list_by_random()\n self._to_use_random = False\n else:\n _multi_batch_list = create_file_list_by_distance()\n # shuffle\n if shuffle is True:\n random.shuffle(_multi_batch_list)\n\n image_file_list = []\n for _batch_list in _multi_batch_list:\n image_file_list.extend(_batch_list)\n\n return image_file_list\n else:\n return self.list_train_image_file()\n\n return self.list_test_image_file()\n\n def _list_file_to_calc_feature(self) -> list:\n if random.random() < self.gen_data_setting[\"ignore_blank_prob\"]:\n all_image_list = list(self.list_train_image_file(ignore_blank=True))\n\n if random.random() < self.gen_data_setting[\"ignore_single_prob\"]:\n _, class_id_vs_image_list, _, anchor_class_id_list, _ = \\\n self.get_basic_info(all_image_list)\n\n all_file_list = []\n anchor_class_id_set = set(anchor_class_id_list)\n for class_id, _file_list in class_id_vs_image_list.items():\n if class_id in anchor_class_id_set:\n all_file_list.extend(_file_list)\n\n self.logger.info(\"list train image file [ignore_single, ignore_blank] to calc feature!\")\n return all_file_list\n else:\n self.logger.info(\"list train image file [ignore_blank] to calc feature!\")\n return all_image_list\n\n self.logger.info(\"list all train image file to calc feature!\")\n return list(self.list_train_image_file())\n\n def calc_feature(self, callback, epoch_num: int = None):\n \"\"\"\n 调用callback计算feature\n Args:\n epoch_num (int): total count\n callback:\n \"\"\"\n if self.gen_data_setting.get(\"stop_calc_feature\", False):\n self.logger.info(\"no need to calc feature: stop_calc_feature is True!\")\n return\n\n if epoch_num is not None and epoch_num == 0 and os.path.exists(self._file + \".feature.h5\"):\n self.logger.info(\"no need to calc feature: feature file exist and epoch_num == 0!\")\n return\n\n if random.random() < self.gen_data_setting.get(\"gen_data_by_random_prob\", 0):\n self.logger.info(\"no need to calc feature: going to gen data by random!\")\n self._to_use_random = True\n return\n\n file_list = self._list_file_to_calc_feature()\n if not file_list:\n self.logger.info(\"no need to calc feature!\")\n return\n\n self.logger.info(\"calculating feature of {} file...\".format(len(file_list)))\n feature_list, feature_file_list = callback(file_list, mode=ProcessMode.train)\n self.logger.info(\"success to calculate feature of {} file!\".format(len(feature_file_list)))\n\n self.update_feature(feature_list, feature_file_list, is_training=True)\n\n def update_feature(self, feature_list, file_list, is_training: bool):\n \"\"\"\n\n Returns:\n object:\n \"\"\"\n if not is_training:\n self.logger.info(\"give up update-feature for no train mode!\")\n return\n\n if not feature_list:\n return\n\n # 保存 feature\n self._save_feature(feature_list, file_list)\n\n def _save_feature(self, feature_list: list, file_list: list):\n # clear exist feature\n _pkl_file = self._file + \".feature.h5\"\n if os.path.exists(_pkl_file):\n os.remove(_pkl_file)\n\n # feature save\n all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n self.get_basic_info(all_image_list=file_list)\n\n all_file_list = []\n for class_id in anchor_class_id_list:\n all_file_list.extend(class_id_vs_image_list[class_id])\n\n for class_id in single_class_id_list:\n all_file_list.extend(class_id_vs_image_list[class_id])\n\n all_file_list.extend(class_id_vs_image_list.get(self.blank_class_id, []))\n assert len(all_image_list) == len(all_file_list)\n\n # sort file index\n sort_index_list = []\n _dict = {}\n for i, file_name in enumerate(file_list):\n _dict[file_name] = i\n for file_name in all_file_list:\n sort_index_list.append(_dict[file_name])\n\n # feature\n file_arr = np.asarray([f.encode(\"utf-8\") for f in all_file_list], dtype=np.string_)\n feature_arr = np.zeros(shape=(len(all_file_list), self.dimension), dtype=np.float32)\n # update feature\n for index, key_index in enumerate(sort_index_list):\n feature_arr[index][:] = feature_list[key_index].reshape((self.dimension,))\n\n _pkl_file = self._file + \".feature.h5\"\n store_data_in_h5file(_pkl_file, [feature_arr, file_arr], key_list=[\"feature\", \"file\"])\n\n def _load_feature(self, create_if_not_exist: bool = True) -> (np.ndarray, np.ndarray):\n \"\"\"\n\n Returns:\n feature, file, is_created\n \"\"\"\n _pkl_file = self._file + \".feature.h5\"\n if os.path.exists(_pkl_file):\n feature, file_arr = None, None\n try:\n feature, file_arr = load_data_from_h5file(_pkl_file, key_list=[\"feature\", \"file\"])\n except Exception as e:\n os.remove(_pkl_file)\n self.logger.error(e)\n\n if feature is not None:\n self.logger.info(\"feature shape is {}, file_arr shape is {}\".format(feature.shape, file_arr.shape))\n assert feature.shape[1] == self.dimension\n assert file_arr.shape[0] == feature.shape[0]\n return feature, file_arr\n\n if not create_if_not_exist:\n return None, None\n\n all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \\\n self.get_basic_info()\n all_file_list = []\n for class_id in anchor_class_id_list:\n all_file_list.extend(class_id_vs_image_list[class_id])\n\n for class_id in single_class_id_list:\n all_file_list.extend(class_id_vs_image_list[class_id])\n\n all_file_list.extend(class_id_vs_image_list[self.blank_class_id])\n assert len(all_image_list) == len(all_file_list)\n\n file_arr = np.asarray([f.encode(\"utf-8\") for f in all_file_list], dtype=np.string_)\n feature = np.zeros(shape=(len(all_file_list), self.dimension))\n\n return feature, file_arr\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f3417e0c9b291d0c846f71aea65731229215dddd"},"cell_type":"markdown","source":"# 3.predict tools "},{"metadata":{"trusted":true,"_uuid":"10ae6a4b20404d42c89abc6781a5568803914380"},"cell_type":"code","source":"\nclass WhaleRankingUtils(object):\n def __init__(self, data_utils: WhaleDataUtils, top_k: int = 5):\n self.logger = logging.getLogger(self.__class__.__name__)\n self.top_k = top_k\n self.data_utils = data_utils\n\n def simple_rank(self, result_list: list, distance_cutoff: float = None, only_distance_fit: bool = False) -> list:\n \"\"\"\n\n Args:\n result_list: list, consist of (distance, class_id, image_file) elements\n distance_cutoff: float\n only_distance_fit: bool\n\n Returns:\n rank_list: list, like, [class_id_1, class_id_2, ... ]\n \"\"\"\n # deal with image_result_list\n _class_id_list = []\n _class_id_set = set()\n\n if distance_cutoff is None:\n for result in result_list:\n class_id = result[1]\n if class_id not in _class_id_set:\n _class_id_list.append(class_id)\n _class_id_set.add(class_id)\n\n if len(_class_id_list) < self.top_k:\n if self.data_utils.blank_class_id not in _class_id_set:\n _class_id_list.append(self.data_utils.blank_class_id)\n _class_id_set.add(self.data_utils.blank_class_id)\n\n else:\n if result_list[0][0] > distance_cutoff:\n _class_id_list.append(self.data_utils.blank_class_id)\n if only_distance_fit:\n return _class_id_list\n else:\n _class_id_list.append(result_list[0][1])\n _class_id_set.add(_class_id_list[0])\n\n for result in result_list[1:]:\n if result[0] <= distance_cutoff:\n class_id = result[1]\n if class_id not in _class_id_set:\n _class_id_list.append(class_id)\n _class_id_set.add(class_id)\n else:\n if only_distance_fit:\n return _class_id_list\n else:\n break\n\n if len(_class_id_list) < self.top_k:\n if self.data_utils.blank_class_id not in _class_id_set:\n _class_id_list.append(self.data_utils.blank_class_id)\n _class_id_set.add(self.data_utils.blank_class_id)\n\n if len(_class_id_list) < self.top_k:\n for result in result_list[1:]:\n if result[0] > distance_cutoff:\n class_id = result[1]\n if class_id not in _class_id_set:\n _class_id_list.append(class_id)\n _class_id_set.add(class_id)\n\n return _class_id_list\n\n def recreate_class_id_list(self, cutoff: float, class_id_list: list, feature_list: list) -> list:\n new_class_id_list = []\n count = 0\n for i, class_list in enumerate(class_id_list):\n _new_class_list = list(class_list)\n if feature_list[i][0][0] <= cutoff or _new_class_list[0] == self.data_utils.blank_class_id:\n new_class_id_list.append(_new_class_list)\n continue\n\n try:\n found_index = _new_class_list.index(self.data_utils.blank_class_id)\n if found_index > -1:\n _new_class_list.pop(found_index)\n except ValueError:\n pass\n\n _new_class_list.insert(0, self.data_utils.blank_class_id)\n new_class_id_list.append(_new_class_list)\n count += 1\n\n self.logger.info(\"recreate_class_id_list change {}/{} record\".format(count, len(class_id_list)))\n return new_class_id_list\n\n def get_similar_cutoff(self, validating_labels: list, validating_feature_list: list) -> float:\n # calc similar distance cutoff\n _distance_list = []\n _, class_id_vs_image_list, _, anchor_class_id_list, _ = \\\n self.data_utils.get_basic_info(self.data_utils.list_train_image_file())\n\n anchor_class_set = set(anchor_class_id_list)\n for i, class_id in enumerate(validating_labels):\n if class_id not in anchor_class_set:\n continue\n\n for (_distance, _class_id, _file_name) in validating_feature_list[i]:\n if class_id == _class_id:\n _distance_list.append(_distance)\n\n self.logger.info(\"distance list: len is {}, first 5 is {}\".format(len(_distance_list), _distance_list[:5]))\n distance_mean, distance_std = show_distance_dense_plot(\n np.asarray(_distance_list), self.data_utils.path_manager.submission_csv + \".distance.jpg\")\n\n distance_similar_cutoff = distance_mean + 3 * distance_std\n self.logger.info(\"distance_similar_cutoff is {}\".format(distance_similar_cutoff))\n\n return distance_similar_cutoff\n\n @staticmethod\n def get_feature_list(feature_file: str) -> (list, list):\n feature_list = []\n file_id_list = []\n with open(feature_file, \"r\", encoding=\"utf-8\") as f:\n for line in f:\n if len(line) < 2:\n continue\n\n line = line.rstrip()\n file_id_list.append(line.split(\",\")[0])\n feature_list.append(eval(line[line.find(\",\") + 1:]))\n\n return file_id_list, feature_list\n\n def analyze_validating_result(self, feature_file: str):\n # read feature list\n file_id_list, feature_list = self.get_feature_list(feature_file)\n\n class_id_result_list = [self.simple_rank(feature)[:self.top_k] for feature in feature_list]\n\n # calc map@5\n labels = self.list_class_id(file_id_list)\n total_map_5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)\n total_map_1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)\n self.logger.info(\"validating data result is: map@5 is {}, map@1 is {}!\".format(total_map_5, total_map_1))\n\n # calc similar distance cutoff\n distance_similar_cutoff = self.get_similar_cutoff(labels, feature_list)\n\n # find error\n for i, image_id in enumerate(file_id_list):\n map_5 = map_per_image(label=labels[i], predictions=class_id_result_list[i], k=5)\n if map_5 < 1.0:\n # error\n self.logger.info(\"map@5 is {}\\nTrue: {}, Predict with cutoff is {}\\n{}\\n\".format(\n map_5, labels[i],\n self.data_utils.blank_class_id if feature_list[i][0][0] > distance_similar_cutoff else\n feature_list[i][0][1],\n feature_list[i])\n )\n\n # create new class id result\n new_class_id_result = self.recreate_class_id_list(\n cutoff=distance_similar_cutoff, class_id_list=class_id_result_list, feature_list=feature_list)\n\n total_map_5 = map_per_set(labels=labels, predictions=new_class_id_result, k=5)\n total_map_1 = map_per_set(labels=labels, predictions=new_class_id_result, k=1)\n self.logger.info(\n \"validating data result with cutoff is: map@5 is {}, map@1 is {}!\".format(total_map_5, total_map_1))\n\n def compare_submission_file(self, file_1, file_2):\n def _compare_same_count(_result_1, _result_2, info):\n assert set(_result_1.keys()) == set(_result_2.keys())\n\n all_same_count = 0\n for image_id, result_list in _result_1.items():\n if \",\".join(result_list[:self.top_k]) == \",\".join(_result_2[image_id][:self.top_k]):\n all_same_count += 1\n\n self.logger.info(\"[{}] same count is {}/{}\".format(info, all_same_count, len(_result_1)))\n\n def _compare_top1_count(_result_1, _result_2, info):\n assert set(_result_1.keys()) == set(_result_2.keys())\n\n all_same_count = 0\n for image_id, result_list in _result_1.items():\n if result_list[0] == _result_2[image_id][0]:\n all_same_count += 1\n\n self.logger.info(\"[{}] same top1 count is {}/{}\".format(info, all_same_count, len(_result_1)))\n\n def _compare_map(_result_1, _result_2, info):\n assert set(_result_1.keys()) == set(_result_2.keys())\n\n _file_id_list = list(_result_1.keys())\n _file_id_list.sort()\n labels = self.list_class_id(_file_id_list)\n\n # 1\n _class_id_result_list_1 = [_result_1[_image_id] for _image_id in _file_id_list]\n map5_1 = map_per_set(labels=labels, predictions=_class_id_result_list_1, k=5)\n map1_1 = map_per_set(labels=labels, predictions=_class_id_result_list_1, k=1)\n\n # 2\n _class_id_result_list_2 = [_result_2[_image_id] for _image_id in _file_id_list]\n map5_2 = map_per_set(labels=labels, predictions=_class_id_result_list_2, k=5)\n map1_2 = map_per_set(labels=labels, predictions=_class_id_result_list_2, k=1)\n\n self.logger.info(\"[{}] map@5 is {}/{}, map@1 is {}/{}\".format(\n info,\n get_pretty_float(map5_1, count=3),\n get_pretty_float(map5_2, count=3),\n get_pretty_float(map1_1, count=3),\n get_pretty_float(map1_2, count=3),\n ))\n\n # true same count\n result_1, result_2 = self.read_submission_file(file_1), self.read_submission_file(file_2)\n result_1_x, result_2_x = self._get_result_from_feature_file(file_1), self._get_result_from_feature_file(file_2)\n result_1_true, result_2_true = self._get_result_by_cutoff(file_1), self._get_result_by_cutoff(file_2)\n self.logger.info(\"Start to compare:\\n\\n\")\n\n # map\n self.logger.info(\"compare map5, map1\")\n _compare_map(result_1, result_2, \"map, raw, resenet vs siamese\")\n _compare_map(result_1, result_1_x, \"map, raw, resenet, csv vs feature\")\n _compare_map(result_2, result_2_x, \"map, raw, siamese, csv vs feature\")\n _compare_map(result_1_true, result_2_true, \"map, true, resenet vs siamese\")\n _compare_map(result_1, result_1_true, \"map, resent, raw vs true\")\n _compare_map(result_2, result_2_true, \"map, siamese, raw vs true\")\n self.logger.info(\"End\\n\\n\")\n\n # top5 same\n self.logger.info(\"compare top5\")\n _compare_same_count(result_1, result_2, \"top5, raw, resenet vs siamese\")\n _compare_same_count(result_1_true, result_2_true, \"top5, true, resenet vs siamese\")\n _compare_same_count(result_1, result_1_true, \"top5, resent, raw vs true\")\n _compare_same_count(result_2, result_2_true, \"top5, siamese, raw vs true\")\n self.logger.info(\"End\\n\\n\")\n\n # top1 same\n self.logger.info(\"compare top1\")\n _compare_top1_count(result_1, result_2, \"top1, raw, resenet vs siamese\")\n _compare_top1_count(result_1_true, result_2_true, \"top1, true, resenet vs siamese\")\n _compare_top1_count(result_1, result_1_true, \"top1, resent, raw vs true\")\n _compare_top1_count(result_2, result_2_true, \"top1, siamese, raw vs true\")\n self.logger.info(\"End\\n\\n\")\n\n @staticmethod\n def read_submission_file(csv_file: str) -> dict:\n submission = pd.read_csv(csv_file)\n _result = {}\n for index, _image_id in enumerate(submission.Image):\n _result[_image_id] = submission.Id[index].rstrip().split(\" \")\n\n return _result\n\n def _get_result_from_feature_file(self, csv_file: str) -> dict:\n feature_file = csv_file + \".feature\"\n assert os.path.exists(feature_file)\n cutoff, _, file_id_list, feature_list = self._read_feature_file(feature_file)\n\n _new_result = {}\n for _index, file_id in enumerate(file_id_list):\n _new_result[file_id] = self.simple_rank(\n feature_list[_index], distance_cutoff=cutoff, only_distance_fit=False)\n\n return _new_result\n\n def _get_result_by_cutoff(self, csv_file: str) -> dict:\n feature_file = csv_file + \".feature\"\n assert os.path.exists(feature_file)\n cutoff, labels, file_id_list, feature_list = self._read_feature_file(feature_file)\n self.logger.info(\"cutoff is {}\".format(get_pretty_float(cutoff, count=3)))\n\n _new_result = {}\n for _index, file_id in enumerate(file_id_list):\n _new_result[file_id] = self.simple_rank(\n feature_list[_index], distance_cutoff=cutoff, only_distance_fit=True)\n\n return _new_result\n\n def _read_feature_file(self, feature_file):\n assert os.path.exists(feature_file)\n file_id_list, feature_list = self.get_feature_list(feature_file)\n labels = self.list_class_id(file_id_list)\n cutoff = self.get_similar_cutoff(labels, feature_list)\n return cutoff, labels, file_id_list, feature_list\n\n @staticmethod\n def _calc_weight_score(feature_list, file_id_list, cutoff) -> dict:\n # todo simple score\n keep_count = 10\n cutoff_score = 5.0\n\n _new_result = {}\n for _index, file_id in enumerate(file_id_list):\n _new_result[file_id] = {}\n\n _sort_num = -1\n for (distance, class_id, file_name) in feature_list[_index]:\n if class_id not in _new_result[file_id]:\n _sort_num += 1\n _new_result[file_id][class_id] = keep_count - _sort_num\n if distance < cutoff:\n _new_result[file_id][class_id] += cutoff_score\n\n if _sort_num >= keep_count:\n break\n\n return _new_result\n\n @staticmethod\n def _get_result_by_weight_score(score_dict: dict, top_k: int = 5) -> dict:\n def _sort_by_value(info_dict: dict) -> list:\n _tuple = sorted(info_dict.items(), key=lambda x: x[1], reverse=True)\n return [_class_id for (_class_id, _weight) in _tuple]\n\n result_dict = {}\n for image_id, score in score_dict.items():\n result_dict[image_id] = _sort_by_value(score)[:top_k]\n\n return result_dict\n\n def combine_submission_file(self, validating_file_list: list, submission_file_list: list, output_file: str):\n def _create_final_result(_result_dict: dict):\n _file_list = []\n _result_list = []\n for _image_id, _class_list in _result_dict.items():\n _file_list.append(_image_id)\n _result_list.append(_class_list)\n return _file_list, _result_list\n\n def _update_weight_score(all_dict: dict, one_dict: dict, model_weight: float):\n for _image_id, _score_dict in one_dict.items():\n if _image_id not in all_dict:\n all_dict[_image_id] = {}\n\n for _class_id, _score in _score_dict.items():\n all_dict[_image_id].setdefault(_class_id, 0.0)\n all_dict[_image_id][_class_id] += _score * model_weight\n\n return all_dict\n\n def eval_result(_result_dict: dict):\n _file_list, _result_list = _create_final_result(_result_dict)\n _labels = self.list_class_id(_file_list)\n _map5 = map_per_set(labels=_labels, predictions=_result_list, k=5)\n _map1 = map_per_set(labels=_labels, predictions=_result_list, k=1)\n return _map5, _map1\n\n # check input\n assert len(validating_file_list) == len(submission_file_list)\n for i, validating_submission_csv in enumerate(validating_file_list):\n predict_submission_csv = submission_file_list[i]\n assert \".\".join(validating_submission_csv.split(\".\")[:3]) == \".\".join(predict_submission_csv.split(\".\")[:3])\n assert os.path.exists(validating_submission_csv + \".feature\")\n assert os.path.exists(predict_submission_csv + \".feature\")\n\n total_predict_score_dict = {}\n total_validate_score_dict = {}\n total_eval_result_list = []\n\n for i, validating_submission_csv in enumerate(validating_file_list):\n predict_submission_csv = submission_file_list[i]\n\n # validate\n cutoff, _, v_file_id_list, v_feature_list = self._read_feature_file(validating_submission_csv + \".feature\")\n v_model_weight = float(\"0.{}\".format(validating_submission_csv.split(\".\")[-3]))\n\n validating_score_dict = self._calc_weight_score(\n feature_list=v_feature_list, file_id_list=v_file_id_list, cutoff=cutoff)\n total_validate_score_dict = _update_weight_score(\n total_validate_score_dict, validating_score_dict, model_weight=v_model_weight)\n\n # predict\n p_file_id_list, p_feature_list = self.get_feature_list(predict_submission_csv + \".feature\")\n predict_score_dict = self._calc_weight_score(\n feature_list=p_feature_list, file_id_list=p_file_id_list, cutoff=cutoff)\n total_predict_score_dict = _update_weight_score(\n total_predict_score_dict, predict_score_dict, model_weight=v_model_weight)\n\n # eval validate\n _validate_result_dict = self.read_submission_file(validating_submission_csv)\n total_eval_result_list.append(eval_result(_validate_result_dict))\n\n # evaluate validating result\n _validate_result_dict = self._get_result_by_weight_score(total_validate_score_dict, top_k=self.top_k)\n total_eval_result_list.append(eval_result(_validate_result_dict))\n self.logger.info(\"after combine {} models: \\nvalidate map@5 is {}/{}, \\nvalidate map@1 is {}/{}\\n\\n\".format(\n len(validating_file_list),\n total_eval_result_list[-1][0], [x[0] for x in total_eval_result_list[:-1]],\n total_eval_result_list[-1][1], [x[1] for x in total_eval_result_list[:-1]],\n ))\n\n # evaluate and save prediction result\n raw_submission = self.data_utils.path_manager.submission_csv\n try:\n predict_result_dict = self._get_result_by_weight_score(total_predict_score_dict, top_k=self.top_k)\n predict_file_list, predict_result_list = _create_final_result(predict_result_dict)\n self.data_utils.path_manager.submission_csv = output_file\n self.data_utils.submit_test_result(test_image_list=predict_file_list, result_list=predict_result_list)\n self.logger.info(\"save final submission result in {}\".format(self.data_utils.path_manager.submission_csv))\n finally:\n self.data_utils.path_manager.submission_csv = raw_submission\n\n def debug_map_csv(self, csv_file):\n self.logger.info(\"debug map set from submission file...\\n\\n\")\n result_dict = self.read_submission_file(csv_file)\n\n class_id_result_list = []\n file_id_list = []\n for _image_id, _result_list in result_dict.items():\n file_id_list.append(_image_id)\n class_id_result_list.append(_result_list)\n\n labels = self.list_class_id(file_id_list)\n\n self.logger.info(\"map5 is {}\".format(map_per_set(labels=labels, predictions=class_id_result_list, k=5)))\n self.logger.info(\"map1 is {}\".format(map_per_set(labels=labels, predictions=class_id_result_list, k=1)))\n\n def list_class_id(self, file_id_list) -> list:\n _, class_id_vs_image_list, _, _, _ = self.data_utils.get_basic_info(self.data_utils.list_train_image_file())\n class_id_vs_image_count = {\n class_id: len(_image_list) for class_id, _image_list in class_id_vs_image_list.items()}\n return [self.get_class_id(class_id_vs_image_count, _image_id) for _image_id in file_id_list]\n\n def get_class_id(self, class_id_vs_image_count, image_file_or_id) -> str:\n # ignore same image\n _class_id = self.data_utils.get_class_id(image_file_or_id)\n if class_id_vs_image_count[_class_id] > 1:\n return _class_id\n else:\n return self.data_utils.blank_class_id\n\n\nclass SimpleSearch(object):\n model = None\n\n def __init__(self, data_utils: WhaleDataUtils, shape: tuple, dimension: int, top_k: int = 5,\n norm_type: NormType = NormType.none, more_search_top_k: int = None):\n self.logger = logging.getLogger(self.__class__.__name__)\n self.data_utils = data_utils\n self.faiss_index_path = os.path.join(self.data_utils.path_manager.working_path, self.__class__.__name__)\n self.shape = shape\n self.dimension = dimension\n self.top_k = top_k\n self._distance_cutoff = None\n self.manager = ImageIndexUtils(\n index_dir=self.faiss_index_path,\n dimension=self.dimension)\n self._norm_type = norm_type\n\n if more_search_top_k is None or more_search_top_k < 0:\n more_search_top_k = max(20, self.top_k)\n self._search_top_k = more_search_top_k + self.top_k\n\n def show_embedding(self, mode: ProcessMode):\n key_name = \"train\" if mode == ProcessMode.train else \"test\"\n list_image_file = self.data_utils.list_train_image_file if mode == ProcessMode.train \\\n else self.data_utils.list_test_image_file\n log_dir = os.path.join(self.faiss_index_path, \"logs_{}\".format(key_name))\n\n self.logger.info(\"trying to show {} embedding...\".format(key_name))\n\n feature_list, image_list = self.calc_or_load_feature(mode)\n self.logger.info(\"got {}-image feature: {}/{}\".format(\n key_name, len(image_list), len(list_image_file()))\n )\n\n labels = []\n if mode == ProcessMode.train:\n for image_file in image_list:\n labels.append(self.data_utils.get_class_id(image_file))\n else:\n for image_file in image_list:\n labels.append(os.path.basename(image_file))\n\n show_embedding(feature_list=feature_list, labels=labels, log_dir=log_dir)\n self.logger.info(\"success to show {} embedding!\".format(key_name))\n\n def list_features(self, mode: ProcessMode) -> (list, list):\n \"\"\"\n input: ?X224X224X3\n feature: list of ?X7X7X320, files: list of str\n \"\"\"\n\n raise NotImplementedError\n\n def calc_or_load_feature(self, mode: ProcessMode) -> (list, list):\n key_name = \"train\" if mode == ProcessMode.train else \"test\"\n\n # calc feature or load image feature\n feature_npy = os.path.join(self.faiss_index_path, \"{}.npy.pkl\".format(key_name))\n\n self.logger.info(\"{}ing image...\".format(key_name))\n\n if not os.path.exists(feature_npy):\n self.logger.info(\"calculating feature of {} image...\".format(key_name))\n feature_list, image_list = self.list_features(mode)\n self.logger.info(\"success to calculate feature of {} image!\".format(key_name))\n with open(feature_npy, \"wb\") as f:\n pickle.dump((feature_list, image_list), f)\n else:\n self.logger.info(\"loading feature of {} image...\".format(key_name))\n with open(feature_npy, \"rb\") as f:\n feature_list, image_list = pickle.load(f)\n self.logger.info(\"success to load feature of {} image!\".format(key_name))\n\n return feature_list, image_list\n\n def train(self):\n if os.path.exists(self.manager.manager.faiss_index_file):\n self.logger.info(\"train before, no need to train again!\")\n return\n\n # train: calc feature or load image feature\n feature_list, image_list = self.calc_or_load_feature(ProcessMode.train)\n self.logger.info(\"got train-image feature: {}/{}\".format(\n len(image_list), len(self.data_utils.list_train_image_file())))\n\n image_info_list = []\n for index, image_file in enumerate(image_list):\n image_info_list.append({\n \"index\": index,\n \"file\": image_file,\n \"class_id\": self.data_utils.get_class_id(image_file)}\n )\n self.manager.add_images(feature_list, image_info_list=image_info_list)\n self.logger.info(\"success to train image!\")\n\n def test(self):\n # calc feature\n feature_list, image_list = self.calc_or_load_feature(ProcessMode.test)\n if len(image_list) != len(self.data_utils.list_test_image_file()):\n self.logger.warning(\"got test-image file: {}/{}\".format(\n len(image_list), len(self.data_utils.list_test_image_file())))\n assert len(image_list) % len(self.data_utils.list_test_image_file()) == 0\n\n self.logger.info(\"got test-image feature: {}/{}\".format(\n len(image_list), len(self.data_utils.list_test_image_file())))\n\n # image search\n num_feature = len(image_list) // len(self.data_utils.list_test_image_file())\n self.logger.info(\"testing image...\")\n test_image_list = [image_list[index * num_feature] for index in\n range(len(self.data_utils.list_test_image_file()))]\n class_id_result_list, test_feature_list = self.search(\n image_list=test_image_list, feature_list=feature_list, return_feature_list=True)\n self.logger.info(\"success to test image!\")\n\n # submit result\n assert len(class_id_result_list) == len(self.data_utils.list_test_image_file())\n self.data_utils.submit_test_result(test_image_list=test_image_list,\n result_list=class_id_result_list,\n feature_list=test_feature_list)\n if self._distance_cutoff is None:\n self.logger.info(\"submit predict result[without distance cutoff] in {}\".format(\n self.data_utils.path_manager.submission_csv))\n else:\n self.logger.info(\"submit predict result[with distance cutoff {}] in {}\".format(\n get_pretty_float(self._distance_cutoff, count=3), self.data_utils.path_manager.submission_csv))\n\n def search(self, image_list: list, feature_list: list, cache_file: str = None, ignore_same: bool = False,\n return_feature_list: bool = True) -> (list, list):\n if cache_file is None:\n cache_file = os.path.join(self.faiss_index_path, \"cache.pkl\")\n\n cache = FileCache(cache_file)\n\n # one_file -> multi feature\n assert len(feature_list) % len(image_list) == 0\n num_feature = len(feature_list) // len(image_list)\n if num_feature > 1:\n self.logger.info(\"1 image vs {} feature when searching!\".format(num_feature))\n\n # cache\n src_list = []\n for index in range(len(image_list)):\n image_file = image_list[index]\n if not cache.get(image_file):\n src_list.append((image_file, feature_list[index * num_feature:(index + 1) * num_feature]))\n\n # image search\n whale_rank_utils = WhaleRankingUtils(data_utils=self.data_utils, top_k=self.top_k)\n extend = False\n _show_log_per_steps = 100 if len(image_list) < 1000 else 1000\n for part_src_list in iter_list_with_size(src_list, size=10240 // num_feature):\n feature_list = []\n for (_, _feature_list) in part_src_list:\n feature_list.extend(_feature_list)\n\n raw_result_iterator = self.manager.image_search_iterator(\n feature_list=feature_list,\n top_k=self._search_top_k,\n extend=extend\n )\n\n index = -1\n cache_list = []\n for raw_image_result_list in raw_result_iterator:\n index += 1\n if index % num_feature == 0:\n # the first result for image file\n _image_file = part_src_list[index // num_feature][0] if ignore_same else None\n image_result_list = []\n\n for (faiss_image_index, faiss_image_extend_list, faiss_distance) in raw_image_result_list:\n image_result = self.manager.get_image_info(faiss_image_index)\n if _image_file and image_result.get(\"file\") and image_result.get(\"file\").find(_image_file) == 0:\n continue\n\n image_result_list.append((faiss_distance, image_result[\"class_id\"], image_result[\"file\"]))\n\n if (index + 1) % num_feature == 0:\n # the last result for image file\n image_result_list = sorted(image_result_list, key=lambda x: x[0], reverse=False)\n _class_id_list = whale_rank_utils.simple_rank(image_result_list,\n distance_cutoff=self._distance_cutoff)\n if len(_class_id_list) < self.top_k:\n self.logger.debug(\"length of class_id_list < {}!\".format(self.top_k))\n\n # save in cache\n if return_feature_list:\n cache_list.append((_class_id_list[:self.top_k], list(image_result_list)))\n else:\n cache_list.append((_class_id_list[:self.top_k], None))\n\n # logging\n if index % _show_log_per_steps == 0:\n self.logger.info(\"got class_id_list of {} images\".format(index))\n\n # bulk save in cache\n for j, _class_id_list_feature_list in enumerate(cache_list):\n cache.unsafe_set(part_src_list[j][0], _class_id_list_feature_list)\n cache.set(part_src_list[0][0], cache_list[0])\n\n # class_id_result_list\n class_id_result_list = []\n search_feature_list = []\n for image_file in image_list:\n _class_id_list, _search_feature_list = cache.get(image_file)\n class_id_result_list.append(_class_id_list)\n search_feature_list.append(_search_feature_list)\n\n self.logger.info(\"success to parse search result for test images!\")\n\n return class_id_result_list, search_feature_list\n\n def _calc_map(self, info, data_percent, labels, class_id_result_list,\n image_list, multi_index_list) -> (float, float):\n fake_map5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)\n fake_map1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)\n\n multi_map_5 = map_per_set(\n labels=[labels[index] for index in multi_index_list],\n predictions=[class_id_result_list[index] for index in multi_index_list],\n k=5)\n multi_map_1 = map_per_set(\n labels=[labels[index] for index in multi_index_list],\n predictions=[class_id_result_list[index] for index in multi_index_list],\n k=1)\n\n # new whale\n new_index_list = []\n for _index, _image_file in enumerate(image_list):\n if self.data_utils.get_class_id(_image_file) == self.data_utils.blank_class_id:\n new_index_list.append(_index)\n\n new_map_5 = map_per_set(\n labels=[labels[index] for index in new_index_list],\n predictions=[class_id_result_list[index] for index in new_index_list],\n k=5)\n new_map_1 = map_per_set(\n labels=[labels[index] for index in new_index_list],\n predictions=[class_id_result_list[index] for index in new_index_list],\n k=1)\n\n # logging\n self.logger.info(\n \"map@5 for validating data[{} of train data, {}] is: standard {}, new-whale {}, all {}!\".format(\n data_percent, info, multi_map_5, new_map_5, fake_map5)\n )\n self.logger.info(\n \"map@1 for validating data[{} of train data, {}] is: standard {}, new-whale {}, all {}!\".format(\n data_percent, info, multi_map_1, new_map_1, fake_map1)\n )\n\n return multi_map_5, multi_map_1\n\n def validate(self, data_percent: float = 1.0) -> (float, float):\n \"\"\"\n validate with training data\n\n Returns:\n (float, float): map@5, map@1\n \"\"\"\n # calc feature\n train_feature_list, train_image_list = self.calc_or_load_feature(ProcessMode.train)\n num_feature = len(train_image_list) // len(self.data_utils.list_train_image_file())\n if len(train_image_list) != len(self.data_utils.list_train_image_file()):\n self.logger.warning(\"got validating-image file: {}/{}\".format(\n len(train_image_list), len(self.data_utils.list_train_image_file())))\n assert len(train_image_list) % len(self.data_utils.list_train_image_file()) == 0\n\n # chosen data\n chosen_index_list = random_choice([index for index in range(len(self.data_utils.list_train_image_file()))],\n k=int(len(self.data_utils.list_train_image_file()) * data_percent),\n unique=True)\n if num_feature == 1:\n image_list = [train_image_list[index] for index in chosen_index_list]\n feature_list = [train_feature_list[index] for index in chosen_index_list]\n else:\n image_list = [train_image_list[index * num_feature] for index in chosen_index_list]\n feature_list = []\n for index in chosen_index_list:\n feature_list.extend(train_feature_list[index * num_feature:(index + 1) * num_feature])\n\n # search\n self.logger.info(\"validating image...\")\n _raw_class_id_result_list, validating_feature_list = self.search(\n image_list=image_list, feature_list=feature_list, ignore_same=True, return_feature_list=True)\n self.logger.info(\"success to validate image!\")\n\n # calc map@5\n image_id_vs_image_file = {os.path.basename(image_file): image_file for image_file in image_list}\n class_id_vs_image_list = {}\n for image_id, class_id in self.data_utils.load_train_info().items():\n class_id_vs_image_list.setdefault(class_id, []).append(image_id_vs_image_file[image_id])\n class_id_vs_image_count = {\n class_id: len(_image_list) for class_id, _image_list in class_id_vs_image_list.items()}\n\n # rand utils\n rank_utils = WhaleRankingUtils(data_utils=self.data_utils, top_k=self.top_k)\n\n # all data\n labels = [rank_utils.get_class_id(class_id_vs_image_count, image_file) for image_file in image_list]\n\n # multi instance\n multi_index_list = []\n for _index, _image_file in enumerate(image_list):\n if class_id_vs_image_count[self.data_utils.get_class_id(_image_file)] > 1:\n multi_index_list.append(_index)\n\n distance_cutoff = rank_utils.get_similar_cutoff(\n validating_labels=labels, validating_feature_list=validating_feature_list)\n self.logger.info(\"got distance_cutoff {} and use it!\".format(get_pretty_float(distance_cutoff, count=3)))\n self._distance_cutoff = distance_cutoff\n\n class_id_result_list = rank_utils.recreate_class_id_list(\n cutoff=self._distance_cutoff, class_id_list=_raw_class_id_result_list, feature_list=validating_feature_list)\n\n total_map_5, total_map_1 = self._calc_map(\n \"with distance cutoff {}\".format(get_pretty_float(self._distance_cutoff, count=3)),\n data_percent, labels, class_id_result_list, image_list, multi_index_list\n )\n\n # baseline\n random_map5, random_map1 = self.data_utils.get_random_baseline()\n self.logger.info(\"random baseline for validating data[{} of train data]: map@5 is {}, map@1 is {}!\".format(\n data_percent, random_map5, random_map1))\n\n # submit result\n raw_submission_csv = self.data_utils.path_manager.submission_csv\n try:\n self.data_utils.path_manager.submission_csv = \\\n raw_submission_csv + \".{}.validate.csv\".format(str((total_map_5 + total_map_1) / 2)[2:])\n self.data_utils.submit_test_result(\n test_image_list=image_list, result_list=class_id_result_list, feature_list=validating_feature_list)\n self.logger.info(\"submit validating result[{} of train data, with distance cutoff {}] in {}\".format(\n data_percent, get_pretty_float(distance_cutoff, count=3), self.data_utils.path_manager.submission_csv))\n finally:\n self.data_utils.path_manager.submission_csv = raw_submission_csv\n\n return total_map_5, total_map_1\n\n\nclass TripletSearch(SimpleSearch):\n \"\"\"\n Name: TripletSearch\n \"\"\"\n model = None\n\n def __init__(self, instance, data_utils: WhaleDataUtils, clear_cache_after_exists: bool,\n norm_type: NormType = NormType.l2):\n self.instance = instance\n super(TripletSearch, self).__init__(\n data_utils, shape=(None,), dimension=instance.params.dimension, top_k=5)\n self.smart_iterator = None\n self.feature_record = {}\n self._clear_cache_after_exists = clear_cache_after_exists\n self._norm_type = norm_type\n\n def list_features(self, mode: ProcessMode) -> (list, list):\n \"\"\"\n input: ?X224X224X3\n feature: list of ?X5005, files: list of str\n \"\"\"\n if \"train\" not in self.feature_record and \"test\" not in self.feature_record:\n train_feature_list, train_feature_file_list = self.instance.list_features(mode=ProcessMode.train)\n test_feature_list, test_feature_file_list = self.instance.list_features(mode=ProcessMode.test)\n\n # l2 norm\n feature = np.vstack(train_feature_list + test_feature_list). \\\n reshape((len(train_feature_list) + len(test_feature_list), self.dimension))\n if self._norm_type is None:\n normalized_feature = feature\n else:\n self.logger.info(\"using {} to normalize feature\".format(self._norm_type))\n normalized_feature = self._norm_type.normalize(feature)\n\n self.feature_record[\"train\"] = train_feature_file_list\n self.feature_record[\"test\"] = test_feature_file_list\n self.feature_record[\"feature\"] = normalized_feature\n\n if mode == ProcessMode.train:\n feature_file_list = self.feature_record[\"train\"]\n feature_list = [f.reshape(1, self.dimension) for f in\n self.feature_record[\"feature\"][:len(feature_file_list)]]\n else:\n feature_file_list = self.feature_record[\"test\"]\n feature_list = [f.reshape(1, self.dimension) for f in\n self.feature_record[\"feature\"][len(self.feature_record[\"train\"]):]]\n\n assert len(feature_list) == len(feature_file_list)\n assert feature_list[0].shape == (1, self.dimension)\n\n return feature_list, feature_file_list\n\n def clear_cache(self, ):\n if os.path.exists(self.faiss_index_path):\n try:\n if not self.data_utils.path_manager.is_kaggle:\n tmp_dir = os.path.join(os.path.dirname(self.faiss_index_path), self.instance.__class__.__name__)\n if os.path.exists(tmp_dir):\n shutil.rmtree(tmp_dir)\n shutil.copytree(self.faiss_index_path, tmp_dir)\n self.logger.info(\"backup Search Cache in {}\".format(os.path.abspath(tmp_dir)))\n shutil.rmtree(self.faiss_index_path)\n except Exception as e:\n self.logger.error(e, exc_info=True)\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n if self._clear_cache_after_exists:\n self.clear_cache()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77384eb4c8bbb4b1a1d1b0d2e5c85b5eca8b3c35"},"cell_type":"markdown","source":"# 4.Triplet loss net + resenet50"},{"metadata":{"trusted":true,"_uuid":"c7bd8fdc3285166b54f3bd6bec578faa82cf6176"},"cell_type":"code","source":"class TripletLossModelCNN(AbstractEstimator):\n def __init__(self, train_ckpt_dir, data_utils: WhaleDataUtils, timeout: int = int(3600 * 5),\n pretrained_ckpt_file: str = None):\n super(TripletLossModelCNN, self).__init__(\n model_name=\"TripletLoss\",\n train_ckpt_dir=train_ckpt_dir,\n pretrained_ckpt_file=pretrained_ckpt_file\n )\n self.data_utils = data_utils\n self.timeout = timeout + time.time()\n self.batch_size = 8\n self._features_images_key = \"images\"\n self._features_filename_key = \"filenames\"\n self._features_embedding_key = \"embeddings\"\n self.params = Params({\n \"num_channels\": 1,\n \"margin\": 1.0,\n \"dimension\": 512,\n \"image_size\": 384,\n \"online_batch_count\": 4,\n })\n self.learning_rate = 64e-5\n self.image_tmp_size = 400\n assert self.image_tmp_size >= self.params.image_size\n\n self.train_loss_hook = LossStepHookForTrain(log_after_run=True, log_after_end=False,\n show_log_per_steps=10, run_after_run_per_steps=5)\n self.data_utils.loss_hook = self.train_loss_hook\n self.using_training_status_of_train_data = None\n self.optimizer_type = OptimizerType.adam\n\n def show_embedding(self, mode: ProcessMode, count: int = -1, remove_log_dir_if_exists: bool = False,\n norm_type: NormType = NormType.all):\n key_name = \"train\" if mode == ProcessMode.train else \"test\"\n log_dir = os.path.join(self.TRAIN_CKPT_DIR, \"logs_{}\".format(key_name))\n if os.path.exists(log_dir) and remove_log_dir_if_exists:\n shutil.rmtree(log_dir)\n\n self.logger.info(\"trying to show {} embedding...\".format(key_name))\n\n if count > 1:\n if mode == ProcessMode.train:\n image_list = self.data_utils.list_debug_image_list(count=count)\n else:\n image_list = self.data_utils.get_file_list(\n online_batch_count=0, is_training=bool(mode == ProcessMode.train))\n image_list = image_list[:count]\n if len(image_list) != count:\n self.logger.warning(\"only have {} files\".format(len(image_list)))\n else:\n image_list = self.data_utils.get_file_list(\n online_batch_count=0, is_training=bool(mode == ProcessMode.train))\n\n if mode == ProcessMode.train:\n label_list = [self.data_utils.get_class_id(image_file) for image_file in image_list]\n else:\n label_list = [os.path.basename(image_file) for image_file in image_list]\n\n feature_list, _ = self.list_features(file_list=image_list)\n\n # normalize\n if norm_type is not None:\n feature = np.vstack(feature_list).reshape((len(feature_list), self.params.dimension))\n self.logger.info(\"using {} to normalize feature\".format(norm_type))\n normalized_feature = norm_type.normalize(feature)\n feature_list = [f.reshape(1, self.params.dimension) for f in normalized_feature]\n\n self.logger.info(\"got {}-image feature: {}/{}\".format(\n key_name, len(feature_list), len(image_list))\n )\n\n show_embedding(feature_list=feature_list, labels=label_list, log_dir=log_dir)\n self.logger.info(\"success to show {} embedding!\".format(key_name))\n\n def show_predict_result(self, count: int = 10, top_k: int = 5):\n tmp_feature_list = glob.glob(\n os.path.join(self.data_utils.path_manager.output_path, \"whale.sub_.*.submission.csv.feature\"))\n tmp_feature_list = sorted(tmp_feature_list, key=lambda x: os.path.getmtime(x), reverse=True)\n file_id_list, feature_list = WhaleRankingUtils.get_feature_list(feature_file=tmp_feature_list[0])\n\n # get index list\n if count < len(file_id_list):\n index_list = random_choice(src_list=[i for i in range(len(file_id_list))], k=count, unique=True)\n else:\n index_list = [i for i in range(len(file_id_list))]\n\n # image list\n image_file_list = []\n for index in index_list:\n _image_file_list = [os.path.join(self.data_utils.path_manager.data_set_test_path, file_id_list[index])]\n _count = 0\n for (_distance, _class_id, _file_name) in feature_list[index]:\n _image_file_list.append(\n os.path.join(self.data_utils.path_manager.data_set_train_path, os.path.basename(_file_name))\n )\n _count += 1\n\n if _count >= top_k:\n break\n\n image_file_list.append(_image_file_list)\n\n # show image\n image_utils.show_images_file(image_list=image_file_list, image_save_file=None, image_size=(200, 200), dpi=100)\n\n def model_fun(self, ):\n \"\"\" 返回func \"\"\"\n\n raise NotImplementedError\n\n def list_features(self, file_list: list = None, mode: ProcessMode = None) -> (list, list):\n if file_list is None and mode is not None:\n if mode == ProcessMode.train:\n input_list = self.data_utils.get_file_list(is_training=True)\n else:\n input_list = self.data_utils.get_file_list(is_training=False)\n else:\n input_list = file_list\n\n input_fn = self.get_dataset_func(\n split_name=DatasetUtils.SPLIT_PREDICT, input_list=input_list,\n num_epochs=1, shuffle=False, batch_size=self.batch_size, num_parallel_calls=2, prefetch_size=2,\n )\n\n # predict\n self.logger.info(\"trying to list feature for {} file...\".format(len(input_list)))\n classifier = self.get_classifier()\n result_list = classifier.predict(input_fn=input_fn)\n\n # parse feature\n feature_list = []\n feature_file_list = []\n count = 0\n _steps_ = 100 if len(input_list) <= 1000 else 1000\n for score in result_list:\n feature_file_list.append(byte_to_string(score[self._features_filename_key]))\n feature_list.append(score[self._features_embedding_key].reshape(1, self.params.dimension))\n count += 1\n if count % _steps_ == 0:\n self.logger.info(\"calculated {} image...\".format(count))\n\n if count % _steps_ != 0:\n self.logger.info(\"calculated {} image...\".format(count))\n\n self.logger.info(\"success to list feature for {} file!\".format(len(input_list)))\n\n return feature_list, feature_file_list\n\n def get_dataset_func(self, split_name, num_epochs=1, shuffle=True, batch_size=64, num_parallel_calls=2,\n prefetch_size=2, shuffle_size=4, input_list=None):\n if self.train_loss_hook is None:\n raise ValueError(\"self.train_loss_hook cannot be null in this class!\")\n\n def tf_decode_with_crop(file_name, label, offset_height, offset_width, target_height, target_width):\n image_str = tf.read_file(file_name)\n image = tf.image.decode_jpeg(image_str, channels=3)\n image = tf_image_crop(image, offset_height, offset_width, target_height, target_width)\n\n image = tf.image.resize_images(tf.image.rgb_to_grayscale(image),\n size=(self.image_tmp_size, self.image_tmp_size))\n\n processed_images = pre_process_utils.whale_gray_preprocess_image(\n image, self.params.image_size, self.params.image_size,\n is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),\n mean_tf_func=pre_process_utils.whale_siamese_image_mean_tf)\n return {self._features_images_key: processed_images, self._features_filename_key: file_name}, label\n\n def input_fn():\n if input_list:\n file_list = input_list\n else:\n file_list = self.data_utils.get_file_list(\n is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),\n shuffle=shuffle, num_epochs=num_epochs, batch_size=batch_size,\n online_batch_count=self.params.online_batch_count)\n\n self.logger.info(\"info of file_list: len is {}.\".format(len(file_list)))\n if split_name == DatasetUtils.SPLIT_TRAIN:\n labels = [self.data_utils.get_label(image_file) for image_file in file_list]\n else:\n labels = [0] * len(file_list)\n\n bounding_boxes = [expend_bounding_box(self.data_utils.get_boxes(image_file)) for\n image_file in file_list]\n offset_height_list, offset_width_list, target_height_list, target_width_list = \\\n parse_bounding_boxes_list(bounding_boxes)\n dataset = tf.data.Dataset.from_tensor_slices(\n (file_list, labels, offset_height_list, offset_width_list, target_height_list, target_width_list))\n dataset = dataset.map(tf_decode_with_crop, num_parallel_calls=num_parallel_calls)\n dataset = dataset.prefetch(buffer_size=prefetch_size * batch_size)\n dataset = dataset.batch(batch_size)\n iterator = dataset.make_one_shot_iterator()\n features, labels = iterator.get_next()\n return features, labels\n\n return input_fn\n\n def map_search(self, only_validate: bool = False, clear_cache_after_exists: bool = True,\n norm_type: NormType = NormType.all):\n # norm_type == NormType.all, 相当于所有元素同时除以一个数值. 与模型一致; 不改变距离顺序\n with TripletSearch(instance=self, data_utils=self.data_utils, norm_type=norm_type,\n clear_cache_after_exists=clear_cache_after_exists) as search:\n self.logger.info(\"trying to search with {}\".format(self.__class__.__name__))\n raw_submission_csv = self.data_utils.path_manager.submission_csv\n try:\n\n _time_start = time.time()\n search.train()\n\n base_name = \"whale.sub_.{}\".format(create_fake_random_string(length=8))\n self.data_utils.path_manager.submission_csv = os.path.join(self.data_utils.path_manager.output_path,\n base_name)\n\n map_5, map_1 = search.validate(data_percent=1.0)\n if only_validate:\n self.logger.info(\"success to validate with {}, time cost {} seconds!\".format(\n self.__class__.__name__, round(time.time() - _time_start, 2)))\n return\n\n # predict\n self.data_utils.path_manager.submission_csv = \\\n self.data_utils.path_manager.submission_csv + \\\n \".{}.submission.csv\".format(str((map_5 + map_1) / 2)[2:])\n search.test()\n self.logger.info(\"success to search with {}, time cost {} seconds, submission save in {}\".format(\n self.__class__.__name__, round(time.time() - _time_start, 2),\n self.data_utils.path_manager.submission_csv))\n finally:\n self.data_utils.path_manager.submission_csv = raw_submission_csv\n\n def combine_submission_csv(self):\n # combine result\n tmp_submission_csv_list = glob.glob(\n os.path.join(self.data_utils.path_manager.output_path, \"whale.sub_.*.submission.csv\"))\n\n if not tmp_submission_csv_list:\n self.logger.warning(\"no submission file found!\")\n return\n self.logger.info(\"found tmp submission files: {}\".format(tmp_submission_csv_list))\n\n result_weight = {}\n for csv_file in tmp_submission_csv_list:\n try:\n result_weight[csv_file] = float(\"0.{}\".format(csv_file.split(\".\")[-3]))\n except Exception as e:\n self.logger.error(e)\n\n combine_csv(result_weight, out_file=os.path.join(self.data_utils.path_manager.output_path, \"sub_ens.csv\"))\n self.logger.info(\"success to merge all submission file into sub_ens.csv\")\n\n if self.data_utils.path_manager.is_kaggle:\n for csv_file in tmp_submission_csv_list:\n os.remove(csv_file)\n\n def train_with_predict(self, safe_max_batch_size=32, calc_every_epoch=1, shuffle_data_every_epoch=1,\n predict_every_epoch=100, max_epoch=100, ignore_error_in_train: bool = True):\n assert max_epoch >= predict_every_epoch >= shuffle_data_every_epoch\n assert max_epoch >= calc_every_epoch >= shuffle_data_every_epoch\n\n self.batch_size = safe_max_batch_size\n\n def end_process_func():\n self.map_search(only_validate=False, clear_cache_after_exists=True)\n\n def loop_process(total_epoch, num_epoch):\n if total_epoch % calc_every_epoch == 0:\n if total_epoch == 0:\n self.train(batch_size=safe_max_batch_size, num_epochs=1, shuffle=True, steps=1)\n\n self.data_utils.calc_feature(self.list_features, epoch_num=total_epoch)\n\n if total_epoch > 0 and not self.data_utils.path_manager.is_kaggle and \\\n os.path.abspath(self.TRAIN_CKPT_DIR).find(\"/content/\") > -1:\n colab_save_file_func(train_dir=self.TRAIN_CKPT_DIR, logger=self.logger, daemon=False,\n only_save_latest_checkpoint=True)\n\n self.train(batch_size=safe_max_batch_size, num_epochs=num_epoch, shuffle=True, steps=None)\n self.set_epoch_num(count=total_epoch + num_epoch)\n\n if total_epoch > 0 and total_epoch % predict_every_epoch == 0:\n end_process_func()\n\n # run\n estimator_iter_process(loop_process, iter_stop_time=self.timeout,\n loop_process_min_epoch=shuffle_data_every_epoch,\n loop_process_start_epoch=self.get_epoch_num(),\n end_process_func=end_process_func, loop_process_max_epoch=max_epoch,\n ignore_error_in_loop_process=ignore_error_in_train, logger=self.logger)\n\n # combine result\n self.combine_submission_csv()\n\n\nclass TripletLossModelResNet50(TripletLossModelCNN):\n def __init__(self, train_ckpt_dir, data_utils: WhaleDataUtils, timeout: int = int(3600 * 5), is_softmax=True):\n if train_ckpt_dir.endswith(\"/\"):\n train_ckpt_dir = train_ckpt_dir[:-1]\n\n softmax_dir = os.path.join(os.path.dirname(train_ckpt_dir), os.path.basename(train_ckpt_dir) + \"_softmax\")\n triplet_dir = os.path.join(os.path.dirname(train_ckpt_dir), os.path.basename(train_ckpt_dir) + \"_triplet\")\n if is_softmax is None:\n if tf.train.latest_checkpoint(triplet_dir):\n is_softmax = False\n else:\n is_softmax = True\n\n _pretrained_ckpt_file = None\n if is_softmax:\n # using softmax net\n _train_ckpt_dir = softmax_dir\n _pretrained_ckpt_file = data_utils.path_manager.ckpt_pretrained_rv250\n if not tf.train.latest_checkpoint(softmax_dir):\n data_utils.path_manager.download_resnet_v2_50()\n else:\n # using triplet loss\n _train_ckpt_dir = triplet_dir\n if tf.train.latest_checkpoint(softmax_dir):\n _pretrained_ckpt_file = tf.train.latest_checkpoint(softmax_dir)\n\n super(TripletLossModelResNet50, self).__init__(\n train_ckpt_dir=_train_ckpt_dir,\n data_utils=data_utils,\n timeout=timeout,\n pretrained_ckpt_file=_pretrained_ckpt_file,\n )\n self.params = Params({\n \"num_channels\": 3,\n \"margin\": 1.0,\n \"dimension\": 512,\n \"image_size\": 224,\n \"online_batch_count\": 4,\n })\n self.image_tmp_size = 250\n\n # softmax setting\n self.is_softmax = is_softmax\n if is_softmax:\n self.use_softmax_net()\n else:\n self.use_triplet_net()\n\n _, class_id_vs_image_list, _, _, _ = \\\n self.data_utils.get_basic_info(self.data_utils.list_train_image_file(ignore_blank=True))\n _class_id_vs_image_count = {\n class_id: len(_image_list) for class_id, _image_list in class_id_vs_image_list.items()}\n if self.data_utils.blank_class_id in _class_id_vs_image_count:\n _class_id_vs_image_count.pop(self.data_utils.blank_class_id)\n\n _image_count_list = list(_class_id_vs_image_count.values())\n _image_count_list.sort()\n _max_count = int(sum(_image_count_list) * 0.2)\n _base_count = 3\n _middle_count = 0\n _total_count = 0\n for _index, _count in enumerate(_image_count_list[::-1]):\n _total_count += _count\n if _total_count >= _max_count:\n _base_count = _count\n _middle_count = _image_count_list[-int(_index / 2)]\n break\n\n anchor_class_id_list = [\n class_id for class_id, _image_count in _class_id_vs_image_count.items() if _image_count >= _base_count]\n\n self.softmax_num_class = len(anchor_class_id_list)\n anchor_class_id_list.sort()\n\n self.softmax_class_id_vs_label = {}\n for index, class_id in enumerate(anchor_class_id_list):\n self.softmax_class_id_vs_label[class_id] = index\n\n self.softmax_image_list = []\n _anchor_class_id_set = set(anchor_class_id_list)\n _class_vs_image = {}\n for class_id, file_list in class_id_vs_image_list.items():\n if class_id in _anchor_class_id_set:\n _class_vs_image[class_id] = file_list\n\n _new_class_image = balance_class_dict(_class_vs_image, target_count=min(100, _middle_count))\n for file_list in _new_class_image.values():\n self.softmax_image_list.extend(file_list)\n\n random.shuffle(self.softmax_image_list)\n\n self.softmax_image_vs_label = {}\n for image_file in self.softmax_image_list:\n self.softmax_image_vs_label[image_file] = self.softmax_class_id_vs_label[\n self.data_utils.get_class_id(image_file)]\n\n self.logger.info(\"train softmax net with {} classes (ignore blank, image_count >= {}), {}/{} files,\".format(\n self.softmax_num_class, _base_count, len(set(self.softmax_image_list)), len(self.softmax_image_list)))\n\n # global setting\n self._net_embedding_key = \"embeddings\"\n\n def model_fun(self, ):\n \"\"\" 返回func \"\"\"\n\n def get_model_net(scope_name, features, params, is_training, labels=None):\n inputs = tf.reshape(features[self._features_images_key],\n [-1, params.image_size, params.image_size, params.num_channels])\n\n with slim.arg_scope(resnet_arg_scope()):\n global_pool, end_points = resnet_v2_50(inputs, num_classes=None, is_training=is_training)\n\n with tf.variable_scope(scope_name, 'triplet_loss', [global_pool]) as sc:\n net = slim.conv2d(global_pool, params.dimension, [1, 1], scope=self._net_embedding_key)\n end_points[sc.name + '/logits'] = net\n net = tf.squeeze(net, [1, 2], name='SpatialSqueeze')\n end_points[sc.name + '/spatial_squeeze'] = net\n end_points[self._net_embedding_key] = net\n\n assert net.shape[-1] == params.dimension\n assert len(net.shape) == 2\n\n return net, end_points\n\n def _softmax_bone_net(scope_name, features, params, is_training, labels=None):\n embeddings, end_points = get_model_net(\n scope_name=scope_name, features=features, params=params, is_training=is_training)\n\n with slim.arg_scope(resnet_arg_scope()):\n with tf.variable_scope(\"softmax_net\", 'softmax_net', [embeddings]):\n net = slim.fully_connected(embeddings, self.softmax_num_class, activation_fn=None,\n normalizer_fn=None, scope='logits')\n return net, end_points\n\n if self.is_softmax:\n return tf_model_fn.tf_softmax_model_fn(\n network=_softmax_bone_net,\n scope_name=self.model_name,\n features_filename_key=self._features_filename_key,\n get_learning_rate_func=self.get_learning_rate,\n optimizer_type=self.optimizer_type,\n logger=self.logger\n )\n\n return tf_model_fn.tf_triplet_loss_model_fn(\n network=get_model_net,\n scope_name=self.model_name,\n features_embedding_key=self._features_embedding_key,\n features_filename_key=self._features_filename_key,\n get_learning_rate_func=self.get_learning_rate,\n optimizer_type=self.optimizer_type,\n logger=self.logger,\n use_l2_normalize=False)\n\n def get_dataset_func(self, split_name, num_epochs=1, shuffle=True, batch_size=64, num_parallel_calls=2,\n prefetch_size=2, shuffle_size=4, input_list=None):\n if self.train_loss_hook is None:\n raise ValueError(\"self.train_loss_hook cannot be null in this class!\")\n\n def tf_decode_with_crop(file_name, label, offset_height, offset_width, target_height, target_width):\n image_str = tf.read_file(file_name)\n image = tf.image.decode_jpeg(image_str, channels=3)\n image = tf_image_crop(image, offset_height, offset_width, target_height, target_width)\n image = tf.image.resize_images(image, size=(self.image_tmp_size, self.image_tmp_size))\n\n processed_images = pre_process_utils.whale_rgb_preprocess_image(\n image, self.params.image_size, self.params.image_size,\n is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN))\n return {self._features_images_key: processed_images, self._features_filename_key: file_name}, label\n\n def input_fn():\n if input_list:\n file_list = input_list\n else:\n file_list = self.data_utils.get_file_list(\n is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),\n shuffle=shuffle, num_epochs=num_epochs, batch_size=batch_size,\n online_batch_count=self.params.online_batch_count)\n\n self.logger.info(\"info of file_list: len is {}.\".format(len(file_list)))\n if split_name == DatasetUtils.SPLIT_TRAIN:\n labels = [self.data_utils.get_label(image_file) for image_file in file_list]\n else:\n labels = [0] * len(file_list)\n\n bounding_boxes = [expend_bounding_box(self.data_utils.get_boxes(image_file)) for\n image_file in file_list]\n offset_height_list, offset_width_list, target_height_list, target_width_list = \\\n parse_bounding_boxes_list(bounding_boxes)\n\n dataset = tf.data.Dataset.from_tensor_slices(\n (file_list, labels, offset_height_list, offset_width_list, target_height_list, target_width_list))\n dataset = dataset.map(tf_decode_with_crop, num_parallel_calls=num_parallel_calls)\n dataset = dataset.prefetch(buffer_size=prefetch_size * batch_size)\n dataset = dataset.batch(batch_size)\n iterator = dataset.make_one_shot_iterator()\n features, labels = iterator.get_next()\n return features, labels\n\n def softmax_input_fn():\n if input_list:\n file_list = input_list\n else:\n file_list = list(self.softmax_image_list) * num_epochs\n if shuffle:\n random.shuffle(file_list)\n\n self.logger.info(\"info of file_list: len is {}.\".format(len(file_list)))\n labels = [self.softmax_image_vs_label[image_file] for image_file in file_list]\n\n bounding_boxes = [expend_bounding_box(self.data_utils.get_boxes(image_file)) for\n image_file in file_list]\n offset_height_list, offset_width_list, target_height_list, target_width_list = \\\n parse_bounding_boxes_list(bounding_boxes)\n dataset = tf.data.Dataset.from_tensor_slices(\n (file_list, labels, offset_height_list, offset_width_list, target_height_list, target_width_list))\n dataset = dataset.map(tf_decode_with_crop, num_parallel_calls=num_parallel_calls)\n dataset = dataset.prefetch(buffer_size=prefetch_size * batch_size)\n dataset = dataset.batch(batch_size)\n iterator = dataset.make_one_shot_iterator()\n features, labels = iterator.get_next()\n return features, labels\n\n if self.is_softmax:\n return softmax_input_fn\n\n return input_fn\n\n def use_softmax_net(self):\n # softmax setting\n self.logger.info(\"using softmax net...\")\n self.is_softmax = True\n self.data_utils.gen_data_setting[\"stop_calc_feature\"] = True\n\n def use_triplet_net(self):\n # softmax setting\n self.logger.info(\"using triplet net...\")\n self.is_softmax = False\n self.data_utils.gen_data_setting[\"stop_calc_feature\"] = False\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3382b9fa20db2bab6ef24da722756daaf53e21a"},"cell_type":"markdown","source":"# 5.training model"},{"metadata":{"trusted":true,"_uuid":"ce7cdd83a72ecdf2ea155f558c912554786f3e01"},"cell_type":"code","source":"!cp ../input/whale-triplet-pretrained-model/tripletresnet/tripletResNet ./triplet_triplet -R\n\n# logger\ninit_logger()\n\n# estimator\npath_manager = PathManager(\"kaggle\")\ndata_utils = WhaleDataUtils(path_manager=path_manager, gen_data_setting={\n \"x_train_num\": 4,\n \"ignore_blank_prob\": 0.9,\n \"ignore_single_prob\": 0.5,\n \"stop_calc_feature\": False,\n \"gen_data_by_random_prob\": 0,\n \"use_norm_when_calc_apn\": True,\n })\n \n# init model with pretrained triplet loss model (finetuning from resnet50)\nestimator = TripletLossModelResNet50(\n train_ckpt_dir=\"./triplet\",\n data_utils=data_utils,\n timeout=int(5 * 3600),\n is_softmax=False,\n )\n \n# predict\nestimator.map_search(only_validate=False, clear_cache_after_exists=True)\nestimator.combine_submission_csv() # create submission csv file: sub_ens.csv\n \n# # show embeddings (you need to download logdata and show it with tensorboard)\n# estimator.show_embedding(\n# mode=ProcessMode.train,\n# count=10000,\n# remove_log_dir_if_exists=True,\n# norm_type=NormType.all,\n# )\n \n# # train and predict\n# try:\n# estimator.train_with_predict(safe_max_batch_size=64, calc_every_epoch=1, shuffle_data_every_epoch=1,\n# predict_every_epoch=100, max_epoch=10000, ignore_error_in_train=True)\n# finally:\n# if data_utils.path_manager.is_kaggle:\n# if os.path.exists(path_manager.bounding_boxes_csv):\n# os.remove(path_manager.bounding_boxes_csv)\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f921ff3e49dfe5cbe0e7d8296e67da9e20db8f9d"},"cell_type":"code","source":"estimator.show_predict_result(count=10, top_k=5)\n\nestimator.show_predict_result(count=10, top_k=3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25df05d8e8db5365c78902ce0333e4709f13cdf9"},"cell_type":"code","source":"# exit\n!rm *.pkl\n\n!pip uninstall aiohttp faiss-prebuilt pyxtools pymltools -y\n!apt remove -y libopenblas-base libomp-dev\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"016708b17db3022a17d59e54b5ac6594ef7c0564"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}