{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nimport torch\nimport torchvision.models as models\nimport math\nfrom torch.nn import init\nfrom torchvision import transforms\nimport torchvision.transforms as transforms\nimport time\nfrom torch.backends import cudnn\nimport numpy as np\nimport json\nimport cv2\n\nimport sys \nsys.path.insert(0, '../input/resnestpy/')\nfrom resnest import resnest50\n\nclass ModelResnest50_Embeding(nn.Module):\n    def __init__(self, enet=\"Resnest\", num_class=203093, out_dim=1):\n        super(ModelResnest50_Embeding, self).__init__()\n        base_model = resnest50(pretrained=False)\n        self.conv = nn.Sequential(*list(base_model.children())[:-2])\n        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) #194390\n        print(\"fc , \", num_class)\n        self.classifier = nn.Linear(in_features=512, out_features=num_class, bias=True)\n\n        nn.init.xavier_uniform_(self.classifier.weight)\n        nn.init.constant_(self.classifier.bias, 0)\n        feature = 2048\n        self.fc5 = nn.Linear(2048, 512)\n\n\n        nn.init.xavier_uniform_(self.fc5.weight)\n        nn.init.constant_(self.fc5.bias, 0)\n        \n        self.bn5 = nn.BatchNorm1d(512)\n        self.bn5.weight.data.normal_(1.0,0.02) #bn层里初始化γ，服从（1，0.02）的正态分布\n        self.bn5.bias.data.fill_(0)  #bn层里初始化β，默认为0\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.avg_pool(x).view(x.size(0), -1)\n        x = self.fc5(x)\n        x = self.bn5(x)\n        feat = x\n        x = self.classifier(x)\n\n        return x, feat\n\ntransform_test  = transforms.Compose(\n    [\n     transforms.ToPILImage(),\n     transforms.Resize([512, 512]),\n     #transforms.CenterCrop((448,448)),\n     transforms.ToTensor(),\n     transforms.Normalize((0.485,0.456,0.406),(0.229,0.224,0.225))\n    ])","metadata":{"execution":{"iopub.status.busy":"2021-09-09T08:16:53.280381Z","iopub.execute_input":"2021-09-09T08:16:53.280733Z","iopub.status.idle":"2021-09-09T08:16:53.301066Z","shell.execute_reply.started":"2021-09-09T08:16:53.280693Z","shell.execute_reply":"2021-09-09T08:16:53.300213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\ndef com_cos_dist(x1, x2):\n    return np.dot(x1, x2)/(np.linalg.norm(x1)*(np.linalg.norm(x2)))\n    \n\n#os.environ['CUDA_VISIBLE_DEVICES'] = '0'\ncropsize = 448\nresize   = 512\nnormalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n\ntransform_test = transforms.Compose([\n        transforms.ToPILImage(),\n        # transforms.Resize(resize),\n        transforms.CenterCrop(cropsize),\n        transforms.ToTensor(),\n        normalize\n        ]\n        )\n\n\ndef get_affine_transform(size1, size2):\n    src = np.zeros((3, 2), dtype=np.float32)\n    dst = np.zeros((3, 2), dtype=np.float32)\n    scale1 = size2[0]*1.0/size1[0]\n    scale2 = size2[1]*1.0/size1[1]\n    scale = min(scale1, scale2)\n    # Center to Center\n    src[0, :] = [size1[0]/2.0 , size1[1]/2.0]\n    dst[0, :] = [size2[0]/2.0 , size2[1]/2.0]\n\n    # Left Center to Left Center Boarder\n    src[1, :] = [0.0 , size1[1]/2.0]\n    dst[1, :] = [size2[0]/2.0 - scale*size1[0]/2.0 , size2[1]/2.0]\n\n    # Top Center to Top Center Boader\n    src[2, :] = [ size1[0]/2.0, 0.0]\n    dst[2, :] = [ size2[0]/2.0 , size2[1]/2.0 - scale*size1[1]/2.0 ]\n    trans = cv2.getAffineTransform(np.float32(src), np.float32(dst))\n    return trans\n\n\nclass Feature_Extract():\n    def __init__(self, ckpt_path=None, transforms = None ):\n        \n        if ckpt_path is None:\n            ckpt_path = os.path.dirname(path) + \"resnest-0038.pth\"\n        NUM_CLASS = 203093\n        model = ModelResnest50_Embeding(num_class=NUM_CLASS)\n        \n        #transform_train, transform_test = get_cub_transform()\n        from collections import OrderedDict\n        #ckpt_path = \"checkpoints_multi/resnest_\" + str(RESUME_EPOACH).zfill(4) + \".pth\"\n        ckpt = torch.load(ckpt_path, map_location=lambda storage, loc: storage)\n        #print(ckpt['model'])\n        state_dict = ckpt['model']\n        new_state_dict = OrderedDict()\n\n        for k, v in state_dict.items():\n            #print(k,v)\n            name = k[7:]  # remove `module.`\n            new_state_dict[name] = v\n        model.load_state_dict(new_state_dict)\n        print(\"finish resume ... \")\n        \n        #self.model_dict = model.cuda().eval()\n        self.model_dict = model.eval()\n        self.Size = (512, 512)\n        self.transforms = transforms\n        \n    def run(self, cv2_img):        \n        img = cv2_img[:, :, ::-1]\n        img_w, img_h = img.shape[1], img.shape[0]\n        trans_input = get_affine_transform((img_w, img_h), self.Size)\n        image = cv2.warpAffine(img, trans_input, self.Size, flags=cv2.INTER_LINEAR)\n        \n        if self.transforms :\n            image = self.transforms(image)\n        #image = image.reshape(1, 3, cropsize, cropsize).cuda()\n        image = image.reshape(1, 3, cropsize, cropsize)\n        res = {}\n        with torch.no_grad():\n            #print(\"image shape is \", image.shape)\n            _, logits_m = self.model_dict(image)\n            #print(\"logits_m shape is \", logits_m)\n            res[\"fea\"] = logits_m\n            return res","metadata":{"execution":{"iopub.status.busy":"2021-09-09T08:17:10.948647Z","iopub.execute_input":"2021-09-09T08:17:10.948991Z","iopub.status.idle":"2021-09-09T08:17:10.976434Z","shell.execute_reply.started":"2021-09-09T08:17:10.948962Z","shell.execute_reply":"2021-09-09T08:17:10.975360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\nimport csv\nimport gc\nimport operator\nimport os\nimport pathlib\nimport shutil\nimport cv2\n\nimport numpy as np\nimport PIL\nimport pydegensac\nfrom scipy import spatial\nimport tensorflow as tf\nimport sys\nimport json\n#sys.path.append(\"/root/mnt/shuhuigao/workspace/backup_from_venus/classification/extract_retrieval_feature/\")\n\nimport torch\n#from extract_retrieval_feature import inference\n#from extract_retrieval_feature import tfs\nimport torchvision.transforms as transforms\n\nNUM_EMBEDDING_DIMENSIONS = 512 \n# Dataset parameters:\nINPUT_DIR = os.path.join('../', 'input/')\n#INPUT_DIR = os.path.join('', '')\n\nDATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2021')\n#DATASET_DIR = os.path.join(INPUT_DIR, '')\nTEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')\nTRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')\nTRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')\n\n# DEBUGGING PARAMS:\nNUM_PUBLIC_TRAIN_IMAGES = 1580470  # Used to detect if in session or re-run.\nMAX_NUM_EMBEDDINGS = -1  # Set to > 1 to subsample dataset while debugging.\n\n# Retrieval & re-ranking parameters:\nNUM_TO_RERANK = 6\nTOP_K = 3  # Number of retrieved images used to make prediction for a test image.\n\n# RANSAC parameters:\nMAX_INLIER_SCORE = 26\nMAX_REPROJECTION_ERROR = 6.0\nMAX_RANSAC_ITERATIONS = 900000\nHOMOGRAPHY_CONFIDENCE = 0.95\n\n# DELG model:\nSAVED_MODEL_DIR = '../input/delg-saved-models/local_and_global'\n#DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)\nDELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])\nDELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)\nDELG_INPUT_TENSOR_NAMES = [\n    'input_image:0', 'input_scales:0', 'input_abs_thres:0'\n]\n\"\"\"\n# Global feature extraction:\nNUM_EMBEDDING_DIMENSIONS = 2048\nGLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,\n                                                ['global_descriptors:0'])\n\n# Local feature extraction:\nLOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)\nLOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(\n    DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],\n    ['boxes:0', 'features:0'])\n\"\"\"\n\ndef to_hex(image_id) -> str:\n    return '{0:0{1}x}'.format(image_id, 16)\n\n\ndef get_image_path(subset, image_id):\n    name = to_hex(image_id)\n    return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],\n                        '{}.jpg'.format(name))\n\n\ndef load_image_tensor(image_path):\n    return tf.convert_to_tensor(\n        np.array(PIL.Image.open(image_path).convert('RGB')))\n\n\ndef extract_global_features(image_root_dir):\n    \"\"\"Extracts embeddings for all the images in given `image_root_dir`.\"\"\"\n    cropsize = 448\n    resize   = 512\n    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n\n\n    #model define  and transformer\n    transform_test = transforms.Compose([\n        transforms.ToPILImage(),\n        # transforms.Resize(resize),\n        transforms.CenterCrop(cropsize),\n        transforms.ToTensor(),\n        normalize\n        ])\n\n    model = Feature_Extract(ckpt_path=\"../input/ckpt-init-true/resnest_0015.pth\", transforms=transform_test)\n    image_paths = [x for x in pathlib.Path(image_root_dir).rglob('*.jpg')]\n    image_paths = image_paths[:100]\n    \n\n    #print( \"image_paths is \", image_paths )\n    num_embeddings = len(image_paths)\n    if MAX_NUM_EMBEDDINGS > 0:\n        num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)\n\n    ids = num_embeddings * [None]\n    embeddings = np.empty((num_embeddings, NUM_EMBEDDING_DIMENSIONS))\n\n    for i, image_path in enumerate(image_paths):\n        if i >= num_embeddings:\n            break\n\n        ids[i] = int(image_path.name.split('.')[0], 16)\n        #print(\" id i\",ids[i] ,image_path, image_path.name )\n        #image_tensor = load_image_tensor(image_path)\n        image_tensor = cv2.imread(str(image_path))\n        features = model.run(image_tensor)\n        feature = features[\"fea\"]\n        feature = feature / (torch.norm(feature, keepdim=True) + 1e-6)\n        feature = feature.view(-1).detach().float().cpu().numpy()\n        embeddings[i, :] = feature\n\n        #features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,\n        #                                        DELG_IMAGE_SCALES_TENSOR,\n        #                                        DELG_SCORE_THRESHOLD_TENSOR)\n\n        #embeddings[i, :] = tf.nn.l2_normalize(\n        #    tf.reduce_sum(features[0], axis=0, name='sum_pooling'),\n        #    axis=0,\n        #    name='final_l2_normalization').numpy()\n\n    return ids, embeddings\n\n\ndef extract_local_features(image_path):\n    \"\"\"Extracts local features for the given `image_path`.\"\"\"\n\n    image_tensor = load_image_tensor(image_path)\n\n    features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,\n                                           DELG_SCORE_THRESHOLD_TENSOR,\n                                           LOCAL_FEATURE_NUM_TENSOR)\n\n    # Shape: (N, 2)\n    keypoints = tf.divide(\n        tf.add(\n            tf.gather(features[0], [0, 1], axis=1),\n            tf.gather(features[0], [2, 3], axis=1)), 2.0).numpy()\n\n    # Shape: (N, 128)\n    descriptors = tf.nn.l2_normalize(\n        features[1], axis=1, name='l2_normalization').numpy()\n\n    return keypoints, descriptors\n\n\ndef get_putative_matching_keypoints(test_keypoints,\n                                    test_descriptors,\n                                    train_keypoints,\n                                    train_descriptors,\n                                    max_distance=0.9):\n    \"\"\"Finds matches from `test_descriptors` to KD-tree of `train_descriptors`.\"\"\"\n\n    train_descriptor_tree = spatial.cKDTree(train_descriptors)\n    _, matches = train_descriptor_tree.query(\n        test_descriptors, distance_upper_bound=max_distance)\n\n    test_kp_count = test_keypoints.shape[0]\n    train_kp_count = train_keypoints.shape[0]\n\n    test_matching_keypoints = np.array([\n        test_keypoints[i,]\n        for i in range(test_kp_count)\n        if matches[i] != train_kp_count\n    ])\n    train_matching_keypoints = np.array([\n        train_keypoints[matches[i],]\n        for i in range(test_kp_count)\n        if matches[i] != train_kp_count\n    ])\n\n    return test_matching_keypoints, train_matching_keypoints\n\n\ndef get_num_inliers(test_keypoints, test_descriptors, train_keypoints,\n                    train_descriptors):\n    \"\"\"Returns the number of RANSAC inliers.\"\"\"\n\n    test_match_kp, train_match_kp = get_putative_matching_keypoints(\n        test_keypoints, test_descriptors, train_keypoints, train_descriptors)\n\n    if test_match_kp.shape[\n        0] <= 4:  # Min keypoints supported by `pydegensac.findHomography()`\n        return 0\n\n    try:\n        _, mask = pydegensac.findHomography(test_match_kp, train_match_kp,\n                                            MAX_REPROJECTION_ERROR,\n                                            HOMOGRAPHY_CONFIDENCE,\n                                            MAX_RANSAC_ITERATIONS)\n    except np.linalg.LinAlgError:  # When det(H)=0, can't invert matrix.\n        return 0\n\n    return int(copy.deepcopy(mask).astype(np.float32).sum())\n\n\ndef get_total_score(num_inliers, global_score):\n    local_score = min(num_inliers, MAX_INLIER_SCORE) / MAX_INLIER_SCORE\n    return local_score + global_score\n\n\ndef rescore_and_rerank_by_num_inliers(test_image_id,\n                                      train_ids_labels_and_scores):\n    \"\"\"Returns rescored and sorted training images by local feature extraction.\"\"\"\n\n    test_image_path = get_image_path('test', test_image_id)\n    test_keypoints, test_descriptors = extract_local_features(test_image_path)\n\n    for i in range(len(train_ids_labels_and_scores)):\n        train_image_id, label, global_score = train_ids_labels_and_scores[i]\n\n        train_image_path = get_image_path('train', train_image_id)\n        train_keypoints, train_descriptors = extract_local_features(\n            train_image_path)\n\n        num_inliers = get_num_inliers(test_keypoints, test_descriptors,\n                                      train_keypoints, train_descriptors)\n        total_score = get_total_score(num_inliers, global_score)\n        train_ids_labels_and_scores[i] = (train_image_id, label, total_score)\n\n    train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)\n\n    return train_ids_labels_and_scores\n\n\ndef load_labelmap():\n    with open(TRAIN_LABELMAP_PATH, mode='r') as csv_file:\n        csv_reader = csv.DictReader(csv_file)\n        labelmap = {row['id']: row['landmark_id'] for row in csv_reader}\n\n    return labelmap\n\n\ndef get_prediction_map(test_ids, train_ids_labels_and_scores):\n    \"\"\"Makes dict from test ids and ranked training ids, labels, scores.\"\"\"\n\n    prediction_map = dict()\n\n    for test_index, test_id in enumerate(test_ids):\n        hex_test_id = to_hex(test_id)\n\n        aggregate_scores = {}\n        for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:\n            if label not in aggregate_scores:\n                aggregate_scores[label] = 0\n            aggregate_scores[label] += score\n\n        label, score = max(aggregate_scores.items(), key=operator.itemgetter(1))\n\n        prediction_map[hex_test_id] = {'score': score, 'class': label}\n\n    return prediction_map\n\ndef load_train_feature(f_in_path):\n\n    f_in = open(f_in_path)\n    line_lib = f_in.readline()\n    list_fea = []\n    list_label = []\n    list_img_dir = []\n    \n    ind_line = 0\n\n    while(line_lib):\n        ind_line += 1\n        if ind_line > 1000:\n            break\n\n        img_id, img_cls, img_dir, fea = line_lib.strip().split(\"\\t\")\n        fea = json.loads(fea)\n        fea = [round(i,8) for i in fea]\n        list_fea.append(np.array(fea).astype(np.float32))\n        current = copy.deepcopy(img_cls)\n        #index.add()\n\n        list_label.append(current)\n        #list_img_dir.append(img_dir)\n        #ids[i] = int(image_path.split(\"/\")[-1].split('.')[0], 16)\n        list_img_dir.append(int(img_dir.split(\"/\")[-1].split('.')[0], 16))\n\n\n        line_lib = f_in.readline()\n        \n    return list_fea, list_label, list_img_dir\n\n\ndef get_predictions(labelmap):\n    \"\"\"Gets predictions using embedding similarity and local feature reranking.\"\"\"\n    \n    # extract test feature\n    test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)\n\n    # load train feature\n    #f_in = \"fea_lib_class_resnest_base_true_15_img_lst_train.txt\"\n    f_in = \"../input/lib-init/fea_lib_class_resnest_base_true_15_img_lst_train_round8.txt\"\n    list_fea, list_label, train_ids_old = load_train_feature(f_in)\n    \n    train_embeddings = np.array(list_fea)\n    train_ids = train_ids_old  #[id_it.split(\"/\")[-1].split(\".\")[0] for id_it in train_ids_old]\n\n    #image_path.name.split('.')[0], 16\n\n    #test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)\n\n    #train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)\n    #--------------------next need change-----------------------\n    train_ids_labels_and_scores = [None] * test_embeddings.shape[0]\n\n    # Using (slow) for-loop, as distance matrix doesn't fit in memory.\n    for test_index in range(test_embeddings.shape[0]):\n        distances = spatial.distance.cdist(\n            test_embeddings[np.newaxis, test_index, :], train_embeddings,\n            'cosine')[0]\n        partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]\n\n        nearest = sorted([(train_ids[p], distances[p]) for p in partition],\n                         key=lambda x: x[1])\n        #print(nearest)\n\n        train_ids_labels_and_scores[test_index] = [\n            (train_id, labelmap[to_hex(train_id)], 1. - cosine_distance)\n            for train_id, cosine_distance in nearest\n        ]\n\n    del test_embeddings\n    del train_embeddings\n    del labelmap\n    gc.collect()\n    \n\n    pre_verification_predictions = get_prediction_map(\n        test_ids, train_ids_labels_and_scores)\n    print(\"pre_verification_predictions is \", pre_verification_predictions )\n\n    return None, pre_verification_predictions\n    \"\"\"\n    for test_index, test_id in enumerate(test_ids):\n        train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(\n            test_id, train_ids_labels_and_scores[test_index])\n\n    post_verification_predictions = get_prediction_map(\n        test_ids, train_ids_labels_and_scores)\n\n    return pre_verification_predictions, post_verification_predictions\"\"\"\n\n\ndef save_submission_csv(predictions=None):\n    \"\"\"Saves optional `predictions` as submission.csv.\n\n  The csv has columns {id, landmarks}. The landmarks column is a string\n  containing the label and score for the id, separated by a ws delimeter.\n\n  If `predictions` is `None` (default), submission.csv is copied from\n  sample_submission.csv in `IMAGE_DIR`.\n\n  Args:\n    predictions: Optional dict of image ids to dicts with keys {class, score}.\n  \"\"\"\n\n    if predictions is None:\n        # Dummy submission!\n        shutil.copyfile(\n            os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')\n        return\n\n    with open('submission.csv', 'w') as submission_csv:\n        csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])\n        csv_writer.writeheader()\n        for image_id, prediction in predictions.items():\n            label = prediction['class']\n            score = prediction['score']\n            csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})\n\n\ndef main():\n    labelmap = load_labelmap()\n    \"\"\"\n    \n    num_training_images = len(labelmap.keys())\n    print(f'Found {num_training_images} training images.')\n\n    if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:\n        print(\n            f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images. Copying sample submission.'\n        )\n        save_submission_csv()\n        return\n\n    _, post_verification_predictions = get_predictions(labelmap)\n    save_submission_csv(post_verification_predictions)\n    \"\"\"\n    _, post_verification_predictions = get_predictions(labelmap)\n    print(\"test_fea shape\", post_verification_predictions )\n    save_submission_csv(post_verification_predictions)\n\n\nif __name__ == '__main__':\n    main()\n","metadata":{"execution":{"iopub.status.busy":"2021-09-09T08:17:22.316739Z","iopub.execute_input":"2021-09-09T08:17:22.317127Z","iopub.status.idle":"2021-09-09T08:23:39.718573Z","shell.execute_reply.started":"2021-09-09T08:17:22.317092Z","shell.execute_reply":"2021-09-09T08:23:39.717478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}