{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch.utils.data as data\nimport os\nimport PIL.Image as Image\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport albumentations as A\nimport torch\nfrom albumentations.pytorch import ToTensorV2\nimport joblib\ntest_image_path = '../input/happywhale-resized-dataset/test_images_512x512'\ntrain_path = \"../input/happywhale-resized-dataset/train_images_512x512\"\ntransform = {\n\n    \"train\": A.Compose([\n\n        # pil.image to tensor\n        #A.HorizontalFlip(p=0.5),\n        #A.VerticalFlip(p=0.5),\n        A.Resize(384,384),\n        #A.OneOf([A.MotionBlur(p=0.2),\n                 #A.MedianBlur(p=0.2),\n                 #A.Blur(blur_limit=3, p=0.2)], p=0.5),\n        #A.GaussNoise(),\n        #A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n        #A.RandomBrightnessContrast(p=0.5),\n        A.Normalize(mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225], ),\n        ToTensorV2(),  # torchvision.transforms.Normalize(mean, std, inplace=False)\n    ]),\n    # tensor([0.4560, 0.4245, 0.3905], device='cuda:0') tensor([0.2755, 0.2671, 0.2602], device='cuda:0')\n    \"valid\": A.Compose([\n\n        A.Resize(384,384),\n        A.Normalize(mean=[0.485, 0.456, 0.406],\n                    std=[0.229, 0.224, 0.225], ),\n        ToTensorV2(),  # torchvision.transforms.Normalize(mean, std, inplace=False)\n    ])\n}\n\n\nclass WhaleDataset(data.Dataset):\n\n    def __init__(self,train_csv, mode, transform=None):\n\n        if mode == 'train':\n\n\n            # print(train_csv.head())\n            train_img = train_csv.image\n            # print(train_img)\n\n            labels = train_csv.individual_key\n            imgs = []\n            for file, label in zip(train_img, labels):\n                imgs.append([file, label])\n\n            self.imgs = imgs\n\n            # self.transform = transform\n            self.mode = mode\n        if mode == 'valid':\n            pass\n        if mode == 'test':\n            train_img = train_csv.image\n            imgs = []\n            for file in train_img:\n                imgs.append(file)\n\n            self.imgs = imgs\n\n            # self.transform = transform\n            self.mode = mode\n\n    def __getitem__(self, idx: int):\n\n        # x是jpg格式，label是png\n        if self.mode == \"train\":\n            x_path, label = self.imgs[idx]\n            pilimage = Image.open(os.path.join(train_path, x_path)).convert(\"RGB\")\n            pilimage = np.array(pilimage)\n            pilimage = transform[\"train\"](image=pilimage)\n            return pilimage[\"image\"], label\n        \n        if self.mode == \"valid\":\n            x_path, label = self.imgs[idx]\n            pilimage = Image.open(os.path.join(opt.train_path, x_path)).convert(\"RGB\")\n            pilimage = np.array(pilimage)\n            pilimage = transform[\"valid\"](image=pilimage)\n            return pilimage[\"image\"], label\n        if self.mode == \"test\":\n            x_path = self.imgs[idx]\n            pilimage = Image.open(os.path.join(test_image_path, x_path)).convert(\"RGB\")\n            pilimage = np.array(pilimage)\n            pilimage = transform[\"valid\"](image=pilimage)\n            return pilimage[\"image\"]\n\n        # if self.transform is not None:\n        # img_x = self.transform(img_x)\n        # plt.imshow(pilimage)\n        # plt.show()\n\n    def __len__(self):\n        return len(self.imgs)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-21T23:57:46.790666Z","iopub.execute_input":"2022-03-21T23:57:46.790983Z","iopub.status.idle":"2022-03-21T23:57:50.675017Z","shell.execute_reply.started":"2022-03-21T23:57:46.790906Z","shell.execute_reply":"2022-03-21T23:57:50.674173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/timm-20220211/pytorch-image-models-master\")\nimport timm\n#sys.path.append('../input/convnext/ConvNeXt')\\\n!pip install faiss-gpu\nimport faiss","metadata":{"execution":{"iopub.status.busy":"2022-03-21T23:57:50.676765Z","iopub.execute_input":"2022-03-21T23:57:50.677016Z","iopub.status.idle":"2022-03-21T23:58:06.122571Z","shell.execute_reply.started":"2022-03-21T23:57:50.676981Z","shell.execute_reply":"2022-03-21T23:58:06.121770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision.models.resnet import resnet50\nimport timm\nimport time\nimport math\nfrom scipy import spatial\nfrom tqdm import tqdm\nimport warnings\nimport cv2\nimport pandas as pd\nimport numpy as np\nfrom numpy import dot, sqrt\nimport seaborn as sns\nimport matplotlib as mpl\nimport matplotlib.patches as patches\nimport matplotlib.pyplot as plt\nfrom IPython.display import display_html\n\nfrom sklearn.model_selection import StratifiedKFold\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.optim import Adam, lr_scheduler\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nnum_classes = 15587\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef l2_norm(input, axis=1):\n    norm = torch.norm(input, 2, axis, True)\n    output = torch.div(input, norm)\n    return output\n\n\ndef softmax_loss(results, labels):\n    labels = labels.view(-1)\n    loss = F.cross_entropy(results, labels, reduce=True)\n    return loss\n\nclass MagrginLinear(nn.Module):\n    # implementation of additive margin softmax loss in https://arxiv.org/abs/1801.05599\n    def __init__(self, embedding_size=512, classnum=10008,  s=64., m=0.5):\n        super(MagrginLinear, self).__init__()\n        self.classnum = classnum\n        self.kernel = nn.Parameter(torch.Tensor(embedding_size,classnum))\n        # initial kernel\n        self.kernel.data.uniform_(-1, 1).renorm_(2,1,1e-5).mul_(1e5)\n        self.m = m # the margin value, default is 0.5\n        self.s = s # scalar value default is 64, see normface https://arxiv.org/abs/1704.06369\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.mm = self.sin_m * m  # issue 1\n        self.threshold = math.cos(math.pi - m)\n\n\n    def forward(self, embbedings, label, is_infer = False):\n        # weights norm\n        nB = len(embbedings)\n        kernel_norm = l2_norm(self.kernel,axis=0)\n        # cos(theta+m)\n        cos_theta = torch.mm(embbedings, kernel_norm)\n#         output = torch.mm(embbedings,kernel_norm)\n        cos_theta = cos_theta.clamp(-1,1) # for numerical stability\n        cos_theta_2 = torch.pow(cos_theta, 2)\n        sin_theta_2 = 1 - cos_theta_2\n        sin_theta = torch.sqrt(sin_theta_2)\n        cos_theta_m = (cos_theta * self.cos_m - sin_theta * self.sin_m)\n\n        # this condition controls the theta+m should in range [0, pi]\n        #      0<=theta+m<=pi\n        #     -m<=theta<=pi-m\n\n        cond_v = cos_theta - self.threshold\n        cond_mask = cond_v <= 0\n        keep_val = (cos_theta - self.mm) # when theta not in [0,pi], use cosface instead\n        cos_theta_m[cond_mask] = keep_val[cond_mask]\n        output = cos_theta * 1.0 # a little bit hacky way to prevent in_place operation on cos_theta\n        idx_ = torch.arange(0, nB, dtype=torch.long)\n\n        if not is_infer:\n            output[idx_, label] = cos_theta_m[idx_, label]\n\n        output *= self.s # scale up in order to make softmax work, first introduced in normface\n        return output\n\n\nclass ArcMarginProduct(nn.Module):\n    r\"\"\"Implement of large margin arc distance: :\n        Args:\n            in_features: size of each input sample\n            out_features: size of each output sample\n            s: norm of input feature\n            m: margin\n            cos(theta + m)\n        \"\"\"\n\n    def __init__(self, in_features, out_features, s=30.0,\n                 m=0.50, easy_margin=False, ls_eps=0.0):\n        super(ArcMarginProduct, self).__init__()\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, input, label):\n        # --------------------------- cos(theta) & phi(theta) ---------------------\n        cosine = F.linear(F.normalize(input), F.normalize(self.weight))\n\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n        # --------------------------- convert label to one-hot ---------------------\n        # one_hot = torch.zeros(cosine.size(), requires_grad=True, device='cuda')\n        one_hot = torch.zeros(cosine.size(), device=device)\n        one_hot.scatter_(1, label.view(-1, 1).long(), 1)\n        if self.ls_eps > 0:\n            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.out_features\n        # -------------torch.where(out_i = {x_i if condition_i else y_i) ------------\n        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n        output *= self.s\n\n        return output\nclass BinaryHead(nn.Module):\n\n    def __init__(self, num_class=15587, emb_size = 2048, s = 16.0):\n        super(BinaryHead,self).__init__()\n        self.s = s\n        self.fc = nn.Sequential(nn.Linear(emb_size, num_class))\n\n    def forward(self, fea):\n        fea = l2_norm(fea)\n        logit = self.fc(fea)*self.s\n        return logit\n\nclass MarginHead(nn.Module):\n\n    def __init__(self, num_class=15587, emb_size = 2048, s=64., m=0.5):\n        super(MarginHead,self).__init__()\n        self.fc = MagrginLinear(embedding_size=emb_size, classnum=num_class , s=s, m=m)\n\n    def forward(self, fea, label, is_infer):\n        fea = l2_norm(fea)\n        logit = self.fc(fea, label, is_infer)\n        return logit\n\n\nclass HappyWhaleModel(nn.Module):\n    def __init__(self, modelName, numClasses, noNeurons, embeddingSize):\n        super(HappyWhaleModel, self).__init__()\n        # @self.fea_extra_layer = [2, 3]\n        self.model = timm.create_model(modelName, pretrained=False, features_only=True,\n                                       # out_indices=self.fea_extra_layer\n                                       )\n        self.embsize = embeddingSize\n        # in_features = self.model.classifier.in_features\n        in_features = 1920\n        # self.model.classifier = nn.Identity()\n        # self.model.global_pool = nn.Identity()\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        self.poolingl2 = nn.AdaptiveAvgPool2d(1)\n        self.poolingl3 = nn.AdaptiveAvgPool2d(1)\n        self.poolingl4 = nn.AdaptiveAvgPool2d(1)\n        self.poolingl5 = nn.AdaptiveAvgPool2d(1)\n\n        self.fc = nn.Sequential(\n            nn.BatchNorm1d(in_features),\n            nn.Dropout(p=0.2, inplace=False),\n            nn.Linear(in_features, embeddingSize),\n            nn.BatchNorm1d(embeddingSize),\n        )\n\n        # self.avgpool = nn.AvgPool1d(kernel_size=3, stride=2, padding=1)\n        self.arc_head = ArcMarginProduct(in_features=embeddingSize, out_features=numClasses, m=0.3)\n        #self.class_head = BinaryHead(numClasses, emb_size=embeddingSize, s=16.0)\n\n    def forward(self, images, labels=None):\n        features = self.model(images)\n\n        # pooled_features = self.pooling(features).flatten(1)\n        # pooled_drop = self.drop(pooled_features)\n        features[0] = self.pooling(features[0])\n        features[1] = self.poolingl2(features[1])\n        features[2] = self.poolingl3(features[2])\n        features[3] = self.poolingl4(features[3])\n        #features[4] = self.poolingl5(features[4])\n        emb = torch.cat(features, dim=1)\n        emb = emb.flatten(1)\n        emb = self.fc(emb)\n\n        if labels != None:\n\n            arc_output = self.arc_head(emb, labels)\n            #class_output = self.class_head(emb)\n            return arc_output, emb\n        else:\n\n            return emb  # feartures","metadata":{"execution":{"iopub.status.busy":"2022-03-21T23:58:06.124484Z","iopub.execute_input":"2022-03-21T23:58:06.124791Z","iopub.status.idle":"2022-03-21T23:58:06.252030Z","shell.execute_reply.started":"2022-03-21T23:58:06.124753Z","shell.execute_reply":"2022-03-21T23:58:06.251396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    import os\n    import gc\n    import pandas as pd\n    import torch\n    from torchvision.transforms import transforms\n\n    from torch.utils.data import DataLoader\n    from tqdm import tqdm\n\n    import matplotlib.pyplot as plt\n    import numpy as np\n    import time\n    from sklearn.model_selection import StratifiedKFold, KFold\n    from sklearn.model_selection import train_test_split\n    from sklearn.neighbors import NearestNeighbors\n    from sklearn.preprocessing import normalize\n    import sklearn.metrics.pairwise as smp\n\n\n    import torch.nn as nn\n    import torch.nn.functional as F\n    from torch.autograd import Variable\n    import joblib\n\n    N_SPLITS = 5\n    MODEL_NAME = 'convnext_base_384_in22ft1k'\n    NUM_CLASSES = 15587\n    NO_NEURONS = 250\n    EMBEDDING_SIZE = 512\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    # os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n    torch.cuda.manual_seed_all(seed=3407)\n    # cross_validation\n    df_train = pd.read_csv(\"../input/train-encoded/train_encoded.csv\")\n    df_test = pd.read_csv(\"../input/happy-whale-and-dolphin/sample_submission.csv\")\n    kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=6)\n    for fold_id, (train_idx, val_idx) in enumerate(kfold.split(df_train[\"image\"], df_train[\"individual_key\"])):\n            if fold_id!=0:\n                break\n            model = HappyWhaleModel(MODEL_NAME, NUM_CLASSES, NO_NEURONS, EMBEDDING_SIZE).to(device)\n            train_data = WhaleDataset(df_train, \"train\")\n            train_loader = DataLoader(train_data, batch_size=64, num_workers=2, shuffle=False)\n            test_data = WhaleDataset(df_test, \"test\")\n            test_loader = DataLoader(test_data, batch_size=64, num_workers=2, shuffle=False)\n\n            model.load_state_dict(torch.load(\"../input/c-base-day322/model_13_0.720611.pth\")[\"net\"])\n\n            model.eval()\n            k = 100\n            thres = 0.7\n            train_embeddings = []\n            trn_lbl_list = []\n\n            threshold = thres\n\n            with torch.no_grad():\n                for data in tqdm(train_loader):\n                    image, target = data\n                    image, target = image.to(device), target.to(device)\n                    embedding = model(image, labels = None)\n\n                    train_embeddings.extend(embedding.detach().cpu().numpy())\n                    trn_lbl_list.append(target.detach().cpu().numpy().tolist())\n\n            image_embeddings = np.array(train_embeddings)\n            trn_lbl_list = np.concatenate(trn_lbl_list)\n            test_embeddings = []\n            with torch.no_grad():\n                for data in tqdm(test_loader):\n                    image = data\n                    image = image.to(device)\n                    embedding = model(image, labels=None)\n\n                    test_embeddings.extend(embedding.detach().cpu().numpy())\n\n            test_image_embeddings = np.array(test_embeddings)\n            \n            \n            train_emb = normalize(image_embeddings, axis=1, norm='l2')\n            test_emb = normalize(test_image_embeddings, axis=1, norm='l2')\n            index = faiss.IndexFlatIP(EMBEDDING_SIZE)\n            train_embeds = train_emb\n            test_embeds = test_emb\n            index.add(train_embeds)\n            dist, idx = index.search(test_embeds, k=100)\n            \n            confs = 1 - dist\n            # === PREDICTION ===\n\n            sum_top5 = 0.0\n            pbar = tqdm(range(len(test_image_embeddings)))\n            map_ = 0.0\n            predictions = []\n            preds_decoded = {}\n            for i in pbar:\n                index = idx[i][:5]\n                conf = confs[i][:5]\n                # preds = np.array(trn_lbl_list)[index]\n                preds = df_train.iloc[index][\"individual_id\"].values\n                #if conf[0] < threshold:\n                #    templist = ['new_individual', preds[1], preds[2], preds[3], preds[4]]\n                #elif conf[1] < threshold:\n                #    templist = [preds[0], 'new_individual', preds[2], preds[3], preds[4]]\n                #elif conf[2] < threshold:\n                #    templist = [preds[0], preds[1], 'new_individual', preds[3], preds[4]]\n                #elif conf[3] < threshold:\n                #    templist = [preds[0], preds[1], preds[2], 'new_individual', preds[4]]\n                if conf[4] < threshold:\n                    templist = [preds[0], preds[1], preds[2], preds[3], 'new_individual']\n                else:\n                    templist = preds\n\n                preds_decoded[df_test.iloc[i][\"image\"]] = templist\n\n            for x in tqdm(preds_decoded):\n                preds_decoded[x] = ' '.join(preds_decoded[x])\n\n\n\n            predictions = pd.Series(preds_decoded).reset_index()\n            predictions.columns = ['image', 'predictions']\n            predictions.to_csv('submission.csv', index=False)\n            predictions.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-21T23:58:06.253959Z","iopub.execute_input":"2022-03-21T23:58:06.254456Z","iopub.status.idle":"2022-03-22T00:26:23.534471Z","shell.execute_reply.started":"2022-03-21T23:58:06.254420Z","shell.execute_reply":"2022-03-22T00:26:23.533680Z"},"trusted":true},"execution_count":null,"outputs":[]}]}