{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm faiss-gpu","metadata":{"execution":{"iopub.status.busy":"2022-03-29T15:34:46.386788Z","iopub.execute_input":"2022-03-29T15:34:46.38713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport os\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport gc\nfrom tqdm import tqdm\nfrom fastai.vision.all import *\nfrom fastai.vision.learner import _update_first_layer\nimport faiss\nfrom timm import create_model\nfrom timm.data.transforms_factory import create_transform\nfrom sklearn.preprocessing import normalize","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = Path(\"..\") / \"input\"\nOUTPUT_DIR = Path(\"/\") / \"kaggle\" / \"working\"\nDATA_ROOT_DIR = INPUT_DIR / \"convert-backfintfrecords\" / \"happy-whale-and-dolphin-backfin\"\nTRAIN_DIR = DATA_ROOT_DIR / \"train_images\"\nTEST_DIR = DATA_ROOT_DIR / \"test_images\"\nTRAIN_CSV_PATH = DATA_ROOT_DIR / \"train.csv\"\nSAMPLE_SUBMISSION_CSV_PATH = DATA_ROOT_DIR / \"sample_submission.csv\"\nPUBLIC_SUBMISSION_CSV_PATH = INPUT_DIR / \"0-720-eff-b5-640-rotate\" / \"submission.csv\"\nIDS_WITHOUT_BACKFIN_PATH = INPUT_DIR / \"ids-without-backfin\" / \"ids_without_backfin.npy\"\nSUBMISSION_CSV_PATH = OUTPUT_DIR / \"submission.csv\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading dataset","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/happy-whale-and-dolphin/train.csv')\nclasses = list(train_csv.individual_id.unique())\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)\ntest_df = pd.read_csv(SAMPLE_SUBMISSION_CSV_PATH)\ntest_df.drop(columns=[\"predictions\"], inplace=True)\ntest_df[\"individual_id\"] = train_df.individual_id.unique()[0]\ntrain_df[\"image_path\"] = train_df[\"image\"].apply(lambda x:TRAIN_DIR/str(x))\ntest_df[\"image_path\"] = test_df[\"image\"].apply(lambda x:TEST_DIR/str(x))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting data","metadata":{}},{"cell_type":"code","source":"N_SPLITS = 5\nencoder = LabelEncoder()\ntrain_df[\"label\"] = encoder.fit_transform(train_df[\"individual_id\"])\nskf = StratifiedKFold(n_splits=N_SPLITS)\n\nfor fold, (_, val_) in enumerate(skf.split(X=train_df, y=train_df.individual_id)):\n    train_df.loc[val_, \"kfold\"] = fold\n    \ntrain_df.drop('label',axis=1,inplace=True)\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df[train_df.kfold == 0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed=999\nset_seed(seed, reproducible=True)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ntorch.backends.cudnn.deterministic = True\ntorch.use_deterministic_algorithms = True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating dataloaders","metadata":{}},{"cell_type":"code","source":"Val_Fold = 0\ndef get_x(r): return r['image_path']\ndef get_y(r): return [r['individual_id']]\ndef splitter(df): \n    train = df.index[df.kfold != Val_Fold].tolist()\n    valid = df.index[df.kfold == Val_Fold].tolist()\n    return [train,valid]\ndef create_dls(df=train_df,bs=64,Val_Fold=0,Image_size=224):\n    dblock = DataBlock(blocks = (ImageBlock,MultiCategoryBlock(vocab=classes)),\n                       get_x = get_x,\n                       get_y = get_y ,\n                       splitter = splitter,\n                       #item_tfms = [Resize(Image_size,method=ResizeMethod.Squish)],\n                       item_tfms = [Resize(600)],\n                       batch_tfms =[*aug_transforms(size=Image_size, max_warp=0), Normalize.from_stats(*imagenet_stats)]\n                       #batch_tfms =[,Normalize.from_stats(*imagenet_stats)]\n                      )\n\n    dls = dblock.dataloaders(train_df,bs=bs)\n    return dls","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = create_dls()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_map = {N:CLASS for N,CLASS in enumerate(dls.vocab)}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Arc Face loss","metadata":{}},{"cell_type":"code","source":"# From https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/blob/master/src/modeling/metric_learning.py\n# Added type annotations, device, and 16bit support\nclass ArcMarginLoss(Module):\n    def __init__(\n        self,\n        in_features: int,\n        out_features: int,\n        s: float,\n        m: float,\n        easy_margin: bool,\n        ls_eps: float,\n    ):\n        self.in_features = in_features\n        self.out_features = out_features\n        self.s = s\n        self.m = m\n        self.ls_eps = ls_eps  # label smoothing\n        self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features))\n        nn.init.xavier_uniform_(self.weight)\n        self.easy_margin = easy_margin\n        self.cos_m = math.cos(m)\n        self.sin_m = math.sin(m)\n        self.th = math.cos(math.pi - m)\n        self.mm = math.sin(math.pi - m) * m\n\n    def forward(self, logits, targ):\n        cosine = F.linear(F.normalize(logits), F.normalize(self.weight))\n        # Enable 16 bit precision\n        cosine = cosine.to(torch.float32)\n        sine = torch.sqrt(1.0 - torch.pow(cosine, 2))\n        phi = cosine * self.cos_m - sine * self.sin_m\n        if self.easy_margin:\n            phi = torch.where(cosine > 0, phi, cosine)\n        else:\n            phi = torch.where(cosine > self.th, phi, cosine - self.mm)\n\n        if self.ls_eps > 0:\n            targ = (1 - self.ls_eps) * targ + self.ls_eps / self.out_features\n        output = (targ * phi) + ((1.0 - targ) * cosine)\n        output *= self.s\n        loss =  F.cross_entropy(output, torch.argmax(targ, dim=1))\n        return loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the model","metadata":{}},{"cell_type":"code","source":"'''\nclass building_model(Module):\n    def __init__(self,embedding_size:int,arch:str ='efficientnet_b0', pretrained:bool=True):\n        self.model = create_model(arch, pretrained=pretrained)\n        self.embedding = nn.Linear(self.model.get_classifier().in_features, embedding_size)\n        self.model.reset_classifier(num_classes=0, global_pool=\"avg\")\n    def forward(self,x):\n        features = self.model(x)\n        embeddings = self.embedding(features)\n        return embeddings\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#emb_size = 512\n#model = building_model(emb_size,'efficientnet_b0',True)\n#model = nn.Sequential(*list(model.children()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_timm_body(arch:str, pretrained=True, drop_rate=0.0, cut=None, n_in=3):\n    \"Creates a body from any model in the `timm` library.\"\n    model = create_model(arch, pretrained=pretrained, drop_rate=drop_rate, \n                         num_classes=0, global_pool='')\n    _update_first_layer(model, n_in, pretrained)\n    if cut is None:\n        ll = list(enumerate(model.children()))\n        cut = next(i for i,o in reversed(ll) if has_pool_type(o))\n    if isinstance(cut, int): return nn.Sequential(*list(model.children())[:cut])\n    elif callable(cut): return cut(model)\n    else: raise NamedError(\"cut must be either integer or function\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emb_size = 512\nbody = create_timm_body(\"convnext_small\", pretrained=True, drop_rate=0.0)\nnf = num_features_model(nn.Sequential(*body.children()))\nhead = nn.Sequential(nn.AdaptiveAvgPool2d(output_size=1),nn.Flatten(),\n                     nn.Linear(nf,emb_size,bias=False))\n\nmodel = nn.Sequential(body, head)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_func = ArcMarginLoss(in_features=emb_size, out_features=dls.c,s=30.0,\n                          m = 0.5, easy_margin=False, ls_eps=0.0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = create_dls(bs=32,Image_size=320)\nlearn = Learner(dls,model, loss_func=loss_func, \n                splitter=default_split).to_fp16()\nlearn.freeze()\nlearn.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef accuracy_multi(inp, targ, thresh=0.5, sigmoid=True):\n     \"Compute accuracy when `inp` and `targ` are the same size.\"\n     if sigmoid: inp = inp.sigmoid()\n     return ((inp>thresh)==targ.bool()).float().mean()\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#learn.lr_find()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(1,lr_max=3e-4,wd=1e-6)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.unfreeze()\nlearn.fit_one_cycle(9,lr_max=3e-4,wd=1e-6)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.export('model.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del learn\ntorch.cuda.empty_cache()\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_eval_learner(learner_path,output_group,dls,device):\n    learn = load_learner(learner_path)\n    learn.model.to(device)\n    learn.dls = dls\n    hook = Hook(learn.model[output_group], lambda m,i,o: o)\n    return learn, hook\n\ndef load_dataloaders(train_df,test_df,val_fold,dls):\n    train_dataloader = dls.test_dl(train_df[train_df.kfold != val_fold],with_labels=True)\n    valid_dataloader = dls.test_dl(train_df[train_df.kfold == val_fold],with_labels=True)\n    test_dataloader  = dls.test_dl(test_df,with_labels=True)\n    return train_dataloader, valid_dataloader, test_dataloader\n\n@torch.inference_mode()\ndef get_embeddings(module, dataloader, device):\n\n    all_image_names = []\n    all_embeddings = []\n    all_targets = []\n\n    for (x, y) in tqdm(dataloader):\n        images =  x.to(device)\n        targets = y.to(device)\n        embeddings = module.model(images)\n        all_embeddings.append(embeddings.cpu().numpy())\n        all_targets.append(targets.cpu().numpy())\n\n    all_image_names = dataloader.items['image'].values\n    all_embeddings = np.vstack(all_embeddings)\n    all_targets = np.concatenate(all_targets)\n    all_targets = L(list(np.argmax(all_targets,axis=1)))\n    all_embeddings = normalize(all_embeddings, axis=1, norm=\"l2\")\n    all_targets = np.array(all_targets.map(target_map.__getitem__))\n    return all_image_names, all_embeddings, all_targets\n\ndef create_and_search_index(embedding_size: int, train_embeddings: np.ndarray, val_embeddings: np.ndarray, k: int):\n    index = faiss.IndexFlatIP(embedding_size)\n    index.add(train_embeddings)\n    D, I = index.search(val_embeddings, k=k)  # noqa: E741\n\n    return D, I\n\ndef create_val_targets_df(\n    train_targets: np.ndarray, val_image_names: np.ndarray, val_targets: np.ndarray\n) -> pd.DataFrame:\n\n    allowed_targets = np.unique(train_targets)\n    val_targets_df = pd.DataFrame(np.stack([val_image_names, val_targets], axis=1), columns=[\"image\", \"target\"])\n    val_targets_df.loc[~val_targets_df.target.isin(allowed_targets), \"target\"] = \"new_individual\"\n\n    return val_targets_df\n\ndef create_distances_df(\n    image_names: np.ndarray, targets: np.ndarray, D: np.ndarray, I: np.ndarray, stage: str  # noqa: E741\n) -> pd.DataFrame:\n    distances_df = []\n    for i, image_name in tqdm(enumerate(image_names), desc=f\"Creating {stage}_df\"):\n        target = targets[I[i]]\n        distances = D[i]\n        subset_preds = pd.DataFrame(np.stack([target, distances], axis=1), columns=[\"target\", \"distances\"])\n        subset_preds[\"image\"] = image_name\n        distances_df.append(subset_preds)\n    distances_df = pd.concat(distances_df).reset_index(drop=True)\n    distances_df = distances_df.groupby([\"image\", \"target\"]).distances.max().reset_index()\n    distances_df = distances_df.sort_values(\"distances\", ascending=False).reset_index(drop=True)\n    return distances_df\n\ndef get_best_threshold(val_targets_df: pd.DataFrame, valid_df: pd.DataFrame):\n    best_th = 0\n    best_cv = 0\n    for th in [0.1 * x for x in range(11)]:\n        all_preds = get_predictions(valid_df, threshold=th)\n\n        cv = 0\n        for i, row in val_targets_df.iterrows():\n            target = row.target\n            preds = all_preds[row.image]\n            val_targets_df.loc[i, th] = map_per_image(target, preds)\n\n        cv = val_targets_df[th].mean()\n\n        print(f\"th={th} cv={cv}\")\n\n        if cv > best_cv:\n            best_th = th\n            best_cv = cv\n\n    print(f\"best_th={best_th}\")\n    print(f\"best_cv={best_cv}\")\n\n    # Adjustment: Since Public lb has nearly 10% 'new_individual' (Be Careful for private LB)\n    val_targets_df[\"is_new_individual\"] = val_targets_df.target == \"new_individual\"\n    val_scores = val_targets_df.groupby(\"is_new_individual\").mean().T\n    val_scores[\"adjusted_cv\"] = val_scores[True] * 0.1 + val_scores[False] * 0.9\n    best_th = val_scores[\"adjusted_cv\"].idxmax()\n    print(f\"best_th_adjusted={best_th}\")\n\n    return best_th, best_cv\n\ndef get_predictions(df: pd.DataFrame, threshold: float = 0.2):\n    sample_list = [\"938b7e931166\", \"5bf17305f073\", \"7593d2aee842\", \"7362d7a01d00\", \"956562ff2888\"]\n    predictions = {}\n    for i, row in tqdm(df.iterrows(), total=len(df), desc=f\"Creating predictions for threshold={threshold}\"):\n        if row.image in predictions:\n            if len(predictions[row.image]) == 5:\n                continue\n            predictions[row.image].append(row.target)\n        elif float(row.distances) > threshold:\n            predictions[row.image] = [row.target, \"new_individual\"]\n        else:\n            predictions[row.image] = [\"new_individual\", row.target]\n\n    for x in tqdm(predictions):\n        if len(predictions[x]) < 5:\n            remaining = [y for y in sample_list if y not in predictions]\n            predictions[x] = predictions[x] + remaining\n            predictions[x] = predictions[x][:5]\n\n    return predictions\n\ndef map_per_image(label, predictions):\n    \"\"\"Computes the precision score of one image.\n\n    Parameters\n    ----------\n    label : string\n            The true label of the image\n    predictions : list\n            A list of predicted elements (order does matter, 5 predictions allowed per image)\n\n    Returns\n    -------\n    score : double\n    \"\"\"\n    try:\n        return 1 / (predictions[:5].index(label) + 1)\n    except ValueError:\n        return 0.0\n\n\ndef create_predictions_df(test_df: pd.DataFrame, best_th: float) -> pd.DataFrame:\n    predictions = get_predictions(test_df, best_th)\n\n    predictions = pd.Series(predictions).reset_index()\n    predictions.columns = [\"image\", \"predictions\"]\n    predictions[\"predictions\"] = predictions[\"predictions\"].apply(lambda x: \" \".join(x))\n\n    return predictions","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer(\n    model_path: str,\n    dls: DataLoader,\n    train_df: pd.DataFrame = train_df,\n    test_df: pd.DataFrame = test_df,\n    val_fold: float = 0.0,\n    k: int = 50,\n    emb_size:int = emb_size\n):\n    (learn, hook) = load_eval_learner(model_path,1,dls,torch.device(\"cuda\"))\n    train_dl, val_dl, test_dl = load_dataloaders(train_df,test_df,val_fold,dls)\n\n    train_image_names, train_embeddings, train_targets = get_embeddings(learn, train_dl,torch.device(\"cuda\"))\n    val_image_names, val_embeddings, val_targets = get_embeddings(learn, val_dl,torch.device(\"cuda\"))\n    test_image_names, test_embeddings, test_targets = get_embeddings(learn, test_dl,torch.device(\"cuda\"))\n\n    D, I = create_and_search_index(emb_size, train_embeddings, val_embeddings, k)  # noqa: E741\n    print(\"Created index with train_embeddings\")\n    \n    val_targets_df = create_val_targets_df(train_targets, val_image_names, val_targets)\n    print(f\"val_targets_df=\\n{val_targets_df.head()}\")\n    print(f\"val_targets_df shape=\\n{val_targets_df.shape}\")\n    val_df = create_distances_df(val_image_names, train_targets, D, I, \"val\")\n    print(f\"val_df=\\n{val_df.head()}\")\n    print(f\"val_df shape=\\n{val_df.shape}\")\n    best_th, best_cv = get_best_threshold(val_targets_df, val_df)\n    print(f\"val_targets_df=\\n{val_targets_df.describe()}\")\n    print(f\"val_targets_df shape=\\n{val_targets_df.shape}\")\n    train_embeddings = np.concatenate([train_embeddings, val_embeddings])\n    train_targets = np.concatenate([train_targets, val_targets])\n    print(\"Updated train_embeddings and train_targets with val data\")\n\n    D, I = create_and_search_index(emb_size, train_embeddings, test_embeddings, k)  # noqa: E741\n    print(\"Created index with train_embeddings\")\n\n    test_df = create_distances_df(test_image_names, train_targets, D, I, \"test\")\n    print(f\"test_df=\\n{test_df.head()}\")\n    print(f\"test_df shape=\\n{test_df.shape}\")\n\n    predictions = create_predictions_df(test_df, best_th)\n    print(f\"predictions.head()={predictions.head()}\")\n    print(f\"predictions shape={predictions.shape}\")\n \n    # Fix missing predictions\n    # From https://www.kaggle.com/code/jpbremer/backfins-arcface-tpu-effnet/notebook\n    public_predictions = pd.read_csv(PUBLIC_SUBMISSION_CSV_PATH)\n    ids_without_backfin = np.load(IDS_WITHOUT_BACKFIN_PATH, allow_pickle=True)\n\n    ids2 = public_predictions[\"image\"][~public_predictions[\"image\"].isin(predictions[\"image\"])]\n\n    predictions = pd.concat(\n        [\n            predictions[~(predictions[\"image\"].isin(ids_without_backfin))],\n            public_predictions[public_predictions[\"image\"].isin(ids_without_backfin)],\n            public_predictions[public_predictions[\"image\"].isin(ids2)],\n        ]\n    )\n    predictions = predictions.drop_duplicates()\n\n    predictions.to_csv(SUBMISSION_CSV_PATH, index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infer(model_path=\"model.pkl\",dls=dls)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nbs = 32\nfor Fold in range(N_SPLITS):\n    print(f'Epoch-{Fold}')\n    dls = create_dls(Val_Fold=Fold,bs=bs)\n    learn = cnn_learner(dls, resnet18, metrics=partial(accuracy_multi,thresh=0.2))\n    learn = learn.to_fp16()\n    learn.fit_one_cycle(1,lr_max=slice(2.2e-6, 2e-4))\n    learn.export(f'learn{Fold}.pkl')\n    del learn\n    torch.cuda.empty_cache()\n    gc.collect()\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}