{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🐋🐬 Lightning⚡Flash 🦈 BackFin & CNN & ArcFace\n\nLet's train [`timm`](https://github.com/rwightman/pytorch-image-models) models with [PyTorch Lightning Flash](https://github.com/PyTorchLightning/lightning-flash)!\n\n## Sources\n- [[Pytorch] ArcFace + GeM Pooling Starter](https://www.kaggle.com/debarshichanda/pytorch-arcface-gem-pooling-starter)\n- [FAISS Pytorch Inference](https://www.kaggle.com/debarshichanda/faiss-pytorch-inference)\n- [backfintfrecrods](https://www.kaggle.com/datasets/jpbremer/backfintfrecords) dataset\n\n## Nice Lightning `Trainer` Flags to try\n- Run a learning rate finder algorithm: `auto_lr_find=True`\n- Automatically try to find the largest batch size that fits into memory: `auto_scale_batch_size=True`\n- Quickly check whether everything runs fine: `fast_dev_run=True`\n- Train on multiple GPUs: `gpus=2` (if you use multiple GPUs, also set `accelerator=ddp`)\n- Train with half precision: `precision=16`\n- Use Stochastic Weight Averaging: `stochastic_weight_avg=True`","metadata":{}},{"cell_type":"markdown","source":"# Installations","metadata":{}},{"cell_type":"code","source":"!pip install -q torch==1.10.1+cu102 torchvision==0.11.2+cu102 torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html\n!pip install -q faiss-gpu 'lightning-flash[image]'\n!pip install -q happywhale -f ../input/-pytorch-lightning-happywhale-pkg\n!pip install -q -U timm segmentation-models-pytorch\n!pip install -q Pillow==9.0.1\n!pip uninstall -y torchtext  # segmentation-models-pytorch\n!pip list | grep torch\n!pip list | grep lightning\n!pip list | grep happywhale","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-19T03:10:59.170879Z","iopub.execute_input":"2022-04-19T03:10:59.171402Z","iopub.status.idle":"2022-04-19T03:16:43.515569Z","shell.execute_reply.started":"2022-04-19T03:10:59.171237Z","shell.execute_reply":"2022-04-19T03:16:43.514475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import math\nfrom typing import Callable, Dict, Optional, Tuple\nfrom pathlib import Path\n\nimport faiss\nimport flash\nimport numpy as np\nimport pandas as pd\nimport pytorch_lightning as pl\nimport timm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom flash.image import ImageClassificationData, ImageClassifier\nfrom PIL import Image\nfrom pytorch_lightning.callbacks.model_checkpoint import ModelCheckpoint\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import normalize, LabelEncoder","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-19T03:16:43.518083Z","iopub.execute_input":"2022-04-19T03:16:43.518436Z","iopub.status.idle":"2022-04-19T03:16:55.543603Z","shell.execute_reply.started":"2022-04-19T03:16:43.518359Z","shell.execute_reply":"2022-04-19T03:16:55.542578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Paths & Settings","metadata":{}},{"cell_type":"code","source":"INPUT_DIR = Path(\"..\") / \"input\"\nOUTPUT_DIR = Path(\"/\") / \"kaggle\" / \"working\"\n\nDATA_ROOT_DIR = INPUT_DIR / \"convert-backfintfrecords\" / \"happy-whale-and-dolphin-backfin\"\nTRAIN_DIR = DATA_ROOT_DIR / \"train_images\"\nTEST_DIR = DATA_ROOT_DIR / \"test_images\"\nTRAIN_CSV_PATH = DATA_ROOT_DIR / \"train.csv\"\nSAMPLE_SUBMISSION_CSV_PATH = DATA_ROOT_DIR / \"sample_submission.csv\"\n\nN_SPLITS = 5\n\nENCODER_CLASSES_PATH = OUTPUT_DIR / \"encoder_classes.npy\"\nTEST_CSV_PATH = OUTPUT_DIR / \"test.csv\"\nTRAIN_CSV_ENCODED_FOLDED_PATH = OUTPUT_DIR / \"train_encoded_folded.csv\"\nCHECKPOINTS_DIR = OUTPUT_DIR / \"checkpoints\"\nSUBMISSION_CSV_PATH = OUTPUT_DIR / \"submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-04-19T03:16:55.547161Z","iopub.execute_input":"2022-04-19T03:16:55.547484Z","iopub.status.idle":"2022-04-19T03:16:55.558469Z","shell.execute_reply.started":"2022-04-19T03:16:55.547438Z","shell.execute_reply":"2022-04-19T03:16:55.555732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare DataFrames","metadata":{}},{"cell_type":"code","source":"def get_image_path(id: str, dir: Path) -> str:\n    return f\"{dir / id}\"","metadata":{"execution":{"iopub.status.busy":"2022-04-19T03:16:55.561562Z","iopub.execute_input":"2022-04-19T03:16:55.561891Z","iopub.status.idle":"2022-04-19T03:16:55.583782Z","shell.execute_reply.started":"2022-04-19T03:16:55.561843Z","shell.execute_reply":"2022-04-19T03:16:55.582720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train DataFrame","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(TRAIN_CSV_PATH)\n\ntrain_df[\"image_path\"] = train_df[\"image\"].apply(get_image_path, dir=TRAIN_DIR)\n\nencoder = LabelEncoder()\ntrain_df[\"individual_id\"] = encoder.fit_transform(train_df[\"individual_id\"])\nnp.save(ENCODER_CLASSES_PATH, encoder.classes_)\n\nskf = StratifiedKFold(n_splits=N_SPLITS)\nfor fold, (_, val_) in enumerate(skf.split(X=train_df, y=train_df.individual_id)):\n    train_df.loc[val_, \"kfold\"] = fold\n    \ntrain_df.to_csv(TRAIN_CSV_ENCODED_FOLDED_PATH, index=False)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T03:16:55.585718Z","iopub.execute_input":"2022-04-19T03:16:55.586103Z","iopub.status.idle":"2022-04-19T03:16:56.898813Z","shell.execute_reply.started":"2022-04-19T03:16:55.586057Z","shell.execute_reply":"2022-04-19T03:16:56.897832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test DataFrame","metadata":{}},{"cell_type":"code","source":"# Use sample submission csv as template\ntest_df = pd.read_csv(SAMPLE_SUBMISSION_CSV_PATH)\ntest_df[\"image_path\"] = test_df[\"image\"].apply(get_image_path, dir=TEST_DIR)\n\ntest_df.drop(columns=[\"predictions\"], inplace=True)\n\n# Dummy id\ntest_df[\"individual_id\"] = 0\ntest_df.to_csv(TEST_CSV_PATH, index=False)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T03:16:56.900258Z","iopub.execute_input":"2022-04-19T03:16:56.900710Z","iopub.status.idle":"2022-04-19T03:16:57.383060Z","shell.execute_reply.started":"2022-04-19T03:16:56.900649Z","shell.execute_reply":"2022-04-19T03:16:57.382127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ArcMargin Loss","metadata":{}},{"cell_type":"code","source":"from happywhale.utils.arc_margin_product import ArcMarginProduct\n\nclass ArcLoss(nn.Module):\n    def __init__(\n        self,\n        embedding_size: int,\n        num_classes: int,\n        arc_s: float,\n        arc_m: float,\n        arc_easy_margin: bool,\n        arc_ls_eps: float,\n    ) -> None:\n        super().__init__()\n        self.arc = ArcMarginProduct(\n            in_features=embedding_size,\n            out_features=num_classes,\n            s=arc_s,\n            m=arc_m,\n            easy_margin=arc_easy_margin,\n            ls_eps=arc_ls_eps,\n        )\n\n    def forward(self, features: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:\n        outputs = self.arc(features, targets)\n        loss = F.cross_entropy(outputs, targets)\n        return loss","metadata":{"execution":{"iopub.status.busy":"2022-04-19T03:16:57.384725Z","iopub.execute_input":"2022-04-19T03:16:57.385291Z","iopub.status.idle":"2022-04-19T03:16:57.396213Z","shell.execute_reply.started":"2022-04-19T03:16:57.385245Z","shell.execute_reply":"2022-04-19T03:16:57.395273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"from pytorch_lightning.loggers import CSVLogger\nfrom pytorch_lightning.callbacks import StochasticWeightAveraging\n\n\ndef train(\n    train_csv_encoded_folded: str = str(TRAIN_CSV_ENCODED_FOLDED_PATH),\n    # val_fold: float = 0.0,\n    val_split: float = 0.05,\n    image_size: int = 384,\n    batch_size: int = 32,\n    num_workers: int = 2,\n    model_name: str = \"convnext_small\",\n    pretrained: bool = True,\n    embedding_size: int = 512,\n    arc_s: float = 30.0,\n    arc_m: float = 0.5,\n    arc_easy_margin: bool = False,\n    arc_ls_eps: float = 0.0,\n    learning_rate: float = 3e-4,\n    weight_decay: float = 1e-6,\n    project: str = \"kaggle-happywhale-flash\",\n    checkpoints_dir: str = str(CHECKPOINTS_DIR),\n    # auto_lr_find: bool = False,\n    # fast_dev_run: bool = False,\n    gpus: int = torch.cuda.device_count(),\n    max_epochs: int = 10,\n    precision: int = 16,\n):\n    # pl.seed_everything(42)\n\n    train_val_df = pd.read_csv(train_csv_encoded_folded)\n    # train_df = train_val_df[train_val_df.kfold != val_fold].reset_index(drop=True)\n    # val_df = train_val_df[train_val_df.kfold == val_fold].reset_index(drop=True)\n\n    datamodule = ImageClassificationData.from_data_frame(\n        input_field=\"image_path\",\n        target_fields=\"individual_id\",\n        train_data_frame=train_val_df,\n        # train_data_frame=train_df,\n        # val_data_frame=val_df,\n        transform_kwargs={\n            \"image_size\": (image_size, image_size),\n            \"mean\": [0.485, 0.456, 0.406],\n            \"std\": [0.229, 0.224, 0.225],\n        },\n        batch_size=batch_size,\n        num_workers=num_workers,\n        val_split=val_split,\n    )\n\n    arc_loss = ArcLoss(\n        embedding_size=embedding_size,\n        num_classes=datamodule.num_classes,\n        arc_s=arc_s,\n        arc_m=arc_m,\n        arc_ls_eps=arc_ls_eps,\n        arc_easy_margin=arc_easy_margin,\n    )\n\n    model = ImageClassifier(\n        num_classes=embedding_size,\n        backbone=model_name,\n        pretrained=pretrained,\n        loss_fn=arc_loss,\n        optimizer=(\"AdamW\", {\"lr\": learning_rate, \"weight_decay\": weight_decay}),\n        learning_rate=learning_rate,\n        metrics=[],\n    )\n\n    model_checkpoint = ModelCheckpoint(\n        checkpoints_dir,\n        filename=f\"{model_name}_{image_size}\",\n        monitor=\"val_arcloss\",\n    )\n    \n    swa = StochasticWeightAveraging(swa_epoch_start=0.6)\n    logger = CSVLogger(save_dir='logs/')\n\n    trainer = flash.Trainer(\n        # benchmark=True,\n        logger=logger,\n        # auto_lr_find=auto_lr_find,\n        # fast_dev_run=fast_dev_run,\n        callbacks=[model_checkpoint],\n        # deterministic=True,\n        gpus=gpus,\n        max_epochs=max_epochs,\n        precision=precision,\n        # limit_train_batches=0.1,\n        # limit_val_batches=0.1,\n    )\n\n    # trainer.tune(model, datamodule=datamodule)\n    trainer.finetune(model, datamodule=datamodule, strategy=\"no_freeze\")\n    # trainer.finetune(model, datamodule=datamodule, strategy=(\"freeze_unfreeze\", 1))\n    # trainer.fit(model, datamodule=datamodule)\n    return model, trainer, logger","metadata":{"execution":{"iopub.status.busy":"2022-04-19T03:49:17.417282Z","iopub.execute_input":"2022-04-19T03:49:17.418596Z","iopub.status.idle":"2022-04-19T03:49:17.462675Z","shell.execute_reply.started":"2022-04-19T03:49:17.418541Z","shell.execute_reply":"2022-04-19T03:49:17.461132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://github.com/rwightman/pytorch-image-models/blob/master/results/results-imagenet.csv","metadata":{}},{"cell_type":"code","source":"MODEL_NAME = \"tf_efficientnet_b4_ns\"\nIMAGE_SIZE = 380\nBATCH_SIZE = 32\n\nmodel, trainer, log = train(model_name=MODEL_NAME, image_size=IMAGE_SIZE, batch_size=BATCH_SIZE, max_epochs=5)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-19T03:49:26.469199Z","iopub.execute_input":"2022-04-19T03:49:26.469505Z","iopub.status.idle":"2022-04-19T04:43:06.847070Z","shell.execute_reply.started":"2022-04-19T03:49:26.469459Z","shell.execute_reply":"2022-04-19T04:43:06.845895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sn\n\nmetrics = pd.read_csv(f'{log.log_dir}/metrics.csv')\ndel metrics[\"step\"]\nmetrics.set_index(\"epoch\", inplace=True)\ndisplay(metrics.dropna(axis=1, how=\"all\").head())\ng = sn.relplot(data=metrics, kind=\"line\")\nplt.gcf().set_size_inches(12, 4)\nplt.grid()","metadata":{"execution":{"iopub.status.busy":"2022-04-19T04:43:06.851602Z","iopub.execute_input":"2022-04-19T04:43:06.851919Z","iopub.status.idle":"2022-04-19T04:43:07.473357Z","shell.execute_reply.started":"2022-04-19T04:43:06.851882Z","shell.execute_reply":"2022-04-19T04:43:07.472338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"from happywhale.inference.infer import (\n    load_eval_module,\n    load_encoder,\n    # get_embeddings,\n    create_and_search_index,\n    create_val_targets_df,\n    create_distances_df,\n    get_best_threshold,\n    create_predictions_df,\n)\n\ndef load_dataloaders(\n    train_csv_encoded_folded: str,\n    test_csv: str,\n    val_fold: float,\n    image_size: int,\n    batch_size: int,\n    num_workers: int,\n) -> Tuple[DataLoader, DataLoader, DataLoader]:\n    \n    train_val_df = pd.read_csv(train_csv_encoded_folded)\n    train_df = train_val_df[train_val_df.kfold != val_fold].reset_index(drop=True)\n    val_df = train_val_df[train_val_df.kfold == val_fold].reset_index(drop=True)\n    test_df = pd.read_csv(test_csv)\n\n    datamodule = ImageClassificationData.from_data_frame(\n        input_field=\"image_path\",\n        target_fields=\"individual_id\",\n        train_data_frame=train_df,\n        val_data_frame=val_df,\n        test_data_frame=test_df,\n        transform_kwargs={\n            \"image_size\": (image_size, image_size),\n            \"mean\": [0.485, 0.456, 0.406],\n            \"std\": [0.229, 0.224, 0.225],\n        },\n        batch_size=batch_size,\n        num_workers=num_workers,\n    )\n    datamodule.setup()\n\n    train_dl = datamodule.train_dataloader()\n    val_dl = datamodule.val_dataloader()\n    test_dl = datamodule.test_dataloader()\n\n    return train_dl, val_dl, test_dl\n\n\n@torch.inference_mode()\ndef get_embeddings(\n    module: ImageClassifier, dataloader: DataLoader, encoder: LabelEncoder, stage: str\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n\n    all_image_names = []\n    all_embeddings = []\n    all_targets = []\n\n    for batch in tqdm(dataloader, desc=f\"Creating {stage} embeddings\"):\n        image_names = [Path(item[\"filepath\"]).name for item in batch[\"metadata\"]]\n        images = batch[\"input\"].to(module.device)\n        targets = batch[\"target\"].to(module.device)\n\n        embeddings = module(images)\n\n        all_image_names.append(image_names)\n        all_embeddings.append(embeddings.cpu().numpy())\n        all_targets.append(targets.cpu().numpy())\n\n    all_image_names = np.concatenate(all_image_names)\n    all_embeddings = np.vstack(all_embeddings)\n    all_targets = np.concatenate(all_targets)\n\n    all_embeddings = normalize(all_embeddings, axis=1, norm=\"l2\")\n    all_targets = encoder.inverse_transform(all_targets)\n\n    return all_image_names, all_embeddings, all_targets","metadata":{"execution":{"iopub.status.busy":"2022-04-19T04:43:07.475357Z","iopub.execute_input":"2022-04-19T04:43:07.475649Z","iopub.status.idle":"2022-04-19T04:43:07.494053Z","shell.execute_reply.started":"2022-04-19T04:43:07.475610Z","shell.execute_reply":"2022-04-19T04:43:07.492854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from happywhale.settings import IDS_WITHOUT_BACKFIN_PATH, PUBLIC_SUBMISSION_CSV_PATH\n\ndef infer(\n    checkpoint_path: str,\n    train_csv_encoded_folded: str = str(TRAIN_CSV_ENCODED_FOLDED_PATH),\n    test_csv: str = str(TEST_CSV_PATH),\n    val_fold: float = 0.0,\n    image_size: int = 256,\n    batch_size: int = 64,\n    num_workers: int = 2,\n    embedding_size: int = 512,\n    k: int = 50,\n):\n    module = load_eval_module(checkpoint_path, torch.device(\"cuda\"), lit_module_cls=ImageClassifier)\n\n    train_dl, val_dl, test_dl = load_dataloaders(\n        train_csv_encoded_folded=train_csv_encoded_folded,\n        test_csv=test_csv,\n        val_fold=val_fold,\n        image_size=image_size,\n        batch_size=batch_size,\n        num_workers=num_workers,\n    )\n\n    encoder = load_encoder(ENCODER_CLASSES_PATH)\n\n    train_image_names, train_embeddings, train_targets = get_embeddings(module, train_dl, encoder, stage=\"train\")\n    val_image_names, val_embeddings, val_targets = get_embeddings(module, val_dl, encoder, stage=\"val\")\n    test_image_names, test_embeddings, test_targets = get_embeddings(module, test_dl, encoder, stage=\"test\")\n\n    D, I = create_and_search_index(embedding_size, train_embeddings, val_embeddings, k)  # noqa: E741\n    print(\"Created index with train_embeddings\")\n\n    val_targets_df = create_val_targets_df(train_targets, val_image_names, val_targets)\n    print(f\"val_targets_df=\\n{val_targets_df.head()}\")\n\n    val_df = create_distances_df(val_image_names, train_targets, D, I, \"val\")\n    print(f\"val_df=\\n{val_df.head()}\")\n\n    best_th, best_cv = get_best_threshold(val_targets_df, val_df, adjust_th=True)\n    print(f\"val_targets_df=\\n{val_targets_df.describe()}\")\n\n    train_embeddings = np.concatenate([train_embeddings, val_embeddings])\n    train_targets = np.concatenate([train_targets, val_targets])\n    print(\"Updated train_embeddings and train_targets with val data\")\n\n    D, I = create_and_search_index(embedding_size, train_embeddings, test_embeddings, k)  # noqa: E741\n    print(\"Created index with train_embeddings\")\n\n    test_df = create_distances_df(test_image_names, train_targets, D, I, \"test\")\n    print(f\"test_df=\\n{test_df.head()}\")\n\n    predictions = create_predictions_df(test_df, best_th)\n    print(f\"predictions.head()={predictions.head()}\")\n    \n    # Fix missing predictions\n    # From https://www.kaggle.com/code/jpbremer/backfins-arcface-tpu-effnet/notebook\n    public_predictions = pd.read_csv(PUBLIC_SUBMISSION_CSV_PATH)\n    ids_without_backfin = np.load(IDS_WITHOUT_BACKFIN_PATH, allow_pickle=True)\n\n    ids2 = public_predictions[\"image\"][~public_predictions[\"image\"].isin(predictions[\"image\"])]\n\n    predictions = pd.concat(\n        [\n            predictions[~(predictions[\"image\"].isin(ids_without_backfin))],\n            public_predictions[public_predictions[\"image\"].isin(ids_without_backfin)],\n            public_predictions[public_predictions[\"image\"].isin(ids2)],\n        ]\n    )\n    predictions = predictions[[\"image\",\"predictions\"]].drop_duplicates()\n    predictions.to_csv(SUBMISSION_CSV_PATH, index=False)","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-04-19T04:52:29.052112Z","iopub.execute_input":"2022-04-19T04:52:29.052508Z","iopub.status.idle":"2022-04-19T04:52:29.073536Z","shell.execute_reply.started":"2022-04-19T04:52:29.052457Z","shell.execute_reply":"2022-04-19T04:52:29.072355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infer(checkpoint_path=CHECKPOINTS_DIR / f\"{MODEL_NAME}_{IMAGE_SIZE}.ckpt\", image_size=IMAGE_SIZE, batch_size=BATCH_SIZE)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-19T04:52:32.870728Z","iopub.execute_input":"2022-04-19T04:52:32.871143Z","iopub.status.idle":"2022-04-19T05:17:18.160427Z","shell.execute_reply.started":"2022-04-19T04:52:32.871068Z","shell.execute_reply":"2022-04-19T05:17:18.159327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2022-04-19T05:17:18.162660Z","iopub.execute_input":"2022-04-19T05:17:18.163007Z","iopub.status.idle":"2022-04-19T05:17:19.062060Z","shell.execute_reply.started":"2022-04-19T05:17:18.162962Z","shell.execute_reply":"2022-04-19T05:17:19.060824Z"},"trusted":true},"execution_count":null,"outputs":[]}]}