{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install ../input/birds-inference-pip-wheels/torchaudio-0.8.1-cp37-cp37m-manylinux1_x86_64.whl ../input/birds-inference-pip-wheels/torch-1.8.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/birds-inference-pip-wheels/timm-0.4.8.zip --no-index --no-deps\n#!pip install ../input/birdclef21trainmeta/timm-0.4.9_23052021/pytorch-image-models-master --no-index --no-deps\n!pip install ../input/birds-inference-pip-wheels/audiomentations-0.16.0-py3-none-any.whl --no-index --no-deps\n!pip install ../input/birds-inference-pip-wheels/torchlibrosa-0.0.9-py3-none-any.whl --no-index --no-deps","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:52:57.06999Z","iopub.execute_input":"2022-02-28T05:52:57.0703Z","iopub.status.idle":"2022-02-28T05:53:12.742186Z","shell.execute_reply.started":"2022-02-28T05:52:57.070264Z","shell.execute_reply":"2022-02-28T05:53:12.741108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\ntimm.__version__","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.745612Z","iopub.execute_input":"2022-02-28T05:53:12.745889Z","iopub.status.idle":"2022-02-28T05:53:12.754925Z","shell.execute_reply.started":"2022-02-28T05:53:12.745856Z","shell.execute_reply":"2022-02-28T05:53:12.754177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport timm\nfrom torch import nn\nimport torch\nimport torchaudio as ta\nfrom torch.cuda.amp import autocast\nimport random\n\nfrom torch.nn import functional as F\nfrom torch.distributions import Beta\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import Dataset\n\nimport numpy as np\nimport librosa\nimport ast\n\nimport os\nfrom types import SimpleNamespace\nimport numpy as np\n\nimport numpy as np\nimport pandas as pd\nimport importlib\nimport sys\nimport random\nfrom tqdm import tqdm\nimport gc\nimport argparse\nimport torch\nfrom torch import optim\nfrom torch.cuda.amp import GradScaler, autocast\nfrom collections import defaultdict\nimport cv2\nfrom copy import copy\nimport os\nfrom transformers import get_cosine_schedule_with_warmup\nfrom torch.utils.data import SequentialSampler, DataLoader\n","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.756399Z","iopub.execute_input":"2022-02-28T05:53:12.756664Z","iopub.status.idle":"2022-02-28T05:53:12.767178Z","shell.execute_reply.started":"2022-02-28T05:53:12.756623Z","shell.execute_reply":"2022-02-28T05:53:12.766184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=1234):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = False\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.770001Z","iopub.execute_input":"2022-02-28T05:53:12.770344Z","iopub.status.idle":"2022-02-28T05:53:12.776875Z","shell.execute_reply.started":"2022-02-28T05:53:12.770247Z","shell.execute_reply":"2022-02-28T05:53:12.776107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"cfg = SimpleNamespace()\n\n# paths\ncfg.data_folder = ''\ncfg.name = \"julian\"\ncfg.data_dir = \"../input/birdclef-2022/\"\ncfg.train_data_folder = cfg.data_dir + \"train_audio/\"\ncfg.val_data_folder = cfg.data_dir + \"train_audio/\"\ncfg.output_dir = \"first_model\"\n\n# dataset\ncfg.dataset = \"base_ds\"\ncfg.min_rating = 0\ncfg.val_df = None\ncfg.batch_size_val = 1\ncfg.train_aug = None\ncfg.val_aug = None\ncfg.test_augs = None\ncfg.wav_len_val = 5  # seconds\n\n# audio\ncfg.window_size = 2048\ncfg.hop_size = 512\ncfg.sample_rate = 32000\ncfg.fmin = 16\ncfg.fmax = 16386\ncfg.power = 2\ncfg.mel_bins = 256\ncfg.top_db = 80.0\n\n# img model\ncfg.backbone = \"resnet18\"\ncfg.pretrained = True\ncfg.pretrained_weights = None\ncfg.train = True\ncfg.val = False\ncfg.in_chans = 1\n\ncfg.alpha = 1\ncfg.eval_epochs = 1\ncfg.eval_train_epochs = 1\ncfg.warmup = 0\n\ncfg.mel_norm = False\n\ncfg.label_smoothing = 0\n\ncfg.remove_pretrained = []\n\n# training\ncfg.seed = 123\ncfg.save_val_data = True\n\n# ressources\ncfg.mixed_precision = True\ncfg.gpu = 0\ncfg.num_workers = 4 # 18\ncfg.drop_last = True \n\ncfg.mixup2 = 0\n\ncfg.label_smoothing = 0\n\ncfg.mixup_2x = False\n\n\ncfg.birds = np.array(['afrsil1', 'akekee', 'akepa1', 'akiapo', 'akikik', 'amewig',\n       'aniani', 'apapan', 'arcter', 'barpet', 'bcnher', 'belkin1',\n       'bkbplo', 'bknsti', 'bkwpet', 'blkfra', 'blknod', 'bongul',\n       'brant', 'brnboo', 'brnnod', 'brnowl', 'brtcur', 'bubsan',\n       'buffle', 'bulpet', 'burpar', 'buwtea', 'cacgoo1', 'calqua',\n       'cangoo', 'canvas', 'caster1', 'categr', 'chbsan', 'chemun',\n       'chukar', 'cintea', 'comgal1', 'commyn', 'compea', 'comsan',\n       'comwax', 'coopet', 'crehon', 'dunlin', 'elepai', 'ercfra',\n       'eurwig', 'fragul', 'gadwal', 'gamqua', 'glwgul', 'gnwtea',\n       'golphe', 'grbher3', 'grefri', 'gresca', 'gryfra', 'gwfgoo',\n       'hawama', 'hawcoo', 'hawcre', 'hawgoo', 'hawhaw', 'hawpet1',\n       'hoomer', 'houfin', 'houspa', 'hudgod', 'iiwi', 'incter1',\n       'jabwar', 'japqua', 'kalphe', 'kauama', 'laugul', 'layalb',\n       'lcspet', 'leasan', 'leater1', 'lessca', 'lesyel', 'lobdow',\n       'lotjae', 'madpet', 'magpet1', 'mallar3', 'masboo', 'mauala',\n       'maupar', 'merlin', 'mitpar', 'moudov', 'norcar', 'norhar2',\n       'normoc', 'norpin', 'norsho', 'nutman', 'oahama', 'omao', 'osprey',\n       'pagplo', 'palila', 'parjae', 'pecsan', 'peflov', 'perfal',\n       'pibgre', 'pomjae', 'puaioh', 'reccar', 'redava', 'redjun',\n       'redpha1', 'refboo', 'rempar', 'rettro', 'ribgul', 'rinduc',\n       'rinphe', 'rocpig', 'rorpar', 'rudtur', 'ruff', 'saffin', 'sander',\n       'semplo', 'sheowl', 'shtsan', 'skylar', 'snogoo', 'sooshe',\n       'sooter1', 'sopsku1', 'sora', 'spodov', 'sposan', 'towsol',\n       'wantat1', 'warwhe1', 'wesmea', 'wessan', 'wetshe', 'whfibi',\n       'whiter', 'whttro', 'wiltur', 'yebcar', 'yefcan', 'zebdov'])\n\n\ncfg.n_classes = len(cfg.birds)\n# dataset\ncfg.min_rating = 2.0\n\ncfg.wav_crop_len = 30  # seconds\n\ncfg.lr = 0.0001\ncfg.epochs = 20\ncfg.batch_size = 64\ncfg.batch_size_val = 64\ncfg.backbone = \"resnet34\"\n\n\ncfg.save_val_data = True\ncfg.mixed_precision = True\n\ncfg.mixup = True\ncfg.mix_beta = 1\n\n\ncfg.train_df1 = \"../input/birdclef-2022/train_metadata.csv\"\ncfg.train_df2 = \"../input/birdclef-2022-df-train-with-durations/df-with-durations.csv\"\n\n\ncfg.device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ncfg.tr_collate_fn = None\ncfg.val_collate_fn = None\ncfg.val = False\n\ncfg.dev = False\n\ncfg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"def batch_to_device(batch, device):\n    batch_dict = {key: batch[key].to(device) for key in batch}\n    return batch_dict\n\n\n\nclass CustomDataset(Dataset):\n    def __init__(self, df, cfg, aug, mode=\"train\"):\n\n        self.cfg = cfg\n        self.mode = mode\n        self.df = df.copy()\n\n        self.bird2id = {bird: idx for idx, bird in enumerate(cfg.birds)}\n        if self.mode == \"train\":\n            self.data_folder = cfg.train_data_folder\n            self.df = self.df[self.df[\"rating\"] >= self.cfg.min_rating]\n        elif self.mode == \"val\":\n            self.data_folder = cfg.val_data_folder\n        elif self.mode == \"test\":\n            self.data_folder = cfg.test_data_folder\n\n        self.fns = self.df[\"filename\"].unique()\n\n        self.df = self.setup_df()\n\n        self.aug_audio = cfg.train_aug\n\n    def setup_df(self):\n        df = self.df.copy()\n\n        if self.mode == \"train\":\n\n            df[\"weight\"] = np.clip(df[\"rating\"] / df[\"rating\"].max(), 0.1, 1.0)\n            df['target'] = df['primary_label'].apply(self.bird2id.get)\n            labels = np.eye(self.cfg.n_classes)[df[\"target\"].astype(int).values]\n            label2 = df[\"secondary_labels\"].apply(lambda x: self.secondary2target(x)).values\n            for i, t in enumerate(label2):\n                labels[i, t] = 1\n        else:\n            targets = df[\"birds\"].apply(lambda x: self.birds2target(x)).values\n            labels = np.zeros((df.shape[0], self.cfg.n_classes))\n            for i, t in enumerate(targets):\n                labels[i, t] = 1\n\n        df[[f\"t{i}\" for i in range(self.cfg.n_classes)]] = labels\n\n        if self.mode != \"train\":\n            df = df.groupby(\"filename\")\n\n        return df\n\n    def __getitem__(self, idx):\n\n        if self.mode == \"train\":\n            row = self.df.iloc[idx]\n            fn = row[\"filename\"]\n            label = row[[f\"t{i}\" for i in range(self.cfg.n_classes)]].values\n            weight = row[\"weight\"]\n            #fold = row[\"fold\"]\n            fold = -1\n\n            #wav_len = row[\"length\"]\n            parts = 1\n        else:\n            fn = self.fns[idx]\n            row = self.df.get_group(fn)\n            label = row[[f\"t{i}\" for i in range(self.cfg.n_classes)]].values\n            wav_len = None\n            parts = label.shape[0]\n            fold = -1\n            weight = 1\n\n        if self.mode == \"train\":\n            #wav_len_sec = wav_len / self.cfg.sample_rate\n            wav_len_sec = row['duration']\n            duration = self.cfg.wav_crop_len\n            max_offset = wav_len_sec - duration\n            max_offset = max(max_offset, 1)\n            offset = np.random.randint(max_offset)\n        else:\n            offset = 0.0\n            duration = None\n\n        wav = self.load_one(fn, offset, duration)\n\n        if wav.shape[0] < (self.cfg.wav_crop_len * self.cfg.sample_rate):\n            pad = self.cfg.wav_crop_len * self.cfg.sample_rate - wav.shape[0]\n            wav = np.pad(wav, (0, pad))\n\n        if self.mode == \"train\":\n            if self.aug_audio:\n                wav = self.aug_audio(samples=wav, sample_rate=self.cfg.sample_rate)\n        else:\n            if self.cfg.val_aug:\n                wav = self.cfg.val_aug(samples=wav, sample_rate=self.cfg.sample_rate)\n\n        wav_tensor = torch.tensor(wav)  # (n_samples)\n        if parts > 1:\n            n_samples = wav_tensor.shape[0]\n            wav_tensor = wav_tensor[: n_samples // parts * parts].reshape(\n                parts, n_samples // parts\n            )\n\n        feature_dict = {\n            \"input\": wav_tensor,\n            \"target\": torch.tensor(label.astype(np.float32)),\n            \"weight\": torch.tensor(weight),\n            \"fold\": torch.tensor(fold),\n        }\n        return feature_dict\n\n    def __len__(self):\n        if cfg.dev:\n            return 256\n        return len(self.fns)\n\n    def load_one(self, id_, offset, duration):\n        fp = self.data_folder + id_\n        try:\n            wav, sr = librosa.load(fp, sr=None, offset=offset, duration=duration)\n        except:\n            print(\"FAIL READING rec\", fp)\n\n        return wav\n\n    def birds2target(self, birds):\n        birds = birds.split()\n        target = [self.bird2id.get(item) for item in birds if not item == \"nocall\"]\n        return target\n\n    def secondary2target(self, secondary_label):\n        birds = ast.literal_eval(secondary_label)\n        target = [self.bird2id.get(item) for item in birds if not item == \"nocall\"]\n        return target\n","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.814006Z","iopub.execute_input":"2022-02-28T05:53:12.814308Z","iopub.status.idle":"2022-02-28T05:53:12.844711Z","shell.execute_reply.started":"2022-02-28T05:53:12.814269Z","shell.execute_reply":"2022-02-28T05:53:12.84379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset related utils","metadata":{}},{"cell_type":"code","source":"def worker_init_fn(worker_id):\n    np.random.seed(np.random.get_state()[1][0] + worker_id)\n\n\ndef get_train_dataloader(train_ds, cfg):\n    train_dataloader = DataLoader(\n        train_ds,\n        sampler=None,\n        shuffle=True,\n        batch_size=cfg.batch_size,\n        num_workers=cfg.num_workers,\n        pin_memory=False,\n        collate_fn=cfg.tr_collate_fn,\n        drop_last=cfg.drop_last,\n        worker_init_fn=worker_init_fn,\n    )\n    print(f\"train: dataset {len(train_ds)}, dataloader {len(train_dataloader)}\")\n    return train_dataloader\n\n\ndef get_scheduler(cfg, optimizer, total_steps):\n    scheduler = get_cosine_schedule_with_warmup(\n        optimizer,\n        num_warmup_steps=cfg.warmup * (total_steps // cfg.batch_size),\n        num_training_steps=cfg.epochs * (total_steps // cfg.batch_size),\n    )\n    return scheduler\n\n\ndef load_df(cfg):\n    train_df1 = pd.read_csv(cfg.train_df1)\n    train_df2 = pd.read_csv(cfg.train_df2)\n    train_df = pd.merge(train_df1[['primary_label', 'secondary_labels', 'rating', 'filename']], train_df2[['filename', 'duration']], how='inner', on='filename')\n    return train_df\n","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.846347Z","iopub.execute_input":"2022-02-28T05:53:12.846729Z","iopub.status.idle":"2022-02-28T05:53:12.856303Z","shell.execute_reply.started":"2022-02-28T05:53:12.846685Z","shell.execute_reply":"2022-02-28T05:53:12.855622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GeM and Mix-up","metadata":{}},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    # Generalized mean: https://arxiv.org/abs/1711.02512\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n    def __repr__(self):\n        return (self.__class__.__name__+ \"(p=\"+ \"{:.4f}\".format(self.p.data.tolist()[0])+ \", eps=\"+ str(self.eps)+ \")\")\n\n\nclass Mixup(nn.Module):\n    def __init__(self, mix_beta):\n\n        super(Mixup, self).__init__()\n        self.beta_distribution = Beta(mix_beta, mix_beta)\n\n    def forward(self, X, Y, weight=None):\n\n        bs = X.shape[0]\n        n_dims = len(X.shape)\n        perm = torch.randperm(bs)\n        coeffs = self.beta_distribution.rsample(torch.Size((bs,))).to(X.device)\n\n        if n_dims == 2:\n            X = coeffs.view(-1, 1) * X + (1 - coeffs.view(-1, 1)) * X[perm]\n        elif n_dims == 3:\n            X = coeffs.view(-1, 1, 1) * X + (1 - coeffs.view(-1, 1, 1)) * X[perm]\n        else:\n            X = coeffs.view(-1, 1, 1, 1) * X + (1 - coeffs.view(-1, 1, 1, 1)) * X[perm]\n\n        Y = coeffs.view(-1, 1) * Y + (1 - coeffs.view(-1, 1)) * Y[perm]\n\n        if weight is None:\n            return X, Y\n        else:\n            weight = coeffs.view(-1) * weight + (1 - coeffs.view(-1)) * weight[perm]\n            return X, Y, weight\n","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.857898Z","iopub.execute_input":"2022-02-28T05:53:12.858809Z","iopub.status.idle":"2022-02-28T05:53:12.873625Z","shell.execute_reply.started":"2022-02-28T05:53:12.858614Z","shell.execute_reply":"2022-02-28T05:53:12.87285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, cfg):\n        super(Net, self).__init__()\n\n        self.cfg = cfg\n\n        self.n_classes = cfg.n_classes\n\n        self.mel_spec = ta.transforms.MelSpectrogram(\n            sample_rate=cfg.sample_rate,\n            n_fft=cfg.window_size,\n            win_length=cfg.window_size,\n            hop_length=cfg.hop_size,\n            f_min=cfg.fmin,\n            f_max=cfg.fmax,\n            pad=0,\n            n_mels=cfg.mel_bins,\n            power=cfg.power,\n            normalized=False,\n        )\n\n        self.amplitude_to_db = ta.transforms.AmplitudeToDB(top_db=cfg.top_db)\n        self.wav2img = torch.nn.Sequential(self.mel_spec, self.amplitude_to_db)\n\n        self.backbone = timm.create_model(\n            cfg.backbone,\n            pretrained=cfg.pretrained,\n            num_classes=0,\n            global_pool=\"\",\n            in_chans=cfg.in_chans,\n        )\n\n        if \"efficientnet\" in cfg.backbone:\n            backbone_out = self.backbone.num_features\n        else:\n            backbone_out = self.backbone.feature_info[-1][\"num_chs\"]\n\n        self.global_pool = GeM()\n\n        self.head = nn.Linear(backbone_out, self.n_classes)\n\n        if cfg.pretrained_weights is not None:\n            sd = torch.load(cfg.pretrained_weights, map_location=\"cpu\")[\"model\"]\n            sd = {k.replace(\"module.\", \"\"): v for k, v in sd.items()}\n            self.load_state_dict(sd, strict=True)\n            print(\"weights loaded from\", cfg.pretrained_weights)\n        self.loss_fn = nn.BCEWithLogitsLoss(reduction=\"none\")\n\n        self.mixup = Mixup(mix_beta=cfg.mix_beta)\n\n        self.factor = int(cfg.wav_crop_len / 5.0)\n\n    def forward(self, batch):\n\n        if not self.training:\n            x = batch[\"input\"]\n            bs, parts, time = x.shape\n            x = x.reshape(parts, time)\n            y = batch[\"target\"]\n            y = y[0]\n        else:\n\n            x = batch[\"input\"]\n            y = batch[\"target\"]\n            bs, time = x.shape\n            x = x.reshape(bs * self.factor, time // self.factor)\n\n        with autocast(enabled=False):\n            x = self.wav2img(x)  # (bs, mel, time)\n            if self.cfg.mel_norm:\n                x = (x + 80) / 80\n\n        x = x.permute(0, 2, 1)\n        x = x[:, None, :, :]\n\n        weight = batch[\"weight\"]\n\n        if self.training:\n            b, c, t, f = x.shape\n            x = x.permute(0, 2, 1, 3)\n            x = x.reshape(b // self.factor, self.factor * t, c, f)\n\n            if self.cfg.mixup:\n                x, y, weight = self.mixup(x, y, weight)\n            if self.cfg.mixup2:\n                x, y, weight = self.mixup(x, y, weight)\n\n            x = x.reshape(b, t, c, f)\n            x = x.permute(0, 2, 1, 3)\n\n        x = self.backbone(x)\n\n        if self.training:\n            b, c, t, f = x.shape\n            x = x.permute(0, 2, 1, 3)\n            x = x.reshape(b // self.factor, self.factor * t, c, f)\n            x = x.permute(0, 2, 1, 3)\n        x = self.global_pool(x)\n        x = x[:, :, 0, 0]\n        logits = self.head(x)\n\n        loss = self.loss_fn(logits, y)\n        loss = (loss.mean(dim=1) * weight) / weight.sum()\n        loss = loss.sum()\n\n        return {\"loss\": loss, \"logits\": logits.sigmoid(), \"logits_raw\": logits, \"target\": y}\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-28T05:53:12.876778Z","iopub.execute_input":"2022-02-28T05:53:12.877092Z","iopub.status.idle":"2022-02-28T05:53:12.899179Z","shell.execute_reply.started":"2022-02-28T05:53:12.877063Z","shell.execute_reply":"2022-02-28T05:53:12.898448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_checkpoint(model, optimizer, epoch, scheduler=None, scaler=None):\n    checkpoint = {\n        \"model\": model.state_dict(),\n        \"optimizer\": optimizer.state_dict(),\n        \"epoch\": epoch,\n    }\n\n    if scheduler is not None:\n        checkpoint[\"scheduler\"] = scheduler.state_dict()\n\n    if scaler is not None:\n        checkpoint[\"scaler\"] = scaler.state_dict()\n    return checkpoint","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.901959Z","iopub.execute_input":"2022-02-28T05:53:12.902314Z","iopub.status.idle":"2022-02-28T05:53:12.909452Z","shell.execute_reply.started":"2022-02-28T05:53:12.902275Z","shell.execute_reply":"2022-02-28T05:53:12.908745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nset_seed(cfg.seed)\n\n\n#val_df = pd.read_csv(cfg.val_df)\n#val_dataset = CustomDataset(val_df, cfg, aug=cfg.val_aug, mode=\"val\")\n\ntrain_df = load_df(cfg)\n\ntrain_dataset = CustomDataset(train_df, cfg, aug=cfg.train_aug, mode=\"train\")\ntrain_dataloader = get_train_dataloader(train_dataset, cfg)\nmodel = Net(cfg)\n\nmodel.to(cfg.device)\n\ntotal_steps = len(train_dataset)\n\nparams = model.parameters()\noptimizer = optim.Adam(params, lr=cfg.lr, weight_decay=0)\nscheduler = get_scheduler(cfg, optimizer, total_steps)\n\n\ndevice = cfg.device","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:12.912375Z","iopub.execute_input":"2022-02-28T05:53:12.912609Z","iopub.status.idle":"2022-02-28T05:53:13.726594Z","shell.execute_reply.started":"2022-02-28T05:53:12.912585Z","shell.execute_reply":"2022-02-28T05:53:13.725792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -l /root/.cache/torch/hub/checkpoints/","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:13.727901Z","iopub.execute_input":"2022-02-28T05:53:13.728159Z","iopub.status.idle":"2022-02-28T05:53:14.440694Z","shell.execute_reply.started":"2022-02-28T05:53:13.728112Z","shell.execute_reply":"2022-02-28T05:53:14.439801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /root/.cache/torch/hub/checkpoints/resnet34-43635321.pth .","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:14.444181Z","iopub.execute_input":"2022-02-28T05:53:14.444438Z","iopub.status.idle":"2022-02-28T05:53:15.299Z","shell.execute_reply.started":"2022-02-28T05:53:14.44441Z","shell.execute_reply":"2022-02-28T05:53:15.297688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    os.makedirs(cfg.output_dir)\nexcept:\n    pass\n\nif cfg.mixed_precision:\n    scaler = GradScaler()\nelse:\n    scaler = None\n\n\ncfg.curr_step = 0\ni = 0\nbest_val_loss = np.inf\noptimizer.zero_grad()\nfor epoch in range(cfg.epochs):\n\n    set_seed(cfg.seed + epoch)\n\n    cfg.curr_epoch = epoch\n\n    print(\"EPOCH:\", epoch)\n\n    progress_bar = tqdm(range(len(train_dataloader)))\n    tr_it = iter(train_dataloader)\n\n    losses = []\n\n    gc.collect()\n\n    if cfg.train:\n        # ==== TRAIN LOOP\n        for itr in progress_bar:\n            i += 1\n\n            cfg.curr_step += cfg.batch_size\n\n            data = next(tr_it)\n\n            model.train()\n            torch.set_grad_enabled(True)\n\n            batch = batch_to_device(data, device)\n\n            if cfg.mixed_precision:\n                with autocast():\n                    output_dict = model(batch)\n            else:\n                output_dict = model(batch)\n\n            loss = output_dict[\"loss\"]\n\n            losses.append(loss.item())\n\n            if cfg.mixed_precision:\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()\n            else:\n                loss.backward()\n                optimizer.step()\n                optimizer.zero_grad()\n\n            if scheduler is not None:\n                scheduler.step()\n\n            if cfg.curr_step % cfg.batch_size == 0:\n                progress_bar.set_description(f\"loss: {np.mean(losses[-10:]):.4f}\")\n\n    if cfg.val:\n        if (epoch + 1) % cfg.eval_epochs == 0 or (epoch + 1) == cfg.epochs:\n            val_loss = run_eval(model, val_dataloader, cfg)\n        else:\n            val_score = 0\n\n    if cfg.epochs > 0:\n        checkpoint = create_checkpoint(\n            model, optimizer, epoch, scheduler=scheduler, scaler=scaler\n        )\n\n        torch.save(checkpoint, f\"{cfg.output_dir}/checkpoint_last_seed{cfg.seed}_{epoch}.pth\")\n\nif cfg.epochs > 0:\n    checkpoint = create_checkpoint(model, optimizer, epoch, scheduler=scheduler, scaler=scaler)\n\n    torch.save(checkpoint, f\"{cfg.output_dir}/checkpoint_last_seed{cfg.seed}.pth\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T05:53:15.300878Z","iopub.execute_input":"2022-02-28T05:53:15.30118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}