{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Objective\n\nIn this notebook i am going to use 8TPU cores for 8Folds training\n\nReference : https://www.kaggle.com/abhishek/super-duper-fast-pytorch-tpu-kernel","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"<h3 style=\"color:red;\"> Please upvote if you like it. It motivates me. Thank you ☺️ .</h3>","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev\n!pip install pretrainedmodels\n!pip install pydub\n!pip install ../input/resnest50-fast-package/resnest-0.0.6b20200701/resnest/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport librosa\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nimport soundfile as sf\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport math\nfrom collections import OrderedDict\n\nfrom PIL import Image\nimport albumentations\nfrom pydub import AudioSegment\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport resnest.torch as resnest_torch\n\nfrom joblib import Parallel, delayed\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom tqdm.autonotebook import tqdm\nimport os\nfrom sklearn.metrics import classification_report\nimport gc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing <a id=\"3\"></a>","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/birdsong-recognition/train.csv\")\ntest = pd.read_csv(\"../input/birdsong-recognition/test.csv\")\nsubmission = pd.read_csv(\"../input/birdsong-recognition/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### e-bird code\n\na code for the bird species. we need to predict `ebird_code` using metadata and audio data \n","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"markdown","source":"###### print(\"Number of Unique birds : \", train.ebird_code.nunique())","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### top20 Birds\nwe are taking top10 birds to build stater model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#top10_birds = list(train.ebird_code.value_counts().index[:10])\n\n#train = train[train.ebird_code.isin(top10_birds)]\n\n# label encoding for target values\ntrain[\"ebird_label\"] = LabelEncoder().fit_transform(train.ebird_code.values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### K-Fold","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"FOLDS = 8\n\ntrain.loc[:, \"kfold\"] = -1\n\ntrain= train.sample(frac=1).reset_index(drop=True)\n\nX = train.filename.values\ny = train.ebird_code.values\n\nkfold = StratifiedKFold(n_splits=FOLDS)\n\nfor fold, (t_idx, v_idx) in enumerate(kfold.split(X, y)):\n    train.loc[v_idx, \"kfold\"] = fold\n\nprint(train.kfold.value_counts())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Arguments","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class args:\n    \n    ROOT_PATH = \"../input/birdsong-recognition/train_audio\"\n    \n    num_classes = 264\n    max_duration= 5 # seconds\n    \n    sample_rate = 32000\n    \n    img_height = 128\n    img_width = 313\n    \n    batch_size = 100\n    num_workers = 4\n    epochs = 66\n    \n    lr = 0.0009\n    wd = 1e-5\n    momentum = 0.9\n    eps = 1e-8\n    betas = (0.9, 0.999)\n    pretrained=False\n    \n    melspectrogram_parameters = {\n        \"n_mels\": 128,\n        \"fmin\": 20,\n        \"fmax\": 16000\n    }\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Audo Albumentations\n\n- check my other notebook [Audio Albumentations](https://www.kaggle.com/gopidurgaprasad/audio-albumentations)","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from albumentations.core.transforms_interface import DualTransform, BasicTransform\n\nclass AudioTransform(BasicTransform):\n    \"\"\"Transform for Audio task\"\"\"\n\n    @property\n    def targets(self):\n        return {\"data\": self.apply}\n    \n    def update_params(self, params, **kwargs):\n        if hasattr(self, \"interpolation\"):\n            params[\"interpolation\"] = self.interpolation\n        if hasattr(self, \"fill_value\"):\n            params[\"fill_value\"] = self.fill_value\n        return params\n\nclass NoiseInjection(AudioTransform):\n    \"\"\"It simply add some random value into data by using numpy\"\"\"\n    def __init__(self, noise_levels=(0, 0.5), always_apply=False, p=0.5):\n        super(NoiseInjection, self).__init__(always_apply, p)\n\n        self.noise_levels = noise_levels\n    \n    def apply(self, data, **params):\n        sound, sr = data\n        noise_level = np.random.uniform(*self.noise_levels)\n        noise = np.random.randn(len(sound))\n        augmented_sound = sound + noise_level * noise\n        # Cast back to same data type\n        augmented_sound = augmented_sound.astype(type(sound[0]))\n\n        return augmented_sound, sr\n\nclass ShiftingTime(AudioTransform):\n    \"\"\"Shifting time axis\"\"\"\n    def __init__(self, always_apply=False, p=0.5):\n        super(ShiftingTime, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        shift_max = np.random.randint(1,len(sound))\n        shift = np.random.randint(int(sr * shift_max))\n        direction = np.random.randint(0,2)\n        if direction == 1:\n            shift = -shift\n\n        augmented_sound = np.roll(sound, shift)\n        # Set to silence for heading/ tailing\n        if shift > 0:\n            augmented_sound[:shift] = 0\n        else:\n            augmented_sound[shift:] = 0\n\n        return augmented_sound, sr\n\nclass PitchShift(AudioTransform):\n    \n    def __init__(self, always_apply=False, p=0.5):\n        super(PitchShift, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        n_steps = np.random.randint(-10, 10)\n        augmented_sound = librosa.effects.pitch_shift(sound, sr, n_steps)\n\n        return augmented_sound, sr\n\nclass TimeStretch(AudioTransform):\n    \n    def __init__(self, always_apply=False, p=0.5):\n        super(TimeStretch, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        rate = np.random.uniform(0, 2)\n        augmented_sound = librosa.effects.time_stretch(sound, rate)\n\n        return augmented_sound, sr\n\nclass RandomAudio(AudioTransform):\n    \n    def __init__(self,  seconds=5, always_apply=False, p=0.5):\n        super(RandomAudio, self).__init__(always_apply, p)\n\n        self.seconds = seconds\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        shift = np.random.randint(len(sound))\n        trim_sound = np.roll(sound, shift)\n\n        min_samples = int(sr * self.seconds)\n\n        if len(trim_sound) < min_samples:\n            padding = min_samples - len(trim_sound)\n            offset = padding // 2\n            trim_sound = np.pad(trim_sound, (offset, padding - offset), \"constant\")\n        else:\n            trim_sound = trim_sound[:min_samples]\n\n        return trim_sound, sr\n\nclass MelSpectrogram(AudioTransform):\n\n    def __init__(self, parameters, always_apply=False, p=0.5):\n        super(MelSpectrogram, self).__init__(always_apply, p)\n\n        self.parameters = parameters\n    \n    def apply(self, data, **params):\n        sound, sr = data\n\n        melspec = librosa.feature.melspectrogram(sound, sr=sr, **self.parameters)\n        melspec = librosa.power_to_db(melspec)\n        melspec = melspec.astype(np.float32)\n\n        return melspec, sr\n\nclass SpecAugment(AudioTransform):\n    \n    def __init__(self, num_mask=2, freq_masking=0.15, time_masking=0.20, always_apply=False, p=0.5):\n        super(SpecAugment, self).__init__(always_apply, p)\n\n        self.num_mask = num_mask\n        self.freq_masking = freq_masking\n        self.time_masking = time_masking\n    \n    def apply(self, data, **params):\n        melspec, sr = data\n\n        spec_aug = self.spec_augment(melspec, \n                                     self.num_mask,\n                                     self.freq_masking,\n                                     self.time_masking,\n                                     melspec.min())\n        \n\n\n        return spec_aug, sr\n    \n    # Source: https://www.kaggle.com/davids1992/specaugment-quick-implementation\n    def spec_augment(self, \n                    spec: np.ndarray,\n                    num_mask=2,\n                    freq_masking=0.15,\n                    time_masking=0.20,\n                    value=0):\n        spec = spec.copy()\n        num_mask = random.randint(1, num_mask)\n        for i in range(num_mask):\n            all_freqs_num, all_frames_num  = spec.shape\n            freq_percentage = random.uniform(0.0, freq_masking)\n\n            num_freqs_to_mask = int(freq_percentage * all_freqs_num)\n            f0 = np.random.uniform(low=0.0, high=all_freqs_num - num_freqs_to_mask)\n            f0 = int(f0)\n            spec[f0:f0 + num_freqs_to_mask, :] = value\n\n            time_percentage = random.uniform(0.0, time_masking)\n\n            num_frames_to_mask = int(time_percentage * all_frames_num)\n            t0 = np.random.uniform(low=0.0, high=all_frames_num - num_frames_to_mask)\n            t0 = int(t0)\n            spec[:, t0:t0 + num_frames_to_mask] = value\n\n        return spec\n\nclass SpectToImage(AudioTransform):\n\n    def __init__(self, always_apply=False, p=0.5):\n        super(SpectToImage, self).__init__(always_apply, p)\n    \n    def apply(self, data, **params):\n        image, sr = data\n        delta = librosa.feature.delta(image)\n        accelerate = librosa.feature.delta(image, order=2)\n        image = np.stack([image, delta, accelerate], axis=0)\n        image = image.astype(np.float32) / 100.0\n\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_audio(code,name):\n    \n    if ord(code[0])<=98:\n        rootpath='../input/birdsong-resampled-train-audio-00'\n    elif ord(code[0])<=102:\n        rootpath='../input/birdsong-resampled-train-audio-01'\n    elif ord(code[0])<=109:\n        rootpath='../input/birdsong-resampled-train-audio-02'\n    elif ord(code[0])<=114:\n        rootpath='../input/birdsong-resampled-train-audio-03'\n    elif ord(code[0])<=121:\n        rootpath='../input/birdsong-resampled-train-audio-04'\n        \n    path='{}/{}/{}'.format(rootpath,code,name.replace(\"mp3\", \"wav\"))\n\n    sound = AudioSegment.from_file(path,format='wav')\n    sound = sound.set_frame_rate(args.sample_rate)\n    sound_array = np.array(sound.get_array_of_samples(), dtype=np.float32)\n        \n    return sound_array,args.sample_rate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Example\n\ntrain_audio_augmentation = albumentations.Compose([\n     RandomAudio(seconds=args.max_duration, always_apply=True),\n     NoiseInjection(p=0.33),\n     MelSpectrogram(parameters=args.melspectrogram_parameters,always_apply=True),\n     SpecAugment(p=0.33),\n     SpectToImage(always_apply=True)\n])\n\nvalid_audio_augmentation = albumentations.Compose([\n     RandomAudio(seconds=args.max_duration, always_apply=True),\n     MelSpectrogram(parameters=args.melspectrogram_parameters,always_apply=True),\n     SpectToImage(always_apply=True)\n])\n\ndata=load_audio('aldfly','XC135454.mp3')\n\nimage = train_audio_augmentation(data=data)['data']\n\nplt.imshow(image.transpose(1,2,0))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pytorch DataLoader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class BirdDataset:\n    def __init__(self, df, valid=False):\n        \n        self.filename = df.filename.values\n        self.ebird_label = df.ebird_label.values\n        self.ebird_code = df.ebird_code.values\n        \n        if valid:\n            self.aug = valid_audio_augmentation\n        else:\n            self.aug = train_audio_augmentation\n        \n    \n    def __len__(self):\n        return len(self.filename)\n    \n    def __getitem__(self, item):\n        \n        filename = self.filename[item]\n        ebird_code = self.ebird_code[item]\n        ebird_label = self.ebird_label[item]\n\n        data = load_audio(ebird_code,filename)\n        spect = self.aug(data=data)[\"data\"]\n        \n        target = ebird_label\n        \n        return {\n            \"spect\" : torch.tensor(spect, dtype=torch.float), \n            \"target\" : torch.tensor(target, dtype=torch.long)\n        }\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ResNet18 Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class line(nn.Module):\n \n    def __init__(self, num_classes=args.num_classes):\n        super(line, self).__init__()\n        self.fc = nn.Sequential(\n        nn.Linear(2*num_classes,400),\n        nn.ReLU(),\n        nn.Dropout(p=0.2),\n        nn.Linear(400,num_classes))\n \n    def forward(self, x):   \n        x = self.fc(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Resnet():\n\n    model = getattr(resnest_torch,'resnest50_fast_1s1x64d')(pretrained=args.pretrained)\n    del model.fc\n    model.fc = nn.Sequential(\n        nn.Linear(2048, 1024),\n        nn.ReLU(),\n        nn.Dropout(p=0.2),\n        nn.Linear(1024, 1024),\n        nn.ReLU(),\n        nn.Dropout(p=0.2),\n        nn.Linear(1024, args.num_classes),\n    )\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Inception(nn.Module):\n \n    def __init__(self, num_classes=args.num_classes, aux_logits=True, transform_input=False):\n        super(Inception, self).__init__()\n        self.aux_logits = aux_logits\n        self.transform_input = transform_input\n        self.Conv2d_1a_3x3 = BasicConv2d(3, 32, kernel_size=3, stride=2)\n        self.Conv2d_2a_3x3 = BasicConv2d(32, 32, kernel_size=3)\n        self.Conv2d_2b_3x3 = BasicConv2d(32, 64, kernel_size=3, padding=1)\n        self.Conv2d_3b_1x1 = BasicConv2d(64, 80, kernel_size=1)\n        self.Conv2d_4a_3x3 = BasicConv2d(80, 192, kernel_size=3)\n        self.Mixed_5b = InceptionA(192, pool_features=32)\n        self.Mixed_5c = InceptionA(256, pool_features=64)\n        self.Mixed_5d = InceptionA(288, pool_features=64)\n        self.Mixed_6a = InceptionB(288)\n        self.Mixed_6b = InceptionC(768, channels_7x7=128)\n        self.Mixed_6c = InceptionC(768, channels_7x7=160)\n        self.Mixed_6d = InceptionC(768, channels_7x7=160)\n        self.Mixed_6e = InceptionC(768, channels_7x7=192)\n        if aux_logits:\n            self.AuxLogits = InceptionAux(768, num_classes)\n        self.Mixed_7a = InceptionD(768)\n        self.Mixed_7b = InceptionE(1280)\n        self.Mixed_7c = InceptionE(2048)\n        self.fc = nn.Linear(2048, num_classes)\n \n    def forward(self, x):\n        if self.transform_input:\n            x_ch0 = torch.unsqueeze(x[:, 0], 1) * (0.229 / 0.5) + (0.485 - 0.5) / 0.5\n            x_ch1 = torch.unsqueeze(x[:, 1], 1) * (0.224 / 0.5) + (0.456 - 0.5) / 0.5\n            x_ch2 = torch.unsqueeze(x[:, 2], 1) * (0.225 / 0.5) + (0.406 - 0.5) / 0.5\n            x = torch.cat((x_ch0, x_ch1, x_ch2), 1)\n        # 299 x 299 x 3\n        x = self.Conv2d_1a_3x3(x)\n        # 149 x 149 x 32\n        x = self.Conv2d_2a_3x3(x)\n        # 147 x 147 x 32\n        x = self.Conv2d_2b_3x3(x)\n        # 147 x 147 x 64\n        x = F.max_pool2d(x, kernel_size=3, stride=2)\n        # 73 x 73 x 64\n        x = self.Conv2d_3b_1x1(x)\n        # 73 x 73 x 80\n        x = self.Conv2d_4a_3x3(x)\n        # 71 x 71 x 192\n        x = F.max_pool2d(x, kernel_size=3, stride=2)\n        # 35 x 35 x 192\n        x = self.Mixed_5b(x)\n        # 35 x 35 x 256\n        x = self.Mixed_5c(x)\n        # 35 x 35 x 288\n        x = self.Mixed_5d(x)\n        # 35 x 35 x 288\n        x = self.Mixed_6a(x)\n        # 17 x 17 x 768\n        x = self.Mixed_6b(x)\n        # 17 x 17 x 768\n        x = self.Mixed_6c(x)\n        # 17 x 17 x 768\n        x = self.Mixed_6d(x)\n        # 17 x 17 x 768\n        x = self.Mixed_6e(x)\n        # 17 x 17 x 768\n        if self.training and self.aux_logits:\n            aux = self.AuxLogits(x)\n        # 17 x 17 x 768\n        x = self.Mixed_7a(x)\n        # 8 x 8 x 1280\n        x = self.Mixed_7b(x)\n        # 8 x 8 x 2048\n        x = self.Mixed_7c(x)\n        # 8 x 8 x 2048\n        x = F.avg_pool2d(x, kernel_size=(2,8))\n        # 1 x 1 x 2048\n        x = F.dropout(x, training=self.training)\n        # 1 x 1 x 2048\n        x = x.view(x.size(0), -1)\n        # 2048\n        x = self.fc(x)\n        # 1000 (num_classes)\n        if self.training and self.aux_logits:\n            return (x+aux)/2\n        return x\n \n \nclass InceptionA(nn.Module):\n \n    def __init__(self, in_channels, pool_features):\n        super(InceptionA, self).__init__()\n        self.branch1x1 = BasicConv2d(in_channels, 64, kernel_size=1)\n \n        self.branch5x5_1 = BasicConv2d(in_channels, 48, kernel_size=1)\n        self.branch5x5_2 = BasicConv2d(48, 64, kernel_size=5, padding=2)\n \n        self.branch3x3dbl_1 = BasicConv2d(in_channels, 64, kernel_size=1)\n        self.branch3x3dbl_2 = BasicConv2d(64, 96, kernel_size=3, padding=1)\n        self.branch3x3dbl_3 = BasicConv2d(96, 96, kernel_size=3, padding=1)\n \n        self.branch_pool = BasicConv2d(in_channels, pool_features, kernel_size=1)\n \n    def forward(self, x):\n        branch1x1 = self.branch1x1(x)\n \n        branch5x5 = self.branch5x5_1(x)\n        branch5x5 = self.branch5x5_2(branch5x5)\n \n        branch3x3dbl = self.branch3x3dbl_1(x)\n        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n        branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)\n \n        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n        branch_pool = self.branch_pool(branch_pool)\n \n        outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool]\n        return torch.cat(outputs, 1)\n \n \nclass InceptionB(nn.Module):\n \n    def __init__(self, in_channels):\n        super(InceptionB, self).__init__()\n        self.branch3x3 = BasicConv2d(in_channels, 384, kernel_size=3, stride=2)\n \n        self.branch3x3dbl_1 = BasicConv2d(in_channels, 64, kernel_size=1)\n        self.branch3x3dbl_2 = BasicConv2d(64, 96, kernel_size=3, padding=1)\n        self.branch3x3dbl_3 = BasicConv2d(96, 96, kernel_size=3, stride=2)\n \n    def forward(self, x):\n        branch3x3 = self.branch3x3(x)\n \n        branch3x3dbl = self.branch3x3dbl_1(x)\n        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n        branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)\n \n        branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)\n \n        outputs = [branch3x3, branch3x3dbl, branch_pool]\n        return torch.cat(outputs, 1)\n \n \nclass InceptionC(nn.Module):\n \n    def __init__(self, in_channels, channels_7x7):\n        super(InceptionC, self).__init__()\n        self.branch1x1 = BasicConv2d(in_channels, 192, kernel_size=1)\n \n        c7 = channels_7x7\n        self.branch7x7_1 = BasicConv2d(in_channels, c7, kernel_size=1)\n        self.branch7x7_2 = BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3))\n        self.branch7x7_3 = BasicConv2d(c7, 192, kernel_size=(7, 1), padding=(3, 0))\n \n        self.branch7x7dbl_1 = BasicConv2d(in_channels, c7, kernel_size=1)\n        self.branch7x7dbl_2 = BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0))\n        self.branch7x7dbl_3 = BasicConv2d(c7, c7, kernel_size=(1, 7), padding=(0, 3))\n        self.branch7x7dbl_4 = BasicConv2d(c7, c7, kernel_size=(7, 1), padding=(3, 0))\n        self.branch7x7dbl_5 = BasicConv2d(c7, 192, kernel_size=(1, 7), padding=(0, 3))\n \n        self.branch_pool = BasicConv2d(in_channels, 192, kernel_size=1)\n \n    def forward(self, x):\n        branch1x1 = self.branch1x1(x)\n \n        branch7x7 = self.branch7x7_1(x)\n        branch7x7 = self.branch7x7_2(branch7x7)\n        branch7x7 = self.branch7x7_3(branch7x7)\n \n        branch7x7dbl = self.branch7x7dbl_1(x)\n        branch7x7dbl = self.branch7x7dbl_2(branch7x7dbl)\n        branch7x7dbl = self.branch7x7dbl_3(branch7x7dbl)\n        branch7x7dbl = self.branch7x7dbl_4(branch7x7dbl)\n        branch7x7dbl = self.branch7x7dbl_5(branch7x7dbl)\n \n        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n        branch_pool = self.branch_pool(branch_pool)\n \n        outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool]\n        return torch.cat(outputs, 1)\n \n \nclass InceptionD(nn.Module):\n \n    def __init__(self, in_channels):\n        super(InceptionD, self).__init__()\n        self.branch3x3_1 = BasicConv2d(in_channels, 192, kernel_size=1)\n        self.branch3x3_2 = BasicConv2d(192, 320, kernel_size=3, stride=2)\n \n        self.branch7x7x3_1 = BasicConv2d(in_channels, 192, kernel_size=1)\n        self.branch7x7x3_2 = BasicConv2d(192, 192, kernel_size=(1, 7), padding=(0, 3))\n        self.branch7x7x3_3 = BasicConv2d(192, 192, kernel_size=(7, 1), padding=(3, 0))\n        self.branch7x7x3_4 = BasicConv2d(192, 192, kernel_size=3, stride=2)\n \n    def forward(self, x):\n        branch3x3 = self.branch3x3_1(x)\n        branch3x3 = self.branch3x3_2(branch3x3)\n \n        branch7x7x3 = self.branch7x7x3_1(x)\n        branch7x7x3 = self.branch7x7x3_2(branch7x7x3)\n        branch7x7x3 = self.branch7x7x3_3(branch7x7x3)\n        branch7x7x3 = self.branch7x7x3_4(branch7x7x3)\n \n        branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)\n        outputs = [branch3x3, branch7x7x3, branch_pool]\n        return torch.cat(outputs, 1)\n \n \nclass InceptionE(nn.Module):\n \n    def __init__(self, in_channels):\n        super(InceptionE, self).__init__()\n        self.branch1x1 = BasicConv2d(in_channels, 320, kernel_size=1)\n \n        self.branch3x3_1 = BasicConv2d(in_channels, 384, kernel_size=1)\n        self.branch3x3_2a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1))\n        self.branch3x3_2b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0))\n \n        self.branch3x3dbl_1 = BasicConv2d(in_channels, 448, kernel_size=1)\n        self.branch3x3dbl_2 = BasicConv2d(448, 384, kernel_size=3, padding=1)\n        self.branch3x3dbl_3a = BasicConv2d(384, 384, kernel_size=(1, 3), padding=(0, 1))\n        self.branch3x3dbl_3b = BasicConv2d(384, 384, kernel_size=(3, 1), padding=(1, 0))\n \n        self.branch_pool = BasicConv2d(in_channels, 192, kernel_size=1)\n \n    def forward(self, x):\n        branch1x1 = self.branch1x1(x)\n \n        branch3x3 = self.branch3x3_1(x)\n        branch3x3 = [\n            self.branch3x3_2a(branch3x3),\n            self.branch3x3_2b(branch3x3),\n        ]\n        branch3x3 = torch.cat(branch3x3, 1)\n \n        branch3x3dbl = self.branch3x3dbl_1(x)\n        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)\n        branch3x3dbl = [\n            self.branch3x3dbl_3a(branch3x3dbl),\n            self.branch3x3dbl_3b(branch3x3dbl),\n        ]\n        branch3x3dbl = torch.cat(branch3x3dbl, 1)\n \n        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)\n        branch_pool = self.branch_pool(branch_pool)\n \n        outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool]\n        return torch.cat(outputs, 1)\n \n \nclass InceptionAux(nn.Module):\n \n    def __init__(self, in_channels, num_classes):\n        super(InceptionAux, self).__init__()\n        self.conv0 = BasicConv2d(in_channels, 128, kernel_size=1)\n        self.conv1 = BasicConv2d(128, 768, kernel_size=(1,5))\n        self.conv1.stddev = 0.01\n        self.fc = nn.Linear(768, num_classes)\n        self.fc.stddev = 0.001\n \n    def forward(self, x):\n        # 17 x 17 x 768\n        x = F.avg_pool2d(x, kernel_size=5, stride=3)\n        # 5 x 5 x 768\n        x = self.conv0(x)\n        # 5 x 5 x 128\n        x = self.conv1(x)\n        # 1 x 1 x 768\n        x = x.view(x.size(0), -1)\n        # 768\n        x = self.fc(x)\n        # 1000\n        return x\n \n \nclass BasicConv2d(nn.Module):\n \n    def __init__(self, in_channels, out_channels, **kwargs):\n        super(BasicConv2d, self).__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)\n        self.bn = nn.BatchNorm2d(out_channels, eps=0.001)\n \n    def forward(self, x):\n        x = self.conv(x)\n        x = self.bn(x)\n        return F.relu(x, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Utility functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_list(tensor):\n    return tensor.detach().cpu().tolist()\n\ndef reduce_fn(vals):\n    return sum(vals) / len(vals)\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current values\"\"\"\n    def __init__(self):\n        self.reset()\n    \n    def __init__(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\ndef get_position_accuracy(logits, labels):\n    predictions = np.argmax(F.softmax(logits, dim=1).cpu().data.numpy(), axis=1)\n    labels = labels.cpu().data.numpy()\n    total_num = 0\n    sum_correct = 0\n    for i in range(len(labels)):\n        if labels[i] >= 0:\n            total_num += 1\n            if predictions[i] == labels[i]:\n                sum_correct += 1\n    if total_num == 0:\n        total_num = 1e-7\n    \n    return np.float32(sum_correct) / total_num, total_num\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Loss function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_fn(preds, labels):\n    loss = nn.CrossEntropyLoss(ignore_index=-1)(preds, labels)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train & validation functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_fn(train_loader, model1,model2,model3, optimizer, device, epoch):\n    total_loss = AverageMeter()\n    accuracies = AverageMeter()\n    f1_score=AverageMeter() \n    \n    model1.eval()\n    model2.eval()\n    model3.train()\n\n    t = tqdm(train_loader)\n    for step, d in enumerate(t):\n        \n        with torch.no_grad():   \n            spect = d[\"spect\"].to(device)\n            targets = d[\"target\"].to(device)\n            outputs1 = F.softmax(model1(spect), dim=1).cpu().data.numpy()\n            outputs2 = F.softmax(model2(spect), dim=1).cpu().data.numpy()\n        outputs3 = model3(torch.tensor(np.append(outputs1,outputs2,axis=1)).to(device))\n\n        loss = loss_fn(outputs3, targets)\n\n        optimizer.zero_grad()\n        \n        loss.backward()\n        xm.optimizer_step(optimizer, barrier=True)\n        \n        acc, n_position = get_position_accuracy(outputs3, targets)\n        \n        total_loss.update(loss.item(), n_position)\n        accuracies.update(acc, n_position)\n   \n        t.set_description(f\"Train E:{epoch+1} - Loss:{total_loss.avg:0.4f} - Acc:{accuracies.avg:0.4f}\")\n        \n    return total_loss.avg, accuracies.avg\n\ndef valid_fn(valid_loader,model1,model2,model3,device, epoch):\n    total_loss = AverageMeter()\n    accuracies = AverageMeter()\n    \n    model1.eval()\n    model2.eval()\n    model3.eval()\n\n    t = tqdm(valid_loader)\n    for step, d in enumerate(t):\n        \n        with torch.no_grad():\n        \n            spect = d[\"spect\"].to(device)\n            targets = d[\"target\"].to(device)\n\n            outputs1 = F.softmax(model1(spect), dim=1).cpu().data.numpy()\n            outputs2 = F.softmax(model2(spect), dim=1).cpu().data.numpy()        \n            outputs3 = model3(torch.tensor(np.append(outputs1,outputs2,axis=1)))\n\n            loss = loss_fn(outputs3, targets)\n\n            acc, n_position = get_position_accuracy(outputs, targets)\n\n            total_loss.update(loss.item(), n_position)\n            accuracies.update(acc, n_position)\n            \n            t.set_description(f\"Train E:{epoch+1} - Loss:{total_loss.avg:0.4f} - Acc:{accuracies.avg:0.4f}\")\n            \n    return total_loss.avg, accuracies.avg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(fold_index):\n\n    MX1 =Resnet()\n    device = xm.xla_device(devkind='TPU')\n    model1 = MX1.to(device)\n    state_dict = torch.load('../input/resnetweights/fold_3.bin')\n    model1.load_state_dict(state_dict)\n    \n    MX2 =Inception()\n    device = xm.xla_device(devkind='TPU')\n    model2 = MX2.to(device)\n    state_dict = torch.load('../input/inceptionnet/fold_5.bin')\n    model2.load_state_dict(state_dict)\n\n    MX3 =line()\n    device = xm.xla_device(devkind='TPU')\n    model3 = MX3.to(device)\n    \n    train_df = train[train.kfold.isin(fold_index)]\n    train_dataset = BirdDataset(df=train_df)\n\n\n    valid_df = train[~train.kfold.isin(fold_index)]\n    valid_dataset = BirdDataset(df=valid_df, valid=True)\n\n    train_loader = DataLoader(\n        dataset = train_dataset,\n        batch_size = args.batch_size,\n        pin_memory = True,\n        drop_last = False\n    )\n\n    valid_loader = DataLoader(\n        dataset = valid_dataset,\n        batch_size = args.batch_size,\n        pin_memory = True,\n        drop_last = False\n    )\n\n    optimizer = torch.optim.AdamW(model3.parameters(),\n                                      lr=args.lr * xm.xrt_world_size(),\n                                      betas=args.betas,\n                                      eps=args.eps,\n                                      weight_decay=args.wd\n                                 )\n\n    best_acc = 0\n\n    for epoch in range(args.epochs):\n        \n        train_loss,train_acc = train_fn(train_loader, model1,model2,model3, optimizer, device, epoch)\n        \n        #valid_loss, valid_acc = valid_fn(valid_loader, model, device, epoch)\n\n        #print(f\"Fold {fold_index} ** Epoch {epoch+1} **==>** Accuracy = {valid_acc} **==>** loss = {train_loss}\")\n        print(f\"Fold {fold_index} ** Epoch {epoch+1} **==>** Accuracy = {train_acc} **==>** loss = {train_loss}\")\n\n        xm.save(model3.state_dict(), f\"fold_{fold_index}.bin\")\n        \n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 8 Folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"run([3,5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Parallel(n_jobs=8, backend=\"threading\")(delayed(run)(i) for i in range(8))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<h2 style=\"color:red;\"> Please upvote if you like it. It motivates me. Thank you ☺️ .</h2>","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}