{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## REMEMBER TO CHANGE ENSEMBLE PART (SOMETIMES ONLY GET RESULT OF 1 MODEL INSTEAD OF AVERAGING)\n## That's in function predict_on_video()\n\n# Version 1: Mobilenet underbalance (with border): LB 0.437\n# Version 2: EffB0 underbalance (with border): LB 0.450 \n# Version 3: Mobilenet all data (with border): LB 0.381\n# Version 4: blend of version 3 (with border) and EffB0 LB0.388 (version 7 of EffB9 kernel, no border): LB ????\n# Version 5: Mobilenet all data (with border) gradual freeze: LB 0.42\n# Version 6: Mobilenet all data (sqrimg, no border): LB 0.37\n# Version 7: Blend 3 models (version 4 and version 6): LB 0.33\n# Version 10: Only Xception (square img size 299): LB 0.42\n# Version 11: Blend 4 models\n# Version 12: NASNet (blackborder): LB 0.42\n# Version 13: Blend 6 models (with last new model EffB1 square)\n# Version 14: LSTM model\n# Version 15: new Xception model\n# Version 16: new LSTM audio model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_DIR = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\nCHECKPOINT = '/kaggle/input/kha-deepfake-dataset/checkpoint_mobilev3_alldata_1903_withfaceforensics_3epochs_.pth' # blackborder, all data\nCHECKPOINT2 = '/kaggle/input/kha-deepfake-dataset/cpt_mbn_sqrimg_2503)2epochs_.pth' # square, all data\nCHECKPOINT3 = '/kaggle/input/kha-deepfake-dataset/checkpoint_b0_1803_0epochs_0.4498354019969702.pth' # square, less data (only old DFDC)\nCHECKPOINT4 = '/kaggle/input/kha-deepfake-dataset/cpt_xception_new_sqrimg_29030_epochs_3_moment.pth' # square, all data\nCHECKPOINT5 = '/kaggle/input/kha-deepfake-dataset/cpt_nasnet_bbd_29032_epochs_0_moment.pth' # black border\nCHECKPOINT6 = '/kaggle/input/kha-deepfake-dataset/cpt_effb1_sqrimg_25032_epochs_1_moment.pth' # black border\nCHECKPOINT7 = '/kaggle/input/kha-deepfake-dataset/cpt_mbn_LSTM16_300313epochs_0.21526583118569945.pth' # square, old dfdc, LSTM\nCHECKPOINT8 = '/kaggle/input/kha-deepfake-dataset/checkpoint_b0_LSTM16_2503_batchfirst1epochs_0.25805166250713446.pth' # square, old dfdc, LSTM with audio\nCHECKPOINT9 = '/kaggle/input/kha-deepfake-dataset/cpt_nasnetnew_bbd_31031_epochs_0_moment.pth' # square, all data\nCHECKPOINT10 = '/kaggle/input/kha-deepfake-dataset/cpt_xception_sqrimg_25031_epochs_0_moment.pth' # square, all data, old xception\nCHECKPOINT11 = '/kaggle/input/kha-deepfake-dataset/cpt_resnet_bbd_31033_epochs_3_moment.pth'  # bbd, resnet\n\nCONVERT_RGB = True\n\nMTCNN_BATCH = 15\nGAP = 5\nIMG_SIZE = 224\nSCALE = 0.5\nimport multiprocessing as mp\nNUM_WORKERS = mp.cpu_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\npackage_path = '../input/kha-efficientnet/EfficientNet-PyTorch/'\nsys.path.append(package_path)\nfrom efficientnet_pytorch import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\n# Install facenet-pytorch (with internet use \"pip install facenet-pytorch\")\n!pip install /kaggle/input/khafacenet/facenet_pytorch-2.2.7-py3-none-any.whl\n!pip install /kaggle/input/imutils/imutils-0.5.3","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# %%capture\n# # Install facenet-pytorch (with internet use \"pip install facenet-pytorch\")\n# !pip install /kaggle/input/facenet-pytorch-vggface2/facenet_pytorch-2.2.7-py3-none-any.whl\n# !pip install /kaggle/input/imutils/imutils-0.5.3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ! tar xvf ../input/ffmpeg-static-build/ffmpeg-git-amd64-static.tar.xz","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os, sys, random, ast, numpy as np, pandas as pd, cv2, random\nimport torch, torchvision.models, torch.nn as nn, torch.nn.functional as F\nfrom torchvision.transforms import Normalize\nfrom tqdm.notebook import tqdm\nimport torchvision.models as models\nfrom facenet_pytorch import MTCNN\nfrom IPython.display import clear_output\nfrom PIL import Image\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport torch.nn as nn\n\n# from pathlib import Path\n# import librosa\n# from scipy.io import wavfile\n# import subprocess\n\n# from sklearn.externals import joblib\n# scaler = joblib.load(\"/kaggle/input/kha-deepfake-dataset/audio_scaler\")\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\n# def audio_feat_extract(aufile):\n#     rate, wave = wavfile.read(aufile)\n#     wave = wave[:,0].astype(float)\n#     feats = []\n#     S = np.abs(librosa.stft(wave))\n#     feats.append(librosa.feature.poly_features(S=S, order=3))\n#     feats.append(librosa.feature.mfcc(S=S, sr=rate)[[0, 1, 3, 7, 15]])\n#     feats.append(librosa.feature.spectral_bandwidth(S=S, sr=rate))\n#     feats.append(librosa.feature.zero_crossing_rate(wave))\n#     feats.append(librosa.onset.onset_strength(S=S, sr=rate))\n#     feats.append(librosa.feature.chroma_stft(S=S, sr=rate))\n#     feats.append(librosa.feature.rms(S=S))\n#     feats.append(librosa.feature.tonnetz(y=wave, sr=rate))\n#     final_feats = []\n#     for f in feats:\n#         m = f.mean(axis=1) if f.ndim==2 else [f.mean()]\n#         v = f.var(axis=1) if f.ndim==2 else [f.var()]\n#         final_feats.extend(m)\n#         final_feats.extend(v)\n#     if len(final_feats) != 62: final_feats = np.zeros(62)\n#     final_feats = scaler.transform([final_feats])[0]\n#     final_feats = torch.tensor(final_feats)\n#     return final_feats\n\n# au_dir = '/kaggle/working/audio_files'\n# os.makedirs(au_dir, exist_ok=True)\n\n# def create_wav(file, output_dir=au_dir, aufile='lalala'):\n#     command = f\"../working/ffmpeg-git-20191209-amd64-static/ffmpeg -i {file} -ab 192000 -ac 2 -ar 44100 -vn {output_dir}/{aufile}.wav\"\n#     subprocess.call(command, shell=True)\n    \n# def get_au_feat(testvid):\n#     aufile = testvid[:-4]\n#     create_wav(TEST_DIR + testvid, aufile=aufile)\n# #     print(os.listdir(au_dir))\n#     fullaufile = au_dir + '/' + aufile + '.wav'\n#     au_feat = audio_feat_extract(fullaufile)\n#     os.remove(fullaufile)\n# #     print(os.listdir(au_dir))\n#     return au_feat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# MOBILENET V3\n\ndef conv_bn(inp, oup, stride, conv_layer=nn.Conv2d, norm_layer=nn.BatchNorm2d, nlin_layer=nn.ReLU):\n    return nn.Sequential(\n        conv_layer(inp, oup, 3, stride, 1, bias=False),\n        norm_layer(oup),\n        nlin_layer(inplace=True)\n    )\n\n\ndef conv_1x1_bn(inp, oup, conv_layer=nn.Conv2d, norm_layer=nn.BatchNorm2d, nlin_layer=nn.ReLU):\n    return nn.Sequential(\n        conv_layer(inp, oup, 1, 1, 0, bias=False),\n        norm_layer(oup),\n        nlin_layer(inplace=True)\n    )\n\nclass Hswish(nn.Module):\n    def __init__(self, inplace=True):\n        super(Hswish, self).__init__()\n        self.inplace = inplace\n\n    def forward(self, x):\n        return x * F.relu6(x + 3., inplace=self.inplace) / 6.\n\n\nclass Hsigmoid(nn.Module):\n    def __init__(self, inplace=True):\n        super(Hsigmoid, self).__init__()\n        self.inplace = inplace\n\n    def forward(self, x):\n        return F.relu6(x + 3., inplace=self.inplace) / 6.\n\n\nclass SEModule(nn.Module):\n    def __init__(self, channel, reduction=4):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(channel, channel // reduction, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Linear(channel // reduction, channel, bias=False),\n            Hsigmoid()\n            # nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y.expand_as(x)\n\n\nclass Identity(nn.Module):\n    def __init__(self, channel):\n        super(Identity, self).__init__()\n\n    def forward(self, x):\n        return x\n\n\ndef make_divisible(x, divisible_by=8):\n    import numpy as np\n    return int(np.ceil(x * 1. / divisible_by) * divisible_by)\n\n\nclass MobileBottleneck(nn.Module):\n    def __init__(self, inp, oup, kernel, stride, exp, se=False, nl='RE'):\n        super(MobileBottleneck, self).__init__()\n        assert stride in [1, 2]\n        assert kernel in [3, 5]\n        padding = (kernel - 1) // 2\n        self.use_res_connect = stride == 1 and inp == oup\n\n        conv_layer = nn.Conv2d\n        norm_layer = nn.BatchNorm2d\n        if nl == 'RE':\n            nlin_layer = nn.ReLU # or ReLU6\n        elif nl == 'HS':\n            nlin_layer = Hswish\n        else:\n            raise NotImplementedError\n        if se:\n            SELayer = SEModule\n        else:\n            SELayer = Identity\n\n        self.conv = nn.Sequential(\n            # pw\n            conv_layer(inp, exp, 1, 1, 0, bias=False),\n            norm_layer(exp),\n            nlin_layer(inplace=True),\n            # dw\n            conv_layer(exp, exp, kernel, stride, padding, groups=exp, bias=False),\n            norm_layer(exp),\n            SELayer(exp),\n            nlin_layer(inplace=True),\n            # pw-linear\n            conv_layer(exp, oup, 1, 1, 0, bias=False),\n            norm_layer(oup),\n        )\n\n    def forward(self, x):\n        if self.use_res_connect:\n            return x + self.conv(x)\n        else:\n            return self.conv(x)\n\n\nclass MobileNetV3(nn.Module):\n    def __init__(self, n_class=1000, input_size=224, dropout=0.8, mode='small', width_mult=1.0):\n        super(MobileNetV3, self).__init__()\n        input_channel = 16\n        last_channel = 1280\n        if mode == 'large':\n            # refer to Table 1 in paper\n            mobile_setting = [\n                # k, exp, c,  se,     nl,  s,\n                [3, 16,  16,  False, 'RE', 1],\n                [3, 64,  24,  False, 'RE', 2],\n                [3, 72,  24,  False, 'RE', 1],\n                [5, 72,  40,  True,  'RE', 2],\n                [5, 120, 40,  True,  'RE', 1],\n                [5, 120, 40,  True,  'RE', 1],\n                [3, 240, 80,  False, 'HS', 2],\n                [3, 200, 80,  False, 'HS', 1],\n                [3, 184, 80,  False, 'HS', 1],\n                [3, 184, 80,  False, 'HS', 1],\n                [3, 480, 112, True,  'HS', 1],\n                [3, 672, 112, True,  'HS', 1],\n                [5, 672, 160, True,  'HS', 2],\n                [5, 960, 160, True,  'HS', 1],\n                [5, 960, 160, True,  'HS', 1],\n            ]\n        elif mode == 'small':\n            # refer to Table 2 in paper\n            mobile_setting = [\n                # k, exp, c,  se,     nl,  s,\n                [3, 16,  16,  True,  'RE', 2],\n                [3, 72,  24,  False, 'RE', 2],\n                [3, 88,  24,  False, 'RE', 1],\n                [5, 96,  40,  True,  'HS', 2],\n                [5, 240, 40,  True,  'HS', 1],\n                [5, 240, 40,  True,  'HS', 1],\n                [5, 120, 48,  True,  'HS', 1],\n                [5, 144, 48,  True,  'HS', 1],\n                [5, 288, 96,  True,  'HS', 2],\n                [5, 576, 96,  True,  'HS', 1],\n                [5, 576, 96,  True,  'HS', 1],\n            ]\n        else:\n            raise NotImplementedError\n\n        # building first layer\n        assert input_size % 32 == 0\n        last_channel = make_divisible(last_channel * width_mult) if width_mult > 1.0 else last_channel\n        self.features = [conv_bn(3, input_channel, 2, nlin_layer=Hswish)]\n        self.classifier = []\n\n        # building mobile blocks\n        for k, exp, c, se, nl, s in mobile_setting:\n            output_channel = make_divisible(c * width_mult)\n            exp_channel = make_divisible(exp * width_mult)\n            self.features.append(MobileBottleneck(input_channel, output_channel, k, s, exp_channel, se, nl))\n            input_channel = output_channel\n\n        # building last several layers\n        if mode == 'large':\n            last_conv = make_divisible(960 * width_mult)\n            self.features.append(conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish))\n            self.features.append(nn.AdaptiveAvgPool2d(1))\n            self.features.append(nn.Conv2d(last_conv, last_channel, 1, 1, 0))\n            self.features.append(Hswish(inplace=True))\n        elif mode == 'small':\n            last_conv = make_divisible(576 * width_mult)\n            self.features.append(conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish))\n            # self.features.append(SEModule(last_conv))  # refer to paper Table2, but I think this is a mistake\n            self.features.append(nn.AdaptiveAvgPool2d(1))\n            self.features.append(nn.Conv2d(last_conv, last_channel, 1, 1, 0))\n            self.features.append(Hswish(inplace=True))\n        else:\n            raise NotImplementedError\n\n        # make it nn.Sequential\n        self.features = nn.Sequential(*self.features)\n\n        # building classifier\n        self.classifier = nn.Sequential(\n            nn.Dropout(p=dropout),    # refer to paper section 6\n            nn.Linear(last_channel, n_class),\n        )\n\n        self._initialize_weights()\n\n    def forward(self, x):\n        x = self.features(x)\n        x = x.mean(3).mean(2)\n        x = self.classifier(x)\n        return x\n\n    def _initialize_weights(self):\n        # weight initialization\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out')\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.ones_(m.weight)\n                nn.init.zeros_(m.bias)\n            elif isinstance(m, nn.Linear):\n                nn.init.normal_(m.weight, 0, 0.01)\n                if m.bias is not None:\n                    nn.init.zeros_(m.bias)\n\n\ndef mobilenetv3(pretrained=False, **kwargs):\n    model = MobileNetV3(**kwargs)\n    if pretrained:\n        state_dict = torch.load(CFG.pretrained)\n        model.load_state_dict(state_dict, strict=True)\n        # raise NotImplementedError\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = mobilenetv3(mode='small', pretrained=False)\nnet.classifier[1] = torch.nn.Linear(in_features=1280, out_features=1)\nnet = net.to(device)\n\nstate_dict = torch.load(CHECKPOINT)\nnet.load_state_dict(state_dict)\nnet.cuda()\nnet.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net2 = mobilenetv3(mode='small', pretrained=False)\nnet2.classifier[1] = torch.nn.Linear(in_features=1280, out_features=1)\nnet2 = net2.to(device)\n\nstate_dict = torch.load(CHECKPOINT2)\nnet2.load_state_dict(state_dict)\nnet2.cuda()\nnet2.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# EFFFICIENTNET\n\nnet3 = EfficientNet.from_name(\"efficientnet-b0\")\nnet3._fc = torch.nn.Linear(in_features=net3._fc.in_features, out_features=1)\nnet3.load_state_dict(torch.load(CHECKPOINT3))\nnet3.cuda()\nnet3.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# EFFFICIENTNET\n\nnet6 = EfficientNet.from_name(\"efficientnet-b1\")\nnet6._fc = torch.nn.Linear(in_features=net6._fc.in_features, out_features=1)\nnet6.load_state_dict(torch.load(CHECKPOINT6))\nnet6.cuda()\nnet6.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# XCEPTION\n\nclass SeparableConv2d(nn.Module):\n    def __init__(self,in_channels,out_channels,kernel_size=1,stride=1,padding=0,dilation=1,bias=False):\n        super(SeparableConv2d,self).__init__()\n\n        self.conv1 = nn.Conv2d(in_channels,in_channels,kernel_size,stride,padding,dilation,groups=in_channels,bias=bias)\n        self.pointwise = nn.Conv2d(in_channels,out_channels,1,1,0,1,1,bias=bias)\n\n    def forward(self,x):\n        x = self.conv1(x)\n        x = self.pointwise(x)\n        return x\n\n\nclass Block(nn.Module):\n    def __init__(self,in_filters,out_filters,reps,strides=1,start_with_relu=True,grow_first=True):\n        super(Block, self).__init__()\n\n        if out_filters != in_filters or strides!=1:\n            self.skip = nn.Conv2d(in_filters,out_filters,1,stride=strides, bias=False)\n            self.skipbn = nn.BatchNorm2d(out_filters)\n        else:\n            self.skip=None\n\n        rep=[]\n\n        filters=in_filters\n        if grow_first:\n            rep.append(nn.ReLU(inplace=True))\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n            filters = out_filters\n\n        for i in range(reps-1):\n            rep.append(nn.ReLU(inplace=True))\n            rep.append(SeparableConv2d(filters,filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(filters))\n\n        if not grow_first:\n            rep.append(nn.ReLU(inplace=True))\n            rep.append(SeparableConv2d(in_filters,out_filters,3,stride=1,padding=1,bias=False))\n            rep.append(nn.BatchNorm2d(out_filters))\n\n        if not start_with_relu:\n            rep = rep[1:]\n        else:\n            rep[0] = nn.ReLU(inplace=False)\n\n        if strides != 1:\n            rep.append(nn.MaxPool2d(3,strides,1))\n        self.rep = nn.Sequential(*rep)\n\n    def forward(self,inp):\n        x = self.rep(inp)\n\n        if self.skip is not None:\n            skip = self.skip(inp)\n            skip = self.skipbn(skip)\n        else:\n            skip = inp\n\n        x+=skip\n        return x\n\n\nclass Xception(nn.Module):\n    \"\"\"\n    Xception optimized for the ImageNet dataset, as specified in\n    https://arxiv.org/pdf/1610.02357.pdf\n    \"\"\"\n    def __init__(self, num_classes=1000):\n        \"\"\" Constructor\n        Args:\n            num_classes: number of classes\n        \"\"\"\n        super(Xception, self).__init__()\n        self.num_classes = num_classes\n\n        self.conv1 = nn.Conv2d(3, 32, 3,2, 0, bias=False)\n        self.bn1 = nn.BatchNorm2d(32)\n        self.relu1 = nn.ReLU(inplace=True)\n\n        self.conv2 = nn.Conv2d(32,64,3,bias=False)\n        self.bn2 = nn.BatchNorm2d(64)\n        self.relu2 = nn.ReLU(inplace=True)\n        #do relu here\n\n        self.block1=Block(64,128,2,2,start_with_relu=False,grow_first=True)\n        self.block2=Block(128,256,2,2,start_with_relu=True,grow_first=True)\n        self.block3=Block(256,728,2,2,start_with_relu=True,grow_first=True)\n\n        self.block4=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block5=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block6=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block7=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block8=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block9=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block10=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n        self.block11=Block(728,728,3,1,start_with_relu=True,grow_first=True)\n\n        self.block12=Block(728,1024,2,2,start_with_relu=True,grow_first=False)\n\n        self.conv3 = SeparableConv2d(1024,1536,3,1,1)\n        self.bn3 = nn.BatchNorm2d(1536)\n        self.relu3 = nn.ReLU(inplace=True)\n\n        #do relu here\n        self.conv4 = SeparableConv2d(1536,2048,3,1,1)\n        self.bn4 = nn.BatchNorm2d(2048)\n\n        self.fc = nn.Linear(2048, num_classes)\n\n        # #------- init weights --------\n        # for m in self.modules():\n        #     if isinstance(m, nn.Conv2d):\n        #         n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n        #         m.weight.data.normal_(0, math.sqrt(2. / n))\n        #     elif isinstance(m, nn.BatchNorm2d):\n        #         m.weight.data.fill_(1)\n        #         m.bias.data.zero_()\n        # #-----------------------------\n\n    def features(self, input):\n        x = self.conv1(input)\n        x = self.bn1(x)\n        x = self.relu1(x)\n\n        x = self.conv2(x)\n        x = self.bn2(x)\n        x = self.relu2(x)\n\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x)\n        x = self.block5(x)\n        x = self.block6(x)\n        x = self.block7(x)\n        x = self.block8(x)\n        x = self.block9(x)\n        x = self.block10(x)\n        x = self.block11(x)\n        x = self.block12(x)\n\n        x = self.conv3(x)\n        x = self.bn3(x)\n        x = self.relu3(x)\n\n        x = self.conv4(x)\n        x = self.bn4(x)\n        return x\n\n    def logits(self, features):\n        x = nn.ReLU(inplace=True)(features)\n\n        x = F.adaptive_avg_pool2d(x, (1, 1))\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, input):\n        x = self.features(input)\n        x = self.logits(x)\n        return x\n\n\ndef xception(num_classes=1000, pretrained='imagenet'):\n    model = Xception(num_classes=num_classes)\n    if pretrained:\n        settings = pretrained_settings['xception'][pretrained]\n        assert num_classes == settings['num_classes'], \\\n            \"num_classes should be {}, but is {}\".format(settings['num_classes'], num_classes)\n\n        model = Xception(num_classes=num_classes)\n        model.load_state_dict(model_zoo.load_url(settings['url']))\n\n        model.input_space = settings['input_space']\n        model.input_size = settings['input_size']\n        model.input_range = settings['input_range']\n        model.mean = settings['mean']\n        model.std = settings['std']\n\n    # TODO: ugly\n    model.last_linear = model.fc\n    del model.fc\n    return model\n\nnet4 = Xception()\n#net.load_state_dict(torch.load('F:/pretrained/xception-43020ad28.pth', map_location=torch.device(device)))\nnet4.last_linear = net4.fc\ndel net4.fc\nnet4.last_linear = nn.Linear(in_features=2048, out_features=1, bias=True)\nnet4.load_state_dict(torch.load(CHECKPOINT4))\nnet4 = net4.to(device)\nnet4.cuda()\nnet4.eval()\n\n\nnet10 = Xception()\n#net.load_state_dict(torch.load('F:/pretrained/xception-43020ad28.pth', map_location=torch.device(device)))\nnet10.last_linear = net10.fc\ndel net10.fc\nnet10.last_linear = nn.Linear(in_features=2048, out_features=1, bias=True)\nnet10.load_state_dict(torch.load(CHECKPOINT10))\nnet10 = net10.to(device)\nnet10.cuda()\nnet10.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# NASNET MOBILE\nclass MaxPoolPad(nn.Module):\n\n    def __init__(self):\n        super(MaxPoolPad, self).__init__()\n        self.pad = nn.ZeroPad2d((1, 0, 1, 0))\n        self.pool = nn.MaxPool2d(3, stride=2, padding=1)\n\n    def forward(self, x):\n        x = self.pad(x)\n        x = self.pool(x)\n        x = x[:, :, 1:, 1:].contiguous()\n        return x\n\n\nclass AvgPoolPad(nn.Module):\n\n    def __init__(self, stride=2, padding=1):\n        super(AvgPoolPad, self).__init__()\n        self.pad = nn.ZeroPad2d((1, 0, 1, 0))\n        self.pool = nn.AvgPool2d(3, stride=stride, padding=padding, count_include_pad=False)\n\n    def forward(self, x):\n        x = self.pad(x)\n        x = self.pool(x)\n        x = x[:, :, 1:, 1:].contiguous()\n        return x\n\n\nclass SeparableConv2d(nn.Module):\n\n    def __init__(self, in_channels, out_channels, dw_kernel, dw_stride, dw_padding, bias=False):\n        super(SeparableConv2d, self).__init__()\n        self.depthwise_conv2d = nn.Conv2d(in_channels, in_channels, dw_kernel,\n                                          stride=dw_stride,\n                                          padding=dw_padding,\n                                          bias=bias,\n                                          groups=in_channels)\n        self.pointwise_conv2d = nn.Conv2d(in_channels, out_channels, 1, stride=1, bias=bias)\n\n    def forward(self, x):\n        x = self.depthwise_conv2d(x)\n        x = self.pointwise_conv2d(x)\n        return x\n\n\nclass BranchSeparables(nn.Module):\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, name=None, bias=False):\n        super(BranchSeparables, self).__init__()\n        self.relu = nn.ReLU()\n        self.separable_1 = SeparableConv2d(in_channels, in_channels, kernel_size, stride, padding, bias=bias)\n        self.bn_sep_1 = nn.BatchNorm2d(in_channels, eps=0.001, momentum=0.1, affine=True)\n        self.relu1 = nn.ReLU()\n        self.separable_2 = SeparableConv2d(in_channels, out_channels, kernel_size, 1, padding, bias=bias)\n        self.bn_sep_2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)\n        self.name = name\n\n    def forward(self, x):\n        x = self.relu(x)\n        if self.name == 'specific':\n            x = nn.ZeroPad2d((1, 0, 1, 0))(x)\n        x = self.separable_1(x)\n        if self.name == 'specific':\n            x = x[:, :, 1:, 1:].contiguous()\n\n        x = self.bn_sep_1(x)\n        x = self.relu1(x)\n        x = self.separable_2(x)\n        x = self.bn_sep_2(x)\n        return x\n\n\nclass BranchSeparablesStem(nn.Module):\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, bias=False):\n        super(BranchSeparablesStem, self).__init__()\n        self.relu = nn.ReLU()\n        self.separable_1 = SeparableConv2d(in_channels, out_channels, kernel_size, stride, padding, bias=bias)\n        self.bn_sep_1 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)\n        self.relu1 = nn.ReLU()\n        self.separable_2 = SeparableConv2d(out_channels, out_channels, kernel_size, 1, padding, bias=bias)\n        self.bn_sep_2 = nn.BatchNorm2d(out_channels, eps=0.001, momentum=0.1, affine=True)\n\n    def forward(self, x):\n        x = self.relu(x)\n        x = self.separable_1(x)\n        x = self.bn_sep_1(x)\n        x = self.relu1(x)\n        x = self.separable_2(x)\n        x = self.bn_sep_2(x)\n        return x\n\n\nclass BranchSeparablesReduction(BranchSeparables):\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride, padding, z_padding=1, bias=False):\n        BranchSeparables.__init__(self, in_channels, out_channels, kernel_size, stride, padding, bias)\n        self.padding = nn.ZeroPad2d((z_padding, 0, z_padding, 0))\n\n    def forward(self, x):\n        x = self.relu(x)\n        x = self.padding(x)\n        x = self.separable_1(x)\n        x = x[:, :, 1:, 1:].contiguous()\n        x = self.bn_sep_1(x)\n        x = self.relu1(x)\n        x = self.separable_2(x)\n        x = self.bn_sep_2(x)\n        return x\n\n\nclass CellStem0(nn.Module):\n    def __init__(self, stem_filters, num_filters=42):\n        super(CellStem0, self).__init__()\n        self.num_filters = num_filters\n        self.stem_filters = stem_filters\n        self.conv_1x1 = nn.Sequential()\n        self.conv_1x1.add_module('relu', nn.ReLU())\n        self.conv_1x1.add_module('conv', nn.Conv2d(self.stem_filters, self.num_filters, 1, stride=1, bias=False))\n        self.conv_1x1.add_module('bn', nn.BatchNorm2d(self.num_filters, eps=0.001, momentum=0.1, affine=True))\n\n        self.comb_iter_0_left = BranchSeparables(self.num_filters, self.num_filters, 5, 2, 2)\n        self.comb_iter_0_right = BranchSeparablesStem(self.stem_filters, self.num_filters, 7, 2, 3, bias=False)\n\n        self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)\n        self.comb_iter_1_right = BranchSeparablesStem(self.stem_filters, self.num_filters, 7, 2, 3, bias=False)\n\n        self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)\n        self.comb_iter_2_right = BranchSeparablesStem(self.stem_filters, self.num_filters, 5, 2, 2, bias=False)\n\n        self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_4_left = BranchSeparables(self.num_filters, self.num_filters, 3, 1, 1, bias=False)\n        self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)\n\n    def forward(self, x):\n        x1 = self.conv_1x1(x)\n\n        x_comb_iter_0_left = self.comb_iter_0_left(x1)\n        x_comb_iter_0_right = self.comb_iter_0_right(x)\n        x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n        x_comb_iter_1_left = self.comb_iter_1_left(x1)\n        x_comb_iter_1_right = self.comb_iter_1_right(x)\n        x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n        x_comb_iter_2_left = self.comb_iter_2_left(x1)\n        x_comb_iter_2_right = self.comb_iter_2_right(x)\n        x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n        x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n        x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n        x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n        x_comb_iter_4_right = self.comb_iter_4_right(x1)\n        x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n        x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)\n        return x_out\n\n\nclass CellStem1(nn.Module):\n\n    def __init__(self, stem_filters, num_filters):\n        super(CellStem1, self).__init__()\n        self.num_filters = num_filters\n        self.stem_filters = stem_filters\n        self.conv_1x1 = nn.Sequential()\n        self.conv_1x1.add_module('relu', nn.ReLU())\n        self.conv_1x1.add_module('conv', nn.Conv2d(2*self.num_filters, self.num_filters, 1, stride=1, bias=False))\n        self.conv_1x1.add_module('bn', nn.BatchNorm2d(self.num_filters, eps=0.001, momentum=0.1, affine=True))\n\n        self.relu = nn.ReLU()\n        self.path_1 = nn.Sequential()\n        self.path_1.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))\n        self.path_1.add_module('conv', nn.Conv2d(self.stem_filters, self.num_filters//2, 1, stride=1, bias=False))\n        self.path_2 = nn.ModuleList()\n        self.path_2.add_module('pad', nn.ZeroPad2d((0, 1, 0, 1)))\n        self.path_2.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))\n        self.path_2.add_module('conv', nn.Conv2d(self.stem_filters, self.num_filters//2, 1, stride=1, bias=False))\n\n        self.final_path_bn = nn.BatchNorm2d(self.num_filters, eps=0.001, momentum=0.1, affine=True)\n\n        self.comb_iter_0_left = BranchSeparables(self.num_filters, self.num_filters, 5, 2, 2, name='specific', bias=False)\n        self.comb_iter_0_right = BranchSeparables(self.num_filters, self.num_filters, 7, 2, 3, name='specific', bias=False)\n\n        # self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)\n        self.comb_iter_1_left = MaxPoolPad()\n        self.comb_iter_1_right = BranchSeparables(self.num_filters, self.num_filters, 7, 2, 3, name='specific', bias=False)\n\n        # self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)\n        self.comb_iter_2_left = AvgPoolPad()\n        self.comb_iter_2_right = BranchSeparables(self.num_filters, self.num_filters, 5, 2, 2, name='specific', bias=False)\n\n        self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_4_left = BranchSeparables(self.num_filters, self.num_filters, 3, 1, 1, name='specific', bias=False)\n        # self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)\n        self.comb_iter_4_right = MaxPoolPad()\n\n    def forward(self, x_conv0, x_stem_0):\n        x_left = self.conv_1x1(x_stem_0)\n\n        x_relu = self.relu(x_conv0)\n        # path 1\n        x_path1 = self.path_1(x_relu)\n        # path 2\n        x_path2 = self.path_2.pad(x_relu)\n        x_path2 = x_path2[:, :, 1:, 1:]\n        x_path2 = self.path_2.avgpool(x_path2)\n        x_path2 = self.path_2.conv(x_path2)\n        # final path\n        x_right = self.final_path_bn(torch.cat([x_path1, x_path2], 1))\n\n        x_comb_iter_0_left = self.comb_iter_0_left(x_left)\n        x_comb_iter_0_right = self.comb_iter_0_right(x_right)\n        x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n        x_comb_iter_1_left = self.comb_iter_1_left(x_left)\n        x_comb_iter_1_right = self.comb_iter_1_right(x_right)\n        x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n        x_comb_iter_2_left = self.comb_iter_2_left(x_left)\n        x_comb_iter_2_right = self.comb_iter_2_right(x_right)\n        x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n        x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n        x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n        x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n        x_comb_iter_4_right = self.comb_iter_4_right(x_left)\n        x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n        x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)\n        return x_out\n\n\nclass FirstCell(nn.Module):\n\n    def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):\n        super(FirstCell, self).__init__()\n        self.conv_1x1 = nn.Sequential()\n        self.conv_1x1.add_module('relu', nn.ReLU())\n        self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))\n        self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))\n\n        self.relu = nn.ReLU()\n        self.path_1 = nn.Sequential()\n        self.path_1.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))\n        self.path_1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))\n        self.path_2 = nn.ModuleList()\n        self.path_2.add_module('pad', nn.ZeroPad2d((0, 1, 0, 1)))\n        self.path_2.add_module('avgpool', nn.AvgPool2d(1, stride=2, count_include_pad=False))\n        self.path_2.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))\n\n        self.final_path_bn = nn.BatchNorm2d(out_channels_left * 2, eps=0.001, momentum=0.1, affine=True)\n\n        self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)\n        self.comb_iter_0_right = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)\n\n        self.comb_iter_1_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)\n        self.comb_iter_1_right = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)\n\n        self.comb_iter_2_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_3_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n        self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)\n\n    def forward(self, x, x_prev):\n        x_relu = self.relu(x_prev)\n        # path 1\n        x_path1 = self.path_1(x_relu)\n        # path 2\n        x_path2 = self.path_2.pad(x_relu)\n        x_path2 = x_path2[:, :, 1:, 1:]\n        x_path2 = self.path_2.avgpool(x_path2)\n        x_path2 = self.path_2.conv(x_path2)\n        # final path\n        x_left = self.final_path_bn(torch.cat([x_path1, x_path2], 1))\n\n        x_right = self.conv_1x1(x)\n\n        x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n        x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n        x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n        x_comb_iter_1_left = self.comb_iter_1_left(x_left)\n        x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n        x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n        x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n        x_comb_iter_2 = x_comb_iter_2_left + x_left\n\n        x_comb_iter_3_left = self.comb_iter_3_left(x_left)\n        x_comb_iter_3_right = self.comb_iter_3_right(x_left)\n        x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right\n\n        x_comb_iter_4_left = self.comb_iter_4_left(x_right)\n        x_comb_iter_4 = x_comb_iter_4_left + x_right\n\n        x_out = torch.cat([x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)\n        return x_out\n\n\nclass NormalCell(nn.Module):\n\n    def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):\n        super(NormalCell, self).__init__()\n        self.conv_prev_1x1 = nn.Sequential()\n        self.conv_prev_1x1.add_module('relu', nn.ReLU())\n        self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))\n        self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))\n\n        self.conv_1x1 = nn.Sequential()\n        self.conv_1x1.add_module('relu', nn.ReLU())\n        self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))\n        self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))\n\n        self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 1, 2, bias=False)\n        self.comb_iter_0_right = BranchSeparables(out_channels_left, out_channels_left, 3, 1, 1, bias=False)\n\n        self.comb_iter_1_left = BranchSeparables(out_channels_left, out_channels_left, 5, 1, 2, bias=False)\n        self.comb_iter_1_right = BranchSeparables(out_channels_left, out_channels_left, 3, 1, 1, bias=False)\n\n        self.comb_iter_2_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_3_left = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n        self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, bias=False)\n\n    def forward(self, x, x_prev):\n        x_left = self.conv_prev_1x1(x_prev)\n        x_right = self.conv_1x1(x)\n\n        x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n        x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n        x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n        x_comb_iter_1_left = self.comb_iter_1_left(x_left)\n        x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n        x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n        x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n        x_comb_iter_2 = x_comb_iter_2_left + x_left\n\n        x_comb_iter_3_left = self.comb_iter_3_left(x_left)\n        x_comb_iter_3_right = self.comb_iter_3_right(x_left)\n        x_comb_iter_3 = x_comb_iter_3_left + x_comb_iter_3_right\n\n        x_comb_iter_4_left = self.comb_iter_4_left(x_right)\n        x_comb_iter_4 = x_comb_iter_4_left + x_right\n\n        x_out = torch.cat([x_left, x_comb_iter_0, x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)\n        return x_out\n\n\nclass ReductionCell0(nn.Module):\n\n    def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):\n        super(ReductionCell0, self).__init__()\n        self.conv_prev_1x1 = nn.Sequential()\n        self.conv_prev_1x1.add_module('relu', nn.ReLU())\n        self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))\n        self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))\n\n        self.conv_1x1 = nn.Sequential()\n        self.conv_1x1.add_module('relu', nn.ReLU())\n        self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))\n        self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))\n\n        self.comb_iter_0_left = BranchSeparablesReduction(out_channels_right, out_channels_right, 5, 2, 2, bias=False)\n        self.comb_iter_0_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 7, 2, 3, bias=False)\n\n        self.comb_iter_1_left = MaxPoolPad()\n        self.comb_iter_1_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 7, 2, 3, bias=False)\n\n        self.comb_iter_2_left = AvgPoolPad()\n        self.comb_iter_2_right = BranchSeparablesReduction(out_channels_right, out_channels_right, 5, 2, 2, bias=False)\n\n        self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_4_left = BranchSeparablesReduction(out_channels_right, out_channels_right, 3, 1, 1, bias=False)\n        self.comb_iter_4_right = MaxPoolPad()\n\n    def forward(self, x, x_prev):\n        x_left = self.conv_prev_1x1(x_prev)\n        x_right = self.conv_1x1(x)\n\n        x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n        x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n        x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n        x_comb_iter_1_left = self.comb_iter_1_left(x_right)\n        x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n        x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n        x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n        x_comb_iter_2_right = self.comb_iter_2_right(x_left)\n        x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n        x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n        x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n        x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n        x_comb_iter_4_right = self.comb_iter_4_right(x_right)\n        x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n        x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)\n        return x_out\n\n\nclass ReductionCell1(nn.Module):\n\n    def __init__(self, in_channels_left, out_channels_left, in_channels_right, out_channels_right):\n        super(ReductionCell1, self).__init__()\n        self.conv_prev_1x1 = nn.Sequential()\n        self.conv_prev_1x1.add_module('relu', nn.ReLU())\n        self.conv_prev_1x1.add_module('conv', nn.Conv2d(in_channels_left, out_channels_left, 1, stride=1, bias=False))\n        self.conv_prev_1x1.add_module('bn', nn.BatchNorm2d(out_channels_left, eps=0.001, momentum=0.1, affine=True))\n\n        self.conv_1x1 = nn.Sequential()\n        self.conv_1x1.add_module('relu', nn.ReLU())\n        self.conv_1x1.add_module('conv', nn.Conv2d(in_channels_right, out_channels_right, 1, stride=1, bias=False))\n        self.conv_1x1.add_module('bn', nn.BatchNorm2d(out_channels_right, eps=0.001, momentum=0.1, affine=True))\n\n        self.comb_iter_0_left = BranchSeparables(out_channels_right, out_channels_right, 5, 2, 2, name='specific', bias=False)\n        self.comb_iter_0_right = BranchSeparables(out_channels_right, out_channels_right, 7, 2, 3, name='specific', bias=False)\n\n        # self.comb_iter_1_left = nn.MaxPool2d(3, stride=2, padding=1)\n        self.comb_iter_1_left = MaxPoolPad()\n        self.comb_iter_1_right = BranchSeparables(out_channels_right, out_channels_right, 7, 2, 3, name='specific', bias=False)\n\n        # self.comb_iter_2_left = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)\n        self.comb_iter_2_left = AvgPoolPad()\n        self.comb_iter_2_right = BranchSeparables(out_channels_right, out_channels_right, 5, 2, 2, name='specific', bias=False)\n\n        self.comb_iter_3_right = nn.AvgPool2d(3, stride=1, padding=1, count_include_pad=False)\n\n        self.comb_iter_4_left = BranchSeparables(out_channels_right, out_channels_right, 3, 1, 1, name='specific', bias=False)\n        # self.comb_iter_4_right = nn.MaxPool2d(3, stride=2, padding=1)\n        self.comb_iter_4_right =MaxPoolPad()\n\n    def forward(self, x, x_prev):\n        x_left = self.conv_prev_1x1(x_prev)\n        x_right = self.conv_1x1(x)\n\n        x_comb_iter_0_left = self.comb_iter_0_left(x_right)\n        x_comb_iter_0_right = self.comb_iter_0_right(x_left)\n        x_comb_iter_0 = x_comb_iter_0_left + x_comb_iter_0_right\n\n        x_comb_iter_1_left = self.comb_iter_1_left(x_right)\n        x_comb_iter_1_right = self.comb_iter_1_right(x_left)\n        x_comb_iter_1 = x_comb_iter_1_left + x_comb_iter_1_right\n\n        x_comb_iter_2_left = self.comb_iter_2_left(x_right)\n        x_comb_iter_2_right = self.comb_iter_2_right(x_left)\n        x_comb_iter_2 = x_comb_iter_2_left + x_comb_iter_2_right\n\n        x_comb_iter_3_right = self.comb_iter_3_right(x_comb_iter_0)\n        x_comb_iter_3 = x_comb_iter_3_right + x_comb_iter_1\n\n        x_comb_iter_4_left = self.comb_iter_4_left(x_comb_iter_0)\n        x_comb_iter_4_right = self.comb_iter_4_right(x_right)\n        x_comb_iter_4 = x_comb_iter_4_left + x_comb_iter_4_right\n\n        x_out = torch.cat([x_comb_iter_1, x_comb_iter_2, x_comb_iter_3, x_comb_iter_4], 1)\n        return x_out\n\n\nclass NASNetAMobile(nn.Module):\n    \"\"\"NASNetAMobile (4 @ 1056) \"\"\"\n\n    def __init__(self, num_classes=1000, stem_filters=32, penultimate_filters=1056, filters_multiplier=2):\n        super(NASNetAMobile, self).__init__()\n        self.num_classes = num_classes\n        self.stem_filters = stem_filters\n        self.penultimate_filters = penultimate_filters\n        self.filters_multiplier = filters_multiplier\n\n        filters = self.penultimate_filters // 24\n        # 24 is default value for the architecture\n\n        self.conv0 = nn.Sequential()\n        self.conv0.add_module('conv', nn.Conv2d(in_channels=3, out_channels=self.stem_filters, kernel_size=3, padding=0, stride=2,\n                                                bias=False))\n        self.conv0.add_module('bn', nn.BatchNorm2d(self.stem_filters, eps=0.001, momentum=0.1, affine=True))\n\n        self.cell_stem_0 = CellStem0(self.stem_filters, num_filters=filters // (filters_multiplier ** 2))\n        self.cell_stem_1 = CellStem1(self.stem_filters, num_filters=filters // filters_multiplier)\n\n        self.cell_0 = FirstCell(in_channels_left=filters, out_channels_left=filters//2, # 1, 0.5\n                                in_channels_right=2*filters, out_channels_right=filters) # 2, 1\n        self.cell_1 = NormalCell(in_channels_left=2*filters, out_channels_left=filters, # 2, 1\n                                 in_channels_right=6*filters, out_channels_right=filters) # 6, 1\n        self.cell_2 = NormalCell(in_channels_left=6*filters, out_channels_left=filters, # 6, 1\n                                 in_channels_right=6*filters, out_channels_right=filters) # 6, 1\n        self.cell_3 = NormalCell(in_channels_left=6*filters, out_channels_left=filters, # 6, 1\n                                 in_channels_right=6*filters, out_channels_right=filters) # 6, 1\n\n        self.reduction_cell_0 = ReductionCell0(in_channels_left=6*filters, out_channels_left=2*filters, # 6, 2\n                                               in_channels_right=6*filters, out_channels_right=2*filters) # 6, 2\n\n        self.cell_6 = FirstCell(in_channels_left=6*filters, out_channels_left=filters, # 6, 1\n                                in_channels_right=8*filters, out_channels_right=2*filters) # 8, 2\n        self.cell_7 = NormalCell(in_channels_left=8*filters, out_channels_left=2*filters, # 8, 2\n                                 in_channels_right=12*filters, out_channels_right=2*filters) # 12, 2\n        self.cell_8 = NormalCell(in_channels_left=12*filters, out_channels_left=2*filters, # 12, 2\n                                 in_channels_right=12*filters, out_channels_right=2*filters) # 12, 2\n        self.cell_9 = NormalCell(in_channels_left=12*filters, out_channels_left=2*filters, # 12, 2\n                                 in_channels_right=12*filters, out_channels_right=2*filters) # 12, 2\n\n        self.reduction_cell_1 = ReductionCell1(in_channels_left=12*filters, out_channels_left=4*filters, # 12, 4\n                                               in_channels_right=12*filters, out_channels_right=4*filters) # 12, 4\n\n        self.cell_12 = FirstCell(in_channels_left=12*filters, out_channels_left=2*filters, # 12, 2\n                                 in_channels_right=16*filters, out_channels_right=4*filters) # 16, 4\n        self.cell_13 = NormalCell(in_channels_left=16*filters, out_channels_left=4*filters, # 16, 4\n                                  in_channels_right=24*filters, out_channels_right=4*filters) # 24, 4\n        self.cell_14 = NormalCell(in_channels_left=24*filters, out_channels_left=4*filters, # 24, 4\n                                  in_channels_right=24*filters, out_channels_right=4*filters) # 24, 4\n        self.cell_15 = NormalCell(in_channels_left=24*filters, out_channels_left=4*filters, # 24, 4\n                                  in_channels_right=24*filters, out_channels_right=4*filters) # 24, 4\n\n        self.relu = nn.ReLU()\n        self.avg_pool = nn.AvgPool2d(7, stride=1, padding=0)\n        self.dropout = nn.Dropout()\n        self.last_linear = nn.Linear(24*filters, self.num_classes)\n\n    def features(self, input):\n        x_conv0 = self.conv0(input)\n        x_stem_0 = self.cell_stem_0(x_conv0)\n        x_stem_1 = self.cell_stem_1(x_conv0, x_stem_0)\n\n        x_cell_0 = self.cell_0(x_stem_1, x_stem_0)\n        x_cell_1 = self.cell_1(x_cell_0, x_stem_1)\n        x_cell_2 = self.cell_2(x_cell_1, x_cell_0)\n        x_cell_3 = self.cell_3(x_cell_2, x_cell_1)\n\n        x_reduction_cell_0 = self.reduction_cell_0(x_cell_3, x_cell_2)\n\n        x_cell_6 = self.cell_6(x_reduction_cell_0, x_cell_3)\n        x_cell_7 = self.cell_7(x_cell_6, x_reduction_cell_0)\n        x_cell_8 = self.cell_8(x_cell_7, x_cell_6)\n        x_cell_9 = self.cell_9(x_cell_8, x_cell_7)\n\n        x_reduction_cell_1 = self.reduction_cell_1(x_cell_9, x_cell_8)\n\n        x_cell_12 = self.cell_12(x_reduction_cell_1, x_cell_9)\n        x_cell_13 = self.cell_13(x_cell_12, x_reduction_cell_1)\n        x_cell_14 = self.cell_14(x_cell_13, x_cell_12)\n        x_cell_15 = self.cell_15(x_cell_14, x_cell_13)\n        return x_cell_15\n\n    def logits(self, features):\n        x = self.relu(features)\n        x = self.avg_pool(x)\n        x = x.view(x.size(0), -1)\n        x = self.dropout(x)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, input):\n        x = self.features(input)\n        x = self.logits(x)\n        return x\n    \nnet5 = NASNetAMobile()\nnet5.last_linear = nn.Linear(in_features=1056, out_features=1, bias=True)    \nnet5.load_state_dict(torch.load(CHECKPOINT5))\nnet5 = net5.to(device)\nnet5.cuda()\nnet5.eval()\n\n\nnet9 = NASNetAMobile()\nnet9.last_linear = nn.Linear(in_features=1056, out_features=1, bias=True)    \nnet9.load_state_dict(torch.load(CHECKPOINT9))\nnet9 = net9.to(device)\nnet9.cuda()\nnet9.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CFG:\n    seq_len=10\n    lstm_in = 16\n    lstm_out = 16\n\nclass LSTM_Model(nn.Module):\n    def __init__(self):\n        super(LSTM_Model, self).__init__()\n        self.cnn_net = mobilenetv3(mode='small', pretrained=False)\n        self.cnn_net.classifier[1] = nn.Linear(in_features=1280, out_features=1)  \n        self.cnn_net.classifier[1] = nn.Linear(in_features=1280, out_features=CFG.lstm_in) \n        self.lstm = nn.LSTM(CFG.lstm_in, CFG.lstm_out, bidirectional=False, batch_first=True)\n        self.reg_layer = nn.Sequential(nn.Dropout(0.5),\n                                       nn.Linear(CFG.lstm_out, 1) )        \n        \n    def forward(self, x):  # x: n_samples x seq_len x 3 x 224 x 224     \n        n_samples = x.shape[0]\n        x = x.view(-1, 3, 224, 224)\n        x = self.cnn_net(x) # shape now: (n_samples x seq_len) x CFG.final_layer_size = (3x15)x24 = 45x24\n        x = x.view(n_samples, CFG.seq_len, -1)\n        x, (h_n, h_c) = self.lstm(x)   # x = seq_len, batch, num_directions*hidden_size  \n        x = x[:,-1]  # Take last time step, which will be BATCH_SIZE * (2*HIDDEN) -> e.g. 12 * 128\n        y = self.reg_layer(x)\n        return y    \n    \nnet7 = LSTM_Model()\nnet7.load_state_dict(torch.load(CHECKPOINT7))\nnet7 = net7.to(device)\nnet7.cuda()\nnet7.eval()\n\nnet8 = LSTM_Model()\nnet8.load_state_dict(torch.load(CHECKPOINT8))\nnet8 = net8.to(device)\nnet8.cuda()\nnet8.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class Audio_LSTM_Model(nn.Module):\n#     def __init__(self):\n#         super(Audio_LSTM_Model, self).__init__()\n#         self.cnn_net = mobilenetv3(mode='small', pretrained=False)\n#         self.cnn_net.classifier[1] = nn.Linear(in_features=1280, out_features=1)  \n#         self.cnn_net.classifier[1] = nn.Linear(in_features=1280, out_features=CFG.lstm_in) \n#         self.lstm = nn.LSTM(CFG.lstm_in, CFG.lstm_out, bidirectional=False, batch_first=True)\n#         self.reg_layer = nn.Sequential(nn.Dropout(0.5),\n#                                        nn.Linear(CFG.lstm_out+62, 1) )        \n        \n#     def forward(self, x, au):  # x: n_samples x seq_len x 3 x 224 x 224\n# #         print('aushape',au.shape)\n#         n_samples = x.shape[0]\n#         x = x.view(-1, 3, 224, 224)\n# #         print('x0',x.shape)\n#         x = self.cnn_net(x) # shape now: (n_samples x seq_len) x CFG.final_layer_size = (3x15)x24 = 45x24\n# #         print('x1',x.shape)\n#         x = x.view(n_samples, CFG.seq_len, -1)\n# #         print('x2',x.shape)\n#         x, (h_n, h_c) = self.lstm(x)   # x = seq_len, batch, num_directions*hidden_size\n# #         print('x3',x.shape)\n#         x = x[:,-1]  # Take last time step, which will be BATCH_SIZE * (2*HIDDEN) -> e.g. 12 * 128\n# #         print('x4',x.shape) # 32 x 16\n#         x = torch.cat((x, au), 1)\n# #         print('x5',x.shape)\n#         y = self.reg_layer(x)\n# #         print('x6',y.shape)\n#         return y    \n\n# net8 = Audio_LSTM_Model()\n# net8.load_state_dict(torch.load(CHECKPOINT8))\n# net8 = net8.to(device)\n# net8.cuda()\n# net8.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net11 = torchvision.models.resnet18(pretrained=False)\nnet11.fc = nn.Linear(in_features=512, out_features=1, bias=True)\nnet11.load_state_dict(torch.load(CHECKPOINT11))\nnet11 = net11.to(device)\nnet11.cuda()\nnet11.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations\nfrom albumentations.augmentations.transforms import ShiftScaleRotate, HorizontalFlip, RandomBrightnessContrast, MotionBlur, Blur, GaussNoise, JpegCompression\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def isotropically_resize_image(img, size, resample=cv2.INTER_AREA):\n    h, w = img.shape[:2]\n    if w > h:\n        h = h * size // w\n        w = size\n    else:\n        w = w * size // h\n        h = size\n\n    resized = cv2.resize(img, (w, h), interpolation=resample)\n    return resized\n\n\ndef make_square_image(img):\n    h, w = img.shape[:2]\n    size = max(h, w)\n    t = (size - h)//2\n    b = (size - h) - (size - h)//2\n    l = (size - w)//2\n    r = (size - w) - (size-w)//2\n    return cv2.copyMakeBorder(img, t, b, l, r, cv2.BORDER_CONSTANT, value=0)\n\n\ndef correct_box_of_one_video(boxes_of_one_video, probs_of_one_video):\n    B = boxes_of_one_video\n    P = probs_of_one_video\n    recent_cluster = []\n    new_B = []\n\n    for i, (boxes, probs) in enumerate(zip(B, P)):\n        if len(boxes) == 0: \n            new_B.append([])\n            continue\n            \n        boxes, probs = np.array(boxes), np.array(probs)\n        boxes = boxes[probs>0.95].tolist()\n        probs = probs[probs>0.95].tolist()\n        \n        if len(boxes) == 0: \n            new_B.append([])\n            continue\n        \n        if len(boxes)!=len(probs): \n            new_B.append([])\n            continue\n\n        boxes = [[item if item>=0 else 0 for item in box] for box in boxes]\n        \n        if len(recent_cluster) == 0: # if the first frame with multiple faces, select the best one\n            best = np.argmax(probs)\n            recent_cluster = boxes[best] \n            new_B.append(boxes[best])\n            continue\n\n        else:\n            best_overlap = 0\n            best_m = -1\n\n            for m, b in enumerate(boxes):\n                iou = IoU(b, recent_cluster)\n                if iou > 0.2: \n                    if iou > best_overlap:\n                        best_m = m\n                        best_overlap = iou\n                        \n            if best_m != -1: \n                new_B.append(boxes[best_m])\n                recent_cluster = boxes[best_m]\n            else: \n                new_B.append([])\n    final_boxes, start_, end_ = rectify_boxes(new_B)\n    return final_boxes, start_, end_             \n\n\ndef rectify_boxes(boxes):\n    \n    start_frame = None\n    for i, b in enumerate(boxes):\n        # Determine proper video start\n        if len(b)!=0:\n            start_frame = i\n            break\n\n    end_frame = None\n    for i, b in enumerate(reversed(boxes)):\n        # Determine proper video end\n        if len(b)!=0:\n            end_frame = len(boxes)-i-1\n            break    \n\n    if start_frame is None or end_frame is None or start_frame==end_frame: \n        boxes = None            \n    else:\n        boxes = boxes[start_frame:end_frame+1]\n\n        for idx in range(len(boxes)):\n            box = boxes[idx]\n            if len(box)!=0: continue\n\n            idx_next_final = -1\n            idx_prev_final = -1\n\n            for idx_next in range(idx+1, len(boxes)): # search for next idx\n                if idx_next < len(boxes):\n                    if len(boxes[idx_next])!=0: \n                        idx_next_final = idx_next\n                        break\n\n            for idx_prev in reversed(range(idx)): # search for prev idx\n                if idx_prev >= 0:\n                    if  len(boxes[idx_prev])!=0: \n                        idx_prev_final = idx_prev\n                        break\n\n            if idx_next_final != -1 and idx_prev_final == -1: \n                boxes[idx] = boxes[idx_next_final]\n\n            elif idx_next_final == -1 and idx_prev_final != -1: \n                boxes[idx] = boxes[idx_prev_final]\n\n            elif idx_next_final != -1 and idx_prev_final != -1: \n                boxes[idx] = regress_box(boxes, idx, idx_prev_final, idx_next_final)\n\n            elif idx_next_final == -1 and idx_prev_final == -1: pass\n\n    return boxes, start_frame, end_frame\n\n    \ndef regress_box(b, i, prev_i, next_i):\n    w1, w2 = abs(i-prev_i), abs(i-next_i)\n    new_box = (np.array(b[prev_i])*w2 + np.array(b[next_i])*w1 ) / (w1+w2)\n    new_box = new_box.astype(int).tolist()\n    return new_box\n\n\ndef IoU(boxA, boxB):\n    xA = max(boxA[0], boxB[0])\n    yA = max(boxA[1], boxB[1])\n    xB = min(boxA[2], boxB[2])\n    yB = min(boxA[3], boxB[3])\n    interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)\n    boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)\n    boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)\n    iou = interArea / float(boxAArea + boxBArea - interArea)\n    return iou\n\ndef incre_counter(counter, idx):\n    if idx not in counter.keys(): counter[idx] = 1\n    else: counter[idx] += 1\n    return counter\n\ndef get_len_cluster(b):\n    return sum([len(item) for item in b])\n\nMAX_FRAMES = 30\n\naugment_ = albumentations.Compose([ShiftScaleRotate(p=0.2, scale_limit=0.1, border_mode=1, rotate_limit=10), # bad\n                                        RandomBrightnessContrast(p=0.25, brightness_limit=0.1, contrast_limit=0.1),\n                                        GaussNoise(p=.1),\n                                        JpegCompression(p=.1, quality_lower=50)])\n\ndef get_augment(img):\n    res = augment_(image=img)\n    img = res['image']\n    return img\n\ndef extract_frames(filepath):\n    v_cap = cv2.VideoCapture(filepath)\n    v_len = int(v_cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    frame_skip = 1\n    this_video_boxes = []\n    this_video_probs = []\n    \n    # Get frames and boxes\n    frames = []\n    frames_RGB = []\n    count_frame_for_mtcnn_batch = 0\n    if v_len > 0:\n        frame_skip = int(np.round(v_len/MAX_FRAMES))\n#         print('vlen', v_len, 'frame_skip', frame_skip)\n        for j in range(v_len):\n            ret = v_cap.grab()\n            if j % frame_skip != 0: continue\n\n            if not ret: continue\n\n            ret, frame = v_cap.retrieve()\n            if not ret or frame is None: continue\n            \n            if CONVERT_RGB: frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n            frames_RGB.append(frame)\n            frame = Image.fromarray(frame)\n            frames.append(frame)\n            count_frame_for_mtcnn_batch += 1\n            \n            if count_frame_for_mtcnn_batch == MTCNN_BATCH or j == v_len - 1:\n                boxes, probs = fast_mtcnn(frames)\n                this_video_boxes.extend(boxes)\n                this_video_probs.extend(probs)\n                count_frame_for_mtcnn_batch = 0\n                frames = []\n\n    v_cap.release()\n\n    boxes, start_, end_ = correct_box_of_one_video(this_video_boxes, this_video_probs)\n    \n    # Final step: build faces\n    if boxes is not None:\n        frames_RGB = frames_RGB[start_:end_+1]\n        for j, (frame, box) in enumerate(zip(frames_RGB, boxes)):\n            \n            ### Box1: with blackborder padding\n            box = [int(item/SCALE) for item in box]\n            face_ori = frame[box[1]:box[3], box[0]:box[2], :]\n            face_ori = make_square_image(isotropically_resize_image(face_ori, IMG_SIZE))\n            face_ori = get_augment(face_ori)\n            \n\n            ########\n    \n            ### Box2: no blackborder\n            box2 = box\n            dim1, dim2 = abs(box2[1]-box2[3]), abs(box2[0]-box2[2])\n            gap = abs(dim1 - dim2)\n            if dim1 < dim2:\n                box2[1] -= gap//2\n                box2[3] += gap//2\n            elif dim1 > dim2:\n                box2[0] -= gap//2\n                box2[2] += gap//2                  \n            box2 = [item if item >= 0 else 0 for item in box2]\n            face_ori_sqr = frame[box2[1]:box2[3], box2[0]:box2[2], :]\n            face_ori_sqr = cv2.resize(face_ori_sqr, (IMG_SIZE, IMG_SIZE))\n            face_lstm_ori = face_ori_sqr.copy()\n            face_ori_sqr = get_augment(face_ori_sqr)\n            face_ori_299_sqr = cv2.resize(face_ori_sqr, (299, 299))\n            face_ori_299_sqr = get_augment(face_ori_299_sqr)\n            #####\n            \n            ### Make batch imgs for frame models\n            for k in range(2):\n                face = face_ori\n                face_sqr = face_ori_sqr\n                face_299_sqr = face_ori_299_sqr\n                face_lstm = face_lstm_ori\n                \n                if k==1: \n                    face = cv2.flip(face, 1)\n                    face_sqr = cv2.flip(face_sqr, 1)\n                    face_299_sqr = cv2.flip(face_299_sqr, 1)\n                    face_lstm = cv2.flip(face_lstm, 1)\n\n#                 plt.figure()\n#                 plt.imshow(face_299_sqr)\n                \n                face = torch.tensor(face).permute((2, 0, 1)).float().div(255.)\n                face_sqr = torch.tensor(face_sqr).permute((2, 0, 1)).float().div(255.)\n                face_299_sqr = torch.tensor(face_299_sqr).permute((2, 0, 1)).float().div(255.)\n                face_lstm = torch.tensor(face_lstm).permute((2, 0, 1)).float().div(255.)\n                \n                normalize_transform = Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n                face = normalize_transform(face)\n                face_sqr = normalize_transform(face_sqr)\n                face_299_sqr = normalize_transform(face_299_sqr)\n                face_lstm = normalize_transform(face_lstm)\n\n                if (j == 0) and (k==0): \n                    faces = face.unsqueeze(0)\n                    faces_sqr = face_sqr.unsqueeze(0)\n                    faces_299_sqr = face_299_sqr.unsqueeze(0)\n                    faces_lstm = face_lstm.unsqueeze(0)\n                else: \n                    faces = torch.cat((faces, face.unsqueeze(0)), 0)\n                    faces_sqr = torch.cat((faces_sqr, face_sqr.unsqueeze(0)), 0)\n                    faces_299_sqr = torch.cat((faces_299_sqr, face_299_sqr.unsqueeze(0)), 0)\n                    faces_lstm = torch.cat((faces_lstm, face_lstm.unsqueeze(0)), 0)\n    \n    else: faces, face_sqr, faces_299_sqr, faces_lstm = None, None, None, None\n    return faces, faces_sqr, faces_299_sqr, faces_lstm, frame_skip\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CHECKPOINT = '/kaggle/input/kha-deepfake-dataset/checkpoint_mobilev3_alldata_1903_withfaceforensics_3epochs_.pth' # blackborder, all data\n# CHECKPOINT2 = '/kaggle/input/kha-deepfake-dataset/cpt_mbn_sqrimg_2503)2epochs_.pth' # square, all data\n# CHECKPOINT3 = '/kaggle/input/kha-deepfake-dataset/checkpoint_b0_1803_0epochs_0.4498354019969702.pth' # square, less data (only old DFDC)\n# CHECKPOINT4 = '/kaggle/input/kha-deepfake-dataset/cpt_xception_sqrimg_25031_epochs_0_moment.pth' # square, all data\n# CHECKPOINT5 = '/kaggle/input/kha-deepfake-dataset/cpt_nasnet_bbd_29032_epochs_0_moment.pth' # black border\n# CHECKPOINT6 = '/kaggle/input/kha-deepfake-dataset/cpt_effb1_sqrimg_25032_epochs_1_moment.pth' # black border\n\ndef predict_on_video(model, model2, model3, model4, model5, model6, model7, model8, model9, model10, model11, video_path):\n    try:\n        x, x_sqr, x_299_sqr, x_lstm, frame_skip = extract_frames(video_path)\n        if x is None or x_sqr is None: return 0.5\n        else:\n            with torch.no_grad():\n                y_pred = model(x.to(device))\n                y_pred = torch.sigmoid(y_pred.squeeze().mean())\n\n                y_pred2 = model2(x_sqr.to(device))\n                y_pred2 = torch.sigmoid(y_pred2.squeeze().mean())\n\n                y_pred3 = model3(x_sqr.to(device))\n                y_pred3 = torch.sigmoid(y_pred3.squeeze().mean())\n\n                y_pred4 = model4(x_299_sqr.to(device))\n                y_pred4 = torch.sigmoid(y_pred4.squeeze().mean())\n\n                y_pred5 = model5(x.to(device))\n                y_pred5 = torch.sigmoid(y_pred5.squeeze().mean())\n\n                y_pred6 = model6(x_sqr.to(device))\n                y_pred6 = torch.sigmoid(y_pred6.squeeze().mean())\n\n                ########################\n\n                ## Create batch of sequences for LSTM model (net7)\n                seq_len = len(x_lstm)//2\n                x_sqr_1, x_sqr_2 = x_lstm[:seq_len],  x_lstm[seq_len:] # 1 is original imgs, 2 is fliped imgs\n    #                 print('s1',x_sqr_1.shape)\n                skip = max(1, seq_len//15) # get about 15 sequences for each of (original imgs, flipped imgs)\n                start_indices = list(range(0, seq_len, skip))\n    #                 print('s2',start_indices)\n\n                gap = max(1, int(np.round(20/frame_skip))) # result of training setting (training: get each 4 frames from original vid, then skip 5 for sequence generation (total gap = 20))\n    #                 print('gap', gap)\n                for ss, start in enumerate(start_indices):\n                    indices = list(range(start, 500, gap))\n                    indices = [i%seq_len for i in indices]\n                    indices = indices[:CFG.seq_len]\n    #                     print('indices', indices)\n\n                    seqs1, seqs2 = x_sqr_1[indices].unsqueeze(0) , x_sqr_2[indices].unsqueeze(0) \n                    if ss == 0: \n                        batch_seqs_1 = seqs1\n                        batch_seqs_2 = seqs2\n                    else: \n                        batch_seqs_1 = torch.cat((batch_seqs_1, seqs1), 0)\n                        batch_seqs_2 = torch.cat((batch_seqs_2, seqs2), 0)\n\n                batch_seqs = torch.cat((batch_seqs_1, batch_seqs_2), 0)\n\n    #                 print('batch_seqs shape', batch_seqs.shape)\n                y_pred7 = model7(batch_seqs.to(device))\n                y_pred7 = torch.sigmoid(y_pred7.squeeze().mean())     \n\n                y_pred8 = model8(batch_seqs.to(device))\n                y_pred8 = torch.sigmoid(y_pred8.squeeze().mean())    \n    #                 ###########\n\n\n    #                 au_feat = get_au_feat(video_path.split('/')[-1])\n    #                 au_feats = torch.FloatTensor(len(batch_seqs), 62)\n    #                 for j in range(len(batch_seqs)): au_feats[j] = au_feat\n    #                 au_feats = au_feats.to(device).float()\n    # #                 print('aushape', au_feats.shape)                \n    #                 y_pred8 = model8(batch_seqs.to(device), au_feats)\n    #                 y_pred8 = torch.sigmoid(y_pred8.squeeze().mean())\n\n\n        ########################\n\n                y_pred9 = model9(x_sqr.to(device))\n                y_pred9 = torch.sigmoid(y_pred9.squeeze().mean())        \n\n                y_pred10 = model10(x_299_sqr.to(device))\n                y_pred10 = torch.sigmoid(y_pred10.squeeze().mean())     \n\n                y_pred11 = model11(x.to(device))\n                y_pred11 = torch.sigmoid(y_pred11.squeeze().mean())  \n\n                w = [3, 2, 8, 2, 2, 2, 4, 4, 2, 2, 3]\n\n                return (w[0]*y_pred.item() + w[1]*y_pred2.item() + w[2]*y_pred3.item() + w[3]*y_pred4.item() \n                        + w[4]*y_pred5.item() + w[5]*y_pred6.item() + \n                        w[6]*y_pred7.item() + w[7]*y_pred8.item() + \n                        w[8]*y_pred9.item() + w[9]*y_pred10.item() + w[10]*y_pred11.item())/sum(w)\n\n                \n    except Exception as e:\n        print(\"Prediction error on video\", e)\n        return 0.5\n    \n    return 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# a=predict_on_video(net, net2, net3, net4, net5, net6, net7, net8, net9, net10, net11, os.path.join(TEST_DIR, test_videos[0]))\n# a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class FastMTCNN(object):\n    def __init__(self, resize=1, *args, **kwargs):\n        self.resize = resize\n        self.mtcnn = MTCNN(*args, **kwargs)\n        \n    def __call__(self, frames):\n        if self.resize != 1:\n            frames = [f.resize([int(d * self.resize) for d in f.size]) for f in frames]    \n        boxes, probs = self.mtcnn.detect(frames)\n        boxes = [b.astype(int).tolist() if b is not None and type(b)==np.ndarray else [] for b in boxes] \n        probs = [b.tolist() if b is not None and type(b)==np.ndarray else [] for b in probs] \n        return boxes, probs\n\n\nfast_mtcnn = FastMTCNN(\n    resize=SCALE,\n    margin=14,\n    keep_all=True,\n    device=device\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_videos = sorted([x for x in os.listdir(TEST_DIR) if x[-4:] == \".mp4\"])\nlen(test_videos)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predictions = []\n# for i in range(len(test_videos)):\n#     filename = test_videos[i]\n#     y_pred = predict_on_video(net, net2, net3, net4, net5, net6, net7, net8, net9, net10, net11, os.path.join(TEST_DIR, filename))\n#     print(i, y_pred)\n#     predictions.append(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from concurrent.futures import ThreadPoolExecutor\n\ndef predict_on_video_set(model, model2, model3, model4, model5, model6, model7, model8, model9, model10, model11, videos, num_workers):\n    def process_file(i):\n        filename = videos[i]\n        y_pred = predict_on_video(model, model2, model3, model4, model5, model6, model7, model8, model9, model10, model11, os.path.join(TEST_DIR, filename))\n        print(i, y_pred)\n        return y_pred\n\n    with ThreadPoolExecutor(max_workers=num_workers) as ex:\n        predictions = ex.map(process_file, range(len(videos)))\n\n    return list(predictions)\n\npredictions = predict_on_video_set(net, net2, net3, net4, net5, net6, net7, net8, net9, net10, net11, test_videos, num_workers=NUM_WORKERS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.clip(predictions, 0.005, 0.995)\nsubmission_df = pd.DataFrame({\"filename\": test_videos, \"label\": predictions})\nsubmission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}