{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":11380964,"sourceType":"datasetVersion","datasetId":7126008}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Preprocess Video & Audio","metadata":{}},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport librosa\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\nimport numpy as np\n\nclass DFDCVideoDataset(Dataset):\n    def __init__(self, video_dir, transform=None):\n        self.video_dir = video_dir\n        self.transform = transform\n        self.data = []\n\n        # loop over all subfolders (dfdc_train_part_0, etc.)\n        for subfolder in os.listdir(video_dir):\n            subfolder_path = os.path.join(video_dir, subfolder)\n            metadata_path = os.path.join(subfolder_path, \"metadata.json\")\n            if not os.path.exists(metadata_path):\n                continue\n            \n            with open(metadata_path, \"r\") as f:\n                metadata = json.load(f)\n\n            for file, meta in metadata.items():\n                if file.endswith(\".mp4\"):\n                    label = 0 if meta[\"label\"] == \"REAL\" else 1\n                    self.data.append((os.path.join(subfolder_path, file), label))\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        video_path, label = self.data[idx]\n        \n        # Read first frame from video\n        cap = cv2.VideoCapture(video_path)\n        ret, frame = cap.read()\n        cap.release()\n\n        if not ret:\n            frame = np.zeros((224, 224, 3), dtype=np.uint8)\n        else:\n            frame = cv2.resize(frame, (224, 224))\n\n        if self.transform:\n            frame = self.transform(frame)\n        else:\n            frame = transforms.ToTensor()(frame)\n\n        # Load audio\n        y, sr = librosa.load(video_path, sr=16000)\n        mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13)  # shape: [13, time]\n        mfcc = mfcc.T.flatten()  # shape: [13*time]\n        if len(mfcc) < 512:\n            pad = np.zeros(512 - len(mfcc))\n            mfcc = np.concatenate([mfcc, pad])\n        else:\n            mfcc = mfcc[:512]\n        \n        mfcc_tensor = torch.tensor(mfcc, dtype=torch.float32)\n\n        return frame, mfcc_tensor, torch.tensor(label)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T04:51:49.45416Z","iopub.execute_input":"2025-04-11T04:51:49.454538Z","iopub.status.idle":"2025-04-11T04:51:57.020526Z","shell.execute_reply.started":"2025-04-11T04:51:49.454517Z","shell.execute_reply":"2025-04-11T04:51:57.019777Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Model Definition","metadata":{}},{"cell_type":"code","source":"\n\nimport torch.nn as nn\nimport torchvision.models as models\n\nclass VisualEncoder(nn.Module):\n    def __init__(self):\n        super().__init__()\n        base = models.resnet18(pretrained=True)\n        self.features = nn.Sequential(*list(base.children())[:-1])\n    def forward(self, x):\n        return self.features(x).squeeze(-1).squeeze(-1)\n\nclass AudioEncoder(nn.Module):\n    def __init__(self, input_dim=13):\n        super().__init__()\n        self.conv = nn.Sequential(\n            nn.Conv1d(input_dim, 64, 3, padding=1),\n            nn.ReLU(),\n            nn.AdaptiveAvgPool1d(1)\n        )\n    def forward(self, x):\n        return self.conv(x.transpose(1, 2)).squeeze(-1)\n\nimport torch.nn as nn\n\nclass AVACL(nn.Module):\n    def __init__(self):\n        super(AVACL, self).__init__()\n\n        # Visual encoder\n        self.visual = nn.Sequential(\n    nn.Flatten(),\n    nn.Linear(3 * 224 * 224, 512),  # ✅ match input size\n    nn.ReLU(),\n    nn.Linear(512, 128)\n)\n\n        # Audio encoder — ✅ THIS is where you replace or update\n        self.audio = nn.Sequential(\n            nn.Linear(512, 128),  # match to processed MFCC feature size\n            nn.ReLU(),\n            nn.Linear(128, 128)\n        )\n\n        # Projection layer (optional)\n        self.proj = nn.Linear(128, 128)\n\n        # Attention layer or fusion logic\n        self.attn = nn.MultiheadAttention(embed_dim=128, num_heads=4, batch_first=True)\n\n        # Classification head\n        self.classifier = nn.Linear(128, 2)  # fake or real (binary)\n\n    def forward(self, img_feat, aud_feat):\n        vis = self.proj(self.visual(img_feat).unsqueeze(1))  # [B, 1, 128]\n        aud = self.proj(self.audio(aud_feat).unsqueeze(1))   # [B, 1, 128]\n        fused, _ = self.attn(vis, aud, aud)                  # attention\n        out = fused.squeeze(1)                               # remove seq dim\n        pred = self.classifier(out)\n        return out, pred\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T04:51:57.021315Z","iopub.execute_input":"2025-04-11T04:51:57.021707Z","iopub.status.idle":"2025-04-11T04:51:57.030107Z","shell.execute_reply.started":"2025-04-11T04:51:57.021682Z","shell.execute_reply":"2025-04-11T04:51:57.029316Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Train and test","metadata":{}},{"cell_type":"code","source":"\ndef contrastive_loss(z1, z2, temperature=0.1):\n    z1 = nn.functional.normalize(z1, dim=1)\n    z2 = nn.functional.normalize(z2, dim=1)\n    logits = z1 @ z2.T / temperature\n    labels = torch.arange(z1.size(0)).to(z1.device)\n    return nn.CrossEntropyLoss()(logits, labels)\n\ndef train_model(model, loader, optimizer):\n    model.train()\n    for frames, mfccs, labels in loader:\n        frames, mfccs, labels = frames.to(device), mfccs.to(device), labels.to(device)\n        z, preds = model(frames, mfccs)\n        loss_cls = nn.CrossEntropyLoss()(preds, labels)\n        loss_cont = contrastive_loss(z, z.clone().detach())\n        loss = loss_cls + 0.1 * loss_cont\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\ndef test_model(model, loader):\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for frames, mfccs, labels in loader:\n            frames, mfccs = frames.to(device), mfccs.to(device)\n            _, preds = model(frames, mfccs)\n            correct += (preds.argmax(1).cpu() == labels).sum().item()\n            total += len(labels)\n    acc = correct / total\n    print(f\"Test Accuracy: {acc:.2f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T04:51:57.031934Z","iopub.execute_input":"2025-04-11T04:51:57.032241Z","iopub.status.idle":"2025-04-11T04:51:57.048403Z","shell.execute_reply.started":"2025-04-11T04:51:57.032217Z","shell.execute_reply":"2025-04-11T04:51:57.047868Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Putting It All Together","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T04:51:57.049032Z","iopub.execute_input":"2025-04-11T04:51:57.049189Z","iopub.status.idle":"2025-04-11T04:51:57.061595Z","shell.execute_reply.started":"2025-04-11T04:51:57.049175Z","shell.execute_reply":"2025-04-11T04:51:57.061016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom torch.utils.data import DataLoader\nfrom torchvision.transforms import ToTensor, Compose, Normalize\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.5]*3, [0.5]*3)\n])\n\nvideo_dir = \"../input/deepfake-detection-challenge\"\ndataset = DFDCVideoDataset(video_dir, transform=transform)\n\nprint(\"Total samples found:\", len(dataset))  # 👈 must be > 0\n\nfrom torch.utils.data import DataLoader\ntrain_loader = DataLoader(dataset, batch_size=4, shuffle=True)\n\nmodel = AVACL().to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n\n# Train for 3 epochs\nfor epoch in range(3):\n    print(f\"Epoch {epoch+1}\")\n    train_model(model, train_loader, optimizer)\n    test_model(model, train_loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T04:51:57.062224Z","iopub.execute_input":"2025-04-11T04:51:57.062455Z","iopub.status.idle":"2025-04-11T05:02:08.684183Z","shell.execute_reply.started":"2025-04-11T04:51:57.062435Z","shell.execute_reply":"2025-04-11T05:02:08.683407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# AVA-CL: Audio-Visual Alignment with Contrastive Learning\n\nimport os\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchaudio\nimport torchvision.transforms as T\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision.models import resnet18\nfrom sklearn.metrics import accuracy_score, roc_auc_score, f1_score\nimport librosa\nimport numpy as np\nimport cv2\n\n# -------------------------------\n# Config\n# -------------------------------\nclass Config:\n    frame_size = (224, 224)\n    audio_sr = 22050\n    audio_duration = 3  # in seconds\n    n_mfcc = 40\n    batch_size = 16\n    lr = 1e-4\n    epochs = 100\n    temperature = 0.1\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n\n# -------------------------------\n# Dataset\n# -------------------------------\nclass AVADataset(Dataset):\n    def __init__(self, video_paths, labels, transform=None):\n        self.video_paths = video_paths\n        self.labels = labels\n        self.transform = transform or T.Compose([\n            T.ToPILImage(),\n            T.Resize(Config.frame_size),\n            T.ToTensor()\n        ])\n\n    def __len__(self):\n        return len(self.video_paths)\n\n    def extract_frames(self, video_path, num_frames=16):\n        cap = cv2.VideoCapture(video_path)\n        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n        frame_idxs = np.linspace(0, total_frames-1, num_frames).astype(int)\n        frames = []\n        for idx in frame_idxs:\n            cap.set(cv2.CAP_PROP_POS_FRAMES, idx)\n            ret, frame = cap.read()\n            if ret:\n                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n                frame = self.transform(frame)\n                frames.append(frame)\n        cap.release()\n        return torch.stack(frames)  # shape: [16, 3, H, W]\n\n    def extract_audio(self, video_path):\n        wav, _ = librosa.load(video_path, sr=Config.audio_sr, duration=Config.audio_duration)\n        mfcc = librosa.feature.mfcc(y=wav, sr=Config.audio_sr, n_mfcc=Config.n_mfcc)\n        return torch.tensor(mfcc).float()  # shape: [n_mfcc, time]\n\n    def __getitem__(self, idx):\n        path = self.video_paths[idx]\n        label = self.labels[idx]\n        frames = self.extract_frames(path)\n        audio = self.extract_audio(path)\n        return frames, audio, torch.tensor(label).float()\n\n\n# -------------------------------\n# Model\n# -------------------------------\nclass AVACL(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.visual_encoder = resnet18(pretrained=True)\n        self.visual_encoder.fc = nn.Identity()\n\n        self.audio_encoder = nn.Sequential(\n            nn.Conv1d(Config.n_mfcc, 128, kernel_size=3, padding=1),\n            nn.ReLU(),\n            nn.AdaptiveAvgPool1d(1),\n            nn.Flatten(),\n        )\n\n        self.fusion_head = nn.Sequential(\n            nn.Linear(512 + 128, 256),\n            nn.ReLU(),\n            nn.Linear(256, 2)  # real/fake\n        )\n\n        self.projection_head = nn.Sequential(\n            nn.Linear(512 + 128, 128),\n            nn.ReLU(),\n            nn.Linear(128, 64)\n        )\n\n    def forward(self, video_frames, audio_mfcc):\n        B, T, C, H, W = video_frames.shape\n        video_frames = video_frames.view(B*T, C, H, W)\n        visual_feat = self.visual_encoder(video_frames)\n        visual_feat = visual_feat.view(B, T, -1).mean(dim=1)  # [B, 512]\n\n        audio_mfcc = audio_mfcc.to(Config.device)\n        audio_feat = self.audio_encoder(audio_mfcc)  # [B, 128]\n\n        fused = torch.cat([visual_feat, audio_feat], dim=1)  # [B, 640]\n        logits = self.fusion_head(fused)\n        proj = self.projection_head(fused)\n        return logits, proj\n\n\n# -------------------------------\n# Contrastive Loss\n# -------------------------------\ndef contrastive_loss(z_i, z_j, temperature):\n    z_i = F.normalize(z_i, dim=1)\n    z_j = F.normalize(z_j, dim=1)\n    logits = torch.matmul(z_i, z_j.T) / temperature\n    labels = torch.arange(z_i.size(0)).to(Config.device)\n    return F.cross_entropy(logits, labels)\n\n\n# -------------------------------\n# Training Loop\n# -------------------------------\ndef train(model, dataloader, optimizer):\n    model.train()\n    for epoch in range(Config.epochs):\n        all_preds, all_labels = [], []\n        for  audio, frames, labels in dataloader:\n            frames, audio, labels = frames.to(Config.device), audio.to(Config.device), labels.to(Config.device)\n            optimizer.zero_grad()\n            logits, proj = model(frames, audio)\n            \n            cls_loss = F.cross_entropy(logits, labels.long())\n            cont_loss = contrastive_loss(proj, proj, Config.temperature)\n            loss = cls_loss + cont_loss\n\n            loss.backward()\n            optimizer.step()\n\n            preds = torch.argmax(logits, dim=1)\n            all_preds.extend(preds.cpu().numpy())\n            all_labels.extend(labels.cpu().numpy())\n\n        acc = accuracy_score(all_labels, all_preds)\n        print(f\"Epoch {epoch+1} | Acc: {acc:.4f} | Loss: {loss.item():.4f}\")\n\n\n# -------------------------------\n# Entry Point\n# -------------------------------\n# if __name__ == '__main__':\n# Placeholder paths and labels\n# video_paths = ['data/video1.mp4', 'data/video2.mp4']  # Replace with your own\n# labels = [0, 1]  # 0 = real, 1 = fake\n\n# dataset = AVADataset(video_paths, labels)\ndataloader = DataLoader(dataset, batch_size=Config.batch_size, shuffle=True)\n\nmodel = AVACL().to(Config.device)\noptimizer = torch.optim.Adam(model.parameters(), lr=Config.lr)\n\ntrain(model, dataloader, optimizer)\n\n    # Evaluation and saving can be added later\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:02:08.68513Z","iopub.execute_input":"2025-04-11T05:02:08.685538Z","iopub.status.idle":"2025-04-11T05:02:13.444545Z","shell.execute_reply.started":"2025-04-11T05:02:08.685518Z","shell.execute_reply":"2025-04-11T05:02:13.443177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dataset[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:02:13.444957Z","iopub.status.idle":"2025-04-11T05:02:13.445175Z","shell.execute_reply.started":"2025-04-11T05:02:13.445075Z","shell.execute_reply":"2025-04-11T05:02:13.445084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport librosa\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torchvision import transforms, models\nfrom sklearn.metrics import classification_report, confusion_matrix, roc_auc_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# ========== Dataset ==========\nclass DFDCVideoDataset(Dataset):\n    def __init__(self, video_dir, transform=None):\n        self.video_dir = video_dir\n        self.transform = transform\n        self.data = []\n\n        for subfolder in os.listdir(video_dir):\n            subfolder_path = os.path.join(video_dir, subfolder)\n            metadata_path = os.path.join(subfolder_path, \"metadata.json\")\n            if not os.path.exists(metadata_path):\n                continue\n            with open(metadata_path, \"r\") as f:\n                metadata = json.load(f)\n\n            for file, meta in metadata.items():\n                if file.endswith(\".mp4\"):\n                    label = 0 if meta[\"label\"] == \"REAL\" else 1\n                    self.data.append((os.path.join(subfolder_path, file), label))\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        video_path, label = self.data[idx]\n        cap = cv2.VideoCapture(video_path)\n        ret, frame = cap.read()\n        cap.release()\n\n        if not ret:\n            frame = np.zeros((224, 224, 3), dtype=np.uint8)\n        else:\n            frame = cv2.resize(frame, (224, 224))\n\n        if self.transform:\n            frame = self.transform(frame)\n        else:\n            frame = transforms.ToTensor()(frame)\n\n        # Audio: MFCC extraction\n        y, sr = librosa.load(video_path, sr=16000)\n        mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=13)\n        mfcc = mfcc.T.flatten()\n        if len(mfcc) < 512:\n            pad = np.zeros(512 - len(mfcc))\n            mfcc = np.concatenate([mfcc, pad])\n        else:\n            mfcc = mfcc[:512]\n\n        mfcc_tensor = torch.tensor(mfcc, dtype=torch.float32)\n\n        return frame, mfcc_tensor, torch.tensor(label)\n\n# ========== Train/Test ==========\ndef train_model(model, loader, optimizer, device):\n    model.train()\n    for frames, mfccs, labels in loader:\n        frames, mfccs, labels = frames.to(device), mfccs.to(device), labels.to(device)\n        z, preds = model(frames, mfccs)\n        loss_cls = F.cross_entropy(preds, labels)\n        loss_cont = contrastive_loss(z, z.clone().detach())\n        loss = loss_cls + 0.1 * loss_cont\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\ndef test_model(model, loader, device):\n    model.eval()\n    all_preds = []\n    all_labels = []\n    with torch.no_grad():\n        for frames, mfccs, labels in loader:\n            frames, mfccs = frames.to(device), mfccs.to(device)\n            _, preds = model(frames, mfccs)\n            all_preds.append(preds.argmax(1).cpu().numpy())\n            all_labels.append(labels.cpu().numpy())\n\n    # Flatten the lists\n    all_preds = np.concatenate(all_preds)\n    all_labels = np.concatenate(all_labels)\n\n    # Print classification report and confusion matrix\n    print(\"Classification Report:\")\n    print(classification_report(all_labels, all_preds))\n    print(\"Confusion Matrix:\")\n    cm = confusion_matrix(all_labels, all_preds)\n    sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\", xticklabels=[\"REAL\", \"FAKE\"], yticklabels=[\"REAL\", \"FAKE\"])\n    plt.xlabel(\"Predicted\")\n    plt.ylabel(\"True\")\n    plt.show()\n\n    # Calculate ROC-AUC score\n    auc = roc_auc_score(all_labels, all_preds)\n    print(f\"ROC-AUC Score: {auc:.4f}\")\n\n# ========== Run Training ==========\n# if __name__ == \"__main__\":\nvideo_dir = \"../input/deepfake-detection-challenge\"\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.5]*3, [0.5]*3)\n])\n\ndataset = DFDCVideoDataset(video_dir, transform=transform)\n\n# Split dataset into train and test sets (80/20)\ntrain_size = int(0.8 * len(dataset))\ntest_size = len(dataset) - train_size\ntrain_dataset, test_dataset = random_split(dataset, [train_size, test_size])\n\n# DataLoaders\ntrain_loader = DataLoader(train_dataset, batch_size=4, shuffle=True)\ntest_loader = DataLoader(test_dataset, batch_size=4, shuffle=False)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nmodel = AVACL().to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n\n# Print dataset info\nprint(f\"Dataset contains {len(dataset)} samples.\")\nprint(f\"Training set contains {len(train_dataset)} samples.\")\nprint(f\"Test set contains {len(test_dataset)} samples.\")\n\n# for epoch in range(3):\n#     print(f\"\\nEpoch {epoch+1}\")\n#     train_model(model, train_loader, optimizer, device)\n#     test_model(model, test_loader, device)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:34:48.151874Z","iopub.execute_input":"2025-04-11T05:34:48.152159Z","iopub.status.idle":"2025-04-11T05:34:48.363496Z","shell.execute_reply.started":"2025-04-11T05:34:48.152139Z","shell.execute_reply":"2025-04-11T05:34:48.362892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torchvision.models import resnet50\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\n\n# Define the Visual Encoder: ResNet + Transformer\nclass VisualEncoder(nn.Module):\n    def __init__(self, embed_dim=512):\n        super(VisualEncoder, self).__init__()\n        self.resnet = resnet50(pretrained=True)\n        self.resnet.fc = nn.Identity()  # Remove the fully connected layer\n        self.transformer_layer = TransformerEncoderLayer(d_model=2048, nhead=8)\n        self.transformer = TransformerEncoder(self.transformer_layer, num_layers=6)\n        self.fc = nn.Linear(2048, embed_dim)\n    \n    def forward(self, x):\n        # x: [batch_size, 16, 3, 224, 224]\n        print(x.shape)\n        batch_size, seq_len, _, _, _ = x.shape\n        x = x.view(batch_size * seq_len, 3, 224, 224)  # Flatten temporal dimension\n        features = self.resnet(x)  # [batch_size * seq_len, 2048]\n        features = features.view(batch_size, seq_len, -1)  # [batch_size, seq_len, 2048]\n        features = self.transformer(features)  # Apply Transformer\n        return self.fc(features.mean(dim=1))  # [batch_size, embed_dim]\n\nclass AudioEncoder(nn.Module):\n    def __init__(self, embed_dim=512):\n        super(AudioEncoder, self).__init__()\n        self.conv1 = nn.Conv1d(1, 64, kernel_size=3, stride=1, padding=1)\n        self.conv2 = nn.Conv1d(64, 128, kernel_size=3, stride=1, padding=1)\n        self.gru = nn.GRU(input_size=128, hidden_size=embed_dim, batch_first=True)\n    \n    def forward(self, x):\n        # x: [batch_size, 1, n_mfcc, time_steps]\n        # Reshape it to [batch_size, 1, time_steps * n_mfcc] to collapse the n_mfcc dimension\n        batch_size, _, n_mfcc, time_steps = x.shape\n        x = x.view(batch_size, 1, n_mfcc * time_steps)  # [batch_size, 1, time_steps * n_mfcc]\n\n        # Apply Conv1d\n        x = F.relu(self.conv1(x))  # [batch_size, 64, time_steps * n_mfcc]\n        x = F.relu(self.conv2(x))  # [batch_size, 128, time_steps * n_mfcc]\n\n        # Pass through GRU\n        x = x.permute(0, 2, 1)  # [batch_size, time_steps * n_mfcc, 128]\n        _, hidden = self.gru(x)  # Get GRU hidden state\n        return hidden.squeeze(0)  # [batch_size, embed_dim]\n\n\n# AV-Attention Module\nclass AVAttention(nn.Module):\n    def __init__(self, embed_dim=512):\n        super(AVAttention, self).__init__()\n        self.cross_attention = nn.MultiheadAttention(embed_dim, num_heads=8)\n    \n    def forward(self, v, a):\n        # v, a: [batch_size, embed_dim]\n        v, a = v.unsqueeze(1), a.unsqueeze(1)  # Add sequence dimension\n        v_out, _ = self.cross_attention(v, a, a)  # Cross-attention\n        a_out, _ = self.cross_attention(a, v, v)  # Cross-attention\n        return v_out.squeeze(1), a_out.squeeze(1)\n\n# Contrastive Learning Loss (InfoNCE)\nclass ContrastiveLoss(nn.Module):\n    def __init__(self, temperature=0.1):\n        super(ContrastiveLoss, self).__init__()\n        self.temperature = temperature\n\n    def forward(self, v, a):\n        # v, a: [batch_size, embed_dim]\n        v = F.normalize(v, dim=-1)\n        a = F.normalize(a, dim=-1)\n        logits = torch.mm(v, a.t()) / self.temperature\n        labels = torch.arange(v.size(0)).to(v.device)\n        return F.cross_entropy(logits, labels)\n\n# Final Model\nclass DeepfakeDetector(nn.Module):\n    def __init__(self, embed_dim=512):\n        super(DeepfakeDetector, self).__init__()\n        self.visual_encoder = VisualEncoder(embed_dim)\n        self.audio_encoder = AudioEncoder(embed_dim)\n        self.attention = AVAttention(embed_dim)\n        self.fc = nn.Sequential(\n            nn.Linear(2 * embed_dim, 512),\n            nn.ReLU(),\n            nn.Linear(512, 2)  # Output: REAL/FAKE\n        )\n        self.contrastive_loss = ContrastiveLoss()\n    \n    def forward(self, visual_input, audio_input):\n        v = self.visual_encoder(visual_input)  # Visual features\n        a = self.audio_encoder(audio_input)   # Audio features\n        v_out, a_out = self.attention(v, a)   # Enhanced features\n        combined = torch.cat([v_out, a_out], dim=-1)  # Fuse features\n        classification = self.fc(combined)   # Classification head\n        contrastive_loss = self.contrastive_loss(v_out, a_out)\n        return classification, contrastive_loss\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:34:51.719455Z","iopub.execute_input":"2025-04-11T05:34:51.719732Z","iopub.status.idle":"2025-04-11T05:34:52.968056Z","shell.execute_reply.started":"2025-04-11T05:34:51.719712Z","shell.execute_reply":"2025-04-11T05:34:52.967427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Define the model\n# model = DeepfakeDetector(embed_dim=512)\n\n# # Dummy input\n# visual_input = torch.randn(8, 16, 3, 224, 224)  # Batch of 8, 16 frames, 224x224\n# audio_input = torch.randn(8, 1, 128)            # Batch of 8, Mel-spectrogram with 128 time steps\n\n# # Forward pass\n# classification, contrastive_loss = model(visual_input, audio_input)\n\n# print(\"Classification Output:\", classification.shape)  # [8, 2]\n# print(\"Contrastive Loss:\", contrastive_loss.item())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:35:36.797756Z","iopub.execute_input":"2025-04-11T05:35:36.798053Z","iopub.status.idle":"2025-04-11T05:35:36.801747Z","shell.execute_reply.started":"2025-04-11T05:35:36.798031Z","shell.execute_reply":"2025-04-11T05:35:36.801014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport os\nimport json\nimport cv2\nimport numpy as np\nimport librosa\nimport librosa.display\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import (\n    accuracy_score,\n    classification_report,\n    confusion_matrix,\n    roc_auc_score,\n    roc_curve,\n)\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\nfrom torchvision.transforms import transforms\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:35:40.899467Z","iopub.execute_input":"2025-04-11T05:35:40.899739Z","iopub.status.idle":"2025-04-11T05:35:40.926445Z","shell.execute_reply.started":"2025-04-11T05:35:40.899717Z","shell.execute_reply":"2025-04-11T05:35:40.925898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom moviepy.editor import VideoFileClip\nimport subprocess\nclass DeepfakeDataset(Dataset):\n    def __init__(self, metadata_file, video_dir, transform=None, sample_rate=16000, n_mfcc=40):\n        self.metadata_file = metadata_file\n        self.video_dir = video_dir\n        self.transform = transform\n        self.sample_rate = sample_rate\n        self.n_mfcc = n_mfcc\n        with open(metadata_file, 'r') as f:\n            self.metadata = json.load(f)\n        self.data = [\n            {\"filename\": key, \"label\": value[\"label\"]}\n            for key, value in self.metadata.items()\n        ]\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        sample = self.data[idx]\n        video_path = os.path.join(self.video_dir, sample[\"filename\"])\n        label = 1 if sample[\"label\"] == \"FAKE\" else 0\n    \n        # Extract video frames\n        cap = cv2.VideoCapture(video_path)\n        frames = []\n        while len(frames) < 16:  # Capture 16 frames\n            ret, frame = cap.read()\n            if not ret:\n                break\n            frame = cv2.resize(frame, (224, 224))\n            if self.transform:\n                frame = self.transform(frame)  # Apply transform on each frame\n            frames.append(frame)\n        cap.release()\n    \n        if len(frames) < 16:\n            frames.extend([frames[-1]] * (16 - len(frames)))\n        \n        frames = torch.stack(frames)  # Combine frames into a 4D tensor (16x3x224x224)\n    \n        # Extract audio features\n        audio = self.extract_audio(video_path)\n        if audio.size == 0:  # If audio extraction failed\n            audio = np.zeros(self.sample_rate)  # 1 second of silence\n        \n        mfcc = librosa.feature.mfcc(y=audio, sr=self.sample_rate, n_mfcc=self.n_mfcc)\n        \n        if mfcc.ndim == 1:  # If MFCC result is 1D, expand it to 2D\n            mfcc = np.expand_dims(mfcc, axis=0)\n        \n        mfcc = np.expand_dims(mfcc, axis=0)  # Add batch dimension (1x40xT) if needed\n    \n        return frames, torch.tensor(mfcc, dtype=torch.float32), torch.tensor(label)\n\n    \n    def extract_audio(self, video_path):\n        \"\"\"Extract audio from video file using ffmpeg.\"\"\"\n        temp_audio_path = \"/tmp/temp_audio.wav\"\n        \n        try:\n            # Extract audio to a temporary WAV file\n            command = [\n                \"ffmpeg\",\n                \"-i\", video_path,\n                \"-ar\", str(self.sample_rate),\n                \"-ac\", \"1\",\n                \"-vn\",  # Ignore video stream\n                temp_audio_path,\n                \"-y\"  # Overwrite if file exists\n            ]\n            subprocess.run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n    \n            # Load the audio data\n            if os.path.exists(temp_audio_path):\n                audio, _ = librosa.load(temp_audio_path, sr=self.sample_rate)\n                os.remove(temp_audio_path)  # Clean up temp file\n                return audio\n            else:\n                print(f\"Audio extraction failed: {video_path}. Returning silence.\")\n                return np.zeros(self.sample_rate)  # 1 second of silence\n            \n        except subprocess.CalledProcessError as e:\n            print(f\"FFmpeg failed for {video_path}: {e}\")\n            return np.zeros(self.sample_rate)  # 1 second of silence\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:35:43.726854Z","iopub.execute_input":"2025-04-11T05:35:43.727559Z","iopub.status.idle":"2025-04-11T05:35:44.568625Z","shell.execute_reply.started":"2025-04-11T05:35:43.727534Z","shell.execute_reply":"2025-04-11T05:35:44.568095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nvideo_dir = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos\"\nmetadata_file = os.path.join(video_dir, \"metadata.json\")\n\n# Apply data augmentation and preprocessing\ntransform = transforms.Compose([\n    transforms.ToPILImage(),  # Convert each frame to PIL Image\n    transforms.ToTensor(),   # Convert PIL Image to Tensor\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])  # Normalize\n])\n\n\ndataset = DeepfakeDataset(metadata_file, video_dir, transform=transform)\ntrain_data, test_data = train_test_split(dataset, test_size=0.2, random_state=42)\ntrain_loader = DataLoader(train_data, batch_size=1, shuffle=True)\ntest_loader = DataLoader(test_data, batch_size=1, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:35:50.418543Z","iopub.execute_input":"2025-04-11T05:35:50.419199Z","iopub.status.idle":"2025-04-11T05:37:51.517757Z","shell.execute_reply.started":"2025-04-11T05:35:50.419176Z","shell.execute_reply":"2025-04-11T05:37:51.517141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:30:27.450456Z","iopub.execute_input":"2025-04-11T05:30:27.450726Z","iopub.status.idle":"2025-04-11T05:30:27.456499Z","shell.execute_reply.started":"2025-04-11T05:30:27.450704Z","shell.execute_reply":"2025-04-11T05:30:27.455698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = DeepfakeDetector(embed_dim=512).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n\nepochs = 10\nscaler = torch.cuda.amp.GradScaler()\n\nfor epoch in range(epochs):\n    model.train()\n    train_loss = 0\n    for audio_input, visual_input, labels in train_loader:\n        visual_input, audio_input, labels = (\n            audio_input.half().to(device),\n            visual_input.to(device),\n            labels.to(device),\n        )\n        optimizer.zero_grad()\n\n        with torch.cuda.amp.autocast():\n            \n            classification, contrastive_loss = model(visual_input, audio_input)\n            loss = criterion(classification, labels) + contrastive_loss\n\n        # Manual backward pass\n        loss.backward()\n\n        # Perform optimizer step\n        optimizer.step()\n\n        train_loss += loss.item()\n\n    print(f\"Epoch {epoch + 1}/{epochs}, Loss: {train_loss / len(train_loader)}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:39:08.275362Z","iopub.execute_input":"2025-04-11T05:39:08.275883Z","iopub.status.idle":"2025-04-11T06:04:00.343237Z","shell.execute_reply.started":"2025-04-11T05:39:08.275861Z","shell.execute_reply":"2025-04-11T06:04:00.342428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.cuda.empty_cache()\n# model.visual_encoder.resnet.requires_grad_(False)  # Freeze ResNet to save memory\nmodel.visual_encoder.to(\"cpu\")\nimport torch\ntorch.cuda.empty_cache()\ntorch.cuda.reset_peak_memory_stats()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:30:27.868377Z","iopub.execute_input":"2025-04-11T05:30:27.86874Z","iopub.status.idle":"2025-04-11T05:30:27.879137Z","shell.execute_reply.started":"2025-04-11T05:30:27.868724Z","shell.execute_reply":"2025-04-11T05:30:27.878522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Audio input shape: {audio_input.shape}\")\nprint(f\"Audio input size (MB): {audio_input.element_size() * audio_input.nelement() / 1e6}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T05:02:13.458167Z","iopub.status.idle":"2025-04-11T05:02:13.458457Z","shell.execute_reply.started":"2025-04-11T05:02:13.458287Z","shell.execute_reply":"2025-04-11T05:02:13.458302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\nall_preds, all_labels = [], []\niteration = 0 \nwith torch.no_grad():\n    for  visual_input, audio_input, labels in test_loader:\n        visual_input, audio_input, labels = (\n            visual_input.to(device),\n            audio_input.to(device),\n            labels.to(device),\n        )\n        classification, _ = model(visual_input, audio_input)\n        preds = torch.argmax(classification, dim=1)\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n        print(iteration)\n        iteration += 1\n# Compute Metrics\naccuracy = accuracy_score(all_labels, all_preds)\nreport = classification_report(all_labels, all_preds, target_names=[\"REAL\", \"FAKE\"])\nconf_matrix = confusion_matrix(all_labels, all_preds)\nroc_auc = roc_auc_score(all_labels, all_preds)\n\nprint(f\"Accuracy: {accuracy}\")\nprint(\"Classification Report:\\n\", report)\nprint(\"Confusion Matrix:\\n\", conf_matrix)\nprint(f\"ROC-AUC: {roc_auc}\")\n\n# Plot ROC Curve\nfpr, tpr, _ = roc_curve(all_labels, all_preds)\nplt.figure()\nplt.plot(fpr, tpr, label=f\"ROC curve (area = {roc_auc:.2f})\")\nplt.xlabel(\"False Positive Rate\")\nplt.ylabel(\"True Positive Rate\")\nplt.title(\"ROC Curve\")\nplt.legend(loc=\"lower right\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T06:16:37.016609Z","iopub.execute_input":"2025-04-11T06:16:37.016879Z","iopub.status.idle":"2025-04-11T06:16:53.259651Z","shell.execute_reply.started":"2025-04-11T06:16:37.01686Z","shell.execute_reply":"2025-04-11T06:16:53.258899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cm = confusion_matrix(all_labels, all_preds)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nsns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Blues\", xticklabels=[\"REAL\", \"FAKE\"], yticklabels=[\"REAL\", \"FAKE\"])\nplt.xlabel(\"Predicted\")\nplt.ylabel(\"True\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-11T06:16:53.261096Z","iopub.execute_input":"2025-04-11T06:16:53.261668Z","iopub.status.idle":"2025-04-11T06:16:53.39962Z","shell.execute_reply.started":"2025-04-11T06:16:53.261647Z","shell.execute_reply":"2025-04-11T06:16:53.399086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}