{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":18147,"sourceType":"datasetVersion","datasetId":13405},{"sourceId":842050,"sourceType":"datasetVersion","datasetId":444558},{"sourceId":893807,"sourceType":"datasetVersion","datasetId":451078},{"sourceId":6358196,"sourceType":"datasetVersion","datasetId":3579787},{"sourceId":8789723,"sourceType":"datasetVersion","datasetId":5284443}],"dockerImageVersionId":29845,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-28T04:52:53.168467Z","iopub.execute_input":"2024-06-28T04:52:53.168871Z","iopub.status.idle":"2024-06-28T04:52:53.832775Z","shell.execute_reply.started":"2024-06-28T04:52:53.168801Z","shell.execute_reply":"2024-06-28T04:52:53.831872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfor i in range(torch.cuda.device_count()):\n   print(torch.cuda.get_device_properties(i).name)","metadata":{"execution":{"iopub.status.busy":"2024-06-28T04:53:12.442638Z","iopub.execute_input":"2024-06-28T04:53:12.443024Z","iopub.status.idle":"2024-06-28T04:53:14.119135Z","shell.execute_reply.started":"2024-06-28T04:53:12.442958Z","shell.execute_reply":"2024-06-28T04:53:14.118034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall torch torchvision numpy -y\n!pip install torch==1.7.1 torchvision==0.8.2 numpy==1.19.5\n!pip install facenet-pytorch==2.5.2\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T04:53:17.103054Z","iopub.execute_input":"2024-06-28T04:53:17.103421Z","iopub.status.idle":"2024-06-28T04:54:53.747574Z","shell.execute_reply.started":"2024-06-28T04:53:17.10335Z","shell.execute_reply":"2024-06-28T04:54:53.746335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install --upgrade albumentations\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T04:54:53.749737Z","iopub.execute_input":"2024-06-28T04:54:53.749986Z","iopub.status.idle":"2024-06-28T04:55:05.073323Z","shell.execute_reply.started":"2024-06-28T04:54:53.749943Z","shell.execute_reply":"2024-06-28T04:55:05.072311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install --upgrade typing_extensions","metadata":{"execution":{"iopub.status.busy":"2024-06-28T04:55:05.075229Z","iopub.execute_input":"2024-06-28T04:55:05.075638Z","iopub.status.idle":"2024-06-28T04:55:13.419279Z","shell.execute_reply.started":"2024-06-28T04:55:05.075566Z","shell.execute_reply":"2024-06-28T04:55:13.418323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport zipfile\nimport numpy as np\nimport pandas as pd\nimport matplotlib\nimport seaborn as sns\nimport torch\nimport matplotlib.pyplot as plt\n# from tqdm import tqdm_notebook\n%matplotlib inline \n# from google.colab.patches import cv2_imshow\nfrom IPython.display import HTML #imports to play videos\nfrom base64 import b64encode \nimport cv2 as cv\nfrom skimage.measure import compare_ssim\nimport glob\nimport time\nfrom PIL import Image\nfrom facenet_pytorch import MTCNN, InceptionResnetV1, extract_face\nfrom tqdm import tqdm\n\nimport math\nimport pickle\nfrom functools import partial\nfrom collections import defaultdict\n\nfrom PIL import Image\nfrom glob import glob\n\nimport cv2\nimport skimage.measure\nimport albumentations as A\nfrom tqdm.notebook import tqdm \n#from albumentations.pytorch import ToTensor \nfrom albumentations.pytorch import ToTensorV2\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torchvision.models.video import mc3_18, r2plus1d_18\n\nfrom facenet_pytorch import MTCNN","metadata":{"execution":{"iopub.status.busy":"2024-06-28T04:55:13.421462Z","iopub.execute_input":"2024-06-28T04:55:13.421826Z","iopub.status.idle":"2024-06-28T04:55:15.589995Z","shell.execute_reply.started":"2024-06-28T04:55:13.421761Z","shell.execute_reply":"2024-06-28T04:55:15.588627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# Load metadata\nmetadata_path = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json'\nmetadata = pd.read_json(metadata_path).transpose()\nmetadata.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T05:03:58.540887Z","iopub.execute_input":"2024-06-28T05:03:58.541214Z","iopub.status.idle":"2024-06-28T05:03:59.352487Z","shell.execute_reply.started":"2024-06-28T05:03:58.541167Z","shell.execute_reply":"2024-06-28T05:03:59.351622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport random\nfrom shutil import copyfile\nfrom collections import defaultdict\n\n# Path to the dataset and metadata\ndataset_path = '/kaggle/input/deepfake-detection-challenge/train_sample_videos'\nmetadata_file = os.path.join(dataset_path, 'metadata.json')\n\n# Load metadata\nwith open(metadata_file, 'r') as f:\n    metadata = json.load(f)\n\n# Categorize videos by label\nreal_videos = [video for video, details in metadata.items() if details['label'] == 'REAL']\nfake_videos = [video for video, details in metadata.items() if details['label'] == 'FAKE']\n\n# Balance the dataset\nif len(real_videos) < len(fake_videos):\n    real_videos = real_videos * (len(fake_videos) // len(real_videos)) + random.sample(real_videos, len(fake_videos) % len(real_videos))\nelse:\n    fake_videos = fake_videos * (len(real_videos) // len(fake_videos)) + random.sample(fake_videos, len(real_videos) % len(fake_videos))\n\n# Check the number of videos in each category after balancing\nprint(f'Balanced Dataset: {len(real_videos)} REAL, {len(fake_videos)} FAKE')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T05:03:51.209294Z","iopub.execute_input":"2024-06-28T05:03:51.209633Z","iopub.status.idle":"2024-06-28T05:03:51.231128Z","shell.execute_reply.started":"2024-06-28T05:03:51.209574Z","shell.execute_reply":"2024-06-28T05:03:51.230339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import shutil\n# import os\n\n# # Define the path to the directory you want to delete\n# directory_to_delete = \"/kaggle/working/facial_organs\"\n\n# # Check if the directory exists before attempting to delete it\n# if os.path.exists(directory_to_delete):\n#     # Remove the directory and all its contents\n#     shutil.rmtree(directory_to_delete)\n#     print(f\"Deleted directory: {directory_to_delete}\")\n# else:\n#     print(f\"Directory does not exist: {directory_to_delete}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T05:53:24.785372Z","iopub.execute_input":"2024-06-28T05:53:24.785741Z","iopub.status.idle":"2024-06-28T05:53:24.798515Z","shell.execute_reply.started":"2024-06-28T05:53:24.78569Z","shell.execute_reply":"2024-06-28T05:53:24.797669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport random\nfrom shutil import copyfile\n\n# Paths\ndataset_path = '/kaggle/input/deepfake-detection-challenge/train_sample_videos'\nmetadata_file = os.path.join(dataset_path, 'metadata.json')\noutput_dir = '/kaggle/working/images'\n\n# Load metadata\nwith open(metadata_file, 'r') as f:\n    metadata = json.load(f)\n\n# Categorize videos by label and ensure they exist in the dataset directory\nreal_videos = [video for video, details in metadata.items() if details['label'] == 'REAL' and os.path.exists(os.path.join(dataset_path, video))]\nfake_videos = [video for video, details in metadata.items() if details['label'] == 'FAKE' and os.path.exists(os.path.join(dataset_path, video))]\n\n# Print the number of real and fake videos available after filtering\nprint(f'Filtered Available Videos: {len(real_videos)} REAL, {len(fake_videos)} FAKE')\n\n# Balance the dataset\nmin_length = min(len(real_videos), len(fake_videos))\nreal_videos = real_videos[:min_length]\nfake_videos = fake_videos[:min_length]\n\n# Print the number of videos in each category after balancing\nprint(f'Balanced Dataset: {len(real_videos)} REAL, {len(fake_videos)} FAKE')\n\n# Split the dataset into train, validation, and test sets\nsplit_ratio = [0.7, 0.2, 0.1]  # Train, validation, test split\n\nreal_train_split = int(len(real_videos) * split_ratio[0])\nreal_val_split = int(len(real_videos) * split_ratio[1])\nfake_train_split = int(len(fake_videos) * split_ratio[0])\nfake_val_split = int(len(fake_videos) * split_ratio[1])\n\ntrain_videos = real_videos[:real_train_split] + fake_videos[:fake_train_split]\nval_videos = real_videos[real_train_split:real_train_split+real_val_split] + fake_videos[fake_train_split:fake_train_split+fake_val_split]\ntest_videos = real_videos[real_train_split+real_val_split:] + fake_videos[fake_train_split+fake_val_split:]\n\n# Ensure the splits are balanced\nprint(f\"Train Set: {len(train_videos)} videos ({len(real_videos[:real_train_split])} REAL, {len(fake_videos[:fake_train_split])} FAKE)\")\nprint(f\"Validation Set: {len(val_videos)} videos ({len(real_videos[real_train_split:real_train_split+real_val_split])} REAL, {len(fake_videos[fake_train_split:fake_train_split+fake_val_split])} FAKE)\")\nprint(f\"Test Set: {len(test_videos)} videos ({len(real_videos[real_train_split+real_val_split:])} REAL, {len(fake_videos[fake_train_split+fake_val_split:])} FAKE)\")\n\ndef extract_frames(video_path, output_path, num_frames=7):\n    cap = cv2.VideoCapture(video_path)\n    frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    interval = frame_count // num_frames\n    \n    os.makedirs(output_path, exist_ok=True)\n    \n    for i in range(num_frames):\n        cap.set(cv2.CAP_PROP_POS_FRAMES, i * interval)\n        ret, frame = cap.read()\n        if ret:\n            frame_filename = os.path.join(output_path, f'frame_{i+1}.jpg')\n            cv2.imwrite(frame_filename, frame)\n        else:\n            print(f'Error reading frame {i} from {video_path}')\n    \n    cap.release()\n\ndef process_videos(videos, split, label, num_frames=7):\n    for video in videos:\n        video_path = os.path.join(dataset_path, video)\n        output_path = os.path.join(output_dir, split, label, video.replace('.mp4', ''))\n        extract_frames(video_path, output_path)\n\n# Process and save frames for each split\nos.makedirs(os.path.join(output_dir, 'train', 'REAL'), exist_ok=True)\nos.makedirs(os.path.join(output_dir, 'train', 'FAKE'), exist_ok=True)\nos.makedirs(os.path.join(output_dir, 'val', 'REAL'), exist_ok=True)\nos.makedirs(os.path.join(output_dir, 'val', 'FAKE'), exist_ok=True)\nos.makedirs(os.path.join(output_dir, 'test', 'REAL'), exist_ok=True)\nos.makedirs(os.path.join(output_dir, 'test', 'FAKE'), exist_ok=True)\n\nprocess_videos(real_videos[:real_train_split], 'train', 'REAL')\nprocess_videos(fake_videos[:fake_train_split], 'train', 'FAKE')\nprocess_videos(real_videos[real_train_split:real_train_split+real_val_split], 'val', 'REAL')\nprocess_videos(fake_videos[fake_train_split:fake_train_split+fake_val_split], 'val', 'FAKE')\nprocess_videos(real_videos[real_train_split+real_val_split:], 'test', 'REAL')\nprocess_videos(fake_videos[fake_train_split+fake_val_split:], 'test', 'FAKE')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T05:14:20.206479Z","iopub.execute_input":"2024-06-28T05:14:20.206847Z","iopub.status.idle":"2024-06-28T05:24:11.236986Z","shell.execute_reply.started":"2024-06-28T05:14:20.206804Z","shell.execute_reply":"2024-06-28T05:24:11.236058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!apt-get update\n!apt-get install -y build-essential cmake\n!pip install wheel\n\n!pip install dlib==19.24.0","metadata":{"execution":{"iopub.status.busy":"2024-06-28T05:25:52.237042Z","iopub.execute_input":"2024-06-28T05:25:52.237419Z","iopub.status.idle":"2024-06-28T05:32:20.733639Z","shell.execute_reply.started":"2024-06-28T05:25:52.237359Z","shell.execute_reply":"2024-06-28T05:32:20.732685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport dlib\n\n# Directories for images and extracted facial organs\nimages_dir = '/kaggle/working/images'\nfacial_organs_dir = '/kaggle/working/facial_organs'\n\n# Load the dlib facial landmark detector\ndetector = dlib.get_frontal_face_detector()\npredictor = dlib.shape_predictor('/kaggle/input/shape-predictor-68-face-landmarks-dat/shape_predictor_68_face_landmarks.dat')\n\n# Define function to extract facial organs\ndef extract_facial_organs(image, shape):\n    organs = {}\n    \n    # Define the indices for each facial organ based on the dlib 68-point model\n    indices = {\n        \"right_eyes\": list(range(36, 42)),\n        \"left_eyes\": list(range(42, 48)),\n        \"right_eyebrow\": list(range(17, 22)),\n        \"left_eyebrow\": list(range(22, 27)),\n        \"jaw\": list(range(0, 17)),\n        \"nose\": list(range(27, 36)),\n        \"mouth\": list(range(48, 68))\n    }\n    \n    for organ, points in indices.items():\n        points = [shape.part(i) for i in points]\n        x_coords = [p.x for p in points]\n        y_coords = [p.y for p in points]\n        x_min, x_max = min(x_coords), max(x_coords)\n        y_min, y_max = min(y_coords), max(y_coords)\n        organs[organ] = image[y_min:y_max, x_min:x_max]\n    \n    return organs\n\n# Function to process frames and save facial organs\ndef process_frames_and_extract_organs(input_dir, output_dir):\n    for root, _, files in os.walk(input_dir):\n        for file in files:\n            if file.endswith('.jpg'):\n                file_path = os.path.join(root, file)\n                image = cv2.imread(file_path)\n                gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n                \n                faces = detector(gray)\n                \n                for face in faces:\n                    shape = predictor(gray, face)\n                    organs = extract_facial_organs(image, shape)\n                    \n                    video_name = os.path.basename(os.path.dirname(file_path))  # Extract video name\n                    frame_number = os.path.splitext(file)[0]  # Extract frame number without extension\n                    base_name = f\"{video_name}_{frame_number}.jpg\"  # Create base name for the output file\n                    \n                    for organ, organ_image in organs.items():\n                        organ_output_dir = os.path.join(output_dir, organ)\n                        os.makedirs(organ_output_dir, exist_ok=True)\n                        organ_file_path = os.path.join(organ_output_dir, base_name)\n                        cv2.imwrite(organ_file_path, organ_image)\n\n# Create necessary directories for facial organs\nsplits = ['train', 'val', 'test']\nlabels = ['REAL', 'FAKE']\norgans = [\"right_eyes\", \"left_eyes\", \"right_eyebrow\", \"left_eyebrow\", \"jaw\", \"nose\", \"mouth\"]\n\nfor split in splits:\n    for label in labels:\n        for organ in organs:\n            os.makedirs(os.path.join(facial_organs_dir, split, label, organ), exist_ok=True)\n\n# Process frames in each split and extract facial organs\nfor split in splits:\n    for label in labels:\n        input_dir = os.path.join(images_dir, split, label)\n        output_dir = os.path.join(facial_organs_dir, split, label)\n        process_frames_and_extract_organs(input_dir, output_dir)\n\nprint(\"Facial organs extraction completed.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T05:54:04.739556Z","iopub.execute_input":"2024-06-28T05:54:04.739914Z","iopub.status.idle":"2024-06-28T05:59:20.530064Z","shell.execute_reply.started":"2024-06-28T05:54:04.739856Z","shell.execute_reply":"2024-06-28T05:59:20.529166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n# Base directory for the facial organs\nfacial_organs_dir = '/kaggle/working/facial_organs'\n\n# Define the splits and labels\nsplits = ['train', 'val', 'test']\nlabels = ['REAL', 'FAKE']\n\n# Dictionary to store the counts\nimage_counts = {split: {label: {} for label in labels} for split in splits}\n\n# Count images in each category\nfor split in splits:\n    for label in labels:\n        total_count = 0\n        for organ in os.listdir(os.path.join(facial_organs_dir, split, label)):\n            organ_dir = os.path.join(facial_organs_dir, split, label, organ)\n            if os.path.isdir(organ_dir):\n                num_images = len([file for file in os.listdir(organ_dir) if file.endswith('.jpg')])\n                image_counts[split][label][organ] = num_images\n                total_count += num_images\n        image_counts[split][label]['total'] = total_count\n\n# Print the image counts\nfor split in splits:\n    print(f\"\\n{split.capitalize()} Split:\")\n    for label in labels:\n        print(f\"  {label}:\")\n        for organ, count in image_counts[split][label].items():\n            print(f\"    {organ}: {count} images\")\n\n# If you want the overall counts as well\noverall_counts = {split: {label: 0 for label in labels} for split in splits}\nfor split in splits:\n    for label in labels:\n        overall_counts[split][label] = image_counts[split][label]['total']\n\nprint(\"\\nOverall Counts:\")\nfor split in splits:\n    for label in labels:\n        print(f\"  {split.capitalize()} {label}: {overall_counts[split][label]} images\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:00:54.081015Z","iopub.execute_input":"2024-06-28T06:00:54.081378Z","iopub.status.idle":"2024-06-28T06:00:54.107736Z","shell.execute_reply.started":"2024-06-28T06:00:54.081333Z","shell.execute_reply":"2024-06-28T06:00:54.106726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming the counts from the previous explanation\nimage_counts = {\n    'train': {\n        'REAL': {'total': 2443},\n        'FAKE': {'total': 2296},\n    },\n    'val': {\n        'REAL': {'total': 714},\n        'FAKE': {'total': 602},\n    },\n    'test': {\n        'REAL': {'total': 385},\n        'FAKE': {'total': 357},\n    }\n}\n\n# Calculate total images for each label\ntotal_real = sum(image_counts[split]['REAL']['total'] for split in image_counts)\ntotal_fake = sum(image_counts[split]['FAKE']['total'] for split in image_counts)\n\n# Calculate and print the percentages\nprint(\"Percentage Split:\")\nfor split in image_counts:\n    real_percentage = (image_counts[split]['REAL']['total'] / total_real) * 100\n    fake_percentage = (image_counts[split]['FAKE']['total'] / total_fake) * 100\n    print(f\"  {split.capitalize()} Split:\")\n    print(f\"    REAL: {real_percentage:.2f}%\")\n    print(f\"    FAKE: {fake_percentage:.2f}%\")\n\n# Print total counts for reference\nprint(\"\\nTotal Counts:\")\nprint(f\"  Total REAL images: {total_real}\")\nprint(f\"  Total FAKE images: {total_fake}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:02:04.799089Z","iopub.execute_input":"2024-06-28T06:02:04.799415Z","iopub.status.idle":"2024-06-28T06:02:04.811337Z","shell.execute_reply.started":"2024-06-28T06:02:04.799373Z","shell.execute_reply":"2024-06-28T06:02:04.810326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install transformers\n!pip install scikit-learn\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:03:09.198211Z","iopub.execute_input":"2024-06-28T06:03:09.198529Z","iopub.status.idle":"2024-06-28T06:03:32.298194Z","shell.execute_reply.started":"2024-06-28T06:03:09.198486Z","shell.execute_reply":"2024-06-28T06:03:32.297413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom PIL import Image\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\nfrom torchvision import transforms\nfrom torchvision.datasets import DatasetFolder\nfrom torch.utils.data import DataLoader, random_split\nfrom transformers import ViTForImageClassification, ViTFeatureExtractor\nimport torch\nfrom torch import nn, optim\nfrom tqdm import tqdm\n\n# Define paths and parameters\ndata_dir = '/kaggle/working/facial_organs'\nbatch_size = 16\nnum_epochs = 10\nlearning_rate = 1e-4\norgans = ['left_eyebrow', 'mouth', 'left_eyes', 'right_eyes', 'jaw', 'right_eyebrow', 'nose']\n\n# Define data transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n])\n\n# Function to load images and labels\nclass OrganDataset(DatasetFolder):\n    def __init__(self, root, organ, split, transform=None):\n        self.organ = organ\n        self.real_dir = os.path.join(root, split, 'REAL', organ)\n        self.fake_dir = os.path.join(root, split, 'FAKE', organ)\n        self.real_images = [(os.path.join(self.real_dir, img), 0) for img in os.listdir(self.real_dir) if img.endswith(('.png', '.jpg', '.jpeg'))]\n        self.fake_images = [(os.path.join(self.fake_dir, img), 1) for img in os.listdir(self.fake_dir) if img.endswith(('.png', '.jpg', '.jpeg'))]\n        self.images = self.real_images + self.fake_images\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        img_path, label = self.images[idx]\n        image = Image.open(img_path).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Function to train the model\ndef train(model, train_loader, val_loader, criterion, optimizer, num_epochs, organ):\n    best_val_acc = 0.0\n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        correct = 0\n        total = 0\n        \n        for inputs, labels in train_loader:\n            inputs = [tensor_to_pil(img) for img in inputs]\n            inputs = feature_extractor(inputs, return_tensors=\"pt\").pixel_values\n            optimizer.zero_grad()\n            outputs = model(inputs).logits\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n        \n        train_acc = correct / total\n        \n        val_acc, val_loss, _, _ = evaluate(model, val_loader, criterion)\n        \n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            torch.save(model.state_dict(), f'best_model_{organ}.pth')\n        \n        print(f'Epoch [{epoch+1}/{num_epochs}], Organ: {organ}, Loss: {running_loss/len(train_loader)}, Train Accuracy: {train_acc}, Val Accuracy: {val_acc}, Val Loss: {val_loss}')\n    \n    print(f'Training complete for {organ}')\n\n# Function to evaluate the model\ndef evaluate(model, data_loader, criterion):\n    model.eval()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    all_labels = []\n    all_preds = []\n    \n    with torch.no_grad():\n        for inputs, labels in data_loader:\n            inputs = [tensor_to_pil(img) for img in inputs]\n            inputs = feature_extractor(inputs, return_tensors=\"pt\").pixel_values\n            outputs = model(inputs).logits\n            loss = criterion(outputs, labels)\n            running_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_preds.extend(predicted.cpu().numpy())\n    \n    acc = correct / total\n    return acc, running_loss / len(data_loader), all_labels, all_preds\n\n# Helper function to convert tensor to PIL image\ndef tensor_to_pil(tensor):\n    unloader = transforms.ToPILImage()\n    return unloader(tensor.cpu())\n\n# Load pretrained Vision Transformer model and feature extractor\nmodel_name = 'google/vit-base-patch16-224'\nfeature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\n\n# Train and save models for each organ\nfor organ in organs:\n    # Load the dataset for the current organ and split\n    train_dataset = OrganDataset(data_dir, organ, 'train', transform=transform)\n    val_dataset = OrganDataset(data_dir, organ, 'val', transform=transform)\n    test_dataset = OrganDataset(data_dir, organ, 'test', transform=transform)\n\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n    # Create a new model for the current organ\n    model = ViTForImageClassification.from_pretrained(model_name)\n    model.classifier = nn.Sequential(\n        nn.Linear(model.config.hidden_size, 2)\n    )\n\n    # Define optimizer and loss function\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    criterion = nn.CrossEntropyLoss()\n\n    # Train the model\n    train(model, train_loader, val_loader, criterion, optimizer, num_epochs, organ)\n\n    # Load the best model for the current organ\n    model.load_state_dict(torch.load(f'best_model_{organ}.pth'))\n\n    # Evaluate on the test set\n    test_acc, test_loss, test_labels, test_preds = evaluate(model, test_loader, criterion)\n    precision = precision_score(test_labels, test_preds, average='binary')\n    recall = recall_score(test_labels, test_preds, average='binary')\n    f1 = f1_score(test_labels, test_preds, average='binary')\n    conf_matrix = confusion_matrix(test_labels, test_preds)\n\n    print(f'Test Accuracy for {organ}: {test_acc}')\n    print(f'Precision for {organ}: {precision}')\n    print(f'Recall for {organ}: {recall}')\n    print(f'F1 Score for {organ}: {f1}')\n    print(f'Confusion Matrix for {organ}:')\n    print(conf_matrix)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T07:23:55.911786Z","iopub.execute_input":"2024-06-28T07:23:55.912193Z","iopub.status.idle":"2024-06-28T08:06:25.701287Z","shell.execute_reply.started":"2024-06-28T07:23:55.91213Z","shell.execute_reply":"2024-06-28T08:06:25.700234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom PIL import Image\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, confusion_matrix\nfrom torchvision import transforms\nfrom torchvision.datasets import DatasetFolder\nfrom torch.utils.data import DataLoader, random_split\nfrom transformers import ViTForImageClassification, ViTFeatureExtractor\nimport torch\nfrom torch import nn, optim\nfrom tqdm import tqdm\n\n# Define paths and parameters\ndata_dir = '/kaggle/working/facial_organs'\nbatch_size = 16\nnum_epochs = 10\nlearning_rate = 1e-4\norgans = ['left_eyebrow', 'mouth', 'left_eyes', 'right_eyes', 'jaw', 'right_eyebrow', 'nose']\n\n# Define data transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n])\n\n# Check if GPU is available\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Function to load images and labels\nclass OrganDataset(DatasetFolder):\n    def __init__(self, root, organ, split, transform=None):\n        self.organ = organ\n        self.real_dir = os.path.join(root, split, 'REAL', organ)\n        self.fake_dir = os.path.join(root, split, 'FAKE', organ)\n        self.real_images = [(os.path.join(self.real_dir, img), 0) for img in os.listdir(self.real_dir) if img.endswith(('.png', '.jpg', '.jpeg'))]\n        self.fake_images = [(os.path.join(self.fake_dir, img), 1) for img in os.listdir(self.fake_dir) if img.endswith(('.png', '.jpg', '.jpeg'))]\n        self.images = self.real_images + self.fake_images\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        img_path, label = self.images[idx]\n        image = Image.open(img_path).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        return image, label\n\n# Function to train the model\ndef train(model, train_loader, val_loader, criterion, optimizer, num_epochs, organ):\n    best_val_acc = 0.0\n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        correct = 0\n        total = 0\n        \n        for inputs, labels in train_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            inputs = [tensor_to_pil(img) for img in inputs]\n            inputs = feature_extractor(inputs, return_tensors=\"pt\").pixel_values.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs).logits\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n        \n        train_acc = correct / total\n        \n        val_acc, val_loss, _, _ = evaluate(model, val_loader, criterion)\n        \n        if val_acc > best_val_acc:\n            best_val_acc = val_acc\n            torch.save(model.state_dict(), f'best_model_{organ}.pth')\n        \n        print(f'Epoch [{epoch+1}/{num_epochs}], Organ: {organ}, Loss: {running_loss/len(train_loader)}, Train Accuracy: {train_acc}, Val Accuracy: {val_acc}, Val Loss: {val_loss}')\n    \n    print(f'Training complete for {organ}')\n\n# Function to evaluate the model\ndef evaluate(model, data_loader, criterion):\n    model.eval()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    all_labels = []\n    all_preds = []\n    \n    with torch.no_grad():\n        for inputs, labels in data_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            inputs = [tensor_to_pil(img) for img in inputs]\n            inputs = feature_extractor(inputs, return_tensors=\"pt\").pixel_values.to(device)\n            outputs = model(inputs).logits\n            loss = criterion(outputs, labels)\n            running_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n            all_labels.extend(labels.cpu().numpy())\n            all_preds.extend(predicted.cpu().numpy())\n    \n    acc = correct / total\n    return acc, running_loss / len(data_loader), all_labels, all_preds\n\n# Helper function to convert tensor to PIL image\ndef tensor_to_pil(tensor):\n    unloader = transforms.ToPILImage()\n    return unloader(tensor.cpu())\n\n# Load pretrained Vision Transformer model and feature extractor\nmodel_name = 'google/vit-base-patch16-224'\nfeature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\n\n# Train and save models for each organ\nfor organ in organs:\n    # Load the dataset for the current organ and split\n    train_dataset = OrganDataset(data_dir, organ, 'train', transform=transform)\n    val_dataset = OrganDataset(data_dir, organ, 'val', transform=transform)\n    test_dataset = OrganDataset(data_dir, organ, 'test', transform=transform)\n\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n    # Create a new model for the current organ\n    model = ViTForImageClassification.from_pretrained(model_name)\n    model.classifier = nn.Sequential(\n        nn.Linear(model.config.hidden_size, 2)\n    )\n    model.to(device)\n\n    # Define optimizer and loss function\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    criterion = nn.CrossEntropyLoss()\n\n    # Train the model\n    train(model, train_loader, val_loader, criterion, optimizer, num_epochs, organ)\n\n    # Load the best model for the current organ\n    model.load_state_dict(torch.load(f'best_model_{organ}.pth'))\n\n    # Evaluate on the test set\n    test_acc, test_loss, test_labels, test_preds = evaluate(model, test_loader, criterion)\n    precision = precision_score(test_labels, test_preds, average='binary')\n    recall = recall_score(test_labels, test_preds, average='binary')\n    f1 = f1_score(test_labels, test_preds, average='binary')\n    conf_matrix = confusion_matrix(test_labels, test_preds)\n\n    print(f'Test Accuracy for {organ}: {test_acc}')\n    print(f'Precision for {organ}: {precision}')\n    print(f'Recall for {organ}: {recall}')\n    print(f'F1 Score for {organ}: {f1}')\n    print(f'Confusion Matrix for {organ}:')\n    print(conf_matrix)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T08:06:30.982092Z","iopub.execute_input":"2024-06-28T08:06:30.982458Z","iopub.status.idle":"2024-06-28T08:43:21.35292Z","shell.execute_reply.started":"2024-06-28T08:06:30.982394Z","shell.execute_reply":"2024-06-28T08:43:21.351694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nfrom transformers import ViTModel, ViTFeatureExtractor\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader, ConcatDataset\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport pandas as pd\nimport numpy as np\nfrom torch import nn, optim\n\n# Directories\nbase_dir = '/kaggle/working/facial_organs'\norgans = ['right_eyes']\nsave_dir = '/kaggle/working/saved_models'\nmetadata_path = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json'\n\n# Create the directory if it doesn't exist\nos.makedirs(save_dir, exist_ok=True)\n\n# Load metadata\nmetadata = pd.read_json(metadata_path).transpose()\n\n# Define the model and feature extractor\nmodel_name = \"google/vit-base-patch16-224\"\n\ntry:\n    model = ViTModel.from_pretrained(model_name)\n    feature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\nexcept OSError:\n    print(f\"Failed to load the model '{model_name}' from Hugging Face. Please check the model name and internet connection.\")\n    raise\n\n# Add a new classifier layer for binary classification\nclass ViTForBinaryClassification(nn.Module):\n    def __init__(self, model):\n        super(ViTForBinaryClassification, self).__init__()\n        self.vit = model\n        self.classifier = nn.Linear(model.config.hidden_size, 2)\n    \n    def forward(self, x):\n        outputs = self.vit(x)\n        logits = self.classifier(outputs.last_hidden_state[:, 0, :])\n        return logits\n\nmodel = ViTForBinaryClassification(model)\n\n# Device configuration\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n# Transform for image preprocessing\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std),\n])\n\n# Function to prepare datasets\ndef prepare_datasets(base_dir, organ, transform, batch_size=16):\n    datasets_dict = {}\n    for split in ['train', 'val', 'test']:\n        data_dir = os.path.join(base_dir, split)\n        real_dir = os.path.join(data_dir, 'REAL', organ)\n        fake_dir = os.path.join(data_dir, 'FAKE', organ)\n        \n        print(f\"Checking directories: {real_dir}, {fake_dir}\")\n        \n        if not os.path.isdir(real_dir) or not os.path.isdir(fake_dir):\n            print(f\"Directory does not exist: {real_dir} or {fake_dir}\")\n            continue\n        \n        real_files = os.listdir(real_dir)\n        fake_files = os.listdir(fake_dir)\n        \n        print(f\"Real files ({len(real_files)}): {real_files}\")\n        print(f\"Fake files ({len(fake_files)}): {fake_files}\")\n        \n        if not real_files:\n            print(f\"No files found in {real_dir}. Please check the directory and file extensions.\")\n        if not fake_files:\n            print(f\"No files found in {fake_dir}. Please check the directory and file extensions.\")\n        \n        real_dataset = datasets.ImageFolder(root=real_dir, transform=transform)\n        fake_dataset = datasets.ImageFolder(root=fake_dir, transform=transform)\n        \n        combined_dataset = ConcatDataset([real_dataset, fake_dataset])\n        loader = DataLoader(combined_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True)\n        datasets_dict[split] = loader\n    return datasets_dict\n\n# Load data for each organ\ndata_loaders = {organ: prepare_datasets(base_dir, organ, transform) for organ in organs}\n\n# Function to train and evaluate model\ndef train_and_evaluate(model, data_loaders, organ, epochs=5):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=1e-4)\n    \n    # Training loop\n    model.train()\n    for epoch in range(epochs):\n        running_loss = 0.0\n        for images, labels in data_loaders['train']:\n            images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n            \n            # Zero the parameter gradients\n            optimizer.zero_grad()\n            \n            # Forward pass\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            \n            # Backward pass and optimize\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item()\n        print(f'Epoch [{epoch+1}/{epochs}], Loss: {running_loss/len(data_loaders[\"train\"]):.4f}')\n    \n    # Save the model after training\n    model_save_path = os.path.join(save_dir, f\"{organ}_vit_model.pth\")\n    torch.save(model.state_dict(), model_save_path)\n    print(f\"Model saved for {organ} at {model_save_path}\")\n    \n    # Evaluation\n    model.eval()\n    metrics = {}\n    for split in ['val', 'test']:\n        all_preds = []\n        all_labels = []\n        with torch.no_grad():\n            for images, labels in data_loaders[split]:\n                images, labels = images.to(device, non_blocking=True), labels.to(device, non_blocking=True)\n                outputs = model(images)\n                _, preds = torch.max(outputs, 1)\n                \n                all_preds.extend(preds.cpu().numpy())\n                all_labels.extend(labels.cpu().numpy())\n        \n        # Calculate performance metrics\n        metrics[split] = classification_report(all_labels, all_preds, target_names=['REAL', 'FAKE'], output_dict=True)\n        metrics[split]['confusion_matrix'] = confusion_matrix(all_labels, all_preds)\n    \n    return metrics\n\n# Train and evaluate the model for each facial organ\norgan_metrics = {}\nfor organ in organs:\n    print(f\"Training and evaluating for {organ}\")\n    organ_metrics[organ] = train_and_evaluate(model, data_loaders[organ])\n\n# Evaluate overall performance\nfor organ, metrics in organ_metrics.items():\n    print(f\"Metrics for {organ}:\")\n    for split in ['val', 'test']:\n        print(f\"  {split.capitalize()} set:\")\n        print(f\"    Classification Report:\\n{classification_report(metrics[split])}\")\n        print(f\"    Confusion Matrix:\\n{metrics[split]['confusion_matrix']}\")\n\nprint(\"Training and evaluation completed.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:52:48.733567Z","iopub.execute_input":"2024-06-28T06:52:48.7339Z","iopub.status.idle":"2024-06-28T06:52:50.162347Z","shell.execute_reply.started":"2024-06-28T06:52:48.733856Z","shell.execute_reply":"2024-06-28T06:52:50.161243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport torch\nimport torchvision.transforms as T\nfrom transformers import ViTFeatureExtractor, ViTForImageClassification\nfrom sklearn.metrics import accuracy_score, confusion_matrix, classification_report\nfrom PIL import Image\n\n# Device configuration (adjust as necessary)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Paths to your data directories\ntrain_dir = \"/kaggle/working/facial_organs/train\"\nval_dir = \"/kaggle/working/facial_organs/val\"\ntest_dir = \"/kaggle/working/facial_organs/test\"\n\n# Define image preprocessing pipeline\npreprocess = T.Compose([\n    T.Resize((224, 224)),  # Resize to match model's expected sizing\n    T.ToTensor(),          # Convert PIL image to tensor\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalize image\n])\n\n# Function to load and preprocess image\ndef load_and_preprocess_image(image_path):\n    image = Image.open(image_path).convert('RGB')\n    image = preprocess(image)\n    return image.unsqueeze(0)  # Add batch dimension\n\n# Function to evaluate model on a dataset\ndef evaluate_model(model, data_dir):\n    predictions = []\n    true_labels = []\n\n    # Iterate through each image in the data directory\n    for label in ['REAL', 'FAKE']:\n        for organ in ['right_eyes']:\n            organ_dir = os.path.join(data_dir, label, organ)\n            for filename in os.listdir(organ_dir):\n                image_path = os.path.join(organ_dir, filename)\n\n                # Load and preprocess image\n                inputs = load_and_preprocess_image(image_path).to(device)\n\n                # Forward pass\n                with torch.no_grad():\n                    outputs = model(inputs)\n\n                # Get predicted label\n                predicted_label = torch.argmax(outputs.logits, dim=1).item()\n                predictions.append(predicted_label)\n                true_labels.append(0 if label == 'REAL' else 1)  # Assuming REAL is class 0, FAKE is class 1\n\n    # Calculate metrics\n    accuracy = accuracy_score(true_labels, predictions)\n    confusion = confusion_matrix(true_labels, predictions)\n    report = classification_report(true_labels, predictions, target_names=['REAL', 'FAKE'])\n\n    return accuracy, confusion, report\n\n# Clear the transformers cache\nos.system(\"transformers-cli cache clear\")\n\n# Load pretrained Vision Transformer model\nmodel_name = 'google/vit-base-patch16-224-in21k'\nfeature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\nmodel = ViTForImageClassification.from_pretrained(model_name, num_labels=2).to(device)\nmodel.eval()  # Set model to evaluation mode\n\n# Evaluate on train set\ntrain_accuracy, train_confusion, train_report = evaluate_model(model, train_dir)\nprint(\"Train Set Evaluation:\")\nprint(f\"Accuracy: {train_accuracy}\")\nprint(\"Confusion Matrix:\")\nprint(train_confusion)\nprint(\"Classification Report:\")\nprint(train_report)\n\n# Evaluate on validation set\nval_accuracy, val_confusion, val_report = evaluate_model(model, val_dir)\nprint(\"\\nValidation Set Evaluation:\")\nprint(f\"Accuracy: {val_accuracy}\")\nprint(\"Confusion Matrix:\")\nprint(val_confusion)\nprint(\"Classification Report:\")\nprint(val_report)\n\n# Evaluate on test set\ntest_accuracy, test_confusion, test_report = evaluate_model(model, test_dir)\nprint(\"\\nTest Set Evaluation:\")\nprint(f\"Accuracy: {test_accuracy}\")\nprint(\"Confusion Matrix:\")\nprint(test_confusion)\nprint(\"Classification Report:\")\nprint(test_report)\n\n# Save the model\nmodel_save_path = \"/kaggle/working/saved_model/right_eyes_vit_model.pth\"\nos.makedirs(os.path.dirname(model_save_path), exist_ok=True)\ntorch.save(model.state_dict(), model_save_path)\nprint(f\"Model saved at {model_save_path}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T07:02:35.15217Z","iopub.execute_input":"2024-06-28T07:02:35.152577Z","iopub.status.idle":"2024-06-28T07:02:42.102799Z","shell.execute_reply.started":"2024-06-28T07:02:35.152518Z","shell.execute_reply":"2024-06-28T07:02:42.1015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport torch\nimport torchvision.transforms as T\nfrom transformers import ViTFeatureExtractor, ViTForImageClassification\nfrom sklearn.metrics import accuracy_score, confusion_matrix, classification_report\nfrom PIL import Image\nimport cv2\n\n# Device configuration (adjust as necessary)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Paths to your data directories\ntrain_dir = \"/kaggle/working/facial_organs/train\"\nval_dir = \"/kaggle/working/facial_organs/val\"\ntest_dir = \"/kaggle/working/facial_organs/test\"\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:55:28.615708Z","iopub.execute_input":"2024-06-28T06:55:28.616093Z","iopub.status.idle":"2024-06-28T06:55:28.622651Z","shell.execute_reply.started":"2024-06-28T06:55:28.616027Z","shell.execute_reply":"2024-06-28T06:55:28.621782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define image preprocessing pipeline\npreprocess = T.Compose([\n    T.Resize((224, 224)),  # Resize to match model's expected sizing\n    T.ToTensor(),          # Convert PIL image to tensor\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalize image\n])\n\n# Function to load and preprocess image\ndef load_and_preprocess_image(image_path):\n    image = Image.open(image_path).convert('RGB')\n    image = preprocess(image)\n    return image.unsqueeze(0)  # Add batch dimension\n\n# Function to evaluate model on a dataset\ndef evaluate_model(model, data_dir):\n    predictions = []\n    true_labels = []\n\n    # Iterate through each image in the data directory\n    for label in ['REAL', 'FAKE']:\n        for organ in ['right_eyes']:\n            organ_dir = os.path.join(data_dir, label, organ)\n            for filename in os.listdir(organ_dir):\n                image_path = os.path.join(organ_dir, filename)\n\n                # Load and preprocess image\n                inputs = load_and_preprocess_image(image_path).to(device)\n\n                # Forward pass\n                with torch.no_grad():\n                    outputs = model(inputs)\n\n                # Get predicted label\n                predicted_label = torch.argmax(outputs[0]).item()\n                predictions.append(predicted_label)\n                true_labels.append(0 if label == 'REAL' else 1)  # Assuming REAL is class 0, FAKE is class 1\n\n    # Calculate metrics\n    accuracy = accuracy_score(true_labels, predictions)\n    confusion = confusion_matrix(true_labels, predictions)\n    report = classification_report(true_labels, predictions, target_names=['REAL', 'FAKE'])\n\n    return accuracy, confusion, report\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:55:54.709938Z","iopub.execute_input":"2024-06-28T06:55:54.710328Z","iopub.status.idle":"2024-06-28T06:55:54.724813Z","shell.execute_reply.started":"2024-06-28T06:55:54.710257Z","shell.execute_reply":"2024-06-28T06:55:54.723887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load pretrained Vision Transformer model\nmodel_name = 'google/vit-base-patch16-224-in21k'\nfeature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\nmodel = ViTForImageClassification.from_pretrained(model_name).to(device)\nmodel.eval()  # Set model to evaluation mode\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:56:05.617989Z","iopub.execute_input":"2024-06-28T06:56:05.618355Z","iopub.status.idle":"2024-06-28T06:56:06.269583Z","shell.execute_reply.started":"2024-06-28T06:56:05.618295Z","shell.execute_reply":"2024-06-28T06:56:06.267975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate on train set\ntrain_accuracy, train_confusion, train_report = evaluate_model(model, train_dir)\nprint(\"Train Set Evaluation:\")\nprint(f\"Accuracy: {train_accuracy}\")\nprint(\"Confusion Matrix:\")\nprint(train_confusion)\nprint(\"Classification Report:\")\nprint(train_report)\n\n# Evaluate on validation set\nval_accuracy, val_confusion, val_report = evaluate_model(model, val_dir)\nprint(\"\\nValidation Set Evaluation:\")\nprint(f\"Accuracy: {val_accuracy}\")\nprint(\"Confusion Matrix:\")\nprint(val_confusion)\nprint(\"Classification Report:\")\nprint(val_report)\n\n# Evaluate on test set\ntest_accuracy, test_confusion, test_report = evaluate_model(model, test_dir)\nprint(\"\\nTest Set Evaluation:\")\nprint(f\"Accuracy: {test_accuracy}\")\nprint(\"Confusion Matrix:\")\nprint(test_confusion)\nprint(\"Classification Report:\")\nprint(test_report)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom transformers import ViTFeatureExtractor, ViTForImageClassification\n\n# Device configuration (adjust as necessary)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Load pretrained Vision Transformer model\nmodel_name = 'google/vit-base-patch16-224-in21k'\nfeature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\nmodel = ViTForImageClassification.from_pretrained(model_name).to(device)\nmodel.eval()  # Set model to evaluation mode\n","metadata":{"execution":{"iopub.status.busy":"2024-06-28T06:57:10.712137Z","iopub.execute_input":"2024-06-28T06:57:10.712469Z","iopub.status.idle":"2024-06-28T06:57:11.204444Z","shell.execute_reply.started":"2024-06-28T06:57:10.712422Z","shell.execute_reply":"2024-06-28T06:57:11.20323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":", 'left_eyes', 'right_eyebrow', 'left_eyebrow', 'jaw', 'nose', 'mouth'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_FOLDER = \"../input/deepfake-detection-challenge\" \nTRAIN_SAMPLE_FOLDER = \"train_sample_videos\"\nTEST_FOLDER = \"test_videos\"","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:14.31608Z","iopub.execute_input":"2024-06-27T05:23:14.316324Z","iopub.status.idle":"2024-06-27T05:23:14.320286Z","shell.execute_reply.started":"2024-06-27T05:23:14.316284Z","shell.execute_reply":"2024-06-27T05:23:14.319385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FACE_DETECTION_FOLDER = '../input/haarcascades'\nprint(f\"Face detection resources: {os.listdir(FACE_DETECTION_FOLDER)}\")    ","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:17.355525Z","iopub.execute_input":"2024-06-27T05:23:17.355864Z","iopub.status.idle":"2024-06-27T05:23:17.370231Z","shell.execute_reply.started":"2024-06-27T05:23:17.355817Z","shell.execute_reply":"2024-06-27T05:23:17.369317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = list(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))\next_dict = []\nfor file in train_list:\n    file_ext = file.split('.')[1]\n    if (file_ext not in ext_dict):\n        ext_dict.append(file_ext)\nprint(f\"Extensions: {ext_dict}\") ","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:20.996713Z","iopub.execute_input":"2024-06-27T05:23:20.997014Z","iopub.status.idle":"2024-06-27T05:23:21.086947Z","shell.execute_reply.started":"2024-06-27T05:23:20.996972Z","shell.execute_reply":"2024-06-27T05:23:21.086013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_list = list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))\next_dict = []\nfor file in test_list:\n    file_ext = file.split('.')[1]\n    if (file_ext not in ext_dict):\n        ext_dict.append(file_ext)\nprint(f\"Extensions: {ext_dict}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:23.62246Z","iopub.execute_input":"2024-06-27T05:23:23.62284Z","iopub.status.idle":"2024-06-27T05:23:23.707213Z","shell.execute_reply.started":"2024-06-27T05:23:23.622772Z","shell.execute_reply":"2024-06-27T05:23:23.706332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_file = [file for file in train_list if  file.endswith('json')][0]\nprint(f\"JSON file: {json_file}\")\n#reading the json file\ndef get_meta_from_json(path):\n    df = pd.read_json(os.path.join(DATA_FOLDER, path, json_file))\n    df = df.T\n    return df\n\nmeta_train_df = get_meta_from_json(TRAIN_SAMPLE_FOLDER)\nmeta_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:26.510714Z","iopub.execute_input":"2024-06-27T05:23:26.51101Z","iopub.status.idle":"2024-06-27T05:23:27.172421Z","shell.execute_reply.started":"2024-06-27T05:23:26.51097Z","shell.execute_reply":"2024-06-27T05:23:27.171552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def missing_data(data):\n    total = data.isnull().sum()\n    percent = (data.isnull().sum()/data.isnull().count()*100)\n    tt = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    types = []\n    for col in data.columns:\n        dtype = str(data[col].dtype)\n        types.append(dtype)\n    tt['Types'] = types\n    return(np.transpose(tt))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:30.660412Z","iopub.execute_input":"2024-06-27T05:23:30.660746Z","iopub.status.idle":"2024-06-27T05:23:30.667878Z","shell.execute_reply.started":"2024-06-27T05:23:30.660696Z","shell.execute_reply":"2024-06-27T05:23:30.667144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_data(meta_train_df)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:33.429198Z","iopub.execute_input":"2024-06-27T05:23:33.429553Z","iopub.status.idle":"2024-06-27T05:23:33.509806Z","shell.execute_reply.started":"2024-06-27T05:23:33.429478Z","shell.execute_reply":"2024-06-27T05:23:33.508939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_data(meta_train_df.loc[meta_train_df.label == 'REAL'])","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:36.754347Z","iopub.execute_input":"2024-06-27T05:23:36.754702Z","iopub.status.idle":"2024-06-27T05:23:36.771855Z","shell.execute_reply.started":"2024-06-27T05:23:36.75464Z","shell.execute_reply":"2024-06-27T05:23:36.771049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unique_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columns = ['Totals']\n    uniques = []\n    for col in data.columns:\n        unique = data[col].nunique() #collect all unique instances\n        uniques.append(unique)\n    tt['Uniques'] = uniques\n    return(np.transpose(tt))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:39.432836Z","iopub.execute_input":"2024-06-27T05:23:39.433138Z","iopub.status.idle":"2024-06-27T05:23:39.440428Z","shell.execute_reply.started":"2024-06-27T05:23:39.433095Z","shell.execute_reply":"2024-06-27T05:23:39.439583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values(meta_train_df)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:43.304897Z","iopub.execute_input":"2024-06-27T05:23:43.305193Z","iopub.status.idle":"2024-06-27T05:23:43.321107Z","shell.execute_reply.started":"2024-06-27T05:23:43.305151Z","shell.execute_reply":"2024-06-27T05:23:43.320388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def most_frequent_values(data):\n    total = data.count()\n    tt = pd.DataFrame(total)\n    tt.columns = ['Total']\n    items = []\n    vals = []\n    for col in data.columns:\n        itm = data[col].value_counts().index[0]\n        val = data[col].value_counts().values[0]\n        items.append(itm)\n        vals.append(val)\n    tt['Most frequent item'] = items\n    tt['Frequence'] = vals\n    tt['Percent from total'] = np.round(vals / total * 100, 3)\n    return(np.transpose(tt))","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:46.94918Z","iopub.execute_input":"2024-06-27T05:23:46.949483Z","iopub.status.idle":"2024-06-27T05:23:46.957913Z","shell.execute_reply.started":"2024-06-27T05:23:46.94944Z","shell.execute_reply":"2024-06-27T05:23:46.956965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_frequent_values(meta_train_df)","metadata":{"execution":{"iopub.status.busy":"2024-06-26T06:15:24.869161Z","iopub.execute_input":"2024-06-26T06:15:24.869505Z","iopub.status.idle":"2024-06-26T06:15:24.894909Z","shell.execute_reply.started":"2024-06-26T06:15:24.869445Z","shell.execute_reply":"2024-06-26T06:15:24.894042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_frequent_values(meta_train_df.loc[meta_train_df.label == 'FAKE'])","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:54.446207Z","iopub.execute_input":"2024-06-27T05:23:54.446525Z","iopub.status.idle":"2024-06-27T05:23:54.470841Z","shell.execute_reply.started":"2024-06-27T05:23:54.446474Z","shell.execute_reply":"2024-06-27T05:23:54.470099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_count(feature, title, df, size=1):\n  '''\n    Plot count of classes / feature\n    param: feature - the feature to analyze\n    param: title - title to add to the graph\n    param: df - dataframe from which we plot feature's classes distribution \n    param: size - default 1.\n  '''  \n  f, ax = plt.subplots(1,1, figsize=(4*size,4))\n  total = float(len(df))\n  g =  sns.countplot(df[feature], order = df[feature].value_counts().index[:20], palette='Set3')\n  g.set_title(\"Number and percentage of {}\".format(title)) \n  if(size > 2):\n    plt.xticks(rotation=90, size=8)\n  for p in ax.patches:\n     height = p.get_height()\n     ax.text(p.get_x()+ p.get_width()/2.,height + 3,'{:1.2f}%'.format(100*height/total),ha=\"center\")\n\n  plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:23:58.964997Z","iopub.execute_input":"2024-06-27T05:23:58.965302Z","iopub.status.idle":"2024-06-27T05:23:58.975113Z","shell.execute_reply.started":"2024-06-27T05:23:58.965257Z","shell.execute_reply":"2024-06-27T05:23:58.974294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_count('split','split(train)',meta_train_df)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:24:02.642483Z","iopub.execute_input":"2024-06-27T05:24:02.642861Z","iopub.status.idle":"2024-06-27T05:24:02.887894Z","shell.execute_reply.started":"2024-06-27T05:24:02.642795Z","shell.execute_reply":"2024-06-27T05:24:02.886414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_count('label','label(train)',meta_train_df)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:24:05.68504Z","iopub.execute_input":"2024-06-27T05:24:05.685338Z","iopub.status.idle":"2024-06-27T05:24:05.913922Z","shell.execute_reply.started":"2024-06-27T05:24:05.685295Z","shell.execute_reply":"2024-06-27T05:24:05.910707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta = np.array(list(meta_train_df.index))\nstorage = np.array([file for file in train_list if  file.endswith('mp4')])\nprint(f\"Metadata: {meta.shape[0]}, Folder: {storage.shape[0]}\")\nprint(f\"Files in metadata and not in folder: {np.setdiff1d(meta,storage,assume_unique=False).shape[0]}\")\nprint(f\"Files in folder and not in metadata: {np.setdiff1d(storage,meta,assume_unique=False).shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:24:08.982838Z","iopub.execute_input":"2024-06-27T05:24:08.983198Z","iopub.status.idle":"2024-06-27T05:24:08.991438Z","shell.execute_reply.started":"2024-06-27T05:24:08.983134Z","shell.execute_reply":"2024-06-27T05:24:08.990459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport random\nfrom shutil import copyfile\n\n# Load metadata\nmetadata_path = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json\"\nwith open(metadata_path, 'r') as f:\n    metadata = json.load(f)\n\n# Split videos into real and fake\nreal_videos = [video for video, info in metadata.items() if info['label'] == 'REAL']\nfake_videos = [video for video, info in metadata.items() if info['label'] == 'FAKE']\n\n# Desired number of videos for balanced dataset\nnum_videos = len(real_videos) + len(fake_videos)\nnum_balanced_videos = num_videos // 2\n\n# Downsample fake videos to match the desired balanced number\ndownsampled_fake_videos = random.sample(fake_videos, num_balanced_videos)\n\n# Upsample real videos to match the desired balanced number\nupsampled_real_videos = random.choices(real_videos, k=num_balanced_videos)\n\n# Combine and shuffle\nbalanced_videos = upsampled_real_videos + downsampled_fake_videos\nrandom.shuffle(balanced_videos)\n\n# Print the result\nprint(f\"Total balanced videos: {len(balanced_videos)}\")\nprint(f\"Number of real videos: {len([video for video in balanced_videos if video in real_videos])}\")\nprint(f\"Number of fake videos: {len([video for video in balanced_videos if video in fake_videos])}\")\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T10:05:35.909712Z","iopub.execute_input":"2024-06-27T10:05:35.910031Z","iopub.status.idle":"2024-06-27T10:05:35.927811Z","shell.execute_reply.started":"2024-06-27T10:05:35.909987Z","shell.execute_reply":"2024-06-27T10:05:35.927092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport random\nfrom shutil import copyfile\nfrom sklearn.model_selection import train_test_split\n# Split into train, validation, and test sets\ntrain_videos, test_videos = train_test_split(balanced_videos, test_size=0.2, random_state=42)\ntrain_videos, val_videos = train_test_split(train_videos, test_size=0.25, random_state=42)  # 0.25 * 0.8 = 0.2\n\nprint(f\"Total training videos: {len(train_videos)}\")\nprint(f\"Total validation videos: {len(val_videos)}\")\nprint(f\"Total testing videos: {len(test_videos)}\")","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:24:20.347722Z","iopub.execute_input":"2024-06-27T05:24:20.348075Z","iopub.status.idle":"2024-06-27T05:24:20.357092Z","shell.execute_reply.started":"2024-06-27T05:24:20.348016Z","shell.execute_reply":"2024-06-27T05:24:20.355869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport random\nimport cv2\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom shutil import copyfile\n\ndef extract_frames(video_path, output_folder, num_frames=7, overwrite=False):\n    try:\n        if os.path.exists(output_folder):\n            if not overwrite:\n                print(f\"Directory {output_folder} already exists. Skipping.\")\n                return\n        else:\n            os.makedirs(output_folder)\n\n        cap = cv2.VideoCapture(video_path)\n        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n        if total_frames < num_frames:\n            print(f\"Skipping {video_path}: Not enough frames.\")\n            return\n\n        frame_interval = max(1, total_frames // num_frames)  # Ensure interval is at least 1\n\n        for i in range(num_frames):\n            frame_id = i * frame_interval\n            cap.set(cv2.CAP_PROP_POS_FRAMES, frame_id)\n            success, frame = cap.read()\n            if success:\n                frame_filename = os.path.join(output_folder, f\"{os.path.basename(video_path).split('.')[0]}_frame_{i}.jpg\")\n                cv2.imwrite(frame_filename, frame)\n            else:\n                print(f\"Failed to read frame {frame_id} from {video_path}.\")\n                break\n\n        cap.release()\n    except Exception as e:\n        print(f\"Error processing video {video_path}: {e}\")\n\n# Load metadata\nmetadata_path = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json\"\nwith open(metadata_path, 'r') as f:\n    metadata = json.load(f)\n\n# Split videos into real and fake\nreal_videos = [video for video, info in metadata.items() if info['label'] == 'REAL']\nfake_videos = [video for video, info in metadata.items() if info['label'] == 'FAKE']\n\n# Desired number of videos for balanced dataset\nnum_videos = len(real_videos) + len(fake_videos)\nnum_balanced_videos = num_videos // 2\n\n# Downsample fake videos to match the desired balanced number\ndownsampled_fake_videos = random.sample(fake_videos, num_balanced_videos)\n\n# Upsample real videos to match the desired balanced number\nupsampled_real_videos = random.choices(real_videos, k=num_balanced_videos)\n\n# Combine and shuffle\nbalanced_videos = upsampled_real_videos + downsampled_fake_videos\nrandom.shuffle(balanced_videos)\n\n# Split into train, validation, and test sets\nfrom sklearn.model_selection import train_test_split\ntrain_videos, test_videos = train_test_split(balanced_videos, test_size=0.2, random_state=42)\ntrain_videos, val_videos = train_test_split(train_videos, test_size=0.25, random_state=42)  # 0.25 * 0.8 = 0.2\n\n# Extract frames from balanced videos in parallel\ninput_video_folder = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos\"\noutput_image_folder = \"/kaggle/working/images\"\noverwrite_existing = False\n\ndef process_video(video):\n    try:\n        video_path = os.path.join(input_video_folder, video)\n        video_output_folder = os.path.join(output_image_folder, os.path.splitext(video)[0])\n        extract_frames(video_path, video_output_folder, overwrite=overwrite_existing)\n    except Exception as e:\n        print(f\"Error processing video {video}: {e}\")\n\nfailed_videos = []\n\ndef process_videos_in_batches(videos, batch_size=50):\n    for i in range(0, len(videos), batch_size):\n        batch_videos = videos[i:i + batch_size]\n        with ThreadPoolExecutor(max_workers=8) as executor:\n            futures = [executor.submit(process_video, video) for video in batch_videos]\n            for future in as_completed(futures):\n                if future.exception():\n                    failed_videos.append(future.result())\n\n        # Log the number of created directories after each batch\n        created_directories = [d for d in os.listdir(output_image_folder) if os.path.isdir(os.path.join(output_image_folder, d))]\n        print(f\"Processed batch {i//batch_size + 1}: Number of created directories so far: {len(created_directories)}\")\n\n# Process training videos\nprocess_videos_in_batches(train_videos)\n\n# Process validation videos\nprocess_videos_in_batches(val_videos)\n\n# Process test videos\nprocess_videos_in_batches(test_videos)\n\nprint(\"Processing complete.\")\n\nif failed_videos:\n    print(f\"Failed to process {len(failed_videos)} videos.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-26T06:55:33.093766Z","iopub.execute_input":"2024-06-26T06:55:33.094075Z","iopub.status.idle":"2024-06-26T06:56:54.362121Z","shell.execute_reply.started":"2024-06-26T06:55:33.09403Z","shell.execute_reply":"2024-06-26T06:56:54.361203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport json\nimport random\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\n\n# Load metadata\nmetadata_path = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json\"\nwith open(metadata_path, 'r') as f:\n    metadata = json.load(f)\n\n# Split videos into real and fake\nreal_videos = [video for video, info in metadata.items() if info['label'] == 'REAL']\nfake_videos = [video for video, info in metadata.items() if info['label'] == 'FAKE']\n\n# Desired number of videos for balanced dataset\nnum_videos = len(real_videos) + len(fake_videos)\nnum_balanced_videos = num_videos // 2\n\n# Downsample fake videos to match the desired balanced number\ndownsampled_fake_videos = random.sample(fake_videos, num_balanced_videos)\n\n# Upsample real videos to match the desired balanced number\nupsampled_real_videos = random.choices(real_videos, k=num_balanced_videos)\n\n# Combine and shuffle\nbalanced_videos = upsampled_real_videos + downsampled_fake_videos\nrandom.shuffle(balanced_videos)\n\n# Split into train, validation, and test sets\nfrom sklearn.model_selection import train_test_split\ntrain_videos, test_videos = train_test_split(balanced_videos, test_size=0.2, random_state=42)\ntrain_videos, val_videos = train_test_split(train_videos, test_size=0.25, random_state=42)  # 0.25 * 0.8 = 0.2\n\n# Log directories that were successfully created\ncreated_directories = []\n\n# Extract frames from videos\ndef extract_frames(video_path, output_folder, num_frames=7, overwrite=False):\n    try:\n        if os.path.exists(output_folder):\n            if not overwrite:\n                print(f\"Directory {output_folder} already exists. Skipping.\")\n                created_directories.append(output_folder)\n                return\n        else:\n            os.makedirs(output_folder)\n\n        cap = cv2.VideoCapture(video_path)\n        total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n        if total_frames < num_frames:\n            print(f\"Skipping {video_path}: Not enough frames.\")\n            return\n\n        frame_interval = max(1, total_frames // num_frames)  # Ensure interval is at least 1\n\n        for i in range(num_frames):\n            frame_id = i * frame_interval\n            cap.set(cv2.CAP_PROP_POS_FRAMES, frame_id)\n            success, frame = cap.read()\n            if success:\n                frame_filename = os.path.join(output_folder, f\"{os.path.basename(video_path).split('.')[0]}_frame_{i}.jpg\")\n                cv2.imwrite(frame_filename, frame)\n            else:\n                print(f\"Failed to read frame {frame_id} from {video_path}.\")\n                break\n\n        cap.release()\n        created_directories.append(output_folder)\n    except Exception as e:\n        print(f\"Error processing video {video_path}: {e}\")\n\n# Function to process each video and handle retries\ndef process_video_with_retry(video, retries=3):\n    video_path = os.path.join(input_video_folder, video)\n    video_output_folder = os.path.join(output_image_folder, os.path.splitext(video)[0])\n    for attempt in range(retries):\n        try:\n            extract_frames(video_path, video_output_folder, overwrite=overwrite_existing)\n            if video_output_folder in created_directories:\n                break\n        except Exception as e:\n            print(f\"Attempt {attempt + 1} failed for {video}: {e}\")\n        print(f\"Retrying {video}...\")\n\n# Extract frames from balanced videos in parallel\ninput_video_folder = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos\"\noutput_image_folder = \"/kaggle/working/images\"\noverwrite_existing = False\n\n# Process videos in batches to manage resources\ndef process_videos_in_batches(videos, batch_size=50):\n    for i in range(0, len(videos), batch_size):\n        batch_videos = videos[i:i + batch_size]\n        with ThreadPoolExecutor(max_workers=8) as executor:\n            futures = [executor.submit(process_video_with_retry, video) for video in batch_videos]\n            for future in as_completed(futures):\n                try:\n                    future.result()\n                except Exception as e:\n                    print(f\"Failed to process a video in batch: {e}\")\n\n        # Log the number of created directories after each batch\n        print(f\"Processed batch {i//batch_size + 1}: Number of created directories so far: {len(created_directories)}\")\n\n# Process training videos\nprocess_videos_in_batches(train_videos)\n\n# Process validation videos\nprocess_videos_in_batches(val_videos)\n\n# Process test videos\nprocess_videos_in_batches(test_videos)\n\n# Ensure all 400 directories are created\nexpected_directories = len(balanced_videos)\ncreated_directories_set = set(created_directories)\nmissing_directories = expected_directories - len(created_directories_set)\n\nprint(f\"Total directories expected: {expected_directories}\")\nprint(f\"Total directories created: {len(created_directories_set)}\")\nprint(f\"Missing directories: {missing_directories}\")\n\nif missing_directories > 0:\n    print(f\"Retrying {missing_directories} missing directories...\")\n\n    missing_videos = [video for video in balanced_videos if os.path.join(output_image_folder, os.path.splitext(video)[0]) not in created_directories_set]\n    process_videos_in_batches(missing_videos)\n\nprint(\"Processing complete.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-26T07:00:03.540043Z","iopub.execute_input":"2024-06-26T07:00:03.540396Z","iopub.status.idle":"2024-06-26T07:00:03.701109Z","shell.execute_reply.started":"2024-06-26T07:00:03.540342Z","shell.execute_reply":"2024-06-26T07:00:03.700411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!apt-get update\n!apt-get install -y build-essential cmake\n!pip install wheel\n\n!pip install dlib==19.24.0","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:24:57.926343Z","iopub.execute_input":"2024-06-27T05:24:57.926684Z","iopub.status.idle":"2024-06-27T05:31:11.820436Z","shell.execute_reply.started":"2024-06-27T05:24:57.926629Z","shell.execute_reply":"2024-06-27T05:31:11.819466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install imutils","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:31:11.823057Z","iopub.execute_input":"2024-06-27T05:31:11.823426Z","iopub.status.idle":"2024-06-27T05:31:20.008715Z","shell.execute_reply.started":"2024-06-27T05:31:11.823363Z","shell.execute_reply":"2024-06-27T05:31:20.007656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport dlib\nfrom imutils import face_utils\nfrom tqdm import tqdm\n\n# Initialize dlib's face detector and landmark predictor\ndetector = dlib.get_frontal_face_detector()\npredictor = dlib.shape_predictor(\"/kaggle/input/shape-predictor-68-face-landmarks-dat/shape_predictor_68_face_landmarks.dat\")\n\n# Define the directories\ninput_folder = \"/kaggle/working/images\"\noutput_folder = \"/kaggle/working/facial_organs\"\norgans = ['right_eye', 'left_eye', 'right_eyebrow', 'left_eyebrow', 'nose', 'mouth', 'jaw']\n\n# Create output directories if they do not exist\nfor organ in organs:\n    os.makedirs(os.path.join(output_folder, organ), exist_ok=True)\n\n# Define a function to extract and save facial organs\ndef extract_and_save_organs(image_path, output_folder):\n    image = cv2.imread(image_path)\n    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    rects = detector(gray, 1)\n\n    for (i, rect) in enumerate(rects):\n        shape = predictor(gray, rect)\n        shape = face_utils.shape_to_np(shape)\n\n        # Extract and save each organ\n        for (i, name) in enumerate(face_utils.FACIAL_LANDMARKS_IDXS.keys()):\n            if name in organs:\n                (j, k) = face_utils.FACIAL_LANDMARKS_IDXS[name]\n                organ_image = image[shape[j:k, 1].min():shape[j:k, 1].max(), shape[j:k, 0].min():shape[j:k, 0].max()]\n                organ_path = os.path.join(output_folder, name, os.path.basename(image_path))\n                cv2.imwrite(organ_path, organ_image)\n\n# Process all images in the input directory\nfor root, _, files in os.walk(input_folder):\n    for file in tqdm(files):\n        if file.endswith('.jpg') or file.endswith('.png'):\n            image_path = os.path.join(root, file)\n            extract_and_save_organs(image_path, output_folder)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-26T07:08:17.75724Z","iopub.execute_input":"2024-06-26T07:08:17.757593Z","iopub.status.idle":"2024-06-26T07:59:34.801178Z","shell.execute_reply.started":"2024-06-26T07:08:17.757532Z","shell.execute_reply":"2024-06-26T07:59:34.800329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\nimport os\n\n# Define the directories to zip\ndirectories_to_zip = ['/kaggle/working/models']\n\n# Path where the zip file will be saved\nzip_file_path = '/kaggle/working/working_directories2.zip'\n\n# Function to zip directories\ndef zip_directories(directories, zip_file):\n    with zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED) as zipf:\n        for directory in directories:\n            for root, _, files in os.walk(directory):\n                for file in files:\n                    zipf.write(os.path.join(root, file), os.path.relpath(os.path.join(root, file), os.path.join(directory, '..')))\n\n# Zip the directories\nzip_directories(directories_to_zip, zip_file_path)\n\nprint(f\"Directories zipped successfully to {zip_file_path}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:41:18.321659Z","iopub.execute_input":"2024-06-27T09:41:18.322022Z","iopub.status.idle":"2024-06-27T09:42:45.355365Z","shell.execute_reply.started":"2024-06-27T09:41:18.321961Z","shell.execute_reply":"2024-06-27T09:42:45.354331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install timm transformers\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T10:09:01.664394Z","iopub.execute_input":"2024-06-27T10:09:01.664724Z","iopub.status.idle":"2024-06-27T10:09:18.482569Z","shell.execute_reply.started":"2024-06-27T10:09:01.664683Z","shell.execute_reply":"2024-06-27T10:09:18.481405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\n# Check GPU availability and print properties\nif torch.cuda.is_available():\n    for i in range(torch.cuda.device_count()):\n        print(f\"GPU {i}: {torch.cuda.get_device_properties(i).name}\")\nelse:\n    print(\"No GPU available. Using CPU.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:31:20.083029Z","iopub.execute_input":"2024-06-27T05:31:20.08325Z","iopub.status.idle":"2024-06-27T05:31:20.09465Z","shell.execute_reply.started":"2024-06-27T05:31:20.083207Z","shell.execute_reply":"2024-06-27T05:31:20.093517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set device to GPU if available\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:28:00.98141Z","iopub.execute_input":"2024-06-27T09:28:00.981763Z","iopub.status.idle":"2024-06-27T09:28:00.986569Z","shell.execute_reply.started":"2024-06-27T09:28:00.981703Z","shell.execute_reply":"2024-06-27T09:28:00.98582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n# from torchvision.models import vit_b_16\nimport os\nimport timm\n\n\n\n\n# Define the list of organs\norgan_dirs = ['right_eye', 'left_eye', 'nose', 'mouth', 'left_eyebrow', 'right_eyebrow', 'jaw']\n\n# Placeholder for dataloaders dictionary\n# Assuming you have a dataloaders dictionary for each organ\n# Example format: {'right_eye': (train_loader_right_eye, val_loader_right_eye, test_loader_right_eye), ...}\n# Replace the following line with your actual data loaders for each organ\ndataloaders = {\n    'right_eye': (train_loader_right_eye, val_loader_right_eye, test_loader_right_eye),\n    'left_eye': (train_loader_left_eye, val_loader_left_eye, test_loader_left_eye),\n    'nose': (train_loader_nose, val_loader_nose, test_loader_nose),\n    'mouth': (train_loader_mouth, val_loader_mouth, test_loader_mouth),\n    'left_eyebrow': (train_loader_left_eyebrow, val_loader_left_eyebrow, test_loader_left_eyebrow),\n    'right_eyebrow': (train_loader_right_eyebrow, val_loader_right_eyebrow, test_loader_right_eyebrow),\n    'jaw': (train_loader_jaw, val_loader_jaw, test_loader_jaw)\n}\n\n# Define the device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Function to train and save a ViT model for a specific organ\ndef train_and_save_vit(organ, train_loader, val_loader, save_dir, num_epochs=10, learning_rate=1e-4):\n    print(f\"Training model for {organ}\")\n    \n    # Load a pretrained ViT model\n#     model = vit_b_16(pretrained=True)\n    # Create a Vision Transformer model (ViT-B/16)\n    model = timm.create_model('vit_base_patch16_224', pretrained=True)\n    \n    # Modify the last layer for binary classification (assuming binary task, adjust as needed)\n    num_ftrs = model.heads.head.in_features\n    model.heads.head = nn.Linear(num_ftrs, 2)  # Adjust the number of classes based on your task\n    \n    model = model.to(device)\n    \n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        \n        for images, labels in train_loader:\n            images, labels = images.to(device), labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item() * images.size(0)\n        \n        epoch_loss = running_loss / len(train_loader.dataset)\n        print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}\")\n        \n        # You can add validation code here to monitor performance on the validation set\n    \n    # Save the model\n    model_save_path = os.path.join(save_dir, f\"{organ}_vit.pth\")\n    torch.save(model.state_dict(), model_save_path)\n    print(f\"Model for {organ} saved at {model_save_path}\")\n    \n    return model\n\n# Define save directory\nsave_dir = \"/kaggle/working/models\"\nos.makedirs(save_dir, exist_ok=True)\n\n# Train and save ViT models for each facial organ\nvit_models = {}\nfor organ in organ_dirs:\n    train_loader, val_loader, _ = dataloaders[organ]\n    vit_models[organ] = train_and_save_vit(organ, train_loader, val_loader, save_dir)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:51:15.882728Z","iopub.execute_input":"2024-06-27T05:51:15.883102Z","iopub.status.idle":"2024-06-27T05:51:15.916383Z","shell.execute_reply.started":"2024-06-27T05:51:15.883039Z","shell.execute_reply":"2024-06-27T05:51:15.915348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nimport random\n\ndef split_data(source_dir, train_dir, val_dir, test_dir, val_split=0.2, test_split=0.1):\n    if not os.path.exists(train_dir):\n        os.makedirs(train_dir)\n    if not os.path.exists(val_dir):\n        os.makedirs(val_dir)\n    if not os.path.exists(test_dir):\n        os.makedirs(test_dir)\n\n    all_files = os.listdir(source_dir)\n    random.shuffle(all_files)\n    \n    val_size = int(len(all_files) * val_split)\n    test_size = int(len(all_files) * test_split)\n    train_size = len(all_files) - val_size - test_size\n    \n    train_files = all_files[:train_size]\n    val_files = all_files[train_size:train_size + val_size]\n    test_files = all_files[train_size + val_size:]\n\n    for file in train_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(train_dir, file))\n    for file in val_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(val_dir, file))\n    for file in test_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(test_dir, file))\n\norgans = ['right_eye', 'left_eye', 'nose', 'mouth', 'jaw', 'left_eyebrow', 'right_eyebrow']\ninput_base_dir = '/kaggle/input/working-dir/facial_organs'\noutput_base_dir = '/kaggle/working/split_data'  # Changed to /kaggle/working/\n\nfor organ in organs:\n    source_dir = os.path.join(input_base_dir, organ)\n    train_dir = os.path.join(output_base_dir, 'train', organ)\n    val_dir = os.path.join(output_base_dir, 'val', organ)\n    test_dir = os.path.join(output_base_dir, 'test', organ)\n    split_data(source_dir, train_dir, val_dir, test_dir)\n\nprint(\"Data split into train, val, and test directories for each organ.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T06:15:05.412425Z","iopub.execute_input":"2024-06-27T06:15:05.412791Z","iopub.status.idle":"2024-06-27T06:15:45.959996Z","shell.execute_reply.started":"2024-06-27T06:15:05.412742Z","shell.execute_reply":"2024-06-27T06:15:45.959114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),  # Resize images to the input size expected by ViT\n    transforms.ToTensor(),          # Convert images to tensors\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalize as per ImageNet\n])\n\n# Function to create data loaders for a given organ\ndef create_dataloaders(organ, batch_size=32):\n    data_dir = output_base_dir\n    \n    train_dataset = datasets.ImageFolder(os.path.join(data_dir, 'train', organ), transform=transform)\n    val_dataset = datasets.ImageFolder(os.path.join(data_dir, 'val', organ), transform=transform)\n    test_dataset = datasets.ImageFolder(os.path.join(data_dir, 'test', organ), transform=transform)\n    \n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n    \n    return train_loader, val_loader, test_loader\n\n# Create data loaders for each organ\ndataloaders = {organ: create_dataloaders(organ) for organ in organs}\n\nprint(\"Data loaders created for each organ.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T06:16:28.195859Z","iopub.execute_input":"2024-06-27T06:16:28.196163Z","iopub.status.idle":"2024-06-27T06:16:28.248742Z","shell.execute_reply.started":"2024-06-27T06:16:28.196122Z","shell.execute_reply":"2024-06-27T06:16:28.247625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\nimport random\n\ndef split_data(source_dir, train_dir, val_dir, test_dir, val_split=0.2, test_split=0.1):\n    if not os.path.exists(train_dir):\n        os.makedirs(train_dir)\n    if not os.path.exists(val_dir):\n        os.makedirs(val_dir)\n    if not os.path.exists(test_dir):\n        os.makedirs(test_dir)\n\n    all_files = os.listdir(source_dir)\n    random.shuffle(all_files)\n    \n    val_size = int(len(all_files) * val_split)\n    test_size = int(len(all_files) * test_split)\n    train_size = len(all_files) - val_size - test_size\n    \n    train_files = all_files[:train_size]\n    val_files = all_files[train_size:train_size + val_size]\n    test_files = all_files[train_size + val_size:]\n\n    for file in train_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(train_dir, file))\n    for file in val_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(val_dir, file))\n    for file in test_files:\n        shutil.copy(os.path.join(source_dir, file), os.path.join(test_dir, file))\n\n    # Print the results for verification\n    print(f\"Organ: {os.path.basename(source_dir)}\")\n    print(f\"Train: {len(train_files)}, Val: {len(val_files)}, Test: {len(test_files)}\")\n\norgans = ['right_eye', 'left_eye', 'nose', 'mouth', 'jaw', 'left_eyebrow', 'right_eyebrow']\ninput_base_dir = '/kaggle/input/working-dir/facial_organs'\noutput_base_dir = '/kaggle/working/split_data'\n\nfor organ in organs:\n    source_dir = os.path.join(input_base_dir, organ)\n    train_dir = os.path.join(output_base_dir, 'train', organ)\n    val_dir = os.path.join(output_base_dir, 'val', organ)\n    test_dir = os.path.join(output_base_dir, 'test', organ)\n    split_data(source_dir, train_dir, val_dir, test_dir)\n\nprint(\"Data split into train, val, and test directories for each organ.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T06:19:56.563478Z","iopub.execute_input":"2024-06-27T06:19:56.563889Z","iopub.status.idle":"2024-06-27T06:20:38.287236Z","shell.execute_reply.started":"2024-06-27T06:19:56.563827Z","shell.execute_reply":"2024-06-27T06:20:38.286482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport shutil\n\n# Define the base directory\noutput_base_dir = '/kaggle/working/split_data'\n\n# List of organs\norgans = ['right_eye', 'left_eye', 'nose', 'mouth', 'jaw', 'left_eyebrow', 'right_eyebrow']\n\n# Function to organize images into a dummy 'class' subdirectory\ndef organize_images_into_class_subdir(base_dir, organ):\n    organ_dir = os.path.join(base_dir, organ)\n    class_dir = os.path.join(organ_dir, 'class')\n    if not os.path.exists(class_dir):\n        os.makedirs(class_dir)\n    \n    # Move images to the 'class' subdirectory\n    for filename in os.listdir(organ_dir):\n        file_path = os.path.join(organ_dir, filename)\n        if os.path.isfile(file_path):\n            shutil.move(file_path, os.path.join(class_dir, filename))\n\n# Organize images for train, val, and test sets\nfor subset in ['train', 'val', 'test']:\n    for organ in organs:\n        organize_images_into_class_subdir(os.path.join(output_base_dir, subset), organ)\n\nprint(\"Images organized into 'class' subdirectories.\")\n\n# Now create the data loaders again\nimport torch\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),  # Resize images to the input size expected by ViT\n    transforms.ToTensor(),          # Convert images to tensors\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalize as per ImageNet\n])\n\n# Function to create data loaders for a given organ\ndef create_dataloaders(organ, batch_size=32):\n    data_dir = output_base_dir\n    \n    train_dir = os.path.join(data_dir, 'train', organ)\n    val_dir = os.path.join(data_dir, 'val', organ)\n    test_dir = os.path.join(data_dir, 'test', organ)\n    \n    # Print the directories and files for debugging\n    print(f\"Creating datasets for {organ}\")\n    print(f\"Train directory: {train_dir}\")\n    print(f\"Val directory: {val_dir}\")\n    print(f\"Test directory: {test_dir}\")\n\n    train_dataset = datasets.ImageFolder(train_dir, transform=transform)\n    val_dataset = datasets.ImageFolder(val_dir, transform=transform)\n    test_dataset = datasets.ImageFolder(test_dir, transform=transform)\n    \n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n    \n    return train_loader, val_loader, test_loader\n\n# Create data loaders for each organ\ndataloaders = {organ: create_dataloaders(organ) for organ in organs}\n\nprint(\"Data loaders created for each organ.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T07:06:21.764819Z","iopub.execute_input":"2024-06-27T07:06:21.765155Z","iopub.status.idle":"2024-06-27T07:06:22.021121Z","shell.execute_reply.started":"2024-06-27T07:06:21.765108Z","shell.execute_reply":"2024-06-27T07:06:22.02028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport timm\nfrom tqdm import tqdm\n\n# Define the Vision Transformer model\nclass ViTModel(nn.Module):\n    def __init__(self, num_classes=2):\n        super(ViTModel, self).__init__()\n        self.vit = timm.create_model('vit_base_patch16_224', pretrained=True)\n        self.vit.head = nn.Linear(self.vit.head.in_features, num_classes)  # Modify for classification\n    \n    def forward(self, x):\n        return self.vit(x)\n\n# Training function\ndef train_model(organ, train_loader, val_loader, num_classes=2, num_epochs=10, learning_rate=1e-4):\n    model = ViTModel(num_classes=num_classes)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n    \n    model = model.to(device)\n    criterion = criterion.to(device)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        for images, labels in tqdm(train_loader):\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item() * images.size(0)\n        \n        epoch_loss = running_loss / len(train_loader.dataset)\n        print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}\")\n        \n        evaluate_model(model, val_loader)\n    \n    # Save the trained model\n    model_save_path = f'/kaggle/working/models/{organ}_vit_model.pth'\n    torch.save(model.state_dict(), model_save_path)\n    print(f\"Saved model for {organ} at {model_save_path}\")\n    \n    return model\n\n# Evaluation function\ndef evaluate_model(model, val_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    accuracy = 100 * correct / total\n    print(f'Validation Accuracy: {accuracy:.2f}%')\n\n# Training and validation for each facial organ\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef train_organ_model(organ):\n    print(f\"\\nTraining model for {organ}\") \n    train_loader, val_loader, _ = dataloaders[organ]\n    trained_model = train_model(organ, train_loader, val_loader)\n    return trained_model\n\nfor organ in organs:\n    train_organ_model(organ)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T07:05:31.793516Z","iopub.execute_input":"2024-06-27T07:05:31.793846Z","iopub.status.idle":"2024-06-27T07:05:39.602973Z","shell.execute_reply.started":"2024-06-27T07:05:31.793802Z","shell.execute_reply":"2024-06-27T07:05:39.601818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\n# Load metadata.json\nmetadata_path = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json'\nwith open(metadata_path, 'r') as f:\n    metadata = json.load(f)\n\n# Extract labels\nvideo_labels = {video: data['label'] for video, data in metadata.items()}\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T06:56:43.236138Z","iopub.execute_input":"2024-06-27T06:56:43.236502Z","iopub.status.idle":"2024-06-27T06:56:43.255505Z","shell.execute_reply.started":"2024-06-27T06:56:43.236423Z","shell.execute_reply":"2024-06-27T06:56:43.254826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom shutil import copyfile\n\ndef organize_frames_by_label(source_dir, dest_dir, video_labels):\n    for root, _, files in os.walk(source_dir):\n        for file in files:\n            if file.endswith('.jpg'):\n                video_id = file.split('_')[0]\n                label = video_labels.get(video_id + '.mp4', 'unknown')\n                if label != 'unknown':\n                    label_dir = os.path.join(dest_dir, label)\n                    if not os.path.exists(label_dir):\n                        os.makedirs(label_dir)\n                    copyfile(os.path.join(root, file), os.path.join(label_dir, file))\n\n# Organize frames\nsource_dir = '/kaggle/input/working-dir/facial_organ'\ndest_dir = '/kaggle/working/organized_frames'\norganize_frames_by_label(source_dir, dest_dir, video_labels)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T06:57:23.473373Z","iopub.execute_input":"2024-06-27T06:57:23.473723Z","iopub.status.idle":"2024-06-27T06:57:23.482703Z","shell.execute_reply.started":"2024-06-27T06:57:23.473673Z","shell.execute_reply":"2024-06-27T06:57:23.481969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\nimport timm\nimport torch.nn as nn\nfrom tqdm import tqdm\n\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\n\n\n# Define the Vision Transformer model\nclass ViTModel(nn.Module):\n    def __init__(self, num_classes=2):\n        super(ViTModel, self).__init__()\n        self.vit = timm.create_model('vit_base_patch16_224', pretrained=True)\n        self.vit.head = nn.Linear(self.vit.head.in_features, num_classes)  # Modify for classification\n    \n    def forward(self, x):\n        return self.vit(x)\n\n# Training function\ndef train_model(organ, train_loader, val_loader, num_classes=2, num_epochs=10, learning_rate=1e-4):\n    model = ViTModel(num_classes=num_classes)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n    \n    model = model.to(device)\n    criterion = criterion.to(device)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        for images, labels in tqdm(train_loader):\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item() * images.size(0)\n        \n        epoch_loss = running_loss / len(train_loader.dataset)\n        print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}\")\n        \n        evaluate_model(model, val_loader)\n    \n    # Save the trained model\n    model_save_path = f'/kaggle/working/models/{organ}_vit_model.pth'\n    torch.save(model.state_dict(), model_save_path)\n    print(f\"Saved model for {organ} at {model_save_path}\")\n    \n    return model\n\n# Evaluation function\ndef evaluate_model(model, val_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    accuracy = 100 * correct / total\n    print(f'Validation Accuracy: {accuracy:.2f}%')\n\n# Training and validation for each facial organ\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef train_organ_model(organ):\n    print(f\"\\nTraining model for {organ}\") \n    train_loader, val_loader, _ = dataloaders[organ]\n    trained_model = train_model(organ, train_loader, val_loader)\n    return trained_model\n\nfor organ in organs:\n    train_organ_model(organ)\n    \n    \n    \n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T07:06:35.767078Z","iopub.execute_input":"2024-06-27T07:06:35.767381Z","iopub.status.idle":"2024-06-27T08:33:34.053064Z","shell.execute_reply.started":"2024-06-27T07:06:35.767339Z","shell.execute_reply":"2024-06-27T08:33:34.05176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport timm\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\n# Define the base directory\noutput_base_dir = '/kaggle/working/split_data'\n\n# List of organs\norgans = ['left_eyebrow', 'right_eyebrow']\n\n# Define transformations\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\n# Function to create data loaders for a given organ\ndef create_dataloaders(organ, batch_size=32):\n    train_dir = os.path.join(output_base_dir, 'train', organ)\n    val_dir = os.path.join(output_base_dir, 'val', organ)\n    test_dir = os.path.join(output_base_dir, 'test', organ)\n\n    train_dataset = datasets.ImageFolder(train_dir, transform=transform)\n    val_dataset = datasets.ImageFolder(val_dir, transform=transform)\n    test_dataset = datasets.ImageFolder(test_dir, transform=transform)\n\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4)\n\n    return train_loader, val_loader, test_loader\n\n# Create data loaders for each organ\ndataloaders = {organ: create_dataloaders(organ) for organ in organs}\n\n# Define the Vision Transformer model\nclass ViTModel(nn.Module):\n    def __init__(self, num_classes=2):\n        super(ViTModel, self).__init__()\n        self.vit = timm.create_model('vit_base_patch16_224', pretrained=True)\n        self.vit.head = nn.Linear(self.vit.head.in_features, num_classes)  # Modify for classification\n    \n    def forward(self, x):\n        return self.vit(x)\n\n# Training function\ndef train_model(organ, train_loader, val_loader, num_classes=2, num_epochs=10, learning_rate=1e-4):\n    model = ViTModel(num_classes=num_classes)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    \n    model = model.to(device)\n    criterion = criterion.to(device)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        for images, labels in tqdm(train_loader):\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item() * images.size(0)\n        \n        epoch_loss = running_loss / len(train_loader.dataset)\n        print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}\")\n        \n        evaluate_model(model, val_loader)\n    \n    # Save the trained model\n    model_save_path = f'/kaggle/working/models/{organ}_vit_model.pth'\n    torch.save(model.state_dict(), model_save_path)\n    print(f\"Saved model for {organ} at {model_save_path}\")\n    \n    return model\n\n# Evaluation function\ndef evaluate_model(model, val_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    accuracy = 100 * correct / total\n    print(f'Validation Accuracy: {accuracy:.2f}%')\n\n# Training and validation for each facial organ\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ndef train_organ_model(organ):\n    print(f\"\\nTraining model for {organ}\") \n    train_loader, val_loader, _ = dataloaders[organ]\n    trained_model = train_model(organ, train_loader, val_loader)\n    return trained_model\n\nfor organ in organs:\n    train_organ_model(organ)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T08:33:59.458675Z","iopub.execute_input":"2024-06-27T08:33:59.458985Z","iopub.status.idle":"2024-06-27T09:07:54.879758Z","shell.execute_reply.started":"2024-06-27T08:33:59.458941Z","shell.execute_reply":"2024-06-27T09:07:54.878797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_state_dict_with_adjusted_keys(model, state_dict_path):\n    state_dict = torch.load(state_dict_path)\n    \n    # Remove any prefix such as 'vit.' from keys if present\n    new_state_dict = {}\n    for k, v in state_dict.items():\n        if k.startswith('vit.'):\n            new_state_dict[k[len('vit.'):]] = v\n        else:\n            new_state_dict[k] = v\n    \n    model.load_state_dict(new_state_dict)\n\n# Load the trained models\nfor organ in organs:\n    hierarchical_model.models[organ].head = nn.Linear(hierarchical_model.models[organ].head.in_features, 2)\n    load_state_dict_with_adjusted_keys(hierarchical_model.models[organ], f'/kaggle/working/models/{organ}_vit_model.pth')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:12:00.565283Z","iopub.execute_input":"2024-06-27T09:12:00.565606Z","iopub.status.idle":"2024-06-27T09:12:00.588685Z","shell.execute_reply.started":"2024-06-27T09:12:00.565561Z","shell.execute_reply":"2024-06-27T09:12:00.587743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the Hierarchical Vision Transformer model\nclass HierarchicalViT(nn.Module):\n    def __init__(self, organs, weights):\n        super(HierarchicalViT, self).__init__()\n        self.organs = organs\n        self.weights = weights\n        self.models = nn.ModuleDict({organ: timm.create_model('vit_base_patch16_224', pretrained=False) for organ in organs})\n        \n        # Load the trained models\n        for organ in organs:\n            self.models[organ].head = nn.Linear(self.models[organ].head.in_features, 2)\n            self.models[organ].load_state_dict(torch.load(f'/kaggle/working/models/{organ}_vit_model.pth'))\n        \n        # Final classifier\n        self.final_classifier = nn.Linear(len(organs) * 2, 2)\n    \n    def forward(self, x):\n        organ_outputs = [self.models[organ](x) * self.weights[i] for i, organ in enumerate(self.organs)]\n        combined_output = torch.cat(organ_outputs, dim=1)\n        return self.final_classifier(combined_output)\n\n# Define weights for each organ\nweights = [1.0] * len(organs)  # Adjust weights as needed\n\n# Instantiate the hierarchical model\nhierarchical_model = HierarchicalViT(organs, weights).to(device)\n\n# Train the hierarchical model\ndef train_hierarchical_model(dataloaders, model, num_epochs=10, learning_rate=0.001):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        \n        for inputs, labels in tqdm(dataloaders['train']):\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n        \n        print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {running_loss / len(dataloaders['train'])}\")\n    \n    # Save the hierarchical model\n    torch.save(model.state_dict(), \"/kaggle/working/models/hierarchical_vit.pth\")\n    print(\"Hierarchical model saved at /kaggle/working/models/hierarchical_vit.pth\")\n\n# Create a combined dataloader for training\ncombined_train_loader = DataLoader(\n    datasets.ImageFolder(os.path.join(output_base_dir, 'train'), transform=transform),\n    batch_size=32, shuffle=True, num_workers=4\n)\n\ncombined_dataloaders = {'train': combined_train_loader}\ntrain_hierarchical_model(combined_dataloaders, hierarchical_model)\n\n# Evaluation function for the hierarchical model\ndef evaluate_model(model, dataloader):\n    model.eval()\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for inputs, labels in dataloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    \n    accuracy = correct / total\n    print(f'Accuracy: {accuracy * 100:.2f}%')\n\n# Evaluate the hierarchical model\ntest_loader = DataLoader(\n    datasets.ImageFolder(os.path.join(output_base_dir, 'test'), transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\n\nevaluate_model(hierarchical_model, test_loader)","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:07:54.88258Z","iopub.execute_input":"2024-06-27T09:07:54.882833Z","iopub.status.idle":"2024-06-27T09:07:57.565413Z","shell.execute_reply.started":"2024-06-27T09:07:54.88279Z","shell.execute_reply":"2024-06-27T09:07:57.564099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport timm\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, transforms\nfrom PIL import Image\nimport os\n\nclass HierarchicalViT(nn.Module):\n    def __init__(self, organs, weights):\n        super(HierarchicalViT, self).__init__()\n        self.organs = organs\n        self.weights = weights\n        self.models = nn.ModuleDict({organ: timm.create_model('vit_base_patch16_224', pretrained=False) for organ in organs})\n        \n        # Load the trained models\n        for organ in organs:\n            self.models[organ].head = nn.Linear(self.models[organ].head.in_features, 2)\n            load_state_dict_with_adjusted_keys(self.models[organ], f'/kaggle/working/models/{organ}_vit_model.pth')\n        \n        # Final classifier\n        self.final_classifier = nn.Linear(len(organs) * 2, 2)\n    \n    def forward(self, x):\n        organ_outputs = [self.models[organ](x) * self.weights[i] for i, organ in enumerate(self.organs)]\n        combined_output = torch.cat(organ_outputs, dim=1)\n        return self.final_classifier(combined_output)\n\n# Define weights for each organ\norgans = ['jaw', 'right_eye', 'left_eye', 'right_eyebrow', 'left_eyebrow', 'mouth', 'nose']  # replace with actual organ names\nweights = [1.0] * len(organs)  # Adjust weights as needed\n\n# Instantiate the hierarchical model\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nhierarchical_model = HierarchicalViT(organs, weights).to(device)\n\n# Define the transforms\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\n# Define data loaders for training and testing\noutput_base_dir = '/kaggle/working'  # Replace with actual directory\ncombined_train_loader = DataLoader(\n    datasets.ImageFolder(os.path.join(output_base_dir, 'train'), transform=transform),\n    batch_size=32, shuffle=True, num_workers=4\n)\n\ncombined_dataloaders = {'train': combined_train_loader}\n\n# Function to train the hierarchical model\ndef train_hierarchical_model(dataloaders, model, num_epochs=10, learning_rate=0.001):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        \n        for inputs, labels in tqdm(dataloaders['train']):\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n        \n        print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {running_loss / len(dataloaders['train'])}\")\n    \n    # Save the hierarchical model\n    torch.save(model.state_dict(), \"/kaggle/working/models/hierarchical_vit.pth\")\n    print(\"Hierarchical model saved at /kaggle/working/models/hierarchical_vit.pth\")\n\n# Train the hierarchical model\ntrain_hierarchical_model(combined_dataloaders, hierarchical_model)\n\n# Function to evaluate the hierarchical model\ndef evaluate_model(model, dataloader):\n    model.eval()\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for inputs, labels in dataloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    \n    accuracy = correct / total\n    print(f'Accuracy: {accuracy * 100:.2f}%')\n\n# Evaluate the hierarchical model\ntest_loader = DataLoader(\n    datasets.ImageFolder(os.path.join(output_base_dir, 'test'), transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\n\nevaluate_model(hierarchical_model, test_loader)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:15:17.687855Z","iopub.execute_input":"2024-06-27T09:15:17.688166Z","iopub.status.idle":"2024-06-27T09:15:28.910635Z","shell.execute_reply.started":"2024-06-27T09:15:17.688121Z","shell.execute_reply":"2024-06-27T09:15:28.90961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_prefix_from_state_dict(state_dict, prefix):\n    \"\"\"\n    Remove a prefix from keys in the state_dict.\n    \"\"\"\n    def strip_prefix(key, prefix):\n        if key.startswith(prefix):\n            return key[len(prefix):]\n        return key\n    \n    return {strip_prefix(key, prefix): value for key, value in state_dict.items()}\n\n# Example usage:\nstate_dict = torch.load(f'/kaggle/input/working-dir1/models/{organ}_vit_model.pth')\nstate_dict = remove_prefix_from_state_dict(state_dict, \"vit.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T10:16:26.803803Z","iopub.execute_input":"2024-06-27T10:16:26.804203Z","iopub.status.idle":"2024-06-27T10:16:26.826275Z","shell.execute_reply.started":"2024-06-27T10:16:26.804137Z","shell.execute_reply":"2024-06-27T10:16:26.824995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport timm\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\ndevice = torch.device('cpu')\n\n# Define transformation\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n])\n\n# Define the base directory\noutput_base_dir = '/kaggle/input/working-dir1/split_data'\n\n# Ensure train, val, and test directories exist\ntrain_dir = os.path.join(output_base_dir, 'train')\nval_dir = os.path.join(output_base_dir, 'val')\ntest_dir = os.path.join(output_base_dir, 'test')\n\nfor dir_path in [train_dir, val_dir, test_dir]:\n    if not os.path.exists(dir_path):\n        print(f\"Error: The directory {dir_path} does not exist.\")\n        raise FileNotFoundError(f\"The directory {dir_path} does not exist.\")\n\n# Create DataLoaders\ntrain_loader = DataLoader(\n    datasets.ImageFolder(train_dir, transform=transform),\n    batch_size=32, shuffle=True, num_workers=4\n)\nval_loader = DataLoader(\n    datasets.ImageFolder(val_dir, transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\ntest_loader = DataLoader(\n    datasets.ImageFolder(test_dir, transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\n\n# Define the Hierarchical Vision Transformer model\nclass HierarchicalViT(nn.Module):\n    def __init__(self, organs, weights):\n        super(HierarchicalViT, self).__init__()\n        self.organs = organs\n        self.weights = weights\n        self.models = nn.ModuleDict({organ: timm.create_model('vit_base_patch16_224', pretrained=False) for organ in organs})\n        \n        # Load the trained models\n        for organ in organs:\n            self.models[organ].head = nn.Linear(self.models[organ].head.in_features, 2)\n            \n            # Load the state dict with adjusted keys\n            state_dict = torch.load(f'/kaggle/input/working-dir1/models/{organ}_vit_model.pth')\n            state_dict = self.remove_prefix_from_state_dict(state_dict, \"vit.\")\n            self.models[organ].load_state_dict(state_dict, strict=False)\n        \n        # Final classifier\n        self.final_classifier = nn.Linear(len(organs) * 2, 2)\n    \n    def remove_prefix_from_state_dict(self, state_dict, prefix):\n        \"\"\"\n        Remove a prefix from keys in the state_dict.\n        \"\"\"\n        def strip_prefix(key, prefix):\n            if key.startswith(prefix):\n                return key[len(prefix):]\n            return key\n        \n        return {strip_prefix(key, prefix): value for key, value in state_dict.items()}\n    \n    def forward(self, x):\n        organ_outputs = [self.models[organ](x) * self.weights[i] for i, organ in enumerate(self.organs)]\n        combined_output = torch.cat(organ_outputs, dim=1)\n        return self.final_classifier(combined_output)\n\n# Define organs and weights\norgans = ['jaw', 'right_eye', 'left_eye', 'right_eyebrow', 'left_eyebrow', 'mouth', 'nose']\nweights = [1.0] * len(organs)  # Adjust weights as needed\n\n# Instantiate the hierarchical model\nhierarchical_model = HierarchicalViT(organs, weights).to(device)\n\n# Train the hierarchical model\ndef train_hierarchical_model(dataloaders, model, num_epochs=10, learning_rate=0.001):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        \n        for inputs, labels in tqdm(dataloaders['train']):\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n        \n        print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {running_loss / len(dataloaders['train'])}\")\n    \n    # Save the hierarchical model\n    torch.save(model.state_dict(), \"/kaggle/working/models/hierarchical_vit.pth\")\n    print(\"Hierarchical model saved at /kaggle/working/models/hierarchical_vit.pth\")\n\n# Combine dataloaders\ncombined_dataloaders = {'train': train_loader, 'val': val_loader, 'test': test_loader}\n\n# Train the hierarchical model\ntrain_hierarchical_model(combined_dataloaders, hierarchical_model)\n\n# Evaluation function for the hierarchical model\ndef evaluate_model(model, dataloader):\n    model.eval()\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for inputs, labels in dataloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    \n    accuracy = correct / total\n    print(f'Accuracy: {accuracy * 100:.2f}%')\n\n# Evaluate the hierarchical model\nevaluate_model(hierarchical_model, combined_dataloaders['test'])\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T10:26:35.542697Z","iopub.execute_input":"2024-06-27T10:26:35.543012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, transforms\nfrom tqdm import tqdm\nfrom torch.cuda.amp import GradScaler, autocast\n\n# Check if CUDA is available and set the device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Function to load a model and remove the prefix from its state_dict\ndef remove_prefix_from_state_dict(state_dict, prefix):\n    new_state_dict = {}\n    for k, v in state_dict.items():\n        if k.startswith(prefix):\n            new_key = k[len(prefix):]  # remove prefix\n            new_state_dict[new_key] = v\n        else:\n            new_state_dict[k] = v\n    return new_state_dict\n\n# Load and prepare individual ViT models for each organ\norgans = ['jaw', 'right_eye', 'left_eye', 'right_eyebrow', 'left_eyebrow', 'mouth', 'nose']\nmodels = {}\n\nfor organ in organs:\n    state_dict = torch.load(f'/kaggle/input/working-dir1/models/{organ}_vit_model.pth')\n    state_dict = remove_prefix_from_state_dict(state_dict, \"vit.\")\n    \n    # Assuming the original ViT model class is called `VisionTransformer`\n    model = VisionTransformer()  # Replace with actual ViT model class and parameters\n    model.load_state_dict(state_dict)\n    models[organ] = model.to(device)\n\n# Define the HierarchicalViT model that integrates individual ViT models\nclass HierarchicalViT(nn.Module):\n    def __init__(self, models, organs):\n        super(HierarchicalViT, self).__init__()\n        self.models = nn.ModuleDict({organ: models[organ] for organ in organs})\n        self.organs = organs\n        self.fc = nn.Linear(len(organs) * 1000, 1000)  # Example linear layer\n\n    def forward(self, x):\n        outputs = []\n        for organ in self.organs:\n            organ_output = self.models[organ](x)\n            outputs.append(organ_output)\n        \n        # Concatenate the outputs\n        concatenated_output = torch.cat(outputs, dim=1)\n        out = self.fc(concatenated_output)\n        return out\n\n# Initialize the hierarchical model\nhierarchical_model = HierarchicalViT(models, organs)\nhierarchical_model = hierarchical_model.to(device)\n\n# Define dataset directories and transforms\ntrain_dir = '/kaggle/input/working-dir1/train'\nval_dir = '/kaggle/input/working-dir1/val'\ntest_dir = '/kaggle/input/working-dir1/test'\n\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n])\n\n# Create data loaders with reduced batch size\nbatch_size = 16  # Reduced batch size to fit memory constraints\n\ntrain_loader = DataLoader(\n    datasets.ImageFolder(train_dir, transform=transform),\n    batch_size=batch_size,\n    shuffle=True, num_workers=4\n)\nval_loader = DataLoader(\n    datasets.ImageFolder(val_dir, transform=transform),\n    batch_size=batch_size,\n    shuffle=False, num_workers=4\n)\ntest_loader = DataLoader(\n    datasets.ImageFolder(test_dir, transform=transform),\n    batch_size=batch_size,\n    shuffle=False, num_workers=4\n)\n\ncombined_dataloaders = {'train': train_loader, 'val': val_loader, 'test': test_loader}\n\n# Function to train the hierarchical model with mixed precision\ndef train_hierarchical_model(dataloaders, model, num_epochs=10, learning_rate=0.001):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    scaler = GradScaler()\n\n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        \n        for inputs, labels in tqdm(dataloaders['train']):\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            \n            with autocast():  # Use mixed precision\n                outputs = model(inputs)\n                loss = criterion(outputs, labels)\n            \n            scaler.scale(loss).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            \n            running_loss += loss.item()\n        \n        print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {running_loss / len(dataloaders['train'])}\")\n    \n    # Save the hierarchical model\n    torch.save(model.state_dict(), \"/kaggle/working/models/hierarchical_vit.pth\")\n    print(\"Hierarchical model saved at /kaggle/working/models/hierarchical_vit.pth\")\n\n# Train the hierarchical model\ntrain_hierarchical_model(combined_dataloaders, hierarchical_model)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T10:25:44.974647Z","iopub.execute_input":"2024-06-27T10:25:44.974963Z","iopub.status.idle":"2024-06-27T10:25:45.205723Z","shell.execute_reply.started":"2024-06-27T10:25:44.974925Z","shell.execute_reply":"2024-06-27T10:25:45.204685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport timm\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Define transformation\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n])\n\n# Define the base directory\noutput_base_dir = '/kaggle/input/working-dir1/split_data'\n\n# Ensure train, val, and test directories exist\ntrain_dir = os.path.join(output_base_dir, 'train')\nval_dir = os.path.join(output_base_dir, 'val')\ntest_dir = os.path.join(output_base_dir, 'test')\n\nfor dir_path in [train_dir, val_dir, test_dir]:\n    if not os.path.exists(dir_path):\n        print(f\"Error: The directory {dir_path} does not exist.\")\n        raise FileNotFoundError(f\"The directory {dir_path} does not exist.\")\n\n# Create DataLoaders\ntrain_loader = DataLoader(\n    datasets.ImageFolder(train_dir, transform=transform),\n    batch_size=32, shuffle=True, num_workers=4\n)\nval_loader = DataLoader(\n    datasets.ImageFolder(val_dir, transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\ntest_loader = DataLoader(\n    datasets.ImageFolder(test_dir, transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\n\n# Define the Hierarchical Vision Transformer model\nclass HierarchicalViT(nn.Module):\n    def __init__(self, organs, weights):\n        super(HierarchicalViT, self).__init__()\n        self.organs = organs\n        self.weights = weights\n        self.models = nn.ModuleDict({organ: timm.create_model('vit_base_patch16_224', pretrained=False) for organ in organs})\n        \n        # Load the trained models  /kaggle/input/working-dir1/models\n        for organ in organs:\n            self.models[organ].head = nn.Linear(self.models[organ].head.in_features, 2)\n            self.models[organ].load_state_dict(torch.load(f'/kaggle/input/working-dir1/models/{organ}_vit_model.pth'))\n        \n        # Final classifier\n        self.final_classifier = nn.Linear(len(organs) * 2, 2)\n    \n    def forward(self, x):\n        organ_outputs = [self.models[organ](x) * self.weights[i] for i, organ in enumerate(self.organs)]\n        combined_output = torch.cat(organ_outputs, dim=1)\n        return self.final_classifier(combined_output)\n\n# Define organs and weights\norgans = ['jaw', 'right_eye', 'left_eye', 'right_eyebrow', 'left_eyebrow', 'mouth', 'nose']\nweights = [1.0] * len(organs)  # Adjust weights as needed\n\n# Instantiate the hierarchical model\nhierarchical_model = HierarchicalViT(organs, weights).to(device)\n\n# Train the hierarchical model\ndef train_hierarchical_model(dataloaders, model, num_epochs=10, learning_rate=0.001):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        \n        for inputs, labels in tqdm(dataloaders['train']):\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n        \n        print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {running_loss / len(dataloaders['train'])}\")\n    \n    # Save the hierarchical model\n    torch.save(model.state_dict(), \"/kaggle/working/models/hierarchical_vit.pth\")\n    print(\"Hierarchical model saved at /kaggle/working/models/hierarchical_vit.pth\")\n\n# Combine dataloaders\ncombined_dataloaders = {'train': train_loader, 'val': val_loader, 'test': test_loader}\n\n# Train the hierarchical model\ntrain_hierarchical_model(combined_dataloaders, hierarchical_model)\n\n# Evaluation function for the hierarchical model\ndef evaluate_model(model, dataloader):\n    model.eval()\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for inputs, labels in dataloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    \n    accuracy = correct / total\n    print(f'Accuracy: {accuracy * 100:.2f}%')\n\n# Evaluate the hierarchical model\nevaluate_model(hierarchical_model, combined_dataloaders['test'])\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T10:11:47.918338Z","iopub.execute_input":"2024-06-27T10:11:47.918715Z","iopub.status.idle":"2024-06-27T10:12:24.286682Z","shell.execute_reply.started":"2024-06-27T10:11:47.918666Z","shell.execute_reply":"2024-06-27T10:12:24.285296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport timm\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Define transformation\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n])\n\n# Define the base directory\noutput_base_dir = '/kaggle/working/split_data'\n\n# Ensure train, val, and test directories exist\ntrain_dir = os.path.join(output_base_dir, 'train')\nval_dir = os.path.join(output_base_dir, 'val')\ntest_dir = os.path.join(output_base_dir, 'test')\n\nfor dir_path in [train_dir, val_dir, test_dir]:\n    if not os.path.exists(dir_path):\n        print(f\"Error: The directory {dir_path} does not exist.\")\n        raise FileNotFoundError(f\"The directory {dir_path} does not exist.\")\n\n# Create DataLoaders\ntrain_loader = DataLoader(\n    datasets.ImageFolder(train_dir, transform=transform),\n    batch_size=32, shuffle=True, num_workers=4\n)\nval_loader = DataLoader(\n    datasets.ImageFolder(val_dir, transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\ntest_loader = DataLoader(\n    datasets.ImageFolder(test_dir, transform=transform),\n    batch_size=32, shuffle=False, num_workers=4\n)\n\n# Define the Hierarchical Vision Transformer model\nclass HierarchicalViT(nn.Module):\n    def __init__(self, organs, weights):\n        super(HierarchicalViT, self).__init__()\n        self.organs = organs\n        self.weights = weights\n        self.models = nn.ModuleDict({organ: timm.create_model('vit_base_patch16_224', pretrained=False) for organ in organs})\n        \n        # Load the trained models\n        for organ in organs:\n            self.models[organ].head = nn.Linear(self.models[organ].head.in_features, 2)\n            state_dict = torch.load(f'/kaggle/working/models/{organ}_vit_model.pth')\n            # Adjust state dict keys\n            new_state_dict = {}\n            for k, v in state_dict.items():\n                new_key = k.replace('vit.', '')\n                new_state_dict[new_key] = v\n            self.models[organ].load_state_dict(new_state_dict)\n        \n        # Final classifier\n        self.final_classifier = nn.Linear(len(organs) * 2, 2)\n    \n    def forward(self, x):\n        organ_outputs = [self.models[organ](x) * self.weights[i] for i, organ in enumerate(self.organs)]\n        combined_output = torch.cat(organ_outputs, dim=1)\n        return self.final_classifier(combined_output)\n\n# Define organs and weights\norgans = ['jaw', 'right_eye', 'left_eye', 'right_eyebrow', 'left_eyebrow', 'mouth', 'nose']\nweights = [1.0] * len(organs)  # Adjust weights as needed\n\n# Instantiate the hierarchical model\nhierarchical_model = HierarchicalViT(organs, weights).to(device)\n\n# Train the hierarchical model\ndef train_hierarchical_model(dataloaders, model, num_epochs=10, learning_rate=0.001):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        \n        for inputs, labels in tqdm(dataloaders['train']):\n            inputs, labels = inputs.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item()\n        \n        print(f\"Epoch {epoch + 1}/{num_epochs}, Loss: {running_loss / len(dataloaders['train'])}\")\n    \n    # Save the hierarchical model\n    torch.save(model.state_dict(), \"/kaggle/working/models/hierarchical_vit.pth\")\n    print(\"Hierarchical model saved at /kaggle/working/models/hierarchical_vit.pth\")\n\n# Combine dataloaders\ncombined_dataloaders = {'train': train_loader, 'val': val_loader, 'test': test_loader}\n\n# Train the hierarchical model\ntrain_hierarchical_model(combined_dataloaders, hierarchical_model)\n\n# Evaluation function for the hierarchical model\ndef evaluate_model(model, dataloader):\n    model.eval()\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for inputs, labels in dataloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    \n    accuracy = correct / total\n    print(f'Accuracy: {accuracy * 100:.2f}%')\n\n# Evaluate the hierarchical model\nevaluate_model(hierarchical_model, combined_dataloaders['test'])\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:29:51.684263Z","iopub.execute_input":"2024-06-27T09:29:51.684649Z","iopub.status.idle":"2024-06-27T09:30:10.210658Z","shell.execute_reply.started":"2024-06-27T09:29:51.684584Z","shell.execute_reply":"2024-06-27T09:30:10.208923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\n\n# Load metadata\nmetadata_path = '/kaggle/working/metadata.json'\nwith open(metadata_path, 'r') as f:\n    metadata = json.load(f)\n\n# Adjust the evaluation function to use metadata\ndef evaluate_model_with_metadata(model, dataloader):\n    model.eval()\n    correct = 0\n    total = 0\n    \n    with torch.no_grad():\n        for inputs, labels in dataloader:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            \n            # Assuming labels in metadata are in a dict with video names as keys\n            video_names = dataloader.dataset.samples\n            for idx, video in enumerate(video_names):\n                true_label = 0 if metadata[video]['label'] == 'REAL' else 1\n                total += 1\n                correct += (predicted[idx].item() == true_label)\n    \n    accuracy = correct / total\n    print(f'Accuracy: {accuracy * 100:.2f}%')\n\n# Evaluate the hierarchical model with metadata\nevaluate_model_with_metadata(hierarchical_model, test_loader)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:07:57.566314Z","iopub.status.idle":"2024-06-27T09:07:57.56689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n\ndef process_video(video_path, model, transform, device):\n    # Open video file\n    cap = cv2.VideoCapture(video_path)\n    frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    frame_rate = int(cap.get(cv2.CAP_PROP_FPS))\n    \n    predictions = []\n    frame_number = 0\n    \n    while frame_number < frame_count:\n        ret, frame = cap.read()\n        if not ret:\n            break\n        \n        # Convert frame to RGB (OpenCV uses BGR by default)\n        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n        frame = Image.fromarray(frame)\n        \n        # Apply transformations\n        frame = transform(frame)\n        frame = frame.unsqueeze(0).to(device)\n        \n        # Get prediction\n        model.eval()\n        with torch.no_grad():\n            output = model(frame)\n            _, predicted = torch.max(output, 1)\n            predictions.append(predicted.item())\n        \n        frame_number += frame_rate  # Skip frames to match desired FPS\n    \n    cap.release()\n    \n    # Aggregate results (e.g., majority vote)\n    final_prediction = max(set(predictions), key=predictions.count)\n    label = 'REAL' if final_prediction == 0 else 'FAKE'\n    \n    return label\n\n# Path to random video\nrandom_video_path = '/kaggle/input/deepfake-detection-challenge/test_videos/aassnaulhq.mp4'\n\n# Predict using the hierarchical model\nlabel = process_video(random_video_path, hierarchical_model, transform, device)\nprint(f'The video is predicted to be: {label}')\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T09:07:57.568051Z","iopub.status.idle":"2024-06-27T09:07:57.568712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\n\ndef process_video(video_path, model, transform, device):\n    # Open video file\n    cap = cv2.VideoCapture(video_path)\n    frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n    frame_rate = int(cap.get(cv2.CAP_PROP_FPS))\n    \n    predictions = []\n    frame_number = 0\n    \n    while frame_number < frame_count:\n        ret, frame = cap.read()\n        if not ret:\n            break\n        \n        # Convert frame to RGB (OpenCV uses BGR by default)\n        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n        frame = Image.fromarray(frame)\n        \n        # Apply transformations\n        frame = transform(frame)\n        frame = frame.unsqueeze(0).to(device)\n        \n        # Get prediction\n        model.eval()\n        with torch.no_grad():\n            output = model(frame)\n            _, predicted = torch.max(output, 1)\n            predictions.append(predicted.item())\n        \n        frame_number += frame_rate  # Skip frames to match desired FPS\n    \n    cap.release()\n    \n    # Aggregate results (e.g., majority vote)\n    final_prediction = max(set(predictions), key=predictions.count)\n    label = 'REAL' if final_prediction == 0 else 'FAKE'\n    \n    return label\n\n# Path to random video\nrandom_video_path = '/kaggle/input/deepfake-detection-challenge/test_videos/aassnaulhq.mp4'\n\n# Predict using the hierarchical model\nlabel = process_video(random_video_path, hierarchical_model, transform, device)\nprint(f'The video is predicted to be: {label}')\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision\nprint(torchvision.__version__)\n","metadata":{"execution":{"iopub.status.busy":"2024-06-27T05:45:52.512483Z","iopub.execute_input":"2024-06-27T05:45:52.512804Z","iopub.status.idle":"2024-06-27T05:45:52.517079Z","shell.execute_reply.started":"2024-06-27T05:45:52.512759Z","shell.execute_reply":"2024-06-27T05:45:52.516311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":", 'left_eye', 'nose', 'mouth', 'jaw', 'left_eyebrow', 'right_eyebrow'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class HierarchicalViTModel(nn.Module):\n    def __init__(self, organ_models, organ_weights):\n        super(HierarchicalViTModel, self).__init__()\n        self.organ_models = organ_models\n        self.organ_weights = organ_weights\n        self.fc = nn.Linear(len(organ_models) * 1000, 2)  # Adjust input dimension accordingly\n\n    def forward(self, images):\n        organ_features = []\n        for organ in images:\n            with torch.no_grad():\n                features = self.organ_models[organ](images[organ])\n                weighted_features = self.organ_weights[organ] * features\n                organ_features.append(weighted_features)\n        \n        combined_features = torch.cat(organ_features, dim=1)\n        output = self.fc(combined_features)\n        return output\n\norgan_weights = {\n    'right_eye': 1.0,\n    'left_eye': 1.0,\n    'nose': 1.0,\n    'mouth': 1.0,\n    'jaw': 1.0,\n    'left_eyebrow': 1.0,\n    'right_eyebrow': 1.0\n}\n\nweighted_hierarchical_model = WeightedHierarchicalViTModel(trained_models, organ_weights).to(device)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming hierarchical_train_loader and hierarchical_val_loader are prepared similarly to the previous data loaders\n\ndef train_hierarchical_model(train_loader, val_loader, hierarchical_model, num_epochs=10):\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(hierarchical_model.parameters(), lr=1e-4)\n    \n    for epoch in range(num_epochs):\n        hierarchical_model.train()\n        running_loss = 0.0\n        for images, labels in train_loader:\n            for organ in images:\n                images[organ] = images[organ].to(device)\n            labels = labels.to(device)\n\n            optimizer.zero_grad()\n            outputs = hierarchical_model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            running_loss += loss.item() * list(images.values())[0].size(0)\n\n        epoch_loss = running_loss / len(train_loader.dataset)\n        print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}')\n        \n        evaluate_model(hierarchical_model, val_loader)\n\n    return hierarchical_model\n\n# Train the hierarchical model\nhierarchical_train_loader, hierarchical_val_loader = load_hierarchical_data(train_videos, val_videos, test_videos, organ_folders)\ntrained_hierarchical_model = train_hierarchical_model(hierarchical_train_loader, hierarchical_val_loader, weighted_hierarchical_model)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate on the test set\nevaluate_model(trained_hierarchical_model, test_loader)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport timm\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\n\n# Load metadata\nmetadata_path = \"/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json\"\nwith open(metadata_path, 'r') as f:\n    metadata = json.load(f)\n\n# Create a DataFrame from the metadata\nmetadata_df = pd.DataFrame.from_dict(metadata, orient='index')\nmetadata_df['filename'] = metadata_df.index\n\n# Function to get label from filename\ndef get_label_from_filename(filename):\n    video_id = filename.split('_')[0] + '.mp4'\n    video_id = video_id.replace('.mp4.mp4', '.mp4')  # Handle cases where multiple '.mp4' are appended\n    try:\n        label = metadata_df.loc[video_id, 'label']\n    except KeyError:\n        print(f\"Warning: Video ID {video_id} not found in metadata.\")\n        return -1  # Return an invalid label to easily detect these cases\n    return 1 if label == 'FAKE' else 0\n\n#Custom dataset for facial organs\nclass FacialOrganDataset(Dataset):\n    def __init__(self, organ_folder, transform=None):\n        self.organ_folder = organ_folder\n        self.transform = transform\n        self.images = [f for f in os.listdir(organ_folder) if os.path.isfile(os.path.join(organ_folder, f))]\n        \n        # Filter out images that do not have valid labels in the metadata\n        self.images = [img for img in self.images if get_label_from_filename(img) != -1]\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        img_name = os.path.join(self.organ_folder, self.images[idx])\n        image = Image.open(img_name).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        label = get_label_from_filename(self.images[idx])\n        if label == -1:\n            raise ValueError(f\"Invalid label for file: {self.images[idx]}\")\n        return image, label\n\n# Load and split data\ndef load_data(organ_folder, test_size=0.2, batch_size=32):\n    transform = transforms.Compose([\n        transforms.Resize((224, 224)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n    ])\n    \n    dataset = FacialOrganDataset(organ_folder, transform=transform)\n    train_indices, val_indices = train_test_split(range(len(dataset)), test_size=test_size, random_state=42)\n    \n    train_loader = DataLoader(dataset, batch_size=batch_size, sampler=torch.utils.data.SubsetRandomSampler(train_indices))\n    val_loader = DataLoader(dataset, batch_size=batch_size, sampler=torch.utils.data.SubsetRandomSampler(val_indices))\n    \n    return train_loader, val_loader\n\n# Define the Vision Transformer model\nclass ViTModel(nn.Module):\n    def __init__(self, num_classes=2):\n        super(ViTModel, self).__init__()\n        self.vit = timm.create_model('vit_base_patch16_224', pretrained=True)\n        self.vit.head = nn.Linear(self.vit.head.in_features, num_classes)  # Modify for classification\n    \n    def forward(self, x):\n        return self.vit(x)\n\n# Training function\ndef train_model(train_loader, val_loader, num_classes=2, num_epochs=10, learning_rate=1e-4):\n    model = ViTModel(num_classes=num_classes)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n    \n    model = model.to(device)\n    criterion = criterion.to(device)\n    \n    for epoch in range(num_epochs):\n        model.train()\n        running_loss = 0.0\n        for images, labels in tqdm(train_loader):\n            images = images.to(device)\n            labels = labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item() * images.size(0)\n        \n        epoch_loss = running_loss / len(train_loader.dataset)\n        print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {epoch_loss:.4f}\")\n        \n        evaluate_model(model, val_loader)\n    \n    return model\n\n# Evaluation function\ndef evaluate_model(model, val_loader):\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = model(images)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n    accuracy = 100 * correct / total\n    print(f'Validation Accuracy: {accuracy:.2f}%')\n\n# Training and validation for each facial organ\ndevice = torch.device('cpu')\n\nfacial_organs = ['right_eye', 'left_eye', 'nose', 'mouth','jaw','left_eyebrow','right_eyebrow']\n\ntrained_models = {}\n\nfor organ in facial_organs:\n    print(f\"\\nTraining model for {organ}\") \n    train_loader, val_loader = load_data(f'/kaggle/input/working-dir/facial_organs/{organ}')\n    trained_model = train_model(train_loader, val_loader)\n    trained_models[organ] = trained_model\n\n","metadata":{"execution":{"iopub.status.busy":"2024-06-26T09:27:31.779325Z","iopub.execute_input":"2024-06-26T09:27:31.779643Z","iopub.status.idle":"2024-06-26T09:43:08.795395Z","shell.execute_reply.started":"2024-06-26T09:27:31.779599Z","shell.execute_reply":"2024-06-26T09:43:08.793859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}