{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":924245,"sourceType":"datasetVersion","datasetId":464091}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Deep Fake Detection using CNN and RNN","metadata":{"id":"Q6tasuafvT2O"}},{"cell_type":"markdown","source":"# Contriubuters - ","metadata":{}},{"cell_type":"markdown","source":"## 1. Rohan Inamdar\n## 2. Kavin Sundarr","metadata":{}},{"cell_type":"markdown","source":"## Importing Required libraries","metadata":{"id":"dFXIv9qNpKzt","tags":[]}},{"cell_type":"code","source":"!pip install tensorflow ","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:06:01.850797Z","iopub.execute_input":"2024-11-15T13:06:01.851432Z","iopub.status.idle":"2024-11-15T13:06:06.227355Z","shell.execute_reply.started":"2024-11-15T13:06:01.851397Z","shell.execute_reply":"2024-11-15T13:06:06.226526Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport sklearn\nimport tensorflow as tf\n\nimport cv2\nimport pandas as pd\nimport numpy as np\n","metadata":{"id":"TFSU3FCOpKzu","execution":{"iopub.status.busy":"2024-11-15T13:06:06.228921Z","iopub.execute_input":"2024-11-15T13:06:06.229181Z","iopub.status.idle":"2024-11-15T13:06:24.831512Z","shell.execute_reply.started":"2024-11-15T13:06:06.229153Z","shell.execute_reply":"2024-11-15T13:06:24.830675Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.test.is_gpu_available()\nstrategy = tf.distribute.MirroredStrategy()\nprint('DEVICES AVAILABLE: {}'.format(strategy.num_replicas_in_sync))","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:06:24.832507Z","iopub.execute_input":"2024-11-15T13:06:24.832979Z","iopub.status.idle":"2024-11-15T13:06:28.903105Z","shell.execute_reply.started":"2024-11-15T13:06:24.832948Z","shell.execute_reply":"2024-11-15T13:06:28.902218Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install plotly\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nfrom matplotlib import pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:06:28.904973Z","iopub.execute_input":"2024-11-15T13:06:28.905346Z","iopub.status.idle":"2024-11-15T13:06:46.474220Z","shell.execute_reply.started":"2024-11-15T13:06:28.905301Z","shell.execute_reply":"2024-11-15T13:06:46.472918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:06:46.475621Z","iopub.execute_input":"2024-11-15T13:06:46.475908Z","iopub.status.idle":"2024-11-15T13:06:46.483973Z","shell.execute_reply.started":"2024-11-15T13:06:46.475877Z","shell.execute_reply":"2024-11-15T13:06:46.482974Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.rc('font', size=14)\nplt.rc('axes', labelsize=14, titlesize=14)\nplt.rc('legend', fontsize=14)\nplt.rc('xtick', labelsize=10)\nplt.rc('ytick', labelsize=10)","metadata":{"id":"8d4TH3NbpKzx","execution":{"iopub.status.busy":"2024-11-15T13:06:46.485154Z","iopub.execute_input":"2024-11-15T13:06:46.485431Z","iopub.status.idle":"2024-11-15T13:06:46.494515Z","shell.execute_reply.started":"2024-11-15T13:06:46.485406Z","shell.execute_reply":"2024-11-15T13:06:46.493503Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Visualisation","metadata":{"id":"NL3Ht4wC9b3n"}},{"cell_type":"code","source":"import os\n\ndef get_data():\n    return pd.read_csv('../input/deepfake-faces/metadata.csv')","metadata":{"id":"jfv9PxSB4tM8","execution":{"iopub.status.busy":"2024-11-15T13:06:46.495613Z","iopub.execute_input":"2024-11-15T13:06:46.495859Z","iopub.status.idle":"2024-11-15T13:06:46.504362Z","shell.execute_reply.started":"2024-11-15T13:06:46.495835Z","shell.execute_reply":"2024-11-15T13:06:46.503360Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta=get_data()\nmeta.head()","metadata":{"id":"tDW7BRph9ehF","outputId":"97de18b5-0a37-4302-8804-8a16a7d2ed2f","execution":{"iopub.status.busy":"2024-11-15T13:06:46.505460Z","iopub.execute_input":"2024-11-15T13:06:46.505717Z","iopub.status.idle":"2024-11-15T13:06:46.690851Z","shell.execute_reply.started":"2024-11-15T13:06:46.505694Z","shell.execute_reply":"2024-11-15T13:06:46.689795Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta.shape","metadata":{"id":"n7FSdDifbZxn","outputId":"5451a127-405a-4c0b-a197-c920b796adbb","execution":{"iopub.status.busy":"2024-11-15T13:06:46.692142Z","iopub.execute_input":"2024-11-15T13:06:46.692476Z","iopub.status.idle":"2024-11-15T13:06:46.698599Z","shell.execute_reply.started":"2024-11-15T13:06:46.692444Z","shell.execute_reply":"2024-11-15T13:06:46.697625Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(meta[meta.label=='FAKE']),len(meta[meta.label=='REAL'])","metadata":{"id":"_FJcz2IthxVG","outputId":"274c3f65-7acb-4f99-8aa9-a5b2a23bf06a","execution":{"iopub.status.busy":"2024-11-15T13:06:46.701474Z","iopub.execute_input":"2024-11-15T13:06:46.701748Z","iopub.status.idle":"2024-11-15T13:06:46.731044Z","shell.execute_reply.started":"2024-11-15T13:06:46.701721Z","shell.execute_reply":"2024-11-15T13:06:46.730068Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"real_df = meta[meta[\"label\"] == \"REAL\"]\nfake_df = meta[meta[\"label\"] == \"FAKE\"]\nsample_size = 15000\nfake_df = fake_df.sample(sample_size, random_state=42)\nsample_meta = pd.concat([real_df, fake_df])","metadata":{"id":"IgMfzY-PjjtH","execution":{"iopub.status.busy":"2024-11-15T13:06:46.732109Z","iopub.execute_input":"2024-11-15T13:06:46.732413Z","iopub.status.idle":"2024-11-15T13:06:46.760011Z","shell.execute_reply.started":"2024-11-15T13:06:46.732384Z","shell.execute_reply":"2024-11-15T13:06:46.759143Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As mentioned instead of using 95k images we will only use 16000 images.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nTrain_set, Test_set = train_test_split(sample_meta,test_size=0.2,random_state=42,stratify=sample_meta['label'])\nTrain_set, Val_set  = train_test_split(Train_set,test_size=0.15,random_state=42,stratify=Train_set['label'])","metadata":{"id":"5eB86S6K-T5Z","execution":{"iopub.status.busy":"2024-11-15T13:06:46.760963Z","iopub.execute_input":"2024-11-15T13:06:46.761200Z","iopub.status.idle":"2024-11-15T13:06:46.961465Z","shell.execute_reply.started":"2024-11-15T13:06:46.761176Z","shell.execute_reply":"2024-11-15T13:06:46.960354Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Train_set.shape,Val_set.shape,Test_set.shape","metadata":{"id":"8p-TONijb4qA","outputId":"56d0b529-9d81-4019-d8fa-618c8cdba90f","execution":{"iopub.status.busy":"2024-11-15T13:06:46.962789Z","iopub.execute_input":"2024-11-15T13:06:46.963488Z","iopub.status.idle":"2024-11-15T13:06:46.969582Z","shell.execute_reply.started":"2024-11-15T13:06:46.963450Z","shell.execute_reply":"2024-11-15T13:06:46.968517Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = dict()\n\ny[0] = []\ny[1] = []\n\nfor set_name in (np.array(Train_set['label']), np.array(Val_set['label']), np.array(Test_set['label'])):\n    y[0].append(np.sum(set_name == 'REAL'))\n    y[1].append(np.sum(set_name == 'FAKE'))\n\ntrace0 = go.Bar(\n    x=['Train Set', 'Validation Set', 'Test Set'],\n    y=y[0],\n    name='REAL',\n    marker=dict(color='#33cc33'),\n    opacity=0.7\n)\ntrace1 = go.Bar(\n    x=['Train Set', 'Validation Set', 'Test Set'],\n    y=y[1],\n    name='FAKE',\n    marker=dict(color='#ff3300'),\n    opacity=0.7\n)\n\ndata = [trace0, trace1]\nlayout = go.Layout(\n    title='Count of classes in each set',\n    xaxis={'title': 'Set'},\n    yaxis={'title': 'Count'}\n)\n\nfig = go.Figure(data, layout)\niplot(fig)","metadata":{"id":"hzNGtCWd-mTk","outputId":"5178c3ed-cbba-4f99-99d0-26bde11a5dab","execution":{"iopub.status.busy":"2024-11-15T13:06:46.970784Z","iopub.execute_input":"2024-11-15T13:06:46.971608Z","iopub.status.idle":"2024-11-15T13:06:47.489577Z","shell.execute_reply.started":"2024-11-15T13:06:46.971576Z","shell.execute_reply":"2024-11-15T13:06:47.488613Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The original image dataset were biased with more fake images than real since we are taking a sample of it its better to take equal proportion of real and fake images.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor cur,i in enumerate(Train_set.index[25:50]):\n    plt.subplot(5,5,cur+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    \n    plt.imshow(cv2.imread('../input/deepfake-faces/faces_224/'+Train_set.loc[i,'videoname'][:-4]+'.jpg'))\n    \n    if(Train_set.loc[i,'label']=='FAKE'):\n        plt.xlabel('FAKE Image')\n    else:\n        plt.xlabel('REAL Image')\n        \nplt.show()","metadata":{"id":"VR7Uly2fcUYi","outputId":"c1f47a82-ef4f-4bcd-b51c-d4738142fc0f","execution":{"iopub.status.busy":"2024-11-15T13:06:47.490673Z","iopub.execute_input":"2024-11-15T13:06:47.490950Z","iopub.status.idle":"2024-11-15T13:06:49.193736Z","shell.execute_reply.started":"2024-11-15T13:06:47.490923Z","shell.execute_reply":"2024-11-15T13:06:49.192604Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Base Model","metadata":{"id":"dOvN_divkl-N"}},{"cell_type":"markdown","source":"### Custom CNN Architecture","metadata":{"id":"oid44Xx-pKz6"}},{"cell_type":"code","source":"def retreive_dataset(set_name):\n    images,labels=[],[]\n    for (img, imclass) in zip(set_name['videoname'], set_name['label']):\n        images.append(cv2.imread('../input/deepfake-faces/faces_224/'+img[:-4]+'.jpg'))\n        if(imclass=='FAKE'):\n            labels.append(1)\n        else:\n            labels.append(0)\n    \n    return np.array(images),np.array(labels)","metadata":{"id":"Hz0ZdQ_fgHhG","execution":{"iopub.status.busy":"2024-11-15T13:06:49.195170Z","iopub.execute_input":"2024-11-15T13:06:49.195541Z","iopub.status.idle":"2024-11-15T13:06:49.200707Z","shell.execute_reply.started":"2024-11-15T13:06:49.195507Z","shell.execute_reply":"2024-11-15T13:06:49.199821Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train,y_train=retreive_dataset(Train_set)\nX_val,y_val=retreive_dataset(Val_set)\nX_test,y_test=retreive_dataset(Test_set)","metadata":{"id":"zeAGRcAbguKU","execution":{"iopub.status.busy":"2024-11-15T13:06:49.201723Z","iopub.execute_input":"2024-11-15T13:06:49.201997Z","iopub.status.idle":"2024-11-15T13:11:29.981786Z","shell.execute_reply.started":"2024-11-15T13:06:49.201970Z","shell.execute_reply":"2024-11-15T13:11:29.980391Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from functools import partial\nimport tensorflow as tf\n\n# Set random seed for reproducibility\ntf.random.set_seed(42)\n\n# Define a default Conv2D configuration\nDefaultConv2D = partial(\n    tf.keras.layers.Conv2D,\n    kernel_size=3,\n    padding=\"same\",\n    activation=\"relu\",\n    kernel_initializer=\"he_normal\"\n)\n\n# Build the model\nmodel = tf.keras.Sequential([\n    tf.keras.layers.Input(shape=[224, 224, 3]),  # Explicitly define the input shape\n    DefaultConv2D(filters=64, kernel_size=7),\n    tf.keras.layers.MaxPool2D(),\n    DefaultConv2D(filters=128),\n    DefaultConv2D(filters=128),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(units=128, activation=\"relu\", kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=64, activation=\"relu\", kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=1, activation=\"sigmoid\")\n])\n","metadata":{"id":"34upiak4pKz6","execution":{"iopub.status.busy":"2024-11-15T13:11:29.983125Z","iopub.execute_input":"2024-11-15T13:11:29.983453Z","iopub.status.idle":"2024-11-15T13:11:30.139889Z","shell.execute_reply.started":"2024-11-15T13:11:29.983425Z","shell.execute_reply":"2024-11-15T13:11:30.138798Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n              metrics=[\"accuracy\"])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:30.141054Z","iopub.execute_input":"2024-11-15T13:11:30.141344Z","iopub.status.idle":"2024-11-15T13:11:30.169810Z","shell.execute_reply.started":"2024-11-15T13:11:30.141304Z","shell.execute_reply":"2024-11-15T13:11:30.168932Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=5,batch_size=64,\n                    validation_data=(X_val, y_val))","metadata":{"id":"KZbWeIBYpKz6","outputId":"deb6f56a-7b93-4241-a1bd-b210c0f2d426","execution":{"iopub.status.busy":"2024-11-15T13:11:30.170800Z","iopub.execute_input":"2024-11-15T13:11:30.171188Z","iopub.status.idle":"2024-11-15T13:11:37.440920Z","shell.execute_reply.started":"2024-11-15T13:11:30.171162Z","shell.execute_reply":"2024-11-15T13:11:37.437565Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_test, y_test)","metadata":{"id":"6HDDr4uehast","execution":{"iopub.status.busy":"2024-11-15T13:11:37.441846Z","iopub.status.idle":"2024-11-15T13:11:37.442180Z","shell.execute_reply.started":"2024-11-15T13:11:37.442016Z","shell.execute_reply":"2024-11-15T13:11:37.442032Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.444274Z","iopub.status.idle":"2024-11-15T13:11:37.444601Z","shell.execute_reply.started":"2024-11-15T13:11:37.444455Z","shell.execute_reply":"2024-11-15T13:11:37.444470Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**A baseline score is set here around ~51%**","metadata":{}},{"cell_type":"markdown","source":"# Pretrained Models for Transfer Learning","metadata":{"id":"hqxnSBJ3pKz8"}},{"cell_type":"markdown","source":"using Xception model for fine-tuning ","metadata":{}},{"cell_type":"code","source":"train_set_raw=tf.data.Dataset.from_tensor_slices((X_train,y_train))\nvalid_set_raw=tf.data.Dataset.from_tensor_slices((X_val,y_val))\ntest_set_raw=tf.data.Dataset.from_tensor_slices((X_test,y_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:47.535911Z","iopub.execute_input":"2024-11-15T13:13:47.537118Z","iopub.status.idle":"2024-11-15T13:13:53.255716Z","shell.execute_reply.started":"2024-11-15T13:13:47.537068Z","shell.execute_reply":"2024-11-15T13:13:53.254473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.keras.backend.clear_session()  # extra code – resets layer name counter\n\nbatch_size_per_replica = 32\nbatch_size = batch_size_per_replica\npreprocess = tf.keras.applications.xception.preprocess_input\ntrain_set = train_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y))\ntrain_set = train_set.shuffle(1000, seed=42).batch(batch_size).prefetch(1)\nvalid_set = valid_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)\ntest_set = test_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)","metadata":{"id":"Bnz0n9XApKz9","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:53.257442Z","iopub.execute_input":"2024-11-15T13:13:53.257738Z","iopub.status.idle":"2024-11-15T13:13:53.684563Z","shell.execute_reply.started":"2024-11-15T13:13:53.257711Z","shell.execute_reply":"2024-11-15T13:13:53.683547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# extra code – displays the first 9 images in the first batch of valid_set\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        plt.imshow((X_batch[index] + 1) / 2)  # rescale to 0–1 for imshow()\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"ZL3c3i4opKz9","outputId":"38847d8d-8822-41a3-cfb2-27479aa5debe","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:53.685620Z","iopub.execute_input":"2024-11-15T13:13:53.685897Z","iopub.status.idle":"2024-11-15T13:13:54.658027Z","shell.execute_reply.started":"2024-11-15T13:13:53.685869Z","shell.execute_reply":"2024-11-15T13:13:54.656840Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_augmentation = tf.keras.Sequential([\n    tf.keras.layers.RandomFlip(mode=\"horizontal\", seed=42),\n    tf.keras.layers.RandomRotation(factor=0.05, seed=42),\n    tf.keras.layers.RandomContrast(factor=0.2, seed=42)\n])","metadata":{"id":"Ib0cA8Y1pKz9","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:54.659971Z","iopub.execute_input":"2024-11-15T13:13:54.660302Z","iopub.status.idle":"2024-11-15T13:13:54.674612Z","shell.execute_reply.started":"2024-11-15T13:13:54.660272Z","shell.execute_reply":"2024-11-15T13:13:54.673747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# extra code – displays the same first 9 images, after augmentation\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    X_batch_augmented = data_augmentation(X_batch, training=True)\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        # We must rescale the images to the 0-1 range for imshow(), and also\n        # clip the result to that range, because data augmentation may\n        # make some values go out of bounds (e.g., RandomContrast in this case).\n        plt.imshow(np.clip((X_batch_augmented[index] + 1) / 2, 0, 1))\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"w6GH5_vupKz-","outputId":"eeb2c924-2f4f-4aa1-bea9-951bebef4bf0","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:54.675661Z","iopub.execute_input":"2024-11-15T13:13:54.675929Z","iopub.status.idle":"2024-11-15T13:13:55.461645Z","shell.execute_reply.started":"2024-11-15T13:13:54.675904Z","shell.execute_reply":"2024-11-15T13:13:55.460718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.random.set_seed(42)  # extra code – ensures reproducibility\nbase_model = tf.keras.applications.xception.Xception(weights=\"imagenet\",\n                                                     include_top=False)\navg = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)\noutput = tf.keras.layers.Dense(1, activation=\"sigmoid\")(avg)\nmodel = tf.keras.Model(inputs=base_model.input, outputs=output)","metadata":{"id":"lRyCgvaKpKz-","outputId":"a825e173-8b1d-4217-a1c4-5491b49c3e82","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:55.462723Z","iopub.execute_input":"2024-11-15T13:13:55.462984Z","iopub.status.idle":"2024-11-15T13:13:57.087292Z","shell.execute_reply.started":"2024-11-15T13:13:55.462958Z","shell.execute_reply":"2024-11-15T13:13:57.086076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for layer in base_model.layers:\n    layer.trainable = False","metadata":{"id":"KBlyG6ElpKz-","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:57.088477Z","iopub.execute_input":"2024-11-15T13:13:57.088764Z","iopub.status.idle":"2024-11-15T13:13:57.094783Z","shell.execute_reply.started":"2024-11-15T13:13:57.088737Z","shell.execute_reply":"2024-11-15T13:13:57.093965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=3,verbose=2)","metadata":{"id":"GGxK2yPcpKz-","outputId":"6b64214a-e104-4b6c-9b7a-3388fc9aa15f","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:13:57.095861Z","iopub.execute_input":"2024-11-15T13:13:57.096103Z","iopub.status.idle":"2024-11-15T13:14:03.870167Z","shell.execute_reply.started":"2024-11-15T13:13:57.096080Z","shell.execute_reply":"2024-11-15T13:14:03.868505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for indices in zip(range(33), range(33, 66), range(66, 99), range(99, 132)):\n    for idx in indices:\n        print(f\"{idx:3}: {base_model.layers[idx].name:22}\", end=\"\")\n    print()","metadata":{"id":"GvGMiJMLpKz-","outputId":"91f2c96c-c058-45e0-e428-66fa6076ad56","trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:14:03.870976Z","iopub.status.idle":"2024-11-15T13:14:03.871379Z","shell.execute_reply.started":"2024-11-15T13:14:03.871170Z","shell.execute_reply":"2024-11-15T13:14:03.871189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T13:14:03.872984Z","iopub.status.idle":"2024-11-15T13:14:03.873298Z","shell.execute_reply.started":"2024-11-15T13:14:03.873149Z","shell.execute_reply":"2024-11-15T13:14:03.873164Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**After Fine tuning The accuracy comes about ~64%**","metadata":{}},{"cell_type":"markdown","source":"Now that the weights of our new top layers are not too bad, we can make the top part of the base model trainable again, and continue training, but with a lower learning rate:","metadata":{"id":"L_bEwL8KpKz_"}},{"cell_type":"code","source":"for layer in base_model.layers[56:]:\n    layer.trainable = True\n\noptimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=10)","metadata":{"id":"GEUNGlhvpKz_","outputId":"c622a91d-f634-4443-b87e-8d46defdb578","execution":{"iopub.status.busy":"2024-11-15T13:11:37.458243Z","iopub.status.idle":"2024-11-15T13:11:37.458574Z","shell.execute_reply.started":"2024-11-15T13:11:37.458413Z","shell.execute_reply":"2024-11-15T13:11:37.458430Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.459963Z","iopub.status.idle":"2024-11-15T13:11:37.460291Z","shell.execute_reply.started":"2024-11-15T13:11:37.460127Z","shell.execute_reply":"2024-11-15T13:11:37.460144Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.461300Z","iopub.status.idle":"2024-11-15T13:11:37.461600Z","shell.execute_reply.started":"2024-11-15T13:11:37.461459Z","shell.execute_reply":"2024-11-15T13:11:37.461473Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**The model accuracy finally reaches to 81.9%**","metadata":{}},{"cell_type":"code","source":"model.save('xception_deepfake_image_3o.h5')","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.462884Z","iopub.status.idle":"2024-11-15T13:11:37.463199Z","shell.execute_reply.started":"2024-11-15T13:11:37.463042Z","shell.execute_reply":"2024-11-15T13:11:37.463058Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Adding Explainability to the model","metadata":{}},{"cell_type":"code","source":"!pip install lime","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.464240Z","iopub.status.idle":"2024-11-15T13:11:37.464541Z","shell.execute_reply.started":"2024-11-15T13:11:37.464395Z","shell.execute_reply":"2024-11-15T13:11:37.464410Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lime import lime_image\n\nexplainer = lime_image.LimeImageExplainer()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.465346Z","iopub.status.idle":"2024-11-15T13:11:37.465673Z","shell.execute_reply.started":"2024-11-15T13:11:37.465515Z","shell.execute_reply":"2024-11-15T13:11:37.465530Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 12))\n\nfor index in range(9):\n    plt.subplot(3, 3, index + 1)\n    plt.imshow((x[index] + 1) / 2)  # rescale to 0–1 for imshow()\n    if(y[index]==1):\n        classt='FAKE'\n    else:\n        classt='REAL'\n    plt.title(f\"Class: {classt}\")\n    plt.axis(\"off\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.466413Z","iopub.status.idle":"2024-11-15T13:11:37.466708Z","shell.execute_reply.started":"2024-11-15T13:11:37.466556Z","shell.execute_reply":"2024-11-15T13:11:37.466572Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data=x[2,:,:,:]\ntest_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.467493Z","iopub.status.idle":"2024-11-15T13:11:37.467795Z","shell.execute_reply.started":"2024-11-15T13:11:37.467640Z","shell.execute_reply":"2024-11-15T13:11:37.467661Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"explanation = explainer.explain_instance(test_data.astype('double'), model.predict,  \n                                         top_labels=3, hide_color=0, num_samples=1000)","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.468855Z","iopub.status.idle":"2024-11-15T13:11:37.469186Z","shell.execute_reply.started":"2024-11-15T13:11:37.469022Z","shell.execute_reply":"2024-11-15T13:11:37.469038Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from skimage.segmentation import mark_boundaries\n\ntemp_1, mask_1 = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=True, num_features=5, hide_rest=True)\ntemp_2, mask_2 = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=False, num_features=10, hide_rest=False)\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))\nax1.imshow(mark_boundaries(temp_1, mask_1))\nax2.imshow(mark_boundaries(temp_2, mask_2))\nax1.axis('off')\nax2.axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.470377Z","iopub.status.idle":"2024-11-15T13:11:37.470689Z","shell.execute_reply.started":"2024-11-15T13:11:37.470527Z","shell.execute_reply":"2024-11-15T13:11:37.470542Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Deep Fake Video Classification","metadata":{}},{"cell_type":"markdown","source":"## Importing required libraries","metadata":{}},{"cell_type":"code","source":"!pip install -U --upgrade tensorflow","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.471880Z","iopub.status.idle":"2024-11-15T13:11:37.472170Z","shell.execute_reply.started":"2024-11-15T13:11:37.472025Z","shell.execute_reply":"2024-11-15T13:11:37.472039Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom tensorflow import keras\n#from imutils import paths\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport imageio\nimport cv2\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.473348Z","iopub.status.idle":"2024-11-15T13:11:37.473665Z","shell.execute_reply.started":"2024-11-15T13:11:37.473502Z","shell.execute_reply":"2024-11-15T13:11:37.473517Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Visualisation","metadata":{}},{"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\nTEST_FOLDER = 'test_videos'\n\nprint(f\"Train samples: {len(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))}\")\nprint(f\"Test samples: {len(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))}\")","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.474370Z","iopub.status.idle":"2024-11-15T13:11:37.474654Z","shell.execute_reply.started":"2024-11-15T13:11:37.474508Z","shell.execute_reply":"2024-11-15T13:11:37.474522Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample_metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\ntrain_sample_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.475923Z","iopub.status.idle":"2024-11-15T13:11:37.476235Z","shell.execute_reply.started":"2024-11-15T13:11:37.476074Z","shell.execute_reply":"2024-11-15T13:11:37.476089Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample_metadata.groupby('label')['label'].count().plot(figsize=(15, 5), kind='bar', title='Distribution of Labels in the Training Set')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.477163Z","iopub.status.idle":"2024-11-15T13:11:37.477482Z","shell.execute_reply.started":"2024-11-15T13:11:37.477326Z","shell.execute_reply":"2024-11-15T13:11:37.477342Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample_metadata.shape","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.478830Z","iopub.status.idle":"2024-11-15T13:11:37.479165Z","shell.execute_reply.started":"2024-11-15T13:11:37.478991Z","shell.execute_reply":"2024-11-15T13:11:37.479007Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Fake videos","metadata":{}},{"cell_type":"code","source":"fake_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.label=='FAKE'].sample(3).index)\nfake_train_sample_video","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.480093Z","iopub.status.idle":"2024-11-15T13:11:37.480440Z","shell.execute_reply.started":"2024-11-15T13:11:37.480260Z","shell.execute_reply":"2024-11-15T13:11:37.480274Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def display_image_from_video(video_path):\n    capture_image = cv2.VideoCapture(video_path) \n    ret, frame = capture_image.read()\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111)\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    ax.imshow(frame)","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.481540Z","iopub.status.idle":"2024-11-15T13:11:37.481853Z","shell.execute_reply.started":"2024-11-15T13:11:37.481697Z","shell.execute_reply":"2024-11-15T13:11:37.481714Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for video_file in fake_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.482678Z","iopub.status.idle":"2024-11-15T13:11:37.482968Z","shell.execute_reply.started":"2024-11-15T13:11:37.482823Z","shell.execute_reply":"2024-11-15T13:11:37.482837Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Real Videos","metadata":{}},{"cell_type":"code","source":"real_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.label=='REAL'].sample(3).index)\nreal_train_sample_video","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.483800Z","iopub.status.idle":"2024-11-15T13:11:37.484076Z","shell.execute_reply.started":"2024-11-15T13:11:37.483940Z","shell.execute_reply":"2024-11-15T13:11:37.483954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for video_file in real_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.484757Z","iopub.status.idle":"2024-11-15T13:11:37.485027Z","shell.execute_reply.started":"2024-11-15T13:11:37.484892Z","shell.execute_reply":"2024-11-15T13:11:37.484906Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Videos with same original","metadata":{}},{"cell_type":"code","source":"train_sample_metadata['original'].value_counts()[0:5]","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.486540Z","iopub.status.idle":"2024-11-15T13:11:37.486857Z","shell.execute_reply.started":"2024-11-15T13:11:37.486699Z","shell.execute_reply":"2024-11-15T13:11:37.486715Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We pick one of the originals with largest number of samples.\n\nWe also modify our visualization function to work with multiple images.","metadata":{}},{"cell_type":"code","source":"def display_image_from_video_list(video_path_list, video_folder=TRAIN_SAMPLE_FOLDER):\n    plt.figure()\n    fig, ax = plt.subplots(2,3,figsize=(16,8))\n    # we only show images extracted from the first 6 videos\n    for i, video_file in enumerate(video_path_list[0:6]):\n        video_path = os.path.join(DATA_FOLDER, video_folder,video_file)\n        capture_image = cv2.VideoCapture(video_path) \n        ret, frame = capture_image.read()\n        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n        ax[i//3, i%3].imshow(frame)\n        ax[i//3, i%3].set_title(f\"Video: {video_file}\")\n        ax[i//3, i%3].axis('on')\n","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.487639Z","iopub.status.idle":"2024-11-15T13:11:37.487924Z","shell.execute_reply.started":"2024-11-15T13:11:37.487783Z","shell.execute_reply":"2024-11-15T13:11:37.487797Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"same_original_fake_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.original=='atvmxvwyns.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.488910Z","iopub.status.idle":"2024-11-15T13:11:37.489239Z","shell.execute_reply.started":"2024-11-15T13:11:37.489078Z","shell.execute_reply":"2024-11-15T13:11:37.489095Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Test video files","metadata":{}},{"cell_type":"code","source":"test_videos = pd.DataFrame(list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER))), columns=['video'])","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.490602Z","iopub.status.idle":"2024-11-15T13:11:37.490895Z","shell.execute_reply.started":"2024-11-15T13:11:37.490754Z","shell.execute_reply":"2024-11-15T13:11:37.490769Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_videos.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.492037Z","iopub.status.idle":"2024-11-15T13:11:37.492356Z","shell.execute_reply.started":"2024-11-15T13:11:37.492186Z","shell.execute_reply":"2024-11-15T13:11:37.492201Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"visual of a videos.","metadata":{}},{"cell_type":"code","source":"display_image_from_video(os.path.join(DATA_FOLDER, TEST_FOLDER, test_videos.iloc[2].video))","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.493386Z","iopub.status.idle":"2024-11-15T13:11:37.493735Z","shell.execute_reply.started":"2024-11-15T13:11:37.493561Z","shell.execute_reply":"2024-11-15T13:11:37.493577Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Play video files","metadata":{}},{"cell_type":"code","source":"fake_videos = list(train_sample_metadata.loc[train_sample_metadata.label=='FAKE'].index)","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.494911Z","iopub.status.idle":"2024-11-15T13:11:37.495231Z","shell.execute_reply.started":"2024-11-15T13:11:37.495069Z","shell.execute_reply":"2024-11-15T13:11:37.495084Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import HTML\nfrom base64 import b64encode\n\ndef play_video(video_file, subset=TRAIN_SAMPLE_FOLDER):\n    video_url = open(os.path.join(DATA_FOLDER, subset,video_file),'rb').read()\n    data_url = \"data:video/mp4;base64,\" + b64encode(video_url).decode()\n    return HTML(\"\"\"<video width=500 controls><source src=\"%s\" type=\"video/mp4\"></video>\"\"\" % data_url)\n\nplay_video(fake_videos[10])","metadata":{"execution":{"iopub.status.busy":"2024-11-15T13:11:37.496270Z","iopub.status.idle":"2024-11-15T13:11:37.496581Z","shell.execute_reply.started":"2024-11-15T13:11:37.496427Z","shell.execute_reply":"2024-11-15T13:11:37.496442Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Conclusion","metadata":{}},{"cell_type":"markdown","source":"**The deep fake image classifier and deep fake video classifier model gave a generalisation error of around ~82%**","metadata":{}},{"cell_type":"markdown","source":"# Reference","metadata":{}},{"cell_type":"markdown","source":"https://keras.io/examples/vision/video_classification/\n\nhttps://www.kaggle.com/code/gpreda/deepfake-starter-kit\n\nhttps://www.kaggle.com/code/robikscube/kaggle-deepfake-detection-introduction\n\nhttps://www.kaggle.com/code/humananalog/binary-image-classifier-training-demo\n\nhttps://www.kaggle.com/datasets/dagnelies/deepfake-faces\n\nhttps://www.kaggle.com/code/gautam20bce1227/fake-detection-on-images","metadata":{}}]}