{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# DeepFake Image detection","metadata":{"id":"vGkKRV1kY8Z-"}},{"cell_type":"code","source":"import sys\nimport sklearn\nimport tensorflow as tf\n\nimport cv2\nimport pandas as pd\nimport numpy as np\n\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nfrom matplotlib import pyplot as plt","metadata":{"id":"TFSU3FCOpKzu","execution":{"iopub.status.busy":"2023-04-07T16:25:13.465456Z","iopub.execute_input":"2023-04-07T16:25:13.467529Z","iopub.status.idle":"2023-04-07T16:25:31.185136Z","shell.execute_reply.started":"2023-04-07T16:25:13.467478Z","shell.execute_reply":"2023-04-07T16:25:31.183861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.test.is_gpu_available()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T16:25:31.186735Z","iopub.execute_input":"2023-04-07T16:25:31.187588Z","iopub.status.idle":"2023-04-07T16:25:35.908357Z","shell.execute_reply.started":"2023-04-07T16:25:31.187546Z","shell.execute_reply":"2023-04-07T16:25:35.907178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2023-04-07T16:25:35.912820Z","iopub.execute_input":"2023-04-07T16:25:35.913766Z","iopub.status.idle":"2023-04-07T16:25:35.926306Z","shell.execute_reply.started":"2023-04-07T16:25:35.913721Z","shell.execute_reply":"2023-04-07T16:25:35.925151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.rc('font', size=14)\nplt.rc('axes', labelsize=14, titlesize=14)\nplt.rc('legend', fontsize=14)\nplt.rc('xtick', labelsize=10)\nplt.rc('ytick', labelsize=10)","metadata":{"id":"8d4TH3NbpKzx","execution":{"iopub.status.busy":"2023-04-07T16:25:35.928935Z","iopub.execute_input":"2023-04-07T16:25:35.929356Z","iopub.status.idle":"2023-04-07T16:25:36.037984Z","shell.execute_reply.started":"2023-04-07T16:25:35.929312Z","shell.execute_reply":"2023-04-07T16:25:36.036748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualisation","metadata":{"id":"NL3Ht4wC9b3n"}},{"cell_type":"code","source":"import os\n\ndef get_data():\n    return pd.read_csv('../input/deepfake-faces/metadata.csv')","metadata":{"id":"jfv9PxSB4tM8","execution":{"iopub.status.busy":"2023-04-07T16:25:36.039670Z","iopub.execute_input":"2023-04-07T16:25:36.040747Z","iopub.status.idle":"2023-04-07T16:25:36.051122Z","shell.execute_reply.started":"2023-04-07T16:25:36.040703Z","shell.execute_reply":"2023-04-07T16:25:36.049864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta=get_data()\nmeta.head()","metadata":{"id":"tDW7BRph9ehF","outputId":"97de18b5-0a37-4302-8804-8a16a7d2ed2f","execution":{"iopub.status.busy":"2023-04-07T16:25:36.052874Z","iopub.execute_input":"2023-04-07T16:25:36.053577Z","iopub.status.idle":"2023-04-07T16:25:36.245155Z","shell.execute_reply.started":"2023-04-07T16:25:36.053534Z","shell.execute_reply":"2023-04-07T16:25:36.244025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.shape","metadata":{"id":"n7FSdDifbZxn","outputId":"5451a127-405a-4c0b-a197-c920b796adbb","execution":{"iopub.status.busy":"2023-04-07T16:25:36.246625Z","iopub.execute_input":"2023-04-07T16:25:36.247072Z","iopub.status.idle":"2023-04-07T16:25:36.255089Z","shell.execute_reply.started":"2023-04-07T16:25:36.247027Z","shell.execute_reply":"2023-04-07T16:25:36.253786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(meta[meta.label=='FAKE']),len(meta[meta.label=='REAL'])","metadata":{"id":"_FJcz2IthxVG","outputId":"274c3f65-7acb-4f99-8aa9-a5b2a23bf06a","execution":{"iopub.status.busy":"2023-04-07T16:25:36.256644Z","iopub.execute_input":"2023-04-07T16:25:36.257885Z","iopub.status.idle":"2023-04-07T16:25:36.295248Z","shell.execute_reply.started":"2023-04-07T16:25:36.257833Z","shell.execute_reply":"2023-04-07T16:25:36.294243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_df = meta[meta[\"label\"] == \"REAL\"]\nfake_df = meta[meta[\"label\"] == \"FAKE\"]\nsample_size = 8000\n\nreal_df = real_df.sample(sample_size, random_state=42)\nfake_df = fake_df.sample(sample_size, random_state=42)\n\nsample_meta = pd.concat([real_df, fake_df])","metadata":{"id":"IgMfzY-PjjtH","execution":{"iopub.status.busy":"2023-04-07T16:25:36.298085Z","iopub.execute_input":"2023-04-07T16:25:36.298825Z","iopub.status.idle":"2023-04-07T16:25:36.333334Z","shell.execute_reply.started":"2023-04-07T16:25:36.298756Z","shell.execute_reply":"2023-04-07T16:25:36.332334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nTrain_set, Test_set = train_test_split(sample_meta,test_size=0.2,random_state=42,stratify=sample_meta['label'])\nTrain_set, Val_set  = train_test_split(Train_set,test_size=0.3,random_state=42,stratify=Train_set['label'])","metadata":{"id":"5eB86S6K-T5Z","execution":{"iopub.status.busy":"2023-04-07T16:25:36.335307Z","iopub.execute_input":"2023-04-07T16:25:36.335818Z","iopub.status.idle":"2023-04-07T16:25:36.531118Z","shell.execute_reply.started":"2023-04-07T16:25:36.335753Z","shell.execute_reply":"2023-04-07T16:25:36.529922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_set.shape,Val_set.shape,Test_set.shape","metadata":{"id":"8p-TONijb4qA","outputId":"56d0b529-9d81-4019-d8fa-618c8cdba90f","execution":{"iopub.status.busy":"2023-04-07T16:25:36.533072Z","iopub.execute_input":"2023-04-07T16:25:36.533980Z","iopub.status.idle":"2023-04-07T16:25:36.544269Z","shell.execute_reply.started":"2023-04-07T16:25:36.533930Z","shell.execute_reply":"2023-04-07T16:25:36.542558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = dict()\n\ny[0] = []\ny[1] = []\n\nfor set_name in (np.array(Train_set['label']), np.array(Val_set['label']), np.array(Test_set['label'])):\n    y[0].append(np.sum(set_name == 'REAL'))\n    y[1].append(np.sum(set_name == 'FAKE'))\n\ntrace0 = go.Bar(\n    x=['Train Set', 'Validation Set', 'Test Set'],\n    y=y[0],\n    name='REAL',\n    marker=dict(color='#33cc33'),\n    opacity=0.7\n)\ntrace1 = go.Bar(\n    x=['Train Set', 'Validation Set', 'Test Set'],\n    y=y[1],\n    name='FAKE',\n    marker=dict(color='#ff3300'),\n    opacity=0.7\n)\n\ndata = [trace0, trace1]\nlayout = go.Layout(\n    title='Count of classes in each set',\n    xaxis={'title': 'Set'},\n    yaxis={'title': 'Count'}\n)\n\nfig = go.Figure(data, layout)\niplot(fig)","metadata":{"id":"hzNGtCWd-mTk","outputId":"5178c3ed-cbba-4f99-99d0-26bde11a5dab","execution":{"iopub.status.busy":"2023-04-07T16:25:36.552039Z","iopub.execute_input":"2023-04-07T16:25:36.552757Z","iopub.status.idle":"2023-04-07T16:25:38.438425Z","shell.execute_reply.started":"2023-04-07T16:25:36.552720Z","shell.execute_reply":"2023-04-07T16:25:38.436854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The original image dataset were biased with more fake images than real since we are taking a sample of it its better to take equal proportion of real and fake images.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor cur,i in enumerate(Train_set.index[25:50]):\n    plt.subplot(5,5,cur+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    \n    plt.imshow(cv2.imread('../input/deepfake-faces/faces_224/'+Train_set.loc[i,'videoname'][:-4]+'.jpg'))\n    \n    if(Train_set.loc[i,'label']=='FAKE'):\n        plt.xlabel('FAKE Image')\n    else:\n        plt.xlabel('REAL Image')\n        \nplt.show()","metadata":{"id":"VR7Uly2fcUYi","outputId":"c1f47a82-ef4f-4bcd-b51c-d4738142fc0f","execution":{"iopub.status.busy":"2023-04-07T16:25:38.441055Z","iopub.execute_input":"2023-04-07T16:25:38.441590Z","iopub.status.idle":"2023-04-07T16:25:40.543455Z","shell.execute_reply.started":"2023-04-07T16:25:38.441530Z","shell.execute_reply":"2023-04-07T16:25:40.541874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modelling","metadata":{"id":"dOvN_divkl-N"}},{"cell_type":"markdown","source":"Before jumping to use pretrained model lets develop some base line model to test how our pretrained model outperforms.","metadata":{}},{"cell_type":"markdown","source":"### Custom CNN Architecture","metadata":{"id":"oid44Xx-pKz6"}},{"cell_type":"code","source":"def retreive_dataset(set_name):\n    images,labels=[],[]\n    for (img, imclass) in zip(set_name['videoname'], set_name['label']):\n        images.append(cv2.imread('../input/deepfake-faces/faces_224/'+img[:-4]+'.jpg'))\n        if(imclass=='FAKE'):\n            labels.append(1)\n        else:\n            labels.append(0)\n    \n    return np.array(images),np.array(labels)","metadata":{"id":"Hz0ZdQ_fgHhG","execution":{"iopub.status.busy":"2023-04-07T16:25:40.544805Z","iopub.execute_input":"2023-04-07T16:25:40.545192Z","iopub.status.idle":"2023-04-07T16:25:40.553550Z","shell.execute_reply.started":"2023-04-07T16:25:40.545155Z","shell.execute_reply":"2023-04-07T16:25:40.552406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,y_train=retreive_dataset(Train_set)\nX_val,y_val=retreive_dataset(Val_set)\nX_test,y_test=retreive_dataset(Test_set)","metadata":{"id":"zeAGRcAbguKU","execution":{"iopub.status.busy":"2023-04-07T16:25:40.555296Z","iopub.execute_input":"2023-04-07T16:25:40.556013Z","iopub.status.idle":"2023-04-07T16:28:09.761431Z","shell.execute_reply.started":"2023-04-07T16:25:40.555975Z","shell.execute_reply":"2023-04-07T16:28:09.760040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\n\ntf.random.set_seed(42) \nDefaultConv2D = partial(tf.keras.layers.Conv2D, kernel_size=3, padding=\"same\",\n                        activation=\"relu\", kernel_initializer=\"he_normal\")\n\nmodel = tf.keras.Sequential([\n    DefaultConv2D(filters=64, kernel_size=7, input_shape=[224, 224, 3]),\n    tf.keras.layers.MaxPool2D(),\n    DefaultConv2D(filters=128),\n    DefaultConv2D(filters=128),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(units=128, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=64, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=1, activation=\"sigmoid\")\n])","metadata":{"id":"34upiak4pKz6","execution":{"iopub.status.busy":"2023-04-07T16:28:09.763881Z","iopub.execute_input":"2023-04-07T16:28:09.764610Z","iopub.status.idle":"2023-04-07T16:28:10.157145Z","shell.execute_reply.started":"2023-04-07T16:28:09.764558Z","shell.execute_reply":"2023-04-07T16:28:10.155932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n              metrics=[\"accuracy\"])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T16:28:10.158549Z","iopub.execute_input":"2023-04-07T16:28:10.158965Z","iopub.status.idle":"2023-04-07T16:28:10.213507Z","shell.execute_reply.started":"2023-04-07T16:28:10.158919Z","shell.execute_reply":"2023-04-07T16:28:10.212521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=5,batch_size=64,\n                    validation_data=(X_val, y_val))","metadata":{"id":"KZbWeIBYpKz6","outputId":"deb6f56a-7b93-4241-a1bd-b210c0f2d426","execution":{"iopub.status.busy":"2023-04-07T16:28:10.214850Z","iopub.execute_input":"2023-04-07T16:28:10.215286Z","iopub.status.idle":"2023-04-07T16:31:57.928965Z","shell.execute_reply.started":"2023-04-07T16:28:10.215238Z","shell.execute_reply":"2023-04-07T16:31:57.927847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model.evaluate(X_test, y_test)","metadata":{"id":"6HDDr4uehast","execution":{"iopub.status.busy":"2023-04-07T16:31:57.930699Z","iopub.execute_input":"2023-04-07T16:31:57.931088Z","iopub.status.idle":"2023-04-07T16:32:09.297484Z","shell.execute_reply.started":"2023-04-07T16:31:57.931058Z","shell.execute_reply":"2023-04-07T16:32:09.296254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T16:32:09.299584Z","iopub.execute_input":"2023-04-07T16:32:09.300077Z","iopub.status.idle":"2023-04-07T16:32:09.836573Z","shell.execute_reply.started":"2023-04-07T16:32:09.300028Z","shell.execute_reply":"2023-04-07T16:32:09.835539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pretrained Models for Transfer Learning","metadata":{"id":"hqxnSBJ3pKz8"}},{"cell_type":"markdown","source":"Here we used Xception model for fine-tuning feel free to try the performance of other pretrained models.","metadata":{}},{"cell_type":"markdown","source":"All three datasets contain individual images. We need to batch them, but for this we first need to ensure they all have the same size, or else batching will not work. We can use a `Resizing` layer for this. We must also call the `tf.keras.applications.xception.preprocess_input()` function to preprocess the images appropriately for the Xception model. We will also add shuffling and prefetching to the training dataset.","metadata":{"id":"gXG6iv8XpKz9"}},{"cell_type":"code","source":"train_set_raw=tf.data.Dataset.from_tensor_slices((X_train,y_train))\nvalid_set_raw=tf.data.Dataset.from_tensor_slices((X_val,y_val))\ntest_set_raw=tf.data.Dataset.from_tensor_slices((X_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T16:32:09.838195Z","iopub.execute_input":"2023-04-07T16:32:09.838536Z","iopub.status.idle":"2023-04-07T16:32:14.761195Z","shell.execute_reply.started":"2023-04-07T16:32:09.838504Z","shell.execute_reply":"2023-04-07T16:32:14.759712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()  # extra code – resets layer name counter\n\nbatch_size = 32\npreprocess = tf.keras.applications.xception.preprocess_input\ntrain_set = train_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y))\ntrain_set = train_set.shuffle(1000, seed=42).batch(batch_size).prefetch(1)\nvalid_set = valid_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)\ntest_set = test_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)","metadata":{"id":"Bnz0n9XApKz9","execution":{"iopub.status.busy":"2023-04-07T16:32:14.764355Z","iopub.execute_input":"2023-04-07T16:32:14.765293Z","iopub.status.idle":"2023-04-07T16:32:14.916870Z","shell.execute_reply.started":"2023-04-07T16:32:14.765241Z","shell.execute_reply":"2023-04-07T16:32:14.915745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extra code – displays the first 9 images in the first batch of valid_set\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        plt.imshow((X_batch[index] + 1) / 2)  # rescale to 0–1 for imshow()\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"ZL3c3i4opKz9","outputId":"38847d8d-8822-41a3-cfb2-27479aa5debe","execution":{"iopub.status.busy":"2023-04-07T16:32:14.918453Z","iopub.execute_input":"2023-04-07T16:32:14.918878Z","iopub.status.idle":"2023-04-07T16:32:16.292175Z","shell.execute_reply.started":"2023-04-07T16:32:14.918836Z","shell.execute_reply":"2023-04-07T16:32:16.291167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augmentation = tf.keras.Sequential([\n    tf.keras.layers.RandomFlip(mode=\"horizontal\", seed=42),\n    tf.keras.layers.RandomRotation(factor=0.05, seed=42),\n    tf.keras.layers.RandomContrast(factor=0.2, seed=42)\n])","metadata":{"id":"Ib0cA8Y1pKz9","execution":{"iopub.status.busy":"2023-04-07T16:32:16.293601Z","iopub.execute_input":"2023-04-07T16:32:16.295083Z","iopub.status.idle":"2023-04-07T16:32:16.313618Z","shell.execute_reply.started":"2023-04-07T16:32:16.295040Z","shell.execute_reply":"2023-04-07T16:32:16.312374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extra code – displays the same first 9 images, after augmentation\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    X_batch_augmented = data_augmentation(X_batch, training=True)\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        # We must rescale the images to the 0-1 range for imshow(), and also\n        # clip the result to that range, because data augmentation may\n        # make some values go out of bounds (e.g., RandomContrast in this case).\n        plt.imshow(np.clip((X_batch_augmented[index] + 1) / 2, 0, 1))\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"w6GH5_vupKz-","outputId":"eeb2c924-2f4f-4aa1-bea9-951bebef4bf0","execution":{"iopub.status.busy":"2023-04-07T16:32:16.315735Z","iopub.execute_input":"2023-04-07T16:32:16.316660Z","iopub.status.idle":"2023-04-07T16:32:19.889764Z","shell.execute_reply.started":"2023-04-07T16:32:16.316617Z","shell.execute_reply":"2023-04-07T16:32:19.888868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's load the pretrained model, without its top layers, and replace them with our own task","metadata":{"id":"kNL9AOsDpKz-"}},{"cell_type":"code","source":"tf.random.set_seed(42)  # extra code – ensures reproducibility\nbase_model = tf.keras.applications.xception.Xception(weights=\"imagenet\",\n                                                     include_top=False)\navg = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)\noutput = tf.keras.layers.Dense(1, activation=\"sigmoid\")(avg)\nmodel = tf.keras.Model(inputs=base_model.input, outputs=output)","metadata":{"id":"lRyCgvaKpKz-","outputId":"a825e173-8b1d-4217-a1c4-5491b49c3e82","execution":{"iopub.status.busy":"2023-04-07T16:32:19.891387Z","iopub.execute_input":"2023-04-07T16:32:19.891979Z","iopub.status.idle":"2023-04-07T16:32:22.087329Z","shell.execute_reply.started":"2023-04-07T16:32:19.891939Z","shell.execute_reply":"2023-04-07T16:32:22.086181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in base_model.layers:\n    layer.trainable = False","metadata":{"id":"KBlyG6ElpKz-","execution":{"iopub.status.busy":"2023-04-07T16:32:22.089079Z","iopub.execute_input":"2023-04-07T16:32:22.089502Z","iopub.status.idle":"2023-04-07T16:32:22.100396Z","shell.execute_reply.started":"2023-04-07T16:32:22.089454Z","shell.execute_reply":"2023-04-07T16:32:22.099065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=3)","metadata":{"id":"GGxK2yPcpKz-","outputId":"6b64214a-e104-4b6c-9b7a-3388fc9aa15f","execution":{"iopub.status.busy":"2023-04-07T16:32:22.102691Z","iopub.execute_input":"2023-04-07T16:32:22.103541Z","iopub.status.idle":"2023-04-07T16:35:52.855849Z","shell.execute_reply.started":"2023-04-07T16:32:22.103492Z","shell.execute_reply":"2023-04-07T16:35:52.854693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for indices in zip(range(33), range(33, 66), range(66, 99), range(99, 132)):\n    for idx in indices:\n        print(f\"{idx:3}: {base_model.layers[idx].name:22}\", end=\"\")\n    print()","metadata":{"id":"GvGMiJMLpKz-","outputId":"91f2c96c-c058-45e0-e428-66fa6076ad56","execution":{"iopub.status.busy":"2023-04-07T16:35:52.859602Z","iopub.execute_input":"2023-04-07T16:35:52.859973Z","iopub.status.idle":"2023-04-07T16:35:52.893636Z","shell.execute_reply.started":"2023-04-07T16:35:52.859940Z","shell.execute_reply":"2023-04-07T16:35:52.892489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T16:35:52.895053Z","iopub.execute_input":"2023-04-07T16:35:52.895707Z","iopub.status.idle":"2023-04-07T16:36:13.783148Z","shell.execute_reply.started":"2023-04-07T16:35:52.895663Z","shell.execute_reply":"2023-04-07T16:36:13.781997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in base_model.layers[56:]:\n    layer.trainable = True\n\noptimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=20)","metadata":{"id":"GEUNGlhvpKz_","outputId":"c622a91d-f634-4443-b87e-8d46defdb578","execution":{"iopub.status.busy":"2023-04-07T16:36:13.784711Z","iopub.execute_input":"2023-04-07T16:36:13.785552Z","iopub.status.idle":"2023-04-07T17:14:00.840426Z","shell.execute_reply.started":"2023-04-07T16:36:13.785503Z","shell.execute_reply":"2023-04-07T17:14:00.839290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:00.842728Z","iopub.execute_input":"2023-04-07T17:14:00.843092Z","iopub.status.idle":"2023-04-07T17:14:01.359358Z","shell.execute_reply.started":"2023-04-07T17:14:00.843058Z","shell.execute_reply":"2023-04-07T17:14:01.358290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:01.361338Z","iopub.execute_input":"2023-04-07T17:14:01.361669Z","iopub.status.idle":"2023-04-07T17:14:15.381659Z","shell.execute_reply.started":"2023-04-07T17:14:01.361639Z","shell.execute_reply":"2023-04-07T17:14:15.380474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('xception_deepfake_image.h5')","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:15.383355Z","iopub.execute_input":"2023-04-07T17:14:15.384313Z","iopub.status.idle":"2023-04-07T17:14:15.966456Z","shell.execute_reply.started":"2023-04-07T17:14:15.384266Z","shell.execute_reply":"2023-04-07T17:14:15.964941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Add Explainability to the model","metadata":{}},{"cell_type":"markdown","source":"Lets try to interpret the trained model on how it finds a image FAKE","metadata":{}},{"cell_type":"code","source":"!pip install lime","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:15.968393Z","iopub.execute_input":"2023-04-07T17:14:15.968740Z","iopub.status.idle":"2023-04-07T17:14:30.014127Z","shell.execute_reply.started":"2023-04-07T17:14:15.968705Z","shell.execute_reply":"2023-04-07T17:14:30.012695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lime import lime_image\nimport matplotlib.pyplot as plt\n\nexplainer = lime_image.LimeImageExplainer()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:30.017301Z","iopub.execute_input":"2023-04-07T17:14:30.017683Z","iopub.status.idle":"2023-04-07T17:14:31.506996Z","shell.execute_reply.started":"2023-04-07T17:14:30.017642Z","shell.execute_reply":"2023-04-07T17:14:31.505819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 12))\nfor X_batch, y_batch in test_set.take(1):\n    X_batch_augmented = data_augmentation(X_batch, training=True)\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        plt.imshow(np.clip((X_batch_augmented[index] + 1) / 2, 0, 1))\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:31.510946Z","iopub.execute_input":"2023-04-07T17:14:31.512932Z","iopub.status.idle":"2023-04-07T17:14:34.293768Z","shell.execute_reply.started":"2023-04-07T17:14:31.512870Z","shell.execute_reply":"2023-04-07T17:14:34.292793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = X_test[2,:,:,:]\nfor X_batch, y_batch in test_set.take(1):\n    test_data = data_augmentation(X_batch, training=True)\n\ntest_data = np.array(test_data[2,:,:,:])\ntest_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:34.295540Z","iopub.execute_input":"2023-04-07T17:14:34.296774Z","iopub.status.idle":"2023-04-07T17:14:36.190837Z","shell.execute_reply.started":"2023-04-07T17:14:34.296730Z","shell.execute_reply":"2023-04-07T17:14:36.189758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"explanation = explainer.explain_instance(test_data.astype('double'), model.predict,  \n                                         top_labels=3, hide_color=0, num_samples=1000)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:36.200499Z","iopub.execute_input":"2023-04-07T17:14:36.200825Z","iopub.status.idle":"2023-04-07T17:14:53.409167Z","shell.execute_reply.started":"2023-04-07T17:14:36.200777Z","shell.execute_reply":"2023-04-07T17:14:53.407627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from skimage.segmentation import mark_boundaries\n\ntemp_1, mask_1 = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=True, num_features=5, hide_rest=True)\ntemp_2, mask_2 = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=False, num_features=10, hide_rest=False)\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))\nax1.imshow(mark_boundaries(temp_1, mask_1))\nax2.imshow(mark_boundaries(temp_2, mask_2))\nax1.axis('off')\nax2.axis('off')","metadata":{"execution":{"iopub.status.busy":"2023-04-07T17:14:53.413685Z","iopub.execute_input":"2023-04-07T17:14:53.416923Z","iopub.status.idle":"2023-04-07T17:14:53.878959Z","shell.execute_reply.started":"2023-04-07T17:14:53.416857Z","shell.execute_reply":"2023-04-07T17:14:53.877835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fake video detection\n","metadata":{}},{"cell_type":"code","source":"from tensorflow import keras\n#from imutils import paths\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport imageio\nimport cv2\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:04.801408Z","iopub.execute_input":"2023-04-07T18:05:04.802448Z","iopub.status.idle":"2023-04-07T18:05:04.948612Z","shell.execute_reply.started":"2023-04-07T18:05:04.802397Z","shell.execute_reply":"2023-04-07T18:05:04.947573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Data Visualisation","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:05.236675Z","iopub.execute_input":"2023-04-07T18:05:05.237285Z","iopub.status.idle":"2023-04-07T18:05:05.242100Z","shell.execute_reply.started":"2023-04-07T18:05:05.237241Z","shell.execute_reply":"2023-04-07T18:05:05.240881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\nTEST_FOLDER = 'test_videos'\n\nprint(f\"Train samples: {len(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))}\")\nprint(f\"Test samples: {len(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))}\")\n\ntrain_sample_metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\ntrain_sample_metadata.head()\n\ntrain_sample_metadata.groupby('label')['label'].count().plot(figsize=(15, 5), kind='bar', title='Distribution of Labels in the Training Set')\nplt.show()\n\ntrain_sample_metadata.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:06.245271Z","iopub.execute_input":"2023-04-07T18:05:06.245717Z","iopub.status.idle":"2023-04-07T18:05:07.020693Z","shell.execute_reply.started":"2023-04-07T18:05:06.245677Z","shell.execute_reply":"2023-04-07T18:05:07.019305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's visualize now the data.\n\n## We select first a list of fake videos.\n\n## Few fake videos","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:17:18.326983Z","iopub.execute_input":"2023-03-29T18:17:18.327683Z","iopub.status.idle":"2023-03-29T18:17:18.332507Z","shell.execute_reply.started":"2023-03-29T18:17:18.327638Z","shell.execute_reply":"2023-03-29T18:17:18.331127Z"}}},{"cell_type":"code","source":"fake_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.label=='FAKE'].sample(3).index)\nfake_train_sample_video\n\ndef display_image_from_video(video_path):\n    '''\n    input: video_path - path for video\n    process:\n    1. perform a video capture from the video\n    2. read the image\n    3. display the image\n    '''\n    capture_image = cv2.VideoCapture(video_path) \n    ret, frame = capture_image.read()\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111)\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    ax.imshow(frame)\n\nfor video_file in fake_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:07.023250Z","iopub.execute_input":"2023-04-07T18:05:07.023671Z","iopub.status.idle":"2023-04-07T18:05:09.889850Z","shell.execute_reply.started":"2023-04-07T18:05:07.023629Z","shell.execute_reply":"2023-04-07T18:05:09.888917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now trying the same for few of the images that are real.","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:17:21.177302Z","iopub.execute_input":"2023-03-29T18:17:21.178072Z","iopub.status.idle":"2023-03-29T18:17:21.182773Z","shell.execute_reply.started":"2023-03-29T18:17:21.178030Z","shell.execute_reply":"2023-03-29T18:17:21.181628Z"}}},{"cell_type":"markdown","source":"## Few Real Videos","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:17:21.184203Z","iopub.execute_input":"2023-03-29T18:17:21.185257Z","iopub.status.idle":"2023-03-29T18:17:21.197029Z","shell.execute_reply.started":"2023-03-29T18:17:21.185211Z","shell.execute_reply":"2023-03-29T18:17:21.195744Z"}}},{"cell_type":"code","source":"real_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.label=='REAL'].sample(3).index)\nreal_train_sample_video\n\nfor video_file in real_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:09.891632Z","iopub.execute_input":"2023-04-07T18:05:09.892322Z","iopub.status.idle":"2023-04-07T18:05:12.275203Z","shell.execute_reply.started":"2023-04-07T18:05:09.892283Z","shell.execute_reply":"2023-04-07T18:05:12.273855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Videos with same original","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:17:23.486876Z","iopub.execute_input":"2023-03-29T18:17:23.487399Z","iopub.status.idle":"2023-03-29T18:17:23.492284Z","shell.execute_reply.started":"2023-03-29T18:17:23.487346Z","shell.execute_reply":"2023-03-29T18:17:23.491154Z"}}},{"cell_type":"markdown","source":"### Let's look now to set of samples with the same original.","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:17:23.494010Z","iopub.execute_input":"2023-03-29T18:17:23.494801Z","iopub.status.idle":"2023-03-29T18:17:23.504630Z","shell.execute_reply.started":"2023-03-29T18:17:23.494711Z","shell.execute_reply":"2023-03-29T18:17:23.502793Z"}}},{"cell_type":"code","source":"train_sample_metadata['original'].value_counts()[0:5]","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:12.636469Z","iopub.execute_input":"2023-04-07T18:05:12.637207Z","iopub.status.idle":"2023-04-07T18:05:12.647115Z","shell.execute_reply.started":"2023-04-07T18:05:12.637167Z","shell.execute_reply":"2023-04-07T18:05:12.645900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We pick one of the originals with largest number of samples.\n\n## We also modify our visualization function to work with multiple images.","metadata":{}},{"cell_type":"code","source":"def display_image_from_video_list(video_path_list, video_folder=TRAIN_SAMPLE_FOLDER):\n    '''\n    input: video_path_list - path for video\n    process:\n    0. for each video in the video path list\n        1. perform a video capture from the video\n        2. read the image\n        3. display the image\n    '''\n    plt.figure()\n    fig, ax = plt.subplots(2,3,figsize=(16,8))\n    # we only show images extracted from the first 6 videos\n    for i, video_file in enumerate(video_path_list[0:6]):\n        video_path = os.path.join(DATA_FOLDER, video_folder,video_file)\n        capture_image = cv2.VideoCapture(video_path) \n        ret, frame = capture_image.read()\n        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n        ax[i//3, i%3].imshow(frame)\n        ax[i//3, i%3].set_title(f\"Video: {video_file}\")\n        ax[i//3, i%3].axis('on')\n\n\nsame_original_fake_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.original=='atvmxvwyns.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:13.614782Z","iopub.execute_input":"2023-04-07T18:05:13.615782Z","iopub.status.idle":"2023-04-07T18:05:17.167321Z","shell.execute_reply.started":"2023-04-07T18:05:13.615743Z","shell.execute_reply":"2023-04-07T18:05:17.166038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test video files","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.787540Z","iopub.execute_input":"2023-03-29T18:18:24.788430Z","iopub.status.idle":"2023-03-29T18:18:24.792911Z","shell.execute_reply.started":"2023-03-29T18:18:24.788383Z","shell.execute_reply":"2023-03-29T18:18:24.791807Z"}}},{"cell_type":"markdown","source":"Let's also look to few of the test data files.","metadata":{}},{"cell_type":"code","source":"test_videos = pd.DataFrame(list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER))), columns=['video'])\ntest_videos.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:17.169555Z","iopub.execute_input":"2023-04-07T18:05:17.170561Z","iopub.status.idle":"2023-04-07T18:05:17.185062Z","shell.execute_reply.started":"2023-04-07T18:05:17.170518Z","shell.execute_reply":"2023-04-07T18:05:17.183853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize now one of the videos.","metadata":{}},{"cell_type":"code","source":"display_image_from_video(os.path.join(DATA_FOLDER, TEST_FOLDER, test_videos.iloc[2].video))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:27.149960Z","iopub.execute_input":"2023-04-07T18:05:27.151092Z","iopub.status.idle":"2023-04-07T18:05:27.990697Z","shell.execute_reply.started":"2023-04-07T18:05:27.151034Z","shell.execute_reply":"2023-04-07T18:05:27.989759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### Play video files","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:27.992561Z","iopub.execute_input":"2023-04-07T18:05:27.993612Z","iopub.status.idle":"2023-04-07T18:05:27.998255Z","shell.execute_reply.started":"2023-04-07T18:05:27.993556Z","shell.execute_reply":"2023-04-07T18:05:27.996741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\nLet's look to few fake videos.","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.811759Z","iopub.status.idle":"2023-03-29T18:18:24.812215Z","shell.execute_reply.started":"2023-03-29T18:18:24.811978Z","shell.execute_reply":"2023-03-29T18:18:24.811998Z"}}},{"cell_type":"code","source":"fake_videos = list(train_sample_metadata.loc[train_sample_metadata.label=='FAKE'].index)\n\nfrom IPython.display import HTML\nfrom base64 import b64encode\n\ndef play_video(video_file, subset=TRAIN_SAMPLE_FOLDER):\n    '''\n    Display video\n    param: video_file - the name of the video file to display\n    param: subset - the folder where the video file is located (can be TRAIN_SAMPLE_FOLDER or TEST_Folder)\n    '''\n    video_url = open(os.path.join(DATA_FOLDER, subset,video_file),'rb').read()\n    data_url = \"data:video/mp4;base64,\" + b64encode(video_url).decode()\n    return HTML(\"\"\"<video width=500 controls><source src=\"%s\" type=\"video/mp4\"></video>\"\"\" % data_url)\n\nplay_video(fake_videos[10])","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:29.077840Z","iopub.execute_input":"2023-04-07T18:05:29.078573Z","iopub.status.idle":"2023-04-07T18:05:29.594491Z","shell.execute_reply.started":"2023-04-07T18:05:29.078532Z","shell.execute_reply":"2023-04-07T18:05:29.592896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From visual inspection of these fakes videos, in some cases is very easy to spot the anomalies created when engineering the deep fake, in some cases is more difficult.","metadata":{}},{"cell_type":"markdown","source":"## Modelling\n\n### A CNN-RNN Architecture","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.817213Z","iopub.status.idle":"2023-03-29T18:18:24.818107Z","shell.execute_reply.started":"2023-03-29T18:18:24.817831Z","shell.execute_reply":"2023-03-29T18:18:24.817873Z"}}},{"cell_type":"code","source":"IMG_SIZE = 224\nBATCH_SIZE = 64\nEPOCHS = 10\n\nMAX_SEQ_LENGTH = 20\nNUM_FEATURES = 2048","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:33.231890Z","iopub.execute_input":"2023-04-07T18:05:33.232633Z","iopub.status.idle":"2023-04-07T18:05:33.237977Z","shell.execute_reply.started":"2023-04-07T18:05:33.232591Z","shell.execute_reply":"2023-04-07T18:05:33.236776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" In this example we will do the following:\n\n* Capture the frames of a video.\n* Extract frames from the videos until a maximum frame count is reached.\n* In the case, where a video's frame count is lesser than the maximum frame count we will pad the video with zeros.","metadata":{}},{"cell_type":"code","source":"def crop_center_square(frame):\n    y, x = frame.shape[0:2]\n    min_dim = min(y, x)\n    start_x = (x // 2) - (min_dim // 2)\n    start_y = (y // 2) - (min_dim // 2)\n    return frame[start_y : start_y + min_dim, start_x : start_x + min_dim]\n\n\ndef load_video(path, max_frames=0, resize=(IMG_SIZE, IMG_SIZE)):\n    cap = cv2.VideoCapture(path)\n    frames = []\n    try:\n        while True:\n            ret, frame = cap.read()\n            if not ret:\n                break\n            frame = crop_center_square(frame)\n            frame = cv2.resize(frame, resize)\n            frame = frame[:, :, [2, 1, 0]]\n            frames.append(frame)\n\n            if len(frames) == max_frames:\n                break\n    finally:\n        cap.release()\n    return np.array(frames)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:33.893635Z","iopub.execute_input":"2023-04-07T18:05:33.894358Z","iopub.status.idle":"2023-04-07T18:05:33.904691Z","shell.execute_reply.started":"2023-04-07T18:05:33.894317Z","shell.execute_reply":"2023-04-07T18:05:33.903496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can use a pre-trained network to extract meaningful features from the extracted frames. The Keras Applications module provides a number of state-of-the-art models pre-trained on the ImageNet-1k dataset. We will be using the InceptionV3 model for this purpose.","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.822096Z","iopub.status.idle":"2023-03-29T18:18:24.822991Z","shell.execute_reply.started":"2023-03-29T18:18:24.822702Z","shell.execute_reply":"2023-03-29T18:18:24.822728Z"}}},{"cell_type":"code","source":"def build_feature_extractor():\n    feature_extractor = keras.applications.InceptionV3(\n        weights=\"imagenet\",\n        include_top=False,\n        pooling=\"avg\",\n        input_shape=(IMG_SIZE, IMG_SIZE, 3),\n    )\n    preprocess_input = keras.applications.inception_v3.preprocess_input\n\n    inputs = keras.Input((IMG_SIZE, IMG_SIZE, 3))\n    preprocessed = preprocess_input(inputs)\n\n    outputs = feature_extractor(preprocessed)\n    return keras.Model(inputs, outputs, name=\"feature_extractor\")\n\n\nfeature_extractor = build_feature_extractor()","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:39.271391Z","iopub.execute_input":"2023-04-07T18:05:39.272120Z","iopub.status.idle":"2023-04-07T18:05:43.169391Z","shell.execute_reply.started":"2023-04-07T18:05:39.272075Z","shell.execute_reply":"2023-04-07T18:05:43.168312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, we can put all the pieces together to create our data processing utility.","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.827610Z","iopub.status.idle":"2023-03-29T18:18:24.828128Z","shell.execute_reply.started":"2023-03-29T18:18:24.827850Z","shell.execute_reply":"2023-03-29T18:18:24.827891Z"}}},{"cell_type":"code","source":"def prepare_all_videos(df, root_dir):\n    num_samples = len(df)\n    video_paths = list(df.index)\n    labels = df[\"label\"].values\n    labels = np.array(labels=='FAKE').astype(np.int)\n\n    # `frame_masks` and `frame_features` are what we will feed to our sequence model.\n    # `frame_masks` will contain a bunch of booleans denoting if a timestep is\n    # masked with padding or not.\n    frame_masks = np.zeros(shape=(num_samples, MAX_SEQ_LENGTH), dtype=\"bool\")\n    frame_features = np.zeros(\n        shape=(num_samples, MAX_SEQ_LENGTH, NUM_FEATURES), dtype=\"float32\"\n    )\n\n    # For each video.\n    for idx, path in enumerate(video_paths):\n        # Gather all its frames and add a batch dimension.\n        frames = load_video(os.path.join(root_dir, path))\n        frames = frames[None, ...]\n\n        # Initialize placeholders to store the masks and features of the current video.\n        temp_frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype=\"bool\")\n        temp_frame_features = np.zeros(\n            shape=(1, MAX_SEQ_LENGTH, NUM_FEATURES), dtype=\"float32\"\n        )\n\n        # Extract features from the frames of the current video.\n        for i, batch in enumerate(frames):\n            video_length = batch.shape[0]\n            length = min(MAX_SEQ_LENGTH, video_length)\n            for j in range(length):\n                temp_frame_features[i, j, :] = feature_extractor.predict(\n                    batch[None, j, :]\n                )\n            temp_frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked\n\n        frame_features[idx,] = temp_frame_features.squeeze()\n        frame_masks[idx,] = temp_frame_mask.squeeze()\n\n    return (frame_features, frame_masks), labels","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:43.868281Z","iopub.execute_input":"2023-04-07T18:05:43.869074Z","iopub.status.idle":"2023-04-07T18:05:43.882239Z","shell.execute_reply.started":"2023-04-07T18:05:43.869033Z","shell.execute_reply":"2023-04-07T18:05:43.880696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we don't have test labels we split the training data to find its performance in unseen data","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.832741Z","iopub.status.idle":"2023-03-29T18:18:24.833263Z","shell.execute_reply.started":"2023-03-29T18:18:24.832996Z","shell.execute_reply":"2023-03-29T18:18:24.833022Z"}}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nTrain_set, Test_set = train_test_split(train_sample_metadata,test_size=0.1,random_state=42,stratify=train_sample_metadata['label'])\n\nprint(Train_set.shape, Test_set.shape )\n\ntrain_data, train_labels = prepare_all_videos(Train_set, \"train\")\ntest_data, test_labels = prepare_all_videos(Test_set, \"test\")\n\nprint(f\"Frame features in train set: {train_data[0].shape}\")\nprint(f\"Frame masks in train set: {train_data[1].shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:05:45.118359Z","iopub.execute_input":"2023-04-07T18:05:45.119609Z","iopub.status.idle":"2023-04-07T18:05:45.208509Z","shell.execute_reply.started":"2023-04-07T18:05:45.119553Z","shell.execute_reply":"2023-04-07T18:05:45.207260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The sequence model","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.837911Z","iopub.status.idle":"2023-03-29T18:18:24.838408Z","shell.execute_reply.started":"2023-03-29T18:18:24.838154Z","shell.execute_reply":"2023-03-29T18:18:24.838179Z"}}},{"cell_type":"markdown","source":"Now, we can feed this data to a sequence model consisting of recurrent layers like GRU.(gated reccurent unit, used for subject detection)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.840445Z","iopub.status.idle":"2023-03-29T18:18:24.840969Z","shell.execute_reply.started":"2023-03-29T18:18:24.840694Z","shell.execute_reply":"2023-03-29T18:18:24.840720Z"}}},{"cell_type":"code","source":"frame_features_input = keras.Input((MAX_SEQ_LENGTH, NUM_FEATURES))\nmask_input = keras.Input((MAX_SEQ_LENGTH,), dtype=\"bool\")\n\n# https://keras.io/api/layers/recurrent_layers/gru/\nx = keras.layers.GRU(16, return_sequences=True)(\n    frame_features_input, mask=mask_input\n)\nx = keras.layers.GRU(8)(x)\nx = keras.layers.Dropout(0.4)(x)\nx = keras.layers.Dense(8, activation=\"relu\")(x)\noutput = keras.layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model([frame_features_input, mask_input], output)\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\nmodel.summary()\n\ncheckpoint = keras.callbacks.ModelCheckpoint('./', save_weights_only=True, save_best_only=True)\nhistory = model.fit(\n        [train_data[0], train_data[1]],\n        train_labels,\n        validation_data=([test_data[0], test_data[1]],test_labels),\n        callbacks=[checkpoint],\n        epochs=EPOCHS,\n        batch_size=8\n    )","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:09:07.066897Z","iopub.execute_input":"2023-04-07T18:09:07.067663Z","iopub.status.idle":"2023-04-07T18:09:55.666396Z","shell.execute_reply.started":"2023-04-07T18:09:07.067619Z","shell.execute_reply":"2023-04-07T18:09:55.665045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:18:24.846484Z","iopub.status.idle":"2023-03-29T18:18:24.846899Z","shell.execute_reply.started":"2023-03-29T18:18:24.846679Z","shell.execute_reply":"2023-03-29T18:18:24.846709Z"}}},{"cell_type":"code","source":"def prepare_single_video(frames):\n    frames = frames[None, ...]\n    frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype=\"bool\")\n    frame_features = np.zeros(shape=(1, MAX_SEQ_LENGTH, NUM_FEATURES), dtype=\"float32\")\n\n    for i, batch in enumerate(frames):\n        video_length = batch.shape[0]\n        length = min(MAX_SEQ_LENGTH, video_length)\n        for j in range(length):\n            frame_features[i, j, :] = feature_extractor.predict(batch[None, j, :])\n        frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked\n\n    return frame_features, frame_mask\n\ndef sequence_prediction(path):\n    frames = load_video(os.path.join(DATA_FOLDER, TEST_FOLDER,path))\n    frame_features, frame_mask = prepare_single_video(frames)\n    return model.predict([frame_features, frame_mask])[0]\n    \n\ntest_video = np.random.choice(test_videos[\"video\"].values.tolist())\nprint(f\"Test video path: {test_video}\")\n\nif(sequence_prediction(test_video)>=0.5):\n    print(f'The predicted class of the video is FAKE')\nelse:\n    print(f'The predicted class of the video is REAL')\n\nplay_video(test_video,TEST_FOLDER)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T18:09:55.669214Z","iopub.execute_input":"2023-04-07T18:09:55.671104Z","iopub.status.idle":"2023-04-07T18:10:05.071350Z","shell.execute_reply.started":"2023-04-07T18:09:55.671054Z","shell.execute_reply":"2023-04-07T18:10:05.069817Z"},"trusted":true},"execution_count":null,"outputs":[]}]}