{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":924245,"sourceType":"datasetVersion","datasetId":464091}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Importing Required libraries","metadata":{"id":"dFXIv9qNpKzt","tags":[]}},{"cell_type":"code","source":"!python --version\n!pip --version","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:51:48.631582Z","iopub.execute_input":"2024-09-06T17:51:48.631917Z","iopub.status.idle":"2024-09-06T17:51:51.256388Z","shell.execute_reply.started":"2024-09-06T17:51:48.631875Z","shell.execute_reply":"2024-09-06T17:51:51.254768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U --upgrade tensorflow","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:51:51.263818Z","iopub.execute_input":"2024-09-06T17:51:51.264505Z","iopub.status.idle":"2024-09-06T17:54:35.178589Z","shell.execute_reply.started":"2024-09-06T17:51:51.264442Z","shell.execute_reply":"2024-09-06T17:54:35.177438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install plotly","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:54:35.180362Z","iopub.execute_input":"2024-09-06T17:54:35.180703Z","iopub.status.idle":"2024-09-06T17:54:48.544408Z","shell.execute_reply.started":"2024-09-06T17:54:35.180669Z","shell.execute_reply":"2024-09-06T17:54:48.543336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install numpy==1.23.0  # Replace 1.23.0 with the version you want to install","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:54:48.547722Z","iopub.execute_input":"2024-09-06T17:54:48.54813Z","iopub.status.idle":"2024-09-06T17:55:06.53857Z","shell.execute_reply.started":"2024-09-06T17:54:48.548082Z","shell.execute_reply":"2024-09-06T17:55:06.537434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport sklearn\nimport tensorflow as tf\n\nimport cv2\nimport pandas as pd\nimport numpy as np\n\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nfrom matplotlib import pyplot as plt","metadata":{"id":"TFSU3FCOpKzu","execution":{"iopub.status.busy":"2024-09-06T17:55:06.539873Z","iopub.execute_input":"2024-09-06T17:55:06.540178Z","iopub.status.idle":"2024-09-06T17:55:13.065073Z","shell.execute_reply.started":"2024-09-06T17:55:06.540142Z","shell.execute_reply":"2024-09-06T17:55:13.064319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.test.is_gpu_available()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:55:13.066215Z","iopub.execute_input":"2024-09-06T17:55:13.06678Z","iopub.status.idle":"2024-09-06T17:55:13.51967Z","shell.execute_reply.started":"2024-09-06T17:55:13.066745Z","shell.execute_reply":"2024-09-06T17:55:13.51839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:55:13.52183Z","iopub.execute_input":"2024-09-06T17:55:13.523024Z","iopub.status.idle":"2024-09-06T17:55:13.538137Z","shell.execute_reply.started":"2024-09-06T17:55:13.522969Z","shell.execute_reply":"2024-09-06T17:55:13.537231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.rc('font', size=14)\nplt.rc('axes', labelsize=14, titlesize=14)\nplt.rc('legend', fontsize=14)\nplt.rc('xtick', labelsize=10)\nplt.rc('ytick', labelsize=10)","metadata":{"id":"8d4TH3NbpKzx","execution":{"iopub.status.busy":"2024-09-06T17:55:13.540397Z","iopub.execute_input":"2024-09-06T17:55:13.540775Z","iopub.status.idle":"2024-09-06T17:55:13.546202Z","shell.execute_reply.started":"2024-09-06T17:55:13.540741Z","shell.execute_reply":"2024-09-06T17:55:13.545465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualisation","metadata":{"id":"NL3Ht4wC9b3n"}},{"cell_type":"code","source":"import os\n\ndef get_data():\n    return pd.read_csv('../input/deepfake-faces/metadata.csv')","metadata":{"id":"jfv9PxSB4tM8","execution":{"iopub.status.busy":"2024-09-06T17:55:13.547406Z","iopub.execute_input":"2024-09-06T17:55:13.548213Z","iopub.status.idle":"2024-09-06T17:55:13.555713Z","shell.execute_reply.started":"2024-09-06T17:55:13.54818Z","shell.execute_reply":"2024-09-06T17:55:13.55487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta=get_data()\nmeta.head()","metadata":{"id":"tDW7BRph9ehF","outputId":"97de18b5-0a37-4302-8804-8a16a7d2ed2f","execution":{"iopub.status.busy":"2024-09-06T17:55:13.559558Z","iopub.execute_input":"2024-09-06T17:55:13.55986Z","iopub.status.idle":"2024-09-06T17:55:13.748716Z","shell.execute_reply.started":"2024-09-06T17:55:13.559819Z","shell.execute_reply":"2024-09-06T17:55:13.747696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta.shape","metadata":{"id":"n7FSdDifbZxn","outputId":"5451a127-405a-4c0b-a197-c920b796adbb","execution":{"iopub.status.busy":"2024-09-06T17:55:13.750183Z","iopub.execute_input":"2024-09-06T17:55:13.750626Z","iopub.status.idle":"2024-09-06T17:55:13.757275Z","shell.execute_reply.started":"2024-09-06T17:55:13.75057Z","shell.execute_reply":"2024-09-06T17:55:13.756199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(meta[meta.label=='FAKE']),len(meta[meta.label=='REAL'])","metadata":{"id":"_FJcz2IthxVG","outputId":"274c3f65-7acb-4f99-8aa9-a5b2a23bf06a","execution":{"iopub.status.busy":"2024-09-06T17:55:13.75887Z","iopub.execute_input":"2024-09-06T17:55:13.759275Z","iopub.status.idle":"2024-09-06T17:55:13.810518Z","shell.execute_reply.started":"2024-09-06T17:55:13.759228Z","shell.execute_reply":"2024-09-06T17:55:13.809577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_df = meta[meta[\"label\"] == \"REAL\"]\nfake_df = meta[meta[\"label\"] == \"FAKE\"]\nsample_size = 8000\n\nreal_df = real_df.sample(sample_size, random_state=42)\nfake_df = fake_df.sample(sample_size, random_state=42)\n\nsample_meta = pd.concat([real_df, fake_df])","metadata":{"id":"IgMfzY-PjjtH","execution":{"iopub.status.busy":"2024-09-06T17:55:13.811838Z","iopub.execute_input":"2024-09-06T17:55:13.812213Z","iopub.status.idle":"2024-09-06T17:55:13.864971Z","shell.execute_reply.started":"2024-09-06T17:55:13.812172Z","shell.execute_reply":"2024-09-06T17:55:13.864136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As mentioned instead of using 95k images we will only use 16000 images.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nTrain_set, Test_set = train_test_split(sample_meta,test_size=0.2,random_state=42,stratify=sample_meta['label'])\nTrain_set, Val_set  = train_test_split(Train_set,test_size=0.3,random_state=42,stratify=Train_set['label'])","metadata":{"id":"5eB86S6K-T5Z","execution":{"iopub.status.busy":"2024-09-06T17:55:13.866056Z","iopub.execute_input":"2024-09-06T17:55:13.866428Z","iopub.status.idle":"2024-09-06T17:55:14.010161Z","shell.execute_reply.started":"2024-09-06T17:55:13.866384Z","shell.execute_reply":"2024-09-06T17:55:14.009325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Train_set.shape,Val_set.shape,Test_set.shape","metadata":{"id":"8p-TONijb4qA","outputId":"56d0b529-9d81-4019-d8fa-618c8cdba90f","execution":{"iopub.status.busy":"2024-09-06T17:55:14.011239Z","iopub.execute_input":"2024-09-06T17:55:14.011552Z","iopub.status.idle":"2024-09-06T17:55:14.018227Z","shell.execute_reply.started":"2024-09-06T17:55:14.011519Z","shell.execute_reply":"2024-09-06T17:55:14.017238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = dict()\n\ny[0] = []\ny[1] = []\n\nfor set_name in (np.array(Train_set['label']), np.array(Val_set['label']), np.array(Test_set['label'])):\n    y[0].append(np.sum(set_name == 'REAL'))\n    y[1].append(np.sum(set_name == 'FAKE'))\n\ntrace0 = go.Bar(\n    x=['Train Set', 'Validation Set', 'Test Set'],\n    y=y[0],\n    name='REAL',\n    marker=dict(color='#33cc33'),\n    opacity=0.7\n)\ntrace1 = go.Bar(\n    x=['Train Set', 'Validation Set', 'Test Set'],\n    y=y[1],\n    name='FAKE',\n    marker=dict(color='#ff3300'),\n    opacity=0.7\n)\n\ndata = [trace0, trace1]\nlayout = go.Layout(\n    title='Count of classes in each set',\n    xaxis={'title': 'Set'},\n    yaxis={'title': 'Count'}\n)\n\nfig = go.Figure(data, layout)\niplot(fig)","metadata":{"id":"hzNGtCWd-mTk","outputId":"5178c3ed-cbba-4f99-99d0-26bde11a5dab","execution":{"iopub.status.busy":"2024-09-06T17:55:14.019585Z","iopub.execute_input":"2024-09-06T17:55:14.019952Z","iopub.status.idle":"2024-09-06T17:55:15.606985Z","shell.execute_reply.started":"2024-09-06T17:55:14.019903Z","shell.execute_reply":"2024-09-06T17:55:15.606065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The original image dataset were biased with more fake images than real since we are taking a sample of it its better to take equal proportion of real and fake images.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor cur,i in enumerate(Train_set.index[25:50]):\n    plt.subplot(5,5,cur+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    \n    plt.imshow(cv2.imread('../input/deepfake-faces/faces_224/'+Train_set.loc[i,'videoname'][:-4]+'.jpg'))\n    \n    if(Train_set.loc[i,'label']=='FAKE'):\n        plt.xlabel('FAKE Image')\n    else:\n        plt.xlabel('REAL Image')\n        \nplt.show()","metadata":{"id":"VR7Uly2fcUYi","outputId":"c1f47a82-ef4f-4bcd-b51c-d4738142fc0f","execution":{"iopub.status.busy":"2024-09-06T17:55:15.608347Z","iopub.execute_input":"2024-09-06T17:55:15.608751Z","iopub.status.idle":"2024-09-06T17:55:17.716542Z","shell.execute_reply.started":"2024-09-06T17:55:15.608704Z","shell.execute_reply":"2024-09-06T17:55:17.715564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modelling","metadata":{"id":"dOvN_divkl-N"}},{"cell_type":"markdown","source":"Before jumping to use pretrained model lets develop some base line model to test how our pretrained model outperforms.","metadata":{}},{"cell_type":"markdown","source":"### Custom CNN Architecture","metadata":{"id":"oid44Xx-pKz6"}},{"cell_type":"code","source":"def retreive_dataset(set_name):\n    images,labels=[],[]\n    for (img, imclass) in zip(set_name['videoname'], set_name['label']):\n        images.append(cv2.imread('../input/deepfake-faces/faces_224/'+img[:-4]+'.jpg'))\n        if(imclass=='FAKE'):\n            labels.append(1)\n        else:\n            labels.append(0)\n    \n    return np.array(images),np.array(labels)","metadata":{"id":"Hz0ZdQ_fgHhG","execution":{"iopub.status.busy":"2024-09-06T17:55:17.71789Z","iopub.execute_input":"2024-09-06T17:55:17.71823Z","iopub.status.idle":"2024-09-06T17:55:17.724753Z","shell.execute_reply.started":"2024-09-06T17:55:17.718194Z","shell.execute_reply":"2024-09-06T17:55:17.723756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,y_train=retreive_dataset(Train_set)\nX_val,y_val=retreive_dataset(Val_set)\nX_test,y_test=retreive_dataset(Test_set)","metadata":{"id":"zeAGRcAbguKU","execution":{"iopub.status.busy":"2024-09-06T17:55:17.72596Z","iopub.execute_input":"2024-09-06T17:55:17.726317Z","iopub.status.idle":"2024-09-06T17:57:07.699295Z","shell.execute_reply.started":"2024-09-06T17:55:17.726253Z","shell.execute_reply":"2024-09-06T17:57:07.698274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\n\ntf.random.set_seed(42) \nDefaultConv2D = partial(tf.keras.layers.Conv2D, kernel_size=3, padding=\"same\",\n                        activation=\"relu\", kernel_initializer=\"he_normal\")\n\nmodel = tf.keras.Sequential([\n    DefaultConv2D(filters=64, kernel_size=7, input_shape=[224, 224, 3]),\n    tf.keras.layers.MaxPool2D(),\n    DefaultConv2D(filters=128),\n    DefaultConv2D(filters=128),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(units=128, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=64, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=1, activation=\"sigmoid\")\n])","metadata":{"id":"34upiak4pKz6","execution":{"iopub.status.busy":"2024-09-06T17:57:07.700883Z","iopub.execute_input":"2024-09-06T17:57:07.701661Z","iopub.status.idle":"2024-09-06T17:57:08.154849Z","shell.execute_reply.started":"2024-09-06T17:57:07.701608Z","shell.execute_reply":"2024-09-06T17:57:08.153928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial\nimport tensorflow as tf\n\ntf.random.set_seed(42) \nDefaultConv2D = partial(tf.keras.layers.Conv2D, kernel_size=3, padding=\"same\",\n                        activation=\"relu\", kernel_initializer=\"he_normal\")\n\nmodel = tf.keras.Sequential([\n    tf.keras.layers.Input(shape=[224, 224, 3]),  # Use Input layer with shape\n    DefaultConv2D(filters=64, kernel_size=7),\n    tf.keras.layers.MaxPool2D(),\n    DefaultConv2D(filters=128),\n    DefaultConv2D(filters=128),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(units=128, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=64, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=1, activation=\"sigmoid\")\n])","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:57:08.156033Z","iopub.execute_input":"2024-09-06T17:57:08.156345Z","iopub.status.idle":"2024-09-06T17:57:08.234944Z","shell.execute_reply.started":"2024-09-06T17:57:08.156307Z","shell.execute_reply":"2024-09-06T17:57:08.233984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n              metrics=[\"accuracy\"])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T17:57:08.236212Z","iopub.execute_input":"2024-09-06T17:57:08.236607Z","iopub.status.idle":"2024-09-06T17:57:08.271594Z","shell.execute_reply.started":"2024-09-06T17:57:08.236565Z","shell.execute_reply":"2024-09-06T17:57:08.270529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=5,batch_size=64,\n                    validation_data=(X_val, y_val))","metadata":{"id":"KZbWeIBYpKz6","outputId":"deb6f56a-7b93-4241-a1bd-b210c0f2d426","execution":{"iopub.status.busy":"2024-09-06T17:57:08.272732Z","iopub.execute_input":"2024-09-06T17:57:08.273022Z","iopub.status.idle":"2024-09-06T18:01:12.32369Z","shell.execute_reply.started":"2024-09-06T17:57:08.272991Z","shell.execute_reply":"2024-09-06T18:01:12.322817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming 'model' is your Keras model\nmodel.save('my_model.keras')\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:12.325455Z","iopub.execute_input":"2024-09-06T18:01:12.326149Z","iopub.status.idle":"2024-09-06T18:01:14.299875Z","shell.execute_reply.started":"2024-09-06T18:01:12.326103Z","shell.execute_reply":"2024-09-06T18:01:14.298999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model.evaluate(X_test, y_test)","metadata":{"id":"6HDDr4uehast","execution":{"iopub.status.busy":"2024-09-06T18:01:14.301021Z","iopub.execute_input":"2024-09-06T18:01:14.30135Z","iopub.status.idle":"2024-09-06T18:01:24.311108Z","shell.execute_reply.started":"2024-09-06T18:01:14.301309Z","shell.execute_reply":"2024-09-06T18:01:24.310117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, f1_score, confusion_matrix, roc_curve, auc\nimport matplotlib.pyplot as plt\n\n# Assuming you have predictions (y_pred) from your model\ny_pred = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:24.312754Z","iopub.execute_input":"2024-09-06T18:01:24.313459Z","iopub.status.idle":"2024-09-06T18:01:29.624624Z","shell.execute_reply.started":"2024-09-06T18:01:24.313411Z","shell.execute_reply":"2024-09-06T18:01:29.623757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming y_pred is a probability score, you may need to convert it to binary predictions\ny_pred_binary = (y_pred > 0.5).astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:29.625911Z","iopub.execute_input":"2024-09-06T18:01:29.626227Z","iopub.status.idle":"2024-09-06T18:01:29.631185Z","shell.execute_reply.started":"2024-09-06T18:01:29.626194Z","shell.execute_reply":"2024-09-06T18:01:29.630145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = accuracy_score(y_test, y_pred_binary)\nprint(f'Accuracy: {accuracy}')","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:29.639952Z","iopub.execute_input":"2024-09-06T18:01:29.640249Z","iopub.status.idle":"2024-09-06T18:01:29.942614Z","shell.execute_reply.started":"2024-09-06T18:01:29.640217Z","shell.execute_reply":"2024-09-06T18:01:29.941483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1 = f1_score(y_test, y_pred_binary)\nprint(f'F1 Score: {f1}')","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:29.944577Z","iopub.execute_input":"2024-09-06T18:01:29.944929Z","iopub.status.idle":"2024-09-06T18:01:29.953766Z","shell.execute_reply.started":"2024-09-06T18:01:29.944893Z","shell.execute_reply":"2024-09-06T18:01:29.952567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate confusion matrix\nconf_matrix = confusion_matrix(y_test, y_pred_binary)\nprint('Confusion Matrix:')\nprint(conf_matrix)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:29.955166Z","iopub.execute_input":"2024-09-06T18:01:29.955557Z","iopub.status.idle":"2024-09-06T18:01:29.962834Z","shell.execute_reply.started":"2024-09-06T18:01:29.955521Z","shell.execute_reply":"2024-09-06T18:01:29.961839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install seaborn","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:29.964081Z","iopub.execute_input":"2024-09-06T18:01:29.964441Z","iopub.status.idle":"2024-09-06T18:01:43.101531Z","shell.execute_reply.started":"2024-09-06T18:01:29.964406Z","shell.execute_reply":"2024-09-06T18:01:43.100361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Assuming y_test and y_pred_binary are defined\n\n# Generate confusion matrix\nconf_matrix = confusion_matrix(y_test, y_pred_binary)\n\n# Plot confusion matrix as a heatmap\nplt.figure(figsize=(8, 6))\nsns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False,\n            xticklabels=['Real', 'Fake'],\n            yticklabels=['Fake', 'Real'])\nplt.title('Confusion Matrix')\nplt.xlabel('Predicted')\nplt.ylabel('Actual')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:43.103252Z","iopub.execute_input":"2024-09-06T18:01:43.103695Z","iopub.status.idle":"2024-09-06T18:01:43.909481Z","shell.execute_reply.started":"2024-09-06T18:01:43.103647Z","shell.execute_reply":"2024-09-06T18:01:43.908103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot ROC curve\nfpr, tpr, thresholds = roc_curve(y_test, y_pred)\nroc_auc = auc(fpr, tpr)\n\nplt.figure(figsize=(8, 6))\nplt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (AUC = {roc_auc:.2f})')\nplt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver Operating Characteristic (ROC) Curve')\nplt.legend(loc='lower right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:43.911795Z","iopub.execute_input":"2024-09-06T18:01:43.91312Z","iopub.status.idle":"2024-09-06T18:01:44.161666Z","shell.execute_reply.started":"2024-09-06T18:01:43.913049Z","shell.execute_reply":"2024-09-06T18:01:44.16076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_curve, auc\n\n# Assuming y_test and y_pred are defined\nfpr, tpr, thresholds = roc_curve(y_test, y_pred)\n\n# Print TPR and FPR values\nfor i, (fpr_value, tpr_value) in enumerate(zip(fpr, tpr)):\n    print(f'Threshold: {thresholds[i]:.4f}, FPR: {fpr_value:.4f}, TPR: {tpr_value:.4f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:44.162958Z","iopub.execute_input":"2024-09-06T18:01:44.163398Z","iopub.status.idle":"2024-09-06T18:01:44.186731Z","shell.execute_reply.started":"2024-09-06T18:01:44.163349Z","shell.execute_reply":"2024-09-06T18:01:44.185814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:44.188097Z","iopub.execute_input":"2024-09-06T18:01:44.188815Z","iopub.status.idle":"2024-09-06T18:01:44.769799Z","shell.execute_reply.started":"2024-09-06T18:01:44.188764Z","shell.execute_reply":"2024-09-06T18:01:44.768753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A baseline score of 50.06% is good to go let's finetune some pretrained model","metadata":{}},{"cell_type":"markdown","source":"# Pretrained Models for Transfer Learning","metadata":{"id":"hqxnSBJ3pKz8"}},{"cell_type":"markdown","source":"Here i used Xception model for fine-tuning feel free to try the performance of other pretrained models.","metadata":{}},{"cell_type":"markdown","source":"All three datasets contain individual images. We need to batch them, but for this we first need to ensure they all have the same size, or else batching will not work. We can use a `Resizing` layer for this. We must also call the `tf.keras.applications.xception.preprocess_input()` function to preprocess the images appropriately for the Xception model. We will also add shuffling and prefetching to the training dataset.","metadata":{"id":"gXG6iv8XpKz9"}},{"cell_type":"code","source":"train_set_raw=tf.data.Dataset.from_tensor_slices((X_train,y_train))\nvalid_set_raw=tf.data.Dataset.from_tensor_slices((X_val,y_val))\ntest_set_raw=tf.data.Dataset.from_tensor_slices((X_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:01:44.770943Z","iopub.execute_input":"2024-09-06T18:01:44.771254Z","iopub.status.idle":"2024-09-06T18:01:50.024907Z","shell.execute_reply.started":"2024-09-06T18:01:44.77122Z","shell.execute_reply":"2024-09-06T18:01:50.024077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.backend.clear_session()  # extra code – resets layer name counter\n\nbatch_size = 32\npreprocess = tf.keras.applications.xception.preprocess_input\ntrain_set = train_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y))\ntrain_set = train_set.shuffle(1000, seed=42).batch(batch_size).prefetch(1)\nvalid_set = valid_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)\ntest_set = test_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)","metadata":{"id":"Bnz0n9XApKz9","execution":{"iopub.status.busy":"2024-09-06T18:01:50.026108Z","iopub.execute_input":"2024-09-06T18:01:50.02645Z","iopub.status.idle":"2024-09-06T18:01:51.659082Z","shell.execute_reply.started":"2024-09-06T18:01:50.026415Z","shell.execute_reply":"2024-09-06T18:01:51.658086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's take a look again at the first 9 images from the validation set: they're all with values ranging from -1 to 1:","metadata":{"id":"ovNEMky-pKz9"}},{"cell_type":"code","source":"# extra code – displays the first 9 images in the first batch of valid_set\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        plt.imshow((X_batch[index] + 1) / 2)  # rescale to 0–1 for imshow()\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"ZL3c3i4opKz9","outputId":"38847d8d-8822-41a3-cfb2-27479aa5debe","execution":{"iopub.status.busy":"2024-09-06T18:01:51.660734Z","iopub.execute_input":"2024-09-06T18:01:51.661781Z","iopub.status.idle":"2024-09-06T18:01:53.236432Z","shell.execute_reply.started":"2024-09-06T18:01:51.661734Z","shell.execute_reply":"2024-09-06T18:01:53.235531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augmentation = tf.keras.Sequential([\n    tf.keras.layers.RandomFlip(mode=\"horizontal\", seed=42),\n    tf.keras.layers.RandomRotation(factor=0.05, seed=42),\n    tf.keras.layers.RandomContrast(factor=0.2, seed=42)\n])","metadata":{"id":"Ib0cA8Y1pKz9","execution":{"iopub.status.busy":"2024-09-06T18:01:53.237639Z","iopub.execute_input":"2024-09-06T18:01:53.237922Z","iopub.status.idle":"2024-09-06T18:01:53.255734Z","shell.execute_reply.started":"2024-09-06T18:01:53.237891Z","shell.execute_reply":"2024-09-06T18:01:53.254869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Try running the following cell multiple times to see different random data augmentations:","metadata":{"id":"G7GrQjsspKz-"}},{"cell_type":"code","source":"# extra code – displays the same first 9 images, after augmentation\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    X_batch_augmented = data_augmentation(X_batch, training=True)\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        # We must rescale the images to the 0-1 range for imshow(), and also\n        # clip the result to that range, because data augmentation may\n        # make some values go out of bounds (e.g., RandomContrast in this case).\n        plt.imshow(np.clip((X_batch_augmented[index] + 1) / 2, 0, 1))\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"w6GH5_vupKz-","outputId":"eeb2c924-2f4f-4aa1-bea9-951bebef4bf0","execution":{"iopub.status.busy":"2024-09-06T18:01:53.256722Z","iopub.execute_input":"2024-09-06T18:01:53.256998Z","iopub.status.idle":"2024-09-06T18:01:56.441733Z","shell.execute_reply.started":"2024-09-06T18:01:53.256967Z","shell.execute_reply":"2024-09-06T18:01:56.440634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's load the pretrained model, without its top layers, and replace them with our own task","metadata":{"id":"kNL9AOsDpKz-"}},{"cell_type":"code","source":"tf.random.set_seed(42)  # extra code – ensures reproducibility\nbase_model = tf.keras.applications.xception.Xception(weights=\"imagenet\",\n                                                     include_top=False)\navg = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)\noutput = tf.keras.layers.Dense(1, activation=\"sigmoid\")(avg)\nmodel = tf.keras.Model(inputs=base_model.input, outputs=output)","metadata":{"id":"lRyCgvaKpKz-","outputId":"a825e173-8b1d-4217-a1c4-5491b49c3e82","execution":{"iopub.status.busy":"2024-09-06T18:01:56.443177Z","iopub.execute_input":"2024-09-06T18:01:56.444043Z","iopub.status.idle":"2024-09-06T18:01:58.335074Z","shell.execute_reply.started":"2024-09-06T18:01:56.443994Z","shell.execute_reply":"2024-09-06T18:01:58.334316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for layer in base_model.layers:\n    layer.trainable = False","metadata":{"id":"KBlyG6ElpKz-","execution":{"iopub.status.busy":"2024-09-06T18:01:58.336264Z","iopub.execute_input":"2024-09-06T18:01:58.336657Z","iopub.status.idle":"2024-09-06T18:01:58.344805Z","shell.execute_reply.started":"2024-09-06T18:01:58.336615Z","shell.execute_reply":"2024-09-06T18:01:58.343848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's train the model for a few epochs, while keeping the base model weights fixed:","metadata":{"id":"WFEFw7GKpKz-"}},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=3)","metadata":{"id":"GGxK2yPcpKz-","outputId":"6b64214a-e104-4b6c-9b7a-3388fc9aa15f","execution":{"iopub.status.busy":"2024-09-06T18:01:58.346262Z","iopub.execute_input":"2024-09-06T18:01:58.346588Z","iopub.status.idle":"2024-09-06T18:04:59.640766Z","shell.execute_reply.started":"2024-09-06T18:01:58.346557Z","shell.execute_reply":"2024-09-06T18:04:59.639685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for indices in zip(range(33), range(33, 66), range(66, 99), range(99, 132)):\n    for idx in indices:\n        print(f\"{idx:3}: {base_model.layers[idx].name:22}\", end=\"\")\n    print()","metadata":{"id":"GvGMiJMLpKz-","outputId":"91f2c96c-c058-45e0-e428-66fa6076ad56","execution":{"iopub.status.busy":"2024-09-06T18:04:59.642Z","iopub.execute_input":"2024-09-06T18:04:59.642369Z","iopub.status.idle":"2024-09-06T18:04:59.661198Z","shell.execute_reply.started":"2024-09-06T18:04:59.642327Z","shell.execute_reply":"2024-09-06T18:04:59.660166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:04:59.662632Z","iopub.execute_input":"2024-09-06T18:04:59.663061Z","iopub.status.idle":"2024-09-06T18:05:13.409476Z","shell.execute_reply.started":"2024-09-06T18:04:59.663017Z","shell.execute_reply":"2024-09-06T18:05:13.408592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('my_model2.h5')","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:05:13.411133Z","iopub.execute_input":"2024-09-06T18:05:13.412051Z","iopub.status.idle":"2024-09-06T18:05:13.789192Z","shell.execute_reply.started":"2024-09-06T18:05:13.412003Z","shell.execute_reply":"2024-09-06T18:05:13.788344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now with the finetuning the top layers of xception model the model performance jumps to 63.8% ","metadata":{}},{"cell_type":"markdown","source":"Now that the weights of our new top layers are not too bad, we can make the top part of the base model trainable again, and continue training, but with a lower learning rate:","metadata":{"id":"L_bEwL8KpKz_"}},{"cell_type":"code","source":"for layer in base_model.layers[56:]:\n    layer.trainable = True\n\noptimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=3)","metadata":{"id":"GEUNGlhvpKz_","outputId":"c622a91d-f634-4443-b87e-8d46defdb578","execution":{"iopub.status.busy":"2024-09-06T18:05:13.790549Z","iopub.execute_input":"2024-09-06T18:05:13.790936Z","iopub.status.idle":"2024-09-06T18:10:29.355499Z","shell.execute_reply.started":"2024-09-06T18:05:13.790891Z","shell.execute_reply":"2024-09-06T18:10:29.354605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('xception_deepfake_image.h5')","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:10:29.356918Z","iopub.execute_input":"2024-09-06T18:10:29.358043Z","iopub.status.idle":"2024-09-06T18:10:29.823813Z","shell.execute_reply.started":"2024-09-06T18:10:29.357997Z","shell.execute_reply":"2024-09-06T18:10:29.823019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:10:29.825274Z","iopub.execute_input":"2024-09-06T18:10:29.825955Z","iopub.status.idle":"2024-09-06T18:10:30.367911Z","shell.execute_reply.started":"2024-09-06T18:10:29.82591Z","shell.execute_reply":"2024-09-06T18:10:30.366941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:10:30.369293Z","iopub.execute_input":"2024-09-06T18:10:30.369947Z","iopub.status.idle":"2024-09-06T18:10:43.834549Z","shell.execute_reply.started":"2024-09-06T18:10:30.3699Z","shell.execute_reply":"2024-09-06T18:10:43.83365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate on the test set\ny_pred_probs = model.predict(test_set)\ny_true_list = [y.numpy() for _, y in test_set_raw]  # Convert TensorFlow tensors to NumPy arrays\n\n# Use np.hstack instead of np.concatenate\ny_true = np.hstack(y_true_list)\n\n# Convert probabilities to binary predictions using a threshold (e.g., 0.5)\ny_pred_binary = (y_pred_probs > 0.5).astype(int)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:10:43.835987Z","iopub.status.idle":"2024-09-06T18:11:00.745676Z","shell.execute_reply.started":"2024-09-06T18:10:43.836402Z","shell.execute_reply":"2024-09-06T18:11:00.744773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\n\n# Assuming you have y_true and y_pred_binary from your previous code\n\n# Compute confusion matrix\nconf_matrix = confusion_matrix(y_true, y_pred_binary)\n\n# Display confusion matrix as a heatmap\nplt.figure(figsize=(8, 6))\nsns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', xticklabels=['Negative', 'Positive'], yticklabels=['Negative', 'Positive'])\nplt.title('Confusion Matrix')\nplt.xlabel('Predicted')\nplt.ylabel('True')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:00.747319Z","iopub.execute_input":"2024-09-06T18:11:00.748098Z","iopub.status.idle":"2024-09-06T18:11:00.964916Z","shell.execute_reply.started":"2024-09-06T18:11:00.748051Z","shell.execute_reply":"2024-09-06T18:11:00.96411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ROC Curve\nfpr, tpr, thresholds = roc_curve(y_true, y_pred_probs)\nroc_auc = auc(fpr, tpr)\n\n# Plot ROC curve\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(8, 6))\nplt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (AUC = {roc_auc:.2f})')\nplt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\nplt.xlabel('False Positive Rate (FPR)')\nplt.ylabel('True Positive Rate (TPR)')\nplt.title('Receiver Operating Characteristic (ROC) Curve')\nplt.legend(loc='lower right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:00.965926Z","iopub.execute_input":"2024-09-06T18:11:00.966194Z","iopub.status.idle":"2024-09-06T18:11:01.194519Z","shell.execute_reply.started":"2024-09-06T18:11:00.966164Z","shell.execute_reply":"2024-09-06T18:11:01.193647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model accuracy finally reaches to 81.9%","metadata":{}},{"cell_type":"markdown","source":"Lets try to interpret the trained model on how it finds a image FAKE","metadata":{}},{"cell_type":"code","source":"!pip install lime","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:01.195639Z","iopub.execute_input":"2024-09-06T18:11:01.195958Z","iopub.status.idle":"2024-09-06T18:11:18.273712Z","shell.execute_reply.started":"2024-09-06T18:11:01.195925Z","shell.execute_reply":"2024-09-06T18:11:18.27261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lime import lime_image\n\nexplainer = lime_image.LimeImageExplainer()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:18.275146Z","iopub.execute_input":"2024-09-06T18:11:18.275468Z","iopub.status.idle":"2024-09-06T18:11:18.537887Z","shell.execute_reply.started":"2024-09-06T18:11:18.275434Z","shell.execute_reply":"2024-09-06T18:11:18.536865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deep Fake Video Classification","metadata":{}},{"cell_type":"markdown","source":"## Importing required libraries","metadata":{}},{"cell_type":"code","source":"!pip install -U --upgrade tensorflow","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:18.539152Z","iopub.execute_input":"2024-09-06T18:11:18.540445Z","iopub.status.idle":"2024-09-06T18:11:32.201299Z","shell.execute_reply.started":"2024-09-06T18:11:18.540409Z","shell.execute_reply":"2024-09-06T18:11:32.200248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install tensorflow-docs\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:32.202932Z","iopub.execute_input":"2024-09-06T18:11:32.203254Z","iopub.status.idle":"2024-09-06T18:11:46.305603Z","shell.execute_reply.started":"2024-09-06T18:11:32.203219Z","shell.execute_reply":"2024-09-06T18:11:46.304456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow_docs.vis import embed\nfrom tensorflow import keras\n#from imutils import paths\nfrom tensorflow_docs.vis import embed\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport imageio\nimport cv2\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.307211Z","iopub.execute_input":"2024-09-06T18:11:46.307571Z","iopub.status.idle":"2024-09-06T18:11:46.372259Z","shell.execute_reply.started":"2024-09-06T18:11:46.307533Z","shell.execute_reply":"2024-09-06T18:11:46.37146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualisation","metadata":{}},{"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\nTEST_FOLDER = 'test_videos'\n\nprint(f\"Train samples: {len(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))}\")\nprint(f\"Test samples: {len(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))}\")","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.373451Z","iopub.execute_input":"2024-09-06T18:11:46.373753Z","iopub.status.idle":"2024-09-06T18:11:46.554609Z","shell.execute_reply.started":"2024-09-06T18:11:46.373723Z","shell.execute_reply":"2024-09-06T18:11:46.553597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sample_metadata = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').T\ntrain_sample_metadata.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.555882Z","iopub.execute_input":"2024-09-06T18:11:46.556175Z","iopub.status.idle":"2024-09-06T18:11:46.681401Z","shell.execute_reply.started":"2024-09-06T18:11:46.556144Z","shell.execute_reply":"2024-09-06T18:11:46.680441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sample_metadata.groupby('label')['label'].count().plot(figsize=(15, 5), kind='bar', title='Distribution of Labels in the Training Set')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.682763Z","iopub.execute_input":"2024-09-06T18:11:46.683196Z","iopub.status.idle":"2024-09-06T18:11:46.962277Z","shell.execute_reply.started":"2024-09-06T18:11:46.683133Z","shell.execute_reply":"2024-09-06T18:11:46.961322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sample_metadata.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.963433Z","iopub.execute_input":"2024-09-06T18:11:46.963715Z","iopub.status.idle":"2024-09-06T18:11:46.969709Z","shell.execute_reply.started":"2024-09-06T18:11:46.963686Z","shell.execute_reply":"2024-09-06T18:11:46.968947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize now the data.\n\nWe select first a list of fake videos.","metadata":{}},{"cell_type":"markdown","source":"### Few fake videos","metadata":{}},{"cell_type":"code","source":"fake_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.label=='FAKE'].sample(3).index)\nfake_train_sample_video","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.970721Z","iopub.execute_input":"2024-09-06T18:11:46.971016Z","iopub.status.idle":"2024-09-06T18:11:46.981554Z","shell.execute_reply.started":"2024-09-06T18:11:46.970985Z","shell.execute_reply":"2024-09-06T18:11:46.980706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_image_from_video(video_path):\n    '''\n    input: video_path - path for video\n    process:\n    1. perform a video capture from the video\n    2. read the image\n    3. display the image\n    '''\n    capture_image = cv2.VideoCapture(video_path) \n    ret, frame = capture_image.read()\n    fig = plt.figure(figsize=(10,10))\n    ax = fig.add_subplot(111)\n    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n    ax.imshow(frame)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.983095Z","iopub.execute_input":"2024-09-06T18:11:46.983757Z","iopub.status.idle":"2024-09-06T18:11:46.98947Z","shell.execute_reply.started":"2024-09-06T18:11:46.983724Z","shell.execute_reply":"2024-09-06T18:11:46.988649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_file in fake_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:46.99073Z","iopub.execute_input":"2024-09-06T18:11:46.991148Z","iopub.status.idle":"2024-09-06T18:11:49.735986Z","shell.execute_reply.started":"2024-09-06T18:11:46.991115Z","shell.execute_reply":"2024-09-06T18:11:49.734997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's try now the same for few of the images that are real.","metadata":{}},{"cell_type":"markdown","source":"### Few Real Videos","metadata":{}},{"cell_type":"code","source":"real_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.label=='REAL'].sample(3).index)\nreal_train_sample_video","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:49.737479Z","iopub.execute_input":"2024-09-06T18:11:49.737919Z","iopub.status.idle":"2024-09-06T18:11:49.746695Z","shell.execute_reply.started":"2024-09-06T18:11:49.737878Z","shell.execute_reply":"2024-09-06T18:11:49.745729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for video_file in real_train_sample_video:\n    display_image_from_video(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER, video_file))","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:49.747907Z","iopub.execute_input":"2024-09-06T18:11:49.748436Z","iopub.status.idle":"2024-09-06T18:11:53.162454Z","shell.execute_reply.started":"2024-09-06T18:11:49.748375Z","shell.execute_reply":"2024-09-06T18:11:53.161416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Videos with same original","metadata":{}},{"cell_type":"markdown","source":"Let's look now to set of samples with the same original.","metadata":{}},{"cell_type":"code","source":"train_sample_metadata['original'].value_counts()[0:5]","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:53.164013Z","iopub.execute_input":"2024-09-06T18:11:53.164383Z","iopub.status.idle":"2024-09-06T18:11:53.176135Z","shell.execute_reply.started":"2024-09-06T18:11:53.164342Z","shell.execute_reply":"2024-09-06T18:11:53.175342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We pick one of the originals with largest number of samples.\n\nWe also modify our visualization function to work with multiple images.","metadata":{}},{"cell_type":"code","source":"def display_image_from_video_list(video_path_list, video_folder=TRAIN_SAMPLE_FOLDER):\n    '''\n    input: video_path_list - path for video\n    process:\n    0. for each video in the video path list\n        1. perform a video capture from the video\n        2. read the image\n        3. display the image\n    '''\n    plt.figure()\n    fig, ax = plt.subplots(2,3,figsize=(16,8))\n    # we only show images extracted from the first 6 videos\n    for i, video_file in enumerate(video_path_list[0:6]):\n        video_path = os.path.join(DATA_FOLDER, video_folder,video_file)\n        capture_image = cv2.VideoCapture(video_path) \n        ret, frame = capture_image.read()\n        frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n        ax[i//3, i%3].imshow(frame)\n        ax[i//3, i%3].set_title(f\"Video: {video_file}\")\n        ax[i//3, i%3].axis('on')\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:53.177307Z","iopub.execute_input":"2024-09-06T18:11:53.177643Z","iopub.status.idle":"2024-09-06T18:11:53.188967Z","shell.execute_reply.started":"2024-09-06T18:11:53.17761Z","shell.execute_reply":"2024-09-06T18:11:53.188053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"same_original_fake_train_sample_video = list(train_sample_metadata.loc[train_sample_metadata.original=='atvmxvwyns.mp4'].index)\ndisplay_image_from_video_list(same_original_fake_train_sample_video)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:53.19032Z","iopub.execute_input":"2024-09-06T18:11:53.190888Z","iopub.status.idle":"2024-09-06T18:11:57.108775Z","shell.execute_reply.started":"2024-09-06T18:11:53.190853Z","shell.execute_reply":"2024-09-06T18:11:57.107848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test video files","metadata":{}},{"cell_type":"markdown","source":"Let's also look to few of the test data files.","metadata":{}},{"cell_type":"code","source":"test_videos = pd.DataFrame(list(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER))), columns=['video'])","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:57.110581Z","iopub.execute_input":"2024-09-06T18:11:57.110945Z","iopub.status.idle":"2024-09-06T18:11:57.116954Z","shell.execute_reply.started":"2024-09-06T18:11:57.110905Z","shell.execute_reply":"2024-09-06T18:11:57.116069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_videos.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:57.118151Z","iopub.execute_input":"2024-09-06T18:11:57.118563Z","iopub.status.idle":"2024-09-06T18:11:57.13034Z","shell.execute_reply.started":"2024-09-06T18:11:57.11853Z","shell.execute_reply":"2024-09-06T18:11:57.129329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize now one of the videos.","metadata":{}},{"cell_type":"code","source":"display_image_from_video(os.path.join(DATA_FOLDER, TEST_FOLDER, test_videos.iloc[2].video))","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:57.131673Z","iopub.execute_input":"2024-09-06T18:11:57.131975Z","iopub.status.idle":"2024-09-06T18:11:57.980693Z","shell.execute_reply.started":"2024-09-06T18:11:57.131943Z","shell.execute_reply":"2024-09-06T18:11:57.979748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Play video files","metadata":{}},{"cell_type":"markdown","source":"Let's look to few fake videos.","metadata":{}},{"cell_type":"code","source":"fake_videos = list(train_sample_metadata.loc[train_sample_metadata.label=='FAKE'].index)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:57.981877Z","iopub.execute_input":"2024-09-06T18:11:57.982186Z","iopub.status.idle":"2024-09-06T18:11:57.987971Z","shell.execute_reply.started":"2024-09-06T18:11:57.982153Z","shell.execute_reply":"2024-09-06T18:11:57.986903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import HTML\nfrom base64 import b64encode\n\ndef play_video(video_file, subset=TRAIN_SAMPLE_FOLDER):\n    '''\n    Display video\n    param: video_file - the name of the video file to display\n    param: subset - the folder where the video file is located (can be TRAIN_SAMPLE_FOLDER or TEST_Folder)\n    '''\n    video_url = open(os.path.join(DATA_FOLDER, subset,video_file),'rb').read()\n    data_url = \"data:video/mp4;base64,\" + b64encode(video_url).decode()\n    return HTML(\"\"\"<video width=500 controls><source src=\"%s\" type=\"video/mp4\"></video>\"\"\" % data_url)\n\nplay_video(fake_videos[10])","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:57.989212Z","iopub.execute_input":"2024-09-06T18:11:57.98962Z","iopub.status.idle":"2024-09-06T18:11:58.366565Z","shell.execute_reply.started":"2024-09-06T18:11:57.989579Z","shell.execute_reply":"2024-09-06T18:11:58.365549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From visual inspection of these fakes videos, in some cases is very easy to spot the anomalies created when engineering the deep fake, in some cases is more difficult.","metadata":{}},{"cell_type":"markdown","source":"## Modelling","metadata":{}},{"cell_type":"markdown","source":"### A CNN-RNN Architecture","metadata":{}},{"cell_type":"code","source":"IMG_SIZE = 224\nBATCH_SIZE = 64\nEPOCHS = 10\n\nMAX_SEQ_LENGTH = 20\nNUM_FEATURES = 2048","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:58.367764Z","iopub.execute_input":"2024-09-06T18:11:58.36807Z","iopub.status.idle":"2024-09-06T18:11:58.373606Z","shell.execute_reply.started":"2024-09-06T18:11:58.368037Z","shell.execute_reply":"2024-09-06T18:11:58.372261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" In this example we will do the following:\n\n* Capture the frames of a video.\n* Extract frames from the videos until a maximum frame count is reached.\n* In the case, where a video's frame count is lesser than the maximum frame count we will pad the video with zeros.","metadata":{}},{"cell_type":"code","source":"def crop_center_square(frame):\n    y, x = frame.shape[0:2]\n    min_dim = min(y, x)\n    start_x = (x // 2) - (min_dim // 2)\n    start_y = (y // 2) - (min_dim // 2)\n    return frame[start_y : start_y + min_dim, start_x : start_x + min_dim]\n\n\ndef load_video(path, max_frames=0, resize=(IMG_SIZE, IMG_SIZE)):\n    cap = cv2.VideoCapture(path)\n    frames = []\n    try:\n        while True:\n            ret, frame = cap.read()\n            if not ret:\n                break\n            frame = crop_center_square(frame)\n            frame = cv2.resize(frame, resize)\n            frame = frame[:, :, [2, 1, 0]]\n            frames.append(frame)\n\n            if len(frames) == max_frames:\n                break\n    finally:\n        cap.release()\n    return np.array(frames)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:58.375847Z","iopub.execute_input":"2024-09-06T18:11:58.37617Z","iopub.status.idle":"2024-09-06T18:11:58.390356Z","shell.execute_reply.started":"2024-09-06T18:11:58.376139Z","shell.execute_reply":"2024-09-06T18:11:58.389211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can use a pre-trained network to extract meaningful features from the extracted frames. The Keras Applications module provides a number of state-of-the-art models pre-trained on the ImageNet-1k dataset. We will be using the InceptionV3 model for this purpose.","metadata":{}},{"cell_type":"code","source":"def build_feature_extractor():\n    feature_extractor = keras.applications.InceptionV3(\n        weights=\"imagenet\",\n        include_top=False,\n        pooling=\"avg\",\n        input_shape=(IMG_SIZE, IMG_SIZE, 3),\n    )\n    preprocess_input = keras.applications.inception_v3.preprocess_input\n\n    inputs = keras.Input((IMG_SIZE, IMG_SIZE, 3))\n    preprocessed = preprocess_input(inputs)\n\n    outputs = feature_extractor(preprocessed)\n    return keras.Model(inputs, outputs, name=\"feature_extractor\")\n\n\nfeature_extractor = build_feature_extractor()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:11:58.391703Z","iopub.execute_input":"2024-09-06T18:11:58.392299Z","iopub.status.idle":"2024-09-06T18:12:00.967092Z","shell.execute_reply.started":"2024-09-06T18:11:58.392238Z","shell.execute_reply":"2024-09-06T18:12:00.966265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, we can put all the pieces together to create our data processing utility.","metadata":{}},{"cell_type":"code","source":"def prepare_all_videos(df, root_dir):\n    num_samples = len(df)\n    video_paths = list(df.index)\n    labels = df[\"label\"].values\n    labels = np.array(labels == 'FAKE').astype(int)\n\n\n    # `frame_masks` and `frame_features` are what we will feed to our sequence model.\n    # `frame_masks` will contain a bunch of booleans denoting if a timestep is\n    # masked with padding or not.\n    frame_masks = np.zeros(shape=(num_samples, MAX_SEQ_LENGTH), dtype=\"bool\")\n    frame_features = np.zeros(\n        shape=(num_samples, MAX_SEQ_LENGTH, NUM_FEATURES), dtype=\"float32\"\n    )\n\n    # For each video.\n    for idx, path in enumerate(video_paths):\n        # Gather all its frames and add a batch dimension.\n        frames = load_video(os.path.join(root_dir, path))\n        frames = frames[None, ...]\n\n        # Initialize placeholders to store the masks and features of the current video.\n        temp_frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype=\"bool\")\n        temp_frame_features = np.zeros(\n            shape=(1, MAX_SEQ_LENGTH, NUM_FEATURES), dtype=\"float32\"\n        )\n\n        # Extract features from the frames of the current video.\n        for i, batch in enumerate(frames):\n            video_length = batch.shape[0]\n            length = min(MAX_SEQ_LENGTH, video_length)\n            for j in range(length):\n                temp_frame_features[i, j, :] = feature_extractor.predict(\n                    batch[None, j, :]\n                )\n            temp_frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked\n\n        frame_features[idx,] = temp_frame_features.squeeze()\n        frame_masks[idx,] = temp_frame_mask.squeeze()\n\n    return (frame_features, frame_masks), labels","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:12:00.970062Z","iopub.execute_input":"2024-09-06T18:12:00.97041Z","iopub.status.idle":"2024-09-06T18:12:00.980546Z","shell.execute_reply.started":"2024-09-06T18:12:00.970376Z","shell.execute_reply":"2024-09-06T18:12:00.979676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since we don't have test labels we split the training data to find its performance in unseen data","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nTrain_set, Test_set = train_test_split(train_sample_metadata,test_size=0.1,random_state=42,stratify=train_sample_metadata['label'])\n\nprint(Train_set.shape, Test_set.shape )","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:12:00.981808Z","iopub.execute_input":"2024-09-06T18:12:00.982167Z","iopub.status.idle":"2024-09-06T18:12:00.999521Z","shell.execute_reply.started":"2024-09-06T18:12:00.982123Z","shell.execute_reply":"2024-09-06T18:12:00.998583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data, train_labels = prepare_all_videos(Train_set, \"train\")\ntest_data, test_labels = prepare_all_videos(Test_set, \"test\")\n\n# Assuming train_data is a tuple with two elements: (frame_features, frame_masks)\nframe_features_shape = train_data[0].shape if train_data and len(train_data) > 0 else None\nframe_masks_shape = train_data[1].shape if train_data and len(train_data) > 1 else None\n\nprint(f\"Frame features in train set: {frame_features_shape}\")\nprint(f\"Frame masks in train set: {frame_masks_shape}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:12:01.000756Z","iopub.execute_input":"2024-09-06T18:12:01.001117Z","iopub.status.idle":"2024-09-06T18:12:01.080449Z","shell.execute_reply.started":"2024-09-06T18:12:01.001077Z","shell.execute_reply":"2024-09-06T18:12:01.079334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The sequence model","metadata":{}},{"cell_type":"markdown","source":"Now, we can feed this data to a sequence model consisting of recurrent layers like GRU.","metadata":{}},{"cell_type":"code","source":"frame_features_input = keras.Input((MAX_SEQ_LENGTH, NUM_FEATURES))\nmask_input = keras.Input((MAX_SEQ_LENGTH,), dtype=\"bool\")\n\n# Refer to the following tutorial to understand the significance of using `mask`:\n# https://keras.io/api/layers/recurrent_layers/gru/\nx = keras.layers.GRU(16, return_sequences=True)(\n    frame_features_input, mask=mask_input\n)\nx = keras.layers.GRU(8)(x)\nx = keras.layers.Dropout(0.4)(x)\nx = keras.layers.Dense(8, activation=\"relu\")(x)\noutput = keras.layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model([frame_features_input, mask_input], output)\n\nmodel.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:12:01.090923Z","iopub.execute_input":"2024-09-06T18:12:01.091214Z","iopub.status.idle":"2024-09-06T18:12:01.268573Z","shell.execute_reply.started":"2024-09-06T18:12:01.091184Z","shell.execute_reply":"2024-09-06T18:12:01.267611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install --upgrade tensorflow\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:12:01.269756Z","iopub.execute_input":"2024-09-06T18:12:01.270142Z","iopub.status.idle":"2024-09-06T18:12:14.931134Z","shell.execute_reply.started":"2024-09-06T18:12:01.270099Z","shell.execute_reply":"2024-09-06T18:12:14.929989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install --upgrade tensorflow\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:15:01.141408Z","iopub.execute_input":"2024-09-06T18:15:01.142109Z","iopub.status.idle":"2024-09-06T18:15:14.911435Z","shell.execute_reply.started":"2024-09-06T18:15:01.142064Z","shell.execute_reply":"2024-09-06T18:15:14.910184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import GRU, Dense, Input\n\n# Define the dimensions based on your data\ntimesteps = 10  # Number of time steps in each sequence\nfeatures = 20   # Number of features per time step\n\n# Create the model\nmodel = Sequential()\nmodel.add(Input(shape=(timesteps, features)))\nmodel.add(GRU(units=64))  # GRU layer with 64 units\nmodel.add(Dense(1))\n\nmodel.compile(optimizer='adam', loss='mse')\n\n# Fit the model\nhistory = model.fit(\n    [train_data[0], train_data[1]],\n    train_labels,\n    validation_data=([test_data[0], test_data[1]], test_labels),\n    callbacks=[checkpoint],\n    epochs=EPOCHS,\n    batch_size=4  # Adjust batch size to prevent possible OOM issues\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:18:49.190551Z","iopub.execute_input":"2024-09-06T18:18:49.190945Z","iopub.status.idle":"2024-09-06T18:18:54.847098Z","shell.execute_reply.started":"2024-09-06T18:18:49.19091Z","shell.execute_reply":"2024-09-06T18:18:54.846094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference","metadata":{}},{"cell_type":"code","source":"def prepare_single_video(frames):\n    frames = frames[None, ...]\n    frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype=\"bool\")\n    frame_features = np.zeros(shape=(1, MAX_SEQ_LENGTH, NUM_FEATURES), dtype=\"float32\")\n\n    for i, batch in enumerate(frames):\n        video_length = batch.shape[0]\n        length = min(MAX_SEQ_LENGTH, video_length)\n        for j in range(length):\n            frame_features[i, j, :] = feature_extractor.predict(batch[None, j, :])\n        frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked\n\n    return frame_features, frame_mask\n\ndef sequence_prediction(path):\n    frames = load_video(os.path.join(DATA_FOLDER, TEST_FOLDER,path))\n    frame_features, frame_mask = prepare_single_video(frames)\n    return model.predict([frame_features, frame_mask])[0]\n    \n\ndef to_gif(images):\n    converted_images = images.astype(np.uint8)\n    imageio.mimsave(\"animation.gif\", converted_images, fps=10)\n    return embed.embed_file(\"animation.gif\")\n\n\ntest_video = input(\"Enter the path of the video: \")\nprint(f\"Test video path: {test_video}\")\n\nif(sequence_prediction(test_video)>=0.5):\n    print(f'The predicted class of the video is FAKE')\nelse:\n    print(f'The predicted class of the video is REAL')\n\nplay_video(test_video,TEST_FOLDER)","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:19:57.241952Z","iopub.execute_input":"2024-09-06T18:19:57.242856Z","iopub.status.idle":"2024-09-06T18:20:47.693575Z","shell.execute_reply.started":"2024-09-06T18:19:57.242813Z","shell.execute_reply":"2024-09-06T18:20:47.691937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport imageio\nfrom tensorflow.keras.models import load_model\n\n# Assuming you have defined MAX_SEQ_LENGTH, NUM_FEATURES, DATA_FOLDER, TEST_FOLDER, and feature_extractor\n\ndef prepare_single_video(frames):\n    frames = frames[None, ...]\n    frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype=\"bool\")\n    frame_features = np.zeros(shape=(1, MAX_SEQ_LENGTH, NUM_FEATURES), dtype=\"float32\")\n\n    for i, batch in enumerate(frames):\n        video_length = batch.shape[0]\n        length = min(MAX_SEQ_LENGTH, video_length)\n        for j in range(length):\n            frame_features[i, j, :] = feature_extractor.predict(batch[None, j, :])\n        frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked\n\n    return frame_features, frame_mask\n\ndef sequence_prediction(path):\n    frames = load_video(os.path.join(DATA_FOLDER, TEST_FOLDER, path))\n    frame_features, frame_mask = prepare_single_video(frames)\n    return model.predict([frame_features, frame_mask])[0]\n\ndef visualize_frames_with_predictions(video_path):\n    frames = load_video(os.path.join(DATA_FOLDER, TEST_FOLDER, video_path))\n\n    for i, frame in enumerate(frames):\n        frame_features, frame_mask = prepare_single_video(np.array([frame]))\n        prediction = sequence_prediction(video_path)\n\n        # Print or save the individual frame along with its prediction\n        print(f\"Frame {i}: {'FAKE' if prediction >= 0.5 else 'REAL'}\")\n        plt.imshow(frame)  # Assuming you have matplotlib for visualization\n        plt.show()\n\ntest_video = input(\"Enter the path of the video: \")\nprint(f\"Test video path: {test_video}\")\n\n# Visualize frames with predictions\nvisualize_frames_with_predictions(test_video)\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T18:12:15.800945Z","iopub.status.idle":"2024-09-06T18:12:15.801518Z","shell.execute_reply.started":"2024-09-06T18:12:15.801202Z","shell.execute_reply":"2024-09-06T18:12:15.801232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass ConvLayer(nn.Module):\n    def __init__(self):\n        super(ConvLayer, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=256, kernel_size=5, stride=1)\n        self.conv2 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=5, stride=2)\n        self.conv3 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=5, stride=1)\n\n    def forward(self, x):\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.conv2(x))\n        x = F.relu(self.conv3(x))\n        return x\n\n\n\nclass PrimaryCaps(nn.Module):\n    def __init__(self):\n        super(PrimaryCaps, self).__init__()\n        self.conv = nn.Conv2d(256, 256, kernel_size=9, stride=2)\n        self.num_capsules = 32\n        self.dim_capsules = 8\n\n    def forward(self, x):\n        x = F.relu(self.conv(x))\n        batch_size, channels, height, width = x.size()\n        x = x.view(batch_size, self.num_capsules, self.dim_capsules, height, width)\n        return x\n\nclass DigitCaps(nn.Module):\n    def __init__(self):\n        super(DigitCaps, self).__init__()\n        self.num_capsules = 10\n        self.dim_capsules = 16\n        self.routings = 3\n\n    def forward(self, x):\n        # Implement routing-by-agreement here\n        # Placeholder implementation\n        return x\n\nclass Decoder(nn.Module):\n    def __init__(self):\n        super(Decoder, self).__init__()\n        self.fc1 = nn.Linear(16 * 10, 512)\n        self.fc2 = nn.Linear(512, 1024)\n        self.fc3 = nn.Linear(1024, 3072)\n        \n    def forward(self, x):\n        x = x.view(x.size(0), -1)\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n\nclass CapsuleNetwork(nn.Module):\n    def __init__(self):\n        super(CapsuleNetwork, self).__init__()\n        self.conv_layer = ConvLayer()\n        self.primary_capsules = PrimaryCaps()\n        self.digit_capsules = DigitCaps()\n        self.decoder = Decoder()\n        \n    def forward(self, images):\n        primary_caps_output = self.primary_capsules(self.conv_layer(images))\n        caps_output = self.digit_capsules(primary_caps_output).squeeze().transpose(0,1)\n        reconstructions = self.decoder(caps_output)\n        return caps_output, reconstructions\n","metadata":{"execution":{"iopub.status.busy":"2024-09-08T08:53:12.685991Z","iopub.execute_input":"2024-09-08T08:53:12.686376Z","iopub.status.idle":"2024-09-08T08:53:12.702744Z","shell.execute_reply.started":"2024-09-08T08:53:12.686342Z","shell.execute_reply":"2024-09-08T08:53:12.701732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\n\ndef train_model(model, train_loader, num_epochs, device):\n    model.to(device)\n    optimizer = optim.Adam(model.parameters())\n    criterion = nn.MSELoss()  # Adapt based on your loss function\n\n    for epoch in range(num_epochs):\n        model.train()\n        for batch_idx, (frames, labels) in enumerate(train_loader):\n            frames, labels = frames.to(device), labels.to(device)\n            optimizer.zero_grad()\n            caps_output, reconstructions = model(frames)\n            loss = criterion(reconstructions, frames)  # Adapt based on your loss function\n            loss.backward()\n            optimizer.step()\n            if batch_idx % 10 == 0:\n                print(f\"Epoch {epoch}, Batch {batch_idx}, Loss: {loss.item()}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T19:37:30.459247Z","iopub.execute_input":"2024-09-06T19:37:30.460289Z","iopub.status.idle":"2024-09-06T19:37:30.467241Z","shell.execute_reply.started":"2024-09-06T19:37:30.460245Z","shell.execute_reply":"2024-09-06T19:37:30.466327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, video_path, device):\n    frames = load_video(os.path.join(DATA_FOLDER, TEST_FOLDER, video_path))\n    frame_features, frame_mask = prepare_single_video(frames)\n    frame_features = torch.tensor(frame_features, dtype=torch.float32).to(device)\n    model.eval()\n    with torch.no_grad():\n        caps_output, _ = model(frame_features)\n    # Process `caps_output` to get predictions\n    return caps_output\n","metadata":{"execution":{"iopub.status.busy":"2024-09-06T19:37:41.843691Z","iopub.execute_input":"2024-09-06T19:37:41.844309Z","iopub.status.idle":"2024-09-06T19:37:41.849739Z","shell.execute_reply.started":"2024-09-06T19:37:41.844271Z","shell.execute_reply":"2024-09-06T19:37:41.848806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\n\n# Define constants\nDATA_FOLDER = '../input/deepfake-detection-challenge'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos'\nTEST_FOLDER = 'test_videos'\nMAX_SEQ_LENGTH = 30\nNUM_FEATURES = 2048\n\n# Define a function to collect video paths and labels\ndef collect_video_paths_labels(data_folder, train_folder):\n    video_paths = []\n    labels = []\n    for video_file in os.listdir(os.path.join(data_folder, train_folder)):\n        if video_file.endswith('.mp4'):\n            video_path = os.path.join(data_folder, train_folder, video_file)\n            # Dummy label assignment: Replace with actual label extraction logic\n            label = 1 if 'fake' in video_file else 0\n            video_paths.append(video_path)\n            labels.append(label)\n    return video_paths, labels\n\n# Collect paths and labels for training data\ntrain_video_paths, train_labels = collect_video_paths_labels(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)\n\ndef prepare_single_video(frames):\n    frames = frames[None, ...]\n    frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype=\"bool\")\n    frame_features = np.zeros(shape=(1, MAX_SEQ_LENGTH, 512), dtype=\"float32\")  # Adjust shape as needed\n\n    for i, batch in enumerate(frames):\n        video_length = batch.shape[0]\n        length = min(MAX_SEQ_LENGTH, video_length)\n        for j in range(length):\n            with torch.no_grad():\n                frame = torch.tensor(batch[j]).permute(2, 0, 1).unsqueeze(0).float()\n                if frame.shape[1] == 1:  # If grayscale, duplicate channels\n                    frame = frame.repeat(1, 3, 1, 1)\n                features = feature_extractor(frame).squeeze().numpy()\n                frame_features[i, j, :] = features\n        frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked\n\n    return frame_features, frame_mask\n\n\n# Define a custom dataset class\nclass VideoDataset(Dataset):\n    def __init__(self, video_paths, labels):\n        self.video_paths = video_paths\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.video_paths)\n\n    def __getitem__(self, idx):\n        video_path = self.video_paths[idx]\n        frames = load_video(video_path)  # Ensure this function is defined\n        frame_features, frame_mask = prepare_single_video(frames)\n        return torch.tensor(frame_features, dtype=torch.float32), torch.tensor(self.labels[idx], dtype=torch.float32)\n\n\n# Prepare data loaders\ntrain_dataset = VideoDataset(train_video_paths, train_labels)\ntrain_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-08T08:43:23.583197Z","iopub.execute_input":"2024-09-08T08:43:23.58368Z","iopub.status.idle":"2024-09-08T08:43:23.720084Z","shell.execute_reply.started":"2024-09-08T08:43:23.583645Z","shell.execute_reply":"2024-09-08T08:43:23.719244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torchvision.models as models\n\nclass FeatureExtractor(nn.Module):\n    def __init__(self):\n        super(FeatureExtractor, self).__init__()\n        # Load a pretrained ResNet-18 model\n        self.resnet = models.resnet18(weights='DEFAULT')\n        # Remove the last fully connected layer\n        self.resnet.fc = nn.Identity()\n        # Add a linear layer to reduce the feature dimension to NUM_FEATURES\n        self.fc = nn.Linear(512, NUM_FEATURES)\n\n    def forward(self, x):\n        features = self.resnet(x)\n        features = self.fc(features)\n        return features\n\n# Initialize the feature extractor\nfeature_extractor = FeatureExtractor()\n","metadata":{"execution":{"iopub.status.busy":"2024-09-08T08:43:42.827127Z","iopub.execute_input":"2024-09-08T08:43:42.82761Z","iopub.status.idle":"2024-09-08T08:43:44.691305Z","shell.execute_reply.started":"2024-09-08T08:43:42.827564Z","shell.execute_reply":"2024-09-08T08:43:44.69034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nfrom torchvision import models\nimport cv2\n\n# Constants\nMAX_SEQ_LENGTH = 20  # Adjust as needed\nNUM_FEATURES = 128  # Adjust as needed\n\n# Feature extractor (example using a pre-trained model)\nfeature_extractor = models.resnet18(pretrained=True)\nfeature_extractor.fc = nn.Identity()  # Remove the final classification layer\n\ndef load_video(video_path):\n    # Load video frames and return as numpy array\n    cap = cv2.VideoCapture(video_path)\n    frames = []\n    while(cap.isOpened()):\n        ret, frame = cap.read()\n        if not ret:\n            break\n        frames.append(frame)\n    cap.release()\n    frames = np.array(frames)\n    return frames\n\ndef prepare_single_video(frames):\n    frames = frames[None, ...]\n    frame_mask = np.zeros(shape=(1, MAX_SEQ_LENGTH,), dtype=\"bool\")\n    frame_features = np.zeros(shape=(1, MAX_SEQ_LENGTH, 512), dtype=\"float32\")  # Change to match the feature size\n\n    for i, batch in enumerate(frames):\n        video_length = batch.shape[0]\n        length = min(MAX_SEQ_LENGTH, video_length)\n        for j in range(length):\n            with torch.no_grad():\n                frame = torch.tensor(batch[j]).permute(2, 0, 1).unsqueeze(0).float()\n                features = feature_extractor(frame).squeeze().numpy()\n                frame_features[i, j, :] = features\n        frame_mask[i, :length] = 1  # 1 = not masked, 0 = masked\n\n    return frame_features, frame_mask\n\n\nclass VideoDataset(Dataset):\n    def __init__(self, video_paths, labels):\n        self.video_paths = video_paths\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.video_paths)\n\n    def __getitem__(self, idx):\n        video_path = self.video_paths[idx]\n        frames = load_video(video_path)  # Ensure this function is defined\n        frame_features, frame_mask = prepare_single_video(frames)\n        return torch.tensor(frame_features, dtype=torch.float32), torch.tensor(self.labels[idx], dtype=torch.float32)\n\ndef collect_video_paths_labels(data_folder, sample_folder):\n    video_paths = []\n    labels = []\n    for filename in os.listdir(os.path.join(data_folder, sample_folder)):\n        video_path = os.path.join(data_folder, sample_folder, filename)\n        video_paths.append(video_path)\n        # Assuming the label is part of the filename or some other logic\n        labels.append(1 if 'fake' in filename else 0)\n    return video_paths, labels\n\n# Prepare data loaders\ntrain_video_paths, train_labels = collect_video_paths_labels(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)\ntrain_dataset = VideoDataset(train_video_paths, train_labels)\ntrain_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-09-08T08:46:38.367873Z","iopub.execute_input":"2024-09-08T08:46:38.368527Z","iopub.status.idle":"2024-09-08T08:46:38.583446Z","shell.execute_reply.started":"2024-09-08T08:46:38.368488Z","shell.execute_reply":"2024-09-08T08:46:38.582606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, train_loader, num_epochs=10, device='cpu'):\n    model.to(device)\n    criterion = nn.BCEWithLogitsLoss()  # Adjust loss function as needed\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n\n    for epoch in range(num_epochs):\n        model.train()\n        for batch_idx, (frames, labels) in enumerate(train_loader):\n            frames = frames.to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(frames)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            if batch_idx % 10 == 0:\n                print(f'Epoch {epoch}/{num_epochs}, Batch {batch_idx}, Loss: {loss.item()}')\n\n# Initialize and train the model\nmodel = CapsuleNetwork()\ntrain_model(model, train_loader)","metadata":{"execution":{"iopub.status.busy":"2024-09-08T08:53:24.328041Z","iopub.execute_input":"2024-09-08T08:53:24.328683Z","iopub.status.idle":"2024-09-08T08:57:19.571195Z","shell.execute_reply.started":"2024-09-08T08:53:24.32864Z","shell.execute_reply":"2024-09-08T08:57:19.569742Z"},"trusted":true},"execution_count":null,"outputs":[]}]}