{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":16880,"databundleVersionId":858837,"sourceType":"competition"},{"sourceId":924245,"sourceType":"datasetVersion","datasetId":464091}],"dockerImageVersionId":29845,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Title","metadata":{"id":"Q6tasuafvT2O"}},{"cell_type":"markdown","source":"Deep Fake Image and Video Detection using CNN's and RNN's","metadata":{"id":"LvJBeHv3vfE_"}},{"cell_type":"markdown","source":"# Problem Statement","metadata":{"id":"xyukrIAev64Y"}},{"cell_type":"markdown","source":"DeepFake is composed from Deep Learning and Fake and means taking one person from an image or video and replacing with someone else likeness using technology such as Deep Artificial Neural Networks. Large companies like Google invest very much in fighting the DeepFake, this including release of large datasets to help training models to counter this threat.The phenomen invades rapidly the film industry and threatens to compromise news agencies. Large digital companies, including content providers and social platforms are in the frontrun of fighting Deep Fakes. GANs that generate DeepFakes becomes better every day and, of course, if you include in a new GAN model all the information we collected until now how to combat various existent models, we create a model that cannot be beatten by the existing ones.\n\nFirst we will work on detecting faces that were forged and we will work on developing a model to detect videos.","metadata":{"id":"MZsX2Wrrv9Op"}},{"cell_type":"markdown","source":"# DeepFake Image detection","metadata":{"id":"vGkKRV1kY8Z-"}},{"cell_type":"markdown","source":"## About the Dataset","metadata":{"id":"ZLLkoLT1wsPL"}},{"cell_type":"markdown","source":"This dataset contains faces extracted from deepfake-detection-challenge. All images were of size 224x224. \n\nDue to memory issue we will only use a sample of the entire dataset for prediction.","metadata":{"id":"y_3tT_2lwv-v"}},{"cell_type":"markdown","source":"## Importing Required libraries","metadata":{"id":"dFXIv9qNpKzt","tags":[]}},{"cell_type":"code","source":"!pip install -U --upgrade tensorflow","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nimport sklearn\nimport tensorflow as tf\n\nimport cv2\nimport pandas as pd\nimport numpy as np\n\nimport plotly.graph_objs as go\nfrom plotly.offline import iplot\nfrom matplotlib import pyplot as plt","metadata":{"id":"TFSU3FCOpKzu","execution":{"iopub.status.busy":"2024-12-05T19:05:48.051644Z","iopub.execute_input":"2024-12-05T19:05:48.051899Z","iopub.status.idle":"2024-12-05T19:05:48.05632Z","shell.execute_reply.started":"2024-12-05T19:05:48.051846Z","shell.execute_reply":"2024-12-05T19:05:48.055649Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.test.is_gpu_available()","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:05:51.051964Z","iopub.execute_input":"2024-12-05T19:05:51.052282Z","iopub.status.idle":"2024-12-05T19:05:51.653016Z","shell.execute_reply.started":"2024-12-05T19:05:51.052222Z","shell.execute_reply":"2024-12-05T19:05:51.652078Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:05:52.777918Z","iopub.execute_input":"2024-12-05T19:05:52.778203Z","iopub.status.idle":"2024-12-05T19:05:52.783256Z","shell.execute_reply.started":"2024-12-05T19:05:52.778165Z","shell.execute_reply":"2024-12-05T19:05:52.78237Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.rc('font', size=14)\nplt.rc('axes', labelsize=14, titlesize=14)\nplt.rc('legend', fontsize=14)\nplt.rc('xtick', labelsize=10)\nplt.rc('ytick', labelsize=10)","metadata":{"id":"8d4TH3NbpKzx","execution":{"iopub.status.busy":"2024-12-05T19:05:53.336581Z","iopub.execute_input":"2024-12-05T19:05:53.336863Z","iopub.status.idle":"2024-12-05T19:05:53.342523Z","shell.execute_reply.started":"2024-12-05T19:05:53.336822Z","shell.execute_reply":"2024-12-05T19:05:53.34158Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Visualisation","metadata":{"id":"NL3Ht4wC9b3n"}},{"cell_type":"code","source":"import os\n\ndef get_data():\n    return pd.read_csv('../input/deepfake-faces/metadata.csv')","metadata":{"id":"jfv9PxSB4tM8","execution":{"iopub.status.busy":"2024-12-05T19:05:54.071086Z","iopub.execute_input":"2024-12-05T19:05:54.071685Z","iopub.status.idle":"2024-12-05T19:05:54.076609Z","shell.execute_reply.started":"2024-12-05T19:05:54.071368Z","shell.execute_reply":"2024-12-05T19:05:54.075644Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta=get_data()\nmeta.head()","metadata":{"id":"tDW7BRph9ehF","outputId":"97de18b5-0a37-4302-8804-8a16a7d2ed2f","execution":{"iopub.status.busy":"2024-12-05T19:05:54.546784Z","iopub.execute_input":"2024-12-05T19:05:54.547068Z","iopub.status.idle":"2024-12-05T19:05:54.66167Z","shell.execute_reply.started":"2024-12-05T19:05:54.547027Z","shell.execute_reply":"2024-12-05T19:05:54.660885Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"meta.shape","metadata":{"id":"n7FSdDifbZxn","outputId":"5451a127-405a-4c0b-a197-c920b796adbb","execution":{"iopub.status.busy":"2024-12-05T19:05:54.793306Z","iopub.execute_input":"2024-12-05T19:05:54.793628Z","iopub.status.idle":"2024-12-05T19:05:54.799063Z","shell.execute_reply.started":"2024-12-05T19:05:54.793579Z","shell.execute_reply":"2024-12-05T19:05:54.798357Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(meta[meta.label=='FAKE']),len(meta[meta.label=='REAL'])","metadata":{"id":"_FJcz2IthxVG","outputId":"274c3f65-7acb-4f99-8aa9-a5b2a23bf06a","execution":{"iopub.status.busy":"2024-12-05T19:05:55.374714Z","iopub.execute_input":"2024-12-05T19:05:55.375001Z","iopub.status.idle":"2024-12-05T19:05:55.400893Z","shell.execute_reply.started":"2024-12-05T19:05:55.374959Z","shell.execute_reply":"2024-12-05T19:05:55.400168Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"real_df = meta[meta[\"label\"] == \"REAL\"]\nfake_df = meta[meta[\"label\"] == \"FAKE\"]\nsample_size = 8000\n\nreal_df = real_df.sample(sample_size, random_state=42)\nfake_df = fake_df.sample(sample_size, random_state=42)\n\nsample_meta = pd.concat([real_df, fake_df])","metadata":{"id":"IgMfzY-PjjtH","execution":{"iopub.status.busy":"2024-12-05T19:05:55.58436Z","iopub.execute_input":"2024-12-05T19:05:55.584714Z","iopub.status.idle":"2024-12-05T19:05:55.617826Z","shell.execute_reply.started":"2024-12-05T19:05:55.584666Z","shell.execute_reply":"2024-12-05T19:05:55.617191Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As mentioned instead of using 95k images we will only use 16000 images.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nTrain_set, Test_set = train_test_split(sample_meta,test_size=0.2,random_state=42,stratify=sample_meta['label'])\nTrain_set, Val_set  = train_test_split(Train_set,test_size=0.3,random_state=42,stratify=Train_set['label'])","metadata":{"id":"5eB86S6K-T5Z","execution":{"iopub.status.busy":"2024-12-05T19:05:56.872437Z","iopub.execute_input":"2024-12-05T19:05:56.872699Z","iopub.status.idle":"2024-12-05T19:05:57.132992Z","shell.execute_reply.started":"2024-12-05T19:05:56.87266Z","shell.execute_reply":"2024-12-05T19:05:57.132415Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Train_set.shape,Val_set.shape,Test_set.shape","metadata":{"id":"8p-TONijb4qA","outputId":"56d0b529-9d81-4019-d8fa-618c8cdba90f","execution":{"iopub.status.busy":"2024-12-05T19:05:57.134849Z","iopub.execute_input":"2024-12-05T19:05:57.13515Z","iopub.status.idle":"2024-12-05T19:05:57.145698Z","shell.execute_reply.started":"2024-12-05T19:05:57.135094Z","shell.execute_reply":"2024-12-05T19:05:57.144851Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor cur,i in enumerate(Train_set.index[25:50]):\n    plt.subplot(5,5,cur+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    \n    plt.imshow(cv2.imread('../input/deepfake-faces/faces_224/'+Train_set.loc[i,'videoname'][:-4]+'.jpg'))\n    \n    if(Train_set.loc[i,'label']=='FAKE'):\n        plt.xlabel('FAKE Image')\n    else:\n        plt.xlabel('REAL Image')\n        \nplt.show()","metadata":{"id":"VR7Uly2fcUYi","outputId":"c1f47a82-ef4f-4bcd-b51c-d4738142fc0f","execution":{"iopub.status.busy":"2024-12-05T19:05:58.310829Z","iopub.execute_input":"2024-12-05T19:05:58.3111Z","iopub.status.idle":"2024-12-05T19:06:00.055348Z","shell.execute_reply.started":"2024-12-05T19:05:58.311059Z","shell.execute_reply":"2024-12-05T19:06:00.054576Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Modelling","metadata":{"id":"dOvN_divkl-N"}},{"cell_type":"markdown","source":"Before jumping to use pretrained model lets develop some base line model to test how our pretrained model outperforms.","metadata":{}},{"cell_type":"markdown","source":"### Custom CNN Architecture","metadata":{"id":"oid44Xx-pKz6"}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"def retreive_dataset(set_name):\n    images,labels=[],[]\n    for (img, imclass) in zip(set_name['videoname'], set_name['label']):\n        images.append(cv2.imread('../input/deepfake-faces/faces_224/'+img[:-4]+'.jpg'))\n        if(imclass=='FAKE'):\n            labels.append(1)\n        else:\n            labels.append(0)\n    \n    return np.array(images),np.array(labels)","metadata":{"id":"Hz0ZdQ_fgHhG","execution":{"iopub.status.busy":"2024-12-05T19:06:03.116486Z","iopub.execute_input":"2024-12-05T19:06:03.11684Z","iopub.status.idle":"2024-12-05T19:06:03.122215Z","shell.execute_reply.started":"2024-12-05T19:06:03.116776Z","shell.execute_reply":"2024-12-05T19:06:03.121529Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train,y_train=retreive_dataset(Train_set)\nX_val,y_val=retreive_dataset(Val_set)\nX_test,y_test=retreive_dataset(Test_set)","metadata":{"id":"zeAGRcAbguKU","execution":{"iopub.status.busy":"2024-12-05T19:06:04.08722Z","iopub.execute_input":"2024-12-05T19:06:04.087536Z","iopub.status.idle":"2024-12-05T19:06:48.041736Z","shell.execute_reply.started":"2024-12-05T19:06:04.087477Z","shell.execute_reply":"2024-12-05T19:06:48.040855Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from functools import partial\n\ntf.random.set_seed(42) \nDefaultConv2D = partial(tf.keras.layers.Conv2D, kernel_size=3, padding=\"same\",\n                        activation=\"relu\", kernel_initializer=\"he_normal\")\n\nmodel = tf.keras.Sequential([\n    DefaultConv2D(filters=64, kernel_size=7, input_shape=[224, 224, 3]),\n    tf.keras.layers.MaxPool2D(),\n    DefaultConv2D(filters=128),\n    DefaultConv2D(filters=128),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(units=128, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=64, activation=\"relu\",\n                          kernel_initializer=\"he_normal\"),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(units=1, activation=\"sigmoid\")\n])","metadata":{"id":"34upiak4pKz6","execution":{"iopub.status.busy":"2024-12-05T19:06:53.749657Z","iopub.execute_input":"2024-12-05T19:06:53.749944Z","iopub.status.idle":"2024-12-05T19:06:54.001193Z","shell.execute_reply.started":"2024-12-05T19:06:53.749902Z","shell.execute_reply":"2024-12-05T19:06:54.000458Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.compile(loss=\"binary_crossentropy\", optimizer=\"nadam\",\n              metrics=[\"accuracy\"])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:07:02.978157Z","iopub.execute_input":"2024-12-05T19:07:02.978481Z","iopub.status.idle":"2024-12-05T19:07:03.024962Z","shell.execute_reply.started":"2024-12-05T19:07:02.978399Z","shell.execute_reply":"2024-12-05T19:07:03.0242Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=5,batch_size=64,\n                    validation_data=(X_val, y_val))","metadata":{"id":"KZbWeIBYpKz6","outputId":"deb6f56a-7b93-4241-a1bd-b210c0f2d426","execution":{"iopub.status.busy":"2024-12-05T19:07:10.042147Z","iopub.execute_input":"2024-12-05T19:07:10.042479Z","iopub.status.idle":"2024-12-05T19:08:01.486161Z","shell.execute_reply.started":"2024-12-05T19:07:10.042419Z","shell.execute_reply":"2024-12-05T19:08:01.484842Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score = model.evaluate(X_test, y_test)","metadata":{"id":"6HDDr4uehast","execution":{"iopub.status.busy":"2024-12-05T19:08:07.651282Z","iopub.execute_input":"2024-12-05T19:08:07.651609Z","iopub.status.idle":"2024-12-05T19:08:11.362749Z","shell.execute_reply.started":"2024-12-05T19:08:07.651558Z","shell.execute_reply":"2024-12-05T19:08:11.361925Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:08:19.470174Z","iopub.execute_input":"2024-12-05T19:08:19.470485Z","iopub.status.idle":"2024-12-05T19:08:19.494948Z","shell.execute_reply.started":"2024-12-05T19:08:19.470439Z","shell.execute_reply":"2024-12-05T19:08:19.493943Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"A baseline score of 50.06% is good to go let's finetune some pretrained model","metadata":{}},{"cell_type":"markdown","source":"# Pretrained Models for Transfer Learning","metadata":{"id":"hqxnSBJ3pKz8"}},{"cell_type":"markdown","source":"Here i used Xception model for fine-tuning feel free to try the performance of other pretrained models.","metadata":{}},{"cell_type":"markdown","source":"All three datasets contain individual images. We need to batch them, but for this we first need to ensure they all have the same size, or else batching will not work. We can use a `Resizing` layer for this. We must also call the `tf.keras.applications.xception.preprocess_input()` function to preprocess the images appropriately for the Xception model. We will also add shuffling and prefetching to the training dataset.","metadata":{"id":"gXG6iv8XpKz9"}},{"cell_type":"code","source":"train_set_raw=tf.data.Dataset.from_tensor_slices((X_train,y_train))\nvalid_set_raw=tf.data.Dataset.from_tensor_slices((X_val,y_val))\ntest_set_raw=tf.data.Dataset.from_tensor_slices((X_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:09:09.447696Z","iopub.execute_input":"2024-12-05T19:09:09.448072Z","iopub.status.idle":"2024-12-05T19:09:11.822773Z","shell.execute_reply.started":"2024-12-05T19:09:09.44801Z","shell.execute_reply":"2024-12-05T19:09:11.821954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tf.keras.backend.clear_session()  # extra code – resets layer name counter\n\nbatch_size = 32\npreprocess = tf.keras.applications.xception.preprocess_input","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:10:11.649168Z","iopub.execute_input":"2024-12-05T19:10:11.649508Z","iopub.status.idle":"2024-12-05T19:10:11.65665Z","shell.execute_reply.started":"2024-12-05T19:10:11.649455Z","shell.execute_reply":"2024-12-05T19:10:11.655755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ntrain_set = train_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y))\ntrain_set = train_set.shuffle(1000, seed=42).batch(batch_size).prefetch(1)\nvalid_set = valid_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)\ntest_set = test_set_raw.map(lambda X, y: (preprocess(tf.cast(X, tf.float32)), y)).batch(batch_size)","metadata":{"id":"Bnz0n9XApKz9","execution":{"iopub.status.busy":"2024-12-05T19:13:49.507052Z","iopub.execute_input":"2024-12-05T19:13:49.507417Z","iopub.status.idle":"2024-12-05T19:13:49.639124Z","shell.execute_reply.started":"2024-12-05T19:13:49.50734Z","shell.execute_reply":"2024-12-05T19:13:49.63831Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's take a look again at the first 9 images from the validation set: they're all with values ranging from -1 to 1:","metadata":{"id":"ovNEMky-pKz9"}},{"cell_type":"code","source":"# extra code – displays the first 9 images in the first batch of valid_set\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        plt.imshow((X_batch[index] + 1) / 2)  # rescale to 0–1 for imshow()\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"ZL3c3i4opKz9","outputId":"38847d8d-8822-41a3-cfb2-27479aa5debe","execution":{"iopub.status.busy":"2024-12-05T19:13:52.897563Z","iopub.execute_input":"2024-12-05T19:13:52.897902Z","iopub.status.idle":"2024-12-05T19:13:54.424597Z","shell.execute_reply.started":"2024-12-05T19:13:52.897855Z","shell.execute_reply":"2024-12-05T19:13:54.42381Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def augment_image(image):\n    # Random horizontal flip\n    image = tf.image.random_flip_left_right(image)\n    \n    # Random rotation\n    image = tf.image.rot90(image, tf.random.uniform([], 0, 4, dtype=tf.int32))  # Rotate by 90, 180, 270, or 0 degrees\n    \n    # Random contrast adjustment\n    image = tf.image.random_contrast(image, lower=0.8, upper=1.2)\n    \n    return image\n\n# Use it in your data pipeline\ndata_augmentation = tf.keras.Sequential([\n    tf.keras.layers.Lambda(lambda x: augment_image(x))\n])\n","metadata":{"id":"Ib0cA8Y1pKz9","execution":{"iopub.status.busy":"2024-12-05T19:14:15.483805Z","iopub.execute_input":"2024-12-05T19:14:15.484101Z","iopub.status.idle":"2024-12-05T19:14:15.493805Z","shell.execute_reply.started":"2024-12-05T19:14:15.48406Z","shell.execute_reply":"2024-12-05T19:14:15.493091Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Try running the following cell multiple times to see different random data augmentations:","metadata":{"id":"G7GrQjsspKz-"}},{"cell_type":"code","source":"# extra code – displays the same first 9 images, after augmentation\n\nplt.figure(figsize=(12, 12))\nfor X_batch, y_batch in valid_set.take(1):\n    X_batch_augmented = data_augmentation(X_batch, training=True)\n    for index in range(9):\n        plt.subplot(3, 3, index + 1)\n        # We must rescale the images to the 0-1 range for imshow(), and also\n        # clip the result to that range, because data augmentation may\n        # make some values go out of bounds (e.g., RandomContrast in this case).\n        plt.imshow(np.clip((X_batch_augmented[index] + 1) / 2, 0, 1))\n        if(y_batch[index]==1):\n            classt='FAKE'\n        else:\n            classt='REAL'\n        plt.title(f\"Class: {classt}\")\n        plt.axis(\"off\")\n\nplt.show()","metadata":{"id":"w6GH5_vupKz-","outputId":"eeb2c924-2f4f-4aa1-bea9-951bebef4bf0","execution":{"iopub.status.busy":"2024-12-05T19:14:58.488071Z","iopub.execute_input":"2024-12-05T19:14:58.488361Z","iopub.status.idle":"2024-12-05T19:14:59.995218Z","shell.execute_reply.started":"2024-12-05T19:14:58.488319Z","shell.execute_reply":"2024-12-05T19:14:59.994467Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now let's load the pretrained model, without its top layers, and replace them with our own task","metadata":{"id":"kNL9AOsDpKz-"}},{"cell_type":"code","source":"tf.random.set_seed(42)  # extra code – ensures reproducibility\nbase_model = tf.keras.applications.xception.Xception(weights=\"imagenet\",\n                                                     include_top=False)\navg = tf.keras.layers.GlobalAveragePooling2D()(base_model.output)\noutput = tf.keras.layers.Dense(1, activation=\"sigmoid\")(avg)\nmodel = tf.keras.Model(inputs=base_model.input, outputs=output)","metadata":{"id":"lRyCgvaKpKz-","outputId":"a825e173-8b1d-4217-a1c4-5491b49c3e82","execution":{"iopub.status.busy":"2024-12-05T19:30:19.036131Z","iopub.execute_input":"2024-12-05T19:30:19.036423Z","iopub.status.idle":"2024-12-05T19:30:21.08548Z","shell.execute_reply.started":"2024-12-05T19:30:19.036368Z","shell.execute_reply":"2024-12-05T19:30:21.084666Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for layer in base_model.layers:\n    layer.trainable = False","metadata":{"id":"KBlyG6ElpKz-","execution":{"iopub.status.busy":"2024-12-05T19:30:29.257329Z","iopub.execute_input":"2024-12-05T19:30:29.257693Z","iopub.status.idle":"2024-12-05T19:30:29.265156Z","shell.execute_reply.started":"2024-12-05T19:30:29.25763Z","shell.execute_reply":"2024-12-05T19:30:29.264238Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's train the model for a few epochs, while keeping the base model weights fixed:","metadata":{"id":"WFEFw7GKpKz-"}},{"cell_type":"code","source":"!pip install timm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:21:27.751394Z","iopub.execute_input":"2024-12-05T19:21:27.75175Z","iopub.status.idle":"2024-12-05T19:22:47.633991Z","shell.execute_reply.started":"2024-12-05T19:21:27.751706Z","shell.execute_reply":"2024-12-05T19:22:47.632979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# how to train  Transformers from Timm library","metadata":{}},{"cell_type":"code","source":"from timm.models.mobilenetv3 import mobilenetv3_large_075","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:22:51.505652Z","iopub.execute_input":"2024-12-05T19:22:51.505959Z","iopub.status.idle":"2024-12-05T19:22:52.436847Z","shell.execute_reply.started":"2024-12-05T19:22:51.505911Z","shell.execute_reply":"2024-12-05T19:22:52.436131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_transfomers=mobilenetv3_large_075()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:29:56.295862Z","iopub.execute_input":"2024-12-05T19:29:56.296192Z","iopub.status.idle":"2024-12-05T19:29:56.38589Z","shell.execute_reply.started":"2024-12-05T19:29:56.296134Z","shell.execute_reply":"2024-12-05T19:29:56.384925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn as nn\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:27:02.546192Z","iopub.execute_input":"2024-12-05T19:27:02.546574Z","iopub.status.idle":"2024-12-05T19:27:02.550313Z","shell.execute_reply.started":"2024-12-05T19:27:02.546512Z","shell.execute_reply":"2024-12-05T19:27:02.549606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_classes=2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:27:51.283113Z","iopub.execute_input":"2024-12-05T19:27:51.283435Z","iopub.status.idle":"2024-12-05T19:27:51.287634Z","shell.execute_reply.started":"2024-12-05T19:27:51.283353Z","shell.execute_reply":"2024-12-05T19:27:51.286492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_transfomers.head=nn.Linear(1280,num_classes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-05T19:30:09.160615Z","iopub.execute_input":"2024-12-05T19:30:09.160904Z","iopub.status.idle":"2024-12-05T19:30:09.165884Z","shell.execute_reply.started":"2024-12-05T19:30:09.160863Z","shell.execute_reply":"2024-12-05T19:30:09.164934Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# End of Example","metadata":{}},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=3)","metadata":{"id":"GGxK2yPcpKz-","outputId":"6b64214a-e104-4b6c-9b7a-3388fc9aa15f","execution":{"iopub.status.busy":"2024-12-05T19:30:56.291136Z","iopub.execute_input":"2024-12-05T19:30:56.291429Z","iopub.status.idle":"2024-12-05T19:32:46.446043Z","shell.execute_reply.started":"2024-12-05T19:30:56.291373Z","shell.execute_reply":"2024-12-05T19:32:46.445245Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for indices in zip(range(33), range(33, 66), range(66, 99), range(99, 132)):\n    for idx in indices:\n        print(f\"{idx:3}: {base_model.layers[idx].name:22}\", end=\"\")\n    print()","metadata":{"id":"GvGMiJMLpKz-","outputId":"91f2c96c-c058-45e0-e428-66fa6076ad56","execution":{"iopub.status.busy":"2024-12-05T19:29:23.901602Z","iopub.execute_input":"2024-12-05T19:29:23.901929Z","iopub.status.idle":"2024-12-05T19:29:23.952347Z","shell.execute_reply.started":"2024-12-05T19:29:23.901884Z","shell.execute_reply":"2024-12-05T19:29:23.951547Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:29:29.968485Z","iopub.execute_input":"2024-12-05T19:29:29.968795Z","iopub.status.idle":"2024-12-05T19:29:29.995197Z","shell.execute_reply.started":"2024-12-05T19:29:29.96875Z","shell.execute_reply":"2024-12-05T19:29:29.993855Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now with the finetuning the top layers of xception model the model performance jumps to 63.8% ","metadata":{}},{"cell_type":"markdown","source":"Now that the weights of our new top layers are not too bad, we can make the top part of the base model trainable again, and continue training, but with a lower learning rate:","metadata":{"id":"L_bEwL8KpKz_"}},{"cell_type":"code","source":"for layer in base_model.layers[56:]:\n    layer.trainable = True\n\noptimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)\nmodel.compile(loss=\"binary_crossentropy\", optimizer=optimizer,\n              metrics=[\"accuracy\"])\nhistory = model.fit(train_set, validation_data=valid_set, epochs=10)","metadata":{"id":"GEUNGlhvpKz_","outputId":"c622a91d-f634-4443-b87e-8d46defdb578","execution":{"iopub.status.busy":"2024-12-05T19:32:46.447679Z","iopub.execute_input":"2024-12-05T19:32:46.447905Z","iopub.status.idle":"2024-12-05T19:33:48.018335Z","shell.execute_reply.started":"2024-12-05T19:32:46.447867Z","shell.execute_reply":"2024-12-05T19:33:48.01671Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot model performance\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs_range = range(1, len(history.epoch) + 1)\n\nplt.figure(figsize=(15,5))\n\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Train Set')\nplt.plot(epochs_range, val_acc, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.title('Model Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Train Set')\nplt.plot(epochs_range, val_loss, label='Val Set')\nplt.legend(loc=\"best\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.title('Model Loss')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.evaluate(test_set)","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:34:06.575714Z","iopub.execute_input":"2024-12-05T19:34:06.576016Z","iopub.status.idle":"2024-12-05T19:34:14.671528Z","shell.execute_reply.started":"2024-12-05T19:34:06.575974Z","shell.execute_reply":"2024-12-05T19:34:14.670715Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The model accuracy finally reaches to 81.9%","metadata":{}},{"cell_type":"code","source":"model.save('xception_deepfake_image.h5')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Add Explainability to the model","metadata":{}},{"cell_type":"markdown","source":"Lets try to interpret the trained model on how it finds a image FAKE","metadata":{}},{"cell_type":"code","source":"!pip install lime","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lime import lime_image\n\nexplainer = lime_image.LimeImageExplainer()","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:35:37.995991Z","iopub.execute_input":"2024-12-05T19:35:37.996287Z","iopub.status.idle":"2024-12-05T19:35:38.793859Z","shell.execute_reply.started":"2024-12-05T19:35:37.996244Z","shell.execute_reply":"2024-12-05T19:35:38.793017Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 12))\n\nfor index in range(9):\n    plt.subplot(3, 3, index + 1)\n    plt.imshow((x[index] + 1) / 2)  # rescale to 0–1 for imshow()\n    if(y[index]==1):\n        classt='FAKE'\n    else:\n        classt='REAL'\n    plt.title(f\"Class: {classt}\")\n    plt.axis(\"off\")\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data=X_test[2,:,:,:]\ntest_data.shape","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:35:29.059872Z","iopub.execute_input":"2024-12-05T19:35:29.060173Z","iopub.status.idle":"2024-12-05T19:35:29.065963Z","shell.execute_reply.started":"2024-12-05T19:35:29.060131Z","shell.execute_reply":"2024-12-05T19:35:29.065152Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"explanation = explainer.explain_instance(test_data.astype('double'), model.predict,  \n                                         top_labels=3, hide_color=0, num_samples=1000)","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:35:41.738789Z","iopub.execute_input":"2024-12-05T19:35:41.739097Z","iopub.status.idle":"2024-12-05T19:35:54.264879Z","shell.execute_reply.started":"2024-12-05T19:35:41.739053Z","shell.execute_reply":"2024-12-05T19:35:54.258369Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from skimage.segmentation import mark_boundaries\n\ntemp_1, mask_1 = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=True, num_features=5, hide_rest=True)\ntemp_2, mask_2 = explanation.get_image_and_mask(explanation.top_labels[0], positive_only=False, num_features=10, hide_rest=False)\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,15))\nax1.imshow(mark_boundaries(temp_1, mask_1))\nax2.imshow(mark_boundaries(temp_2, mask_2))\nax1.axis('off')\nax2.axis('off')","metadata":{"execution":{"iopub.status.busy":"2024-12-05T19:35:57.615578Z","iopub.execute_input":"2024-12-05T19:35:57.615887Z","iopub.status.idle":"2024-12-05T19:35:57.974767Z","shell.execute_reply.started":"2024-12-05T19:35:57.615841Z","shell.execute_reply":"2024-12-05T19:35:57.973826Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}