{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n### Install imutils\n!pip install imutils #need to have internet on- see Settings\n\n#### Import packages\nimport os\nimport numpy as np\nimport shutil\nimport cv2\nimport pandas as pd\nimport matplotlib\nmatplotlib.use(\"Agg\")\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pickle\nfrom imutils import paths\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications import VGG16\nfrom keras.layers.core import Dropout\nfrom keras.layers.core import Flatten\nfrom keras.layers.core import Dense\nfrom keras.layers import Input\nfrom keras.models import Model\nfrom keras.optimizers import SGD\nfrom sklearn.metrics import classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#CONFIG \n\n# initialize the base path to the *new* directory that will contain\n# our images after computing the training and testing split\nBASE_PATH = \"/kaggle/working/finetuningkeras/dataset\"\n\n# define the names of the training, testing, and validation\n# directories\nTRAIN = \"training\"\nTEST = \"evaluation\"\nVAL = \"validation\"\n\nREAL = 'REAL'\nFAKE = 'FAKE'\n\n# initialize the list of class label names\nCLASSES = [\"FAKE\", \"REAL\"]\n\n\n# set the batch size when fine-tuning\nBATCH_SIZE = 32\n\ntrainEpochs = 10\nepochsFineTune = 10\nmaxVids = 5\n\n# set the path to the serialized model after training\nMODEL_PATH = os.path.sep.join([\"/kaggle/working/finetuningkeras\",\"output\", \"Deepfake.model\"])\n\n# define the path to the output training history plots\nUNFROZEN_PLOT_PATH = os.path.sep.join([\"/kaggle/working/finetuningkeras\",\"output\", \"unfrozen.png\"])\nWARMUP_PLOT_PATH = os.path.sep.join([\"/kaggle/working/finetuningkeras\",\"output\", \"warmup.png\"])\n\nfile = '/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json'\nimg_path = '/kaggle/input/deepfake-detection-challenge/train_sample_videos'\ndata_path = '/kaggle/working/finetuningkeras/real_fake'\ndir_fake_frames = '/kaggle/working/FAKE_frames'\ndir_real_frames = '/kaggle/working/REAL_frames'\ndir_output = '/kaggle/working/finetuningkeras/output'\n\ndir_data_path_real = os.path.join(data_path, REAL)\ndir_data_path_fake = os.path.join(data_path, FAKE)\n\ndir_train_real = os.path.join(BASE_PATH, TRAIN, REAL)\ndir_train_fake = os.path.join(BASE_PATH, TRAIN, FAKE)\ndir_valid_real = os.path.join(BASE_PATH, VAL, REAL)\ndir_valid_fake = os.path.join(BASE_PATH, VAL, FAKE)\ndir_test_real = os.path.join(BASE_PATH, TEST, REAL)\ndir_test_fake = os.path.join(BASE_PATH, TEST, FAKE)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nBreak mp4 files into individual images/frames (jpg)\ninput_dir: input/source directory containing one or more mp4 files\noutput_dir: target directory for saving individual frames (format: filename_frame#.jpg)\nmaxN: maximum number of mp4 files to explode\n\"\"\"\ninput_dir = '/kaggle/working/finetuningkeras/real_fake/FAKE'\noutput_dir = '/kaggle/working/FAKE_frames/'\ndef explode_frames(input_dir, output_dir, maxN):\n\n    mp4_filenames = [f for f in os.listdir(input_dir) if f.endswith('.mp4')]\n    n = 0\n    \n    for mp4fn in mp4_filenames:\n        \n        if(n < maxN):\n            n += 1 \n            mp4fp = os.path.join(input_dir, mp4fn)\n            cam = cv2.VideoCapture(mp4fp) \n            if(cam.isOpened()):\n                print('Processing file #'+ str(n) + ' (' + mp4fn + ')...')\n            else: \n                print('Problem opening file #'+ str(n) + ' (' + mp4fn + ')...')\n                continue \n            \n            nframe = 0\n            while(True): #continue until ret = False then break\n                nframe += 1\n                ret,frame = cam.read()\n                \n                if ret: \n                    # if video is still left continue creating images \n                    out_filename = os.path.splitext(mp4fn)[0]+  '_frame' + str(nframe) + '.jpg'\n                    out_filepath =  os.path.join(output_dir, out_filename)\n                    \n                    # writing the extracted images \n                    cv2.imwrite(out_filepath, frame) \n                else: \n                    break\n\n            # Release all space and windows once done\n            print(' - created ' + str(nframe-1) + ' images') # -1 bc count incremented before exit\n            cam.release() \n            cv2.destroyAllWindows()\n            \n        else: \n            break\n            \n\"\"\"\nDistribute files/images from a source directory into training, validation, and testing directories. \nsrc_dir = source/input directory\ntrain_dir, val_dir, test_dir = target training/validation/testing directory\nvalperc = fraction of dataset to use for validation (0-1)\ntestperc = fraction of dataset to use for testing (0-1)\n\"\"\"\n            \ndef trainvaltest_split(src_dir, train_dir, val_dir, test_dir, valperc = 0.15, testperc = 0.15):\n    \n    filenames = os.listdir(src_dir) #get all filenames in random order\n    np.random.shuffle(filenames)\n    \n    n = len(filenames)\n    split1 = int(n*(1 - (valperc + testperc)))\n    split2 = int(n*(1 - (testperc)))\n    \n    fn_train, fn_val, fn_test = np.split(np.array(filenames), [split1, split2])\n    \n    fn_lists = [fn_train, fn_val, fn_test]\n    targetdirs = [train_dir, val_dir, test_dir]\n    \n    print('Total images: ', n)\n    print('Training: ', len(fn_train))\n    print('Validation: ', len(fn_val))\n    print('Testing: ', len(fn_test))\n    \n    all_fp = [os.path.join(src_dir, fn) for fn in filenames]\n    \n    #move files\n    for i, fn_list in enumerate(fn_lists):\n        for fn in fn_list: \n            target_dir = targetdirs[i]\n            fp_from = os.path.join(src_dir, fn)\n            fp_to = os.path.join(target_dir, fn)\n            \n            shutil.move(fp_from, fp_to)\n\n            \n\"\"\"\nConstruct a plot that plots and saves the training history\n\"\"\"           \ndef plot_training(H, N, plotPath):\n\tplt.style.use(\"ggplot\")\n\tplt.figure()\n\tplt.plot(np.arange(0, N), H.history[\"loss\"], label=\"train_loss\")\n\tplt.plot(np.arange(0, N), H.history[\"val_loss\"], label=\"val_loss\")\n\tplt.plot(np.arange(0, N), H.history[\"accuracy\"], label=\"train_acc\")\n\tplt.plot(np.arange(0, N), H.history[\"val_accuracy\"], label=\"val_acc\")\n\tplt.title(\"Training Loss and Accuracy\")\n\tplt.xlabel(\"Epoch #\")\n\tplt.ylabel(\"Loss/Accuracy\")\n\tplt.legend(loc=\"lower left\")\n\tplt.savefig(plotPath)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"######### CREATING FOLDERS AND DATA STRUCTURE ########\n# # Creating Train / Val / Test folders (One time use)\n\nos.makedirs(dir_train_real, exist_ok = True)\nos.makedirs(dir_train_fake, exist_ok = True)\nos.makedirs(dir_valid_real, exist_ok = True)\nos.makedirs(dir_valid_fake, exist_ok = True)\nos.makedirs(dir_test_real, exist_ok = True)\nos.makedirs(dir_test_fake, exist_ok = True)\n\nos.makedirs(dir_data_path_real, exist_ok = True)\nos.makedirs(dir_data_path_fake, exist_ok = True)\nos.makedirs(dir_fake_frames, exist_ok = True) \nos.makedirs(dir_real_frames, exist_ok = True) \nos.makedirs(dir_output, exist_ok = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in labels from json file\ndf = pd.read_json(file)\ndf = df.T\n\n# %% [code]\nlabel = df[['label']] #df with .mp4 filename index and label field","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# copy training sample videos into /kaggle/working/finetuningkeras/real_fake/[FAKE or REAL]\n# target dir\n\nfor fn, row in label.iterrows():\n    src = os.path.join(img_path, fn)\n    dest = os.path.join(data_path, row['label'], fn)\n    shutil.copy(src, dest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract individual frames from fake vids\nexplode_frames(dir_data_path_fake, dir_fake_frames, maxN= maxVids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract individual frames from real vids\nexplode_frames(dir_data_path_real, dir_real_frames, maxN= maxVids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Split fake frames into training, validation, test sets\ntrainvaltest_split(src_dir = dir_fake_frames,\n                   train_dir = dir_train_fake, \n                   val_dir = dir_valid_fake, \n                   test_dir = dir_test_fake)\n\n## Split real frames into training, validation, test sets\ntrainvaltest_split(src_dir = dir_real_frames,\n                   train_dir = dir_train_real, \n                   val_dir = dir_valid_real, \n                   test_dir = dir_test_real)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### IMAGE CLASSIFCATION ###\n# initialize the training data augmentation object\n\ntrainAug = ImageDataGenerator(\n\trotation_range=30,\n\tzoom_range=0.15,\n\twidth_shift_range=0.2,\n\theight_shift_range=0.2,\n\tshear_range=0.15,\n\thorizontal_flip=True,\n\tfill_mode=\"nearest\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# initialize the validation/testing data augmentation object (which\n# we'll be adding mean subtraction to)\nvalAug = ImageDataGenerator()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define the ImageNet mean subtraction (in RGB order) and set the\n# the mean subtraction value for each of the data augmentation\n# objects\nmean = np.array([123.68, 116.779, 103.939], dtype=\"float32\")\ntrainAug.mean = mean\nvalAug.mean = mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ImageDataGenerator.flow_from_direcotry treats subdir names as classes\n\n# initialize the training generator\ntrainPath = os.path.join(BASE_PATH, TRAIN)\ntrainGen = trainAug.flow_from_directory(\n\ttrainPath,\n\tclass_mode=\"categorical\",\n\ttarget_size=(224, 224),\n\tcolor_mode=\"rgb\",\n\tshuffle=True,\n\tbatch_size=BATCH_SIZE)\n\n# initialize the validation generator\nvalPath = os.path.join(BASE_PATH, VAL)\nvalGen = valAug.flow_from_directory(\n\tvalPath,\n\tclass_mode=\"categorical\",\n\ttarget_size=(224, 224),\n\tcolor_mode=\"rgb\",\n\tshuffle=False,\n\tbatch_size=BATCH_SIZE)\n\n# initialize the testing generator\ntestPath = os.path.join(BASE_PATH, TEST)\ntestGen = valAug.flow_from_directory(\n\ttestPath,\n\tclass_mode=\"categorical\",\n\ttarget_size=(224, 224),\n\tcolor_mode=\"rgb\",\n\tshuffle=False,\n\tbatch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load the VGG16 network, ensuring the head FC layer sets are left off\nbaseModel = VGG16(weights=\"imagenet\", include_top=False,\n\tinput_tensor=Input(shape=(224, 224, 3)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# construct the head of the model that will be placed on top of the\n# the base model\nheadModel = baseModel.output\nheadModel = Flatten(name=\"flatten\")(headModel)\nheadModel = Dense(512, activation=\"relu\")(headModel)\nheadModel = Dropout(0.5)(headModel)\nheadModel = Dense(len(CLASSES), activation=\"softmax\")(headModel)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# place the head FC model on top of the base model (this will become\n# the actual model we will train)\nmodel = Model(inputs=baseModel.input, outputs=headModel)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loop over all layers in the base model and freeze them so they will\n# *not* be updated during the first training process\nfor layer in baseModel.layers:\n\tlayer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# compile our model (this needs to be done after our setting our\n# layers to being non-trainable\nprint(\"[INFO] compiling model...\")\nopt = SGD(lr=1e-4, momentum=0.9)\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=opt,\n\tmetrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train the head of the network for a few epochs (all other layers\n# are frozen) -- this will allow the new FC layers to start to become\n# initialized with actual \"learned\" values versus pure random\n# and testing directories\ntotalTrain = len(list(paths.list_images(trainPath)))\ntotalVal = len(list(paths.list_images(valPath)))\ntotalTest = len(list(paths.list_images(testPath)))\n\nprint(\"[INFO] training head...\")\nH = model.fit_generator(\n\ttrainGen,\n\tsteps_per_epoch=totalTrain // BATCH_SIZE,\n\tvalidation_data=valGen,\n\tvalidation_steps=totalVal // BATCH_SIZE,\n\tepochs= trainEpochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reset the testing generator and evaluate the network after\n# fine-tuning just the network head\nprint(\"[INFO] evaluating after fine-tuning network head...\")\ntestGen.reset()\npredIdxs = model.predict_generator(testGen,\n\tsteps=(totalTest // BATCH_SIZE) + 1)\npredIdxs = np.argmax(predIdxs, axis=1)\nprint(classification_report(testGen.classes, predIdxs,\n\ttarget_names=testGen.class_indices.keys()))\n\n\nplot_training(H, trainEpochs, WARMUP_PLOT_PATH)\nplt.show()  #plot not showning in kaggle notebook.. does it support ploting?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reset our data generators\ntrainGen.reset()\nvalGen.reset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# now that the head FC layers have been trained/initialized, lets\n# unfreeze the final set of CONV layers and make them trainable\nfor layer in baseModel.layers[15:]:\n\tlayer.trainable = True\n\n# loop over the layers in the model and show which ones are trainable\n# or not\nfor layer in baseModel.layers:\n\tprint(\"{}: {}\".format(layer, layer.trainable))\n\n# for the changes to the model to take affect we need to recompile\n# the model, this time using SGD with a *very* small learning rate\nprint(\"[INFO] re-compiling model...\")\nopt = SGD(lr=1e-4, momentum=0.9)\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=opt,\n\tmetrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train the model again, this time fine-tuning *both* the final set\n# of CONV layers along with our set of FC layers\nH = model.fit_generator(\n\ttrainGen,\n\tsteps_per_epoch=totalTrain // BATCH_SIZE,\n\tvalidation_data=valGen,\n\tvalidation_steps=totalVal // BATCH_SIZE,\n\tepochs= epochsFineTune)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reset the testing generator and then use our trained model to\n# make predictions on the data\nprint(\"[INFO] evaluating after fine-tuning network...\")\ntestGen.reset()\npredIdxs = model.predict_generator(testGen,\n\tsteps=(totalTest // BATCH_SIZE) + 1)\npredIdxs = np.argmax(predIdxs, axis=1)\nprint(classification_report(testGen.classes, predIdxs,\n\ttarget_names=testGen.class_indices.keys()))\nplot_training(H, epochsFineTune, UNFROZEN_PLOT_PATH)\n\n# serialize the model to disk\nprint(\"[INFO] serializing network...\")\nmodel.save(MODEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### fast.ai section ####","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision import *\nfrom fastai.metrics import error_rate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntfms = get_transforms(do_flip=False)\n\n\n# fastai subdirs need to be named train, valid, test (test is optional)\nos.rename('/kaggle/working/finetuningkeras/dataset/training', \n          '/kaggle/working/finetuningkeras/dataset/train')\nos.rename('/kaggle/working/finetuningkeras/dataset/validation', \n          '/kaggle/working/finetuningkeras/dataset/valid')\nos.rename('/kaggle/working/finetuningkeras/dataset/evaluation', \n          '/kaggle/working/finetuningkeras/dataset/test')\n\n\ndata = ImageDataBunch.from_folder(BASE_PATH, ds_tfms=tfms, size=224)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.show_batch(rows=3, figsize=(5,5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn = cnn_learner(data, models.resnet18, metrics=accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit(2)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}