{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport subprocess\nimport glob\nimport os\nfrom pathlib import Path\nimport shutil\nfrom zipfile import ZipFile\nfrom scipy import signal\nfrom scipy.io import wavfile\nfrom skopt import gp_minimize\nfrom skopt.space import Real\nfrom functools import partial\nimport librosa.display\nimport librosa.filters\nimport matplotlib.pyplot as plt\nimport skimage","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-30T17:43:17.428102Z","iopub.execute_input":"2023-03-30T17:43:17.428624Z","iopub.status.idle":"2023-03-30T17:43:21.487581Z","shell.execute_reply.started":"2023-03-30T17:43:17.428568Z","shell.execute_reply":"2023-03-30T17:43:21.486394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport random\nfrom tqdm import tqdm\nimport xgboost as xgb\nimport tensorflow as tf\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.models import Model\nfrom keras.preprocessing import image\nfrom keras.applications.resnet50 import preprocess_input, decode_predictions\nfrom keras.layers import Flatten, Input\nimport scipy\nfrom sklearn.metrics import fbeta_score","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:21.489983Z","iopub.execute_input":"2023-03-30T17:43:21.490283Z","iopub.status.idle":"2023-03-30T17:43:26.361225Z","shell.execute_reply.started":"2023-03-30T17:43:21.490225Z","shell.execute_reply":"2023-03-30T17:43:26.360515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using the Static Build of ffmpeg from https://johnvansickle.com/ffmpeg/ because internet is not available. <br>\nThe public data set can be found here:\nhttps://www.kaggle.com/rakibilly/ffmpeg-static-build\n","metadata":{}},{"cell_type":"code","source":"! tar xvf ../input/ffmpeg-static-build/ffmpeg-git-amd64-static.tar.xz","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-03-30T17:43:26.362464Z","iopub.execute_input":"2023-03-30T17:43:26.362696Z","iopub.status.idle":"2023-03-30T17:43:31.858101Z","shell.execute_reply.started":"2023-03-30T17:43:26.362659Z","shell.execute_reply":"2023-03-30T17:43:31.857094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Specify output format and create a directory for the output Audio files\nFor 400 mp3 files, the directory is approx 94 MB.<br>\nFor 400 wav files, the directory is approx 673 MB.","metadata":{}},{"cell_type":"code","source":"DATA_FOLDER = '../input/deepfake-detection-challenge/'\nTRAIN_SAMPLE_FOLDER = 'train_sample_videos/'\nTEST_FOLDER = 'test_videos/'\nDATA_PATH = os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER)\nos.makedirs('/kaggle/working/output', exist_ok=True)\nos.makedirs('/kaggle/working/test_output', exist_ok=True)\nOUTPUT_PATH = '/kaggle/working/output'\nTEST_OUTPUT_PATH = '/kaggle/working/test_output/'\nprint(f\"Train samples: {len(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))}\")\nprint(f\"Test samples: {len(os.listdir(os.path.join(DATA_FOLDER, TEST_FOLDER)))}\")\nSPLIT='00'","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:31.859452Z","iopub.execute_input":"2023-03-30T17:43:31.859733Z","iopub.status.idle":"2023-03-30T17:43:32.033278Z","shell.execute_reply.started":"2023-03-30T17:43:31.859682Z","shell.execute_reply":"2023-03-30T17:43:32.032292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_list = list(os.listdir(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER)))\next_dict = []\nfor file in train_list:\n    file_ext = file.split('.')[1]\n    if (file_ext not in ext_dict):\n        ext_dict.append(file_ext)\nprint(f\"Extensions: {ext_dict}\")      ","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.036523Z","iopub.execute_input":"2023-03-30T17:43:32.037099Z","iopub.status.idle":"2023-03-30T17:43:32.045669Z","shell.execute_reply.started":"2023-03-30T17:43:32.037034Z","shell.execute_reply":"2023-03-30T17:43:32.044392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_file = [file for file in train_list if  file.endswith('json')][0]\nprint(f\"JSON file: {json_file}\")","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.047942Z","iopub.execute_input":"2023-03-30T17:43:32.048383Z","iopub.status.idle":"2023-03-30T17:43:32.055501Z","shell.execute_reply.started":"2023-03-30T17:43:32.048283Z","shell.execute_reply":"2023-03-30T17:43:32.054356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_meta_from_json(path):\n    df = pd.read_json(os.path.join(DATA_FOLDER, path, json_file))\n    df = df.T\n    return df\n\nmeta_train_df = get_meta_from_json(TRAIN_SAMPLE_FOLDER)\nmeta_train_df.head(20)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.056780Z","iopub.execute_input":"2023-03-30T17:43:32.057222Z","iopub.status.idle":"2023-03-30T17:43:32.537567Z","shell.execute_reply.started":"2023-03-30T17:43:32.057177Z","shell.execute_reply":"2023-03-30T17:43:32.536612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_format = 'wav'  # can also use aac, wav, etc\n\noutput_dir = Path(f\"{output_format}s\")\nPath(output_dir).mkdir(exist_ok=True, parents=True)\nfake_name ='aaeflzzhvy'\nreal_name = 'flqgmnetsg'","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.539351Z","iopub.execute_input":"2023-03-30T17:43:32.539742Z","iopub.status.idle":"2023-03-30T17:43:32.546063Z","shell.execute_reply.started":"2023-03-30T17:43:32.539668Z","shell.execute_reply":"2023-03-30T17:43:32.544675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta = (list(meta_train_df.index))","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.547838Z","iopub.execute_input":"2023-03-30T17:43:32.548280Z","iopub.status.idle":"2023-03-30T17:43:32.555704Z","shell.execute_reply.started":"2023-03-30T17:43:32.548154Z","shell.execute_reply":"2023-03-30T17:43:32.554610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get the list of videos to extract audio from","metadata":{}},{"cell_type":"code","source":"INPUT_PATH = '../input/realfake045/assorted/'\nWAV_PATH = './wavs/'","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.557112Z","iopub.execute_input":"2023-03-30T17:43:32.557429Z","iopub.status.idle":"2023-03-30T17:43:32.567151Z","shell.execute_reply.started":"2023-03-30T17:43:32.557373Z","shell.execute_reply":"2023-03-30T17:43:32.566383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_of_files = []\nfor file in os.listdir(os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER)):\n    filename = os.path.join(DATA_FOLDER,TRAIN_SAMPLE_FOLDER)+file\n    list_of_files.append(filename)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.568840Z","iopub.execute_input":"2023-03-30T17:43:32.569536Z","iopub.status.idle":"2023-03-30T17:43:32.578003Z","shell.execute_reply.started":"2023-03-30T17:43:32.569477Z","shell.execute_reply":"2023-03-30T17:43:32.577314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract the audio from files","metadata":{}},{"cell_type":"code","source":"def create_wav(list_of_files):\n    for file in list_of_files:\n        command = f\"../working/ffmpeg-git-20191209-amd64-static/ffmpeg -i {file} -ab 192000 -ac 2 -ar 44100 -vn {output_dir/file[-14:-4]}.{output_format}\"\n        subprocess.call(command, shell=True)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.579477Z","iopub.execute_input":"2023-03-30T17:43:32.579838Z","iopub.status.idle":"2023-03-30T17:43:32.588172Z","shell.execute_reply.started":"2023-03-30T17:43:32.579770Z","shell.execute_reply":"2023-03-30T17:43:32.586923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncreate_wav(list_of_files)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:43:32.589526Z","iopub.execute_input":"2023-03-30T17:43:32.589987Z","iopub.status.idle":"2023-03-30T17:44:27.605931Z","shell.execute_reply.started":"2023-03-30T17:43:32.589929Z","shell.execute_reply":"2023-03-30T17:44:27.604585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_spectogram(name,sr):\n    audio_array, sample_rate = librosa.load(WAV_PATH+f'{name}', sr=sr)\n    trim_audio_array, index = librosa.effects.trim(audio_array)\n    S = librosa.feature.melspectrogram(y=trim_audio_array, sr=sr, n_mels=128, fmax=8000)\n    S_dB = np.log(S + 1e-9)\n    # min-max scale to fit inside 8-bit range\n    img = scale_minmax(S_dB, 0, 255).astype(np.uint8)\n    img = np.flip(img, axis=0) # put low frequencies at the bottom in image\n    img = 255-img # invert. make black==more energy\n    #S_dB = librosa.power_to_db(S, ref=np.max)\n    return S_dB ,img\n\ndef scale_minmax(X, min=0.0, max=1.0):\n    X_std = (X - X.min()) / (X.max() - X.min())\n    X_scaled = X_std * (max - min) + min\n    return X_scaled","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-03-30T17:47:21.363093Z","iopub.execute_input":"2023-03-30T17:47:21.363445Z","iopub.status.idle":"2023-03-30T17:47:21.372815Z","shell.execute_reply.started":"2023-03-30T17:47:21.363395Z","shell.execute_reply":"2023-03-30T17:47:21.371780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ni=0\nsr=20000\nfake_images = []\nreal_images = []\nfor index,row in meta_train_df.iterrows():\n    if row.label == 'FAKE':\n        if os.path.exists(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER,row.original)):\n              if os.path.exists(os.path.join(DATA_FOLDER, TRAIN_SAMPLE_FOLDER,index)):\n                    fake_name = index.split('.')[0]+'.wav'\n                    real_name =row.original.split('.')[0]+'.wav'\n                    S_fake,img_fake =create_spectogram(fake_name,sr)\n                    S_real,img_real =create_spectogram(real_name,sr)\n                    fake_images.append(img_fake)\n                    real_images.append(img_real)\n                    if not(np.array_equal(S_fake,S_real)):\n                        diff = np.sum(np.abs(S_real - S_fake))\n                        print(f\"There is a difference in Audio : {diff}\")\n                        plt.figure(figsize=(10, 4))\n                        plt.axis('off')\n                        #librosa.display.specshow(S_fake, x_axis='time',\n                        #          y_axis='mel', sr=sr,\n                        #          fmax=8000)\n                        plt.imshow(img_fake,cmap='gray')\n                        #image.append(img_fake)\n                        plt.colorbar(format='%+2.0f dB')\n                        plt.title(f'Mel-frequency spectrogram Fake name {fake_name}')\n                        plt.tight_layout()\n                        plt.show()\n                        plt.figure(figsize=(10, 4))\n                        plt.axis('off')\n                        #librosa.display.specshow(S_real, x_axis='time',\n                        #          y_axis='mel', sr=sr,\n                        #          fmax=8000)\n                        plt.imshow(img_real,cmap='gray')\n                        plt.colorbar(format='%+2.0f dB')\n                        plt.title(f'Mel-frequency spectrogram Real name {real_name}')\n                        #image1.append(img_real)\n                        plt.tight_layout()\n                        plt.show()\n            \n    i=i+1 ","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:47:23.298361Z","iopub.execute_input":"2023-03-30T17:47:23.298854Z","iopub.status.idle":"2023-03-30T17:48:24.080757Z","shell.execute_reply.started":"2023-03-30T17:47:23.298807Z","shell.execute_reply":"2023-03-30T17:48:24.076756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saving spectograms in a directory as images","metadata":{}},{"cell_type":"code","source":"import os\nos.mkdir(\"spectogram_images\")","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:50:06.999080Z","iopub.execute_input":"2023-03-30T17:50:06.999616Z","iopub.status.idle":"2023-03-30T17:50:07.004230Z","shell.execute_reply.started":"2023-03-30T17:50:06.999559Z","shell.execute_reply":"2023-03-30T17:50:07.003262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = len(real_images)\nfor i in range(l):\n    plt.imshow(real_images[i])\n    plt.savefig(\"/kaggle/working/spectogram_images/\"+str(i+1)+\"_real.png\")\n    plt.close()\nfor i in range(l):\n    plt.imshow(fake_images[i])\n    plt.savefig(\"/kaggle/working/spectogram_images/\"+str(i+1)+\"_fake.png\")\n    plt.close()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T17:59:26.614311Z","iopub.execute_input":"2023-03-30T17:59:26.614617Z","iopub.status.idle":"2023-03-30T17:59:49.345921Z","shell.execute_reply.started":"2023-03-30T17:59:26.614570Z","shell.execute_reply":"2023-03-30T17:59:49.344482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with ZipFile(f'all_{output_format}s.zip', 'w') as zipObj:\n   # Iterate over all the files in directory\n   for folderName, subfolders, filenames in os.walk(f'./{output_format}s/'):\n       for filename in filenames:\n           #create complete filepath of file in directory\n           filePath = os.path.join(folderName, filename)\n           # Add file to zip\n           zipObj.write(filePath)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:16:01.183262Z","iopub.execute_input":"2023-03-30T16:16:01.183981Z","iopub.status.idle":"2023-03-30T16:16:03.875918Z","shell.execute_reply.started":"2023-03-30T16:16:01.183715Z","shell.execute_reply":"2023-03-30T16:16:03.874883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Cleanup","metadata":{}},{"cell_type":"code","source":"#Remove FFMPEG directory from output\nshutil.rmtree(\"../working/ffmpeg-git-20191209-amd64-static\")\n#Remove directory of output files\nshutil.rmtree(f'./{output_format}s/')","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:16:03.877393Z","iopub.execute_input":"2023-03-30T16:16:03.877657Z","iopub.status.idle":"2023-03-30T16:16:04.055787Z","shell.execute_reply.started":"2023-03-30T16:16:03.877619Z","shell.execute_reply":"2023-03-30T16:16:04.054894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(image[0])","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:16:04.056923Z","iopub.execute_input":"2023-03-30T16:16:04.057319Z","iopub.status.idle":"2023-03-30T16:16:04.063451Z","shell.execute_reply.started":"2023-03-30T16:16:04.057275Z","shell.execute_reply":"2023-03-30T16:16:04.062581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(image1[0])","metadata":{"execution":{"iopub.status.busy":"2023-03-30T16:16:04.064791Z","iopub.execute_input":"2023-03-30T16:16:04.065170Z","iopub.status.idle":"2023-03-30T16:16:04.077690Z","shell.execute_reply.started":"2023-03-30T16:16:04.065109Z","shell.execute_reply":"2023-03-30T16:16:04.076408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}