{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# friendly reminder：Meta-information detection has been officially banned\n#鸡肋，食之无味，弃之可惜","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print(os.listdir('../input'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('../input/ddocuments'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install mfc_video_utils\n!yes|apt-get install --no-install-recommends ffmpeg && pip install ffmpeg scikit-video","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom mfc_video_utils import MfcVideoProcessor, BasicTransformer, compute_roc, save_object, load_object, grid_search_forest, grid_search_svm\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import mfc_video_utils\nfrom mfc_video_utils import MfcVideoProcessor, BasicTransformer, compute_roc, save_object, load_object, grid_search_forest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gs_forest_50 = load_object('../input/ddocuments/my_forest50_icml.pkl')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MfcVideoProcessor:\n    ''' Class to process MFC video datasets easily'''\n    frac_list = ['video_@codec_time_base', 'video_@r_frame_rate', 'video_@avg_frame_rate', \n             'video_@time_base', 'audio_@r_frame_rate','audio_@codec_time_base', \n             'audio_@time_base', 'subtitle_@r_frame_rate', 'subtitle_@time_base', \n             'data_@r_frame_rate', 'data_@avg_frame_rate', 'data_@time_base']\n    time_list = ['video_tag_@creation_time', 'audio_tag_@creation_time', 'video_tag_@DateTime',\n             'video_tag_@DateTimeOriginal', 'video_tag_@DateTimeDigitized', 'data_tag_@creation_time']\n    \n    def __init__(self, name: str, dataset_abs_path: Path, ref_avail: bool=False, ref_folder: bool=False):\n        '''__init__ constructor'''\n        if not dataset_abs_path.exists():\n            raise Exception(\"Dataset path does not exist\")\n        else:\n            self.name = name\n            self.basepath = Path(dataset_abs_path)\n            self.probes = self.basepath / \"probe\"\n            self.ref_avail = ref_avail\n            if ref_avail and ref_folder:\n                self.reference_basepath = Path(str(dataset_abs_path) + '-Reference')\n                if not self.reference_basepath.exists():\n                    raise Exception(\"Path to reference folder does not exist\")\n                else:\n                    self.csv_path = [item for item in (self.reference_basepath / \"reference/manipulation-video/\").glob(\"*-manipulation-video-ref.csv\")][0]\n                    self.csv = pd.read_csv(self.csv_path, sep=\"|\")\n                    self.labels = [self.get_video_label(i) for i in range(len(self.csv))]\n            elif ref_avail:\n                self.csv_path = [item for item in self.basepath.glob(\"reference/manipulation-video/*-manipulation-video-ref.csv\")][0]\n                self.csv = pd.read_csv(self.csv_path, sep=\"|\")\n                self.labels = [self.get_video_label(i) for i in range(len(self.csv))]\n            else:\n                pass\n#                 self.csv_path = [item for item in self.basepath.glob(\"indexes/manipulation-video/*-manipulation-video-ref.csv\")][0]\n#                 self.csv = pd.read_csv(self.csv_path, sep=\"|\")\n                \n            self.ffprobe_df = self._generate_ffprobe_dataset_dataframe()\n\n    def get_video_path(self, video_id: int) -> str:\n        video_name = str(self.csv.at[video_id, 'ProbeFileName'])\n        return str(self.basepath / video_name)\n\n    def get_video_label(self, video_id: int) -> int:\n        if self.ref_avail:\n            return 1 if self.csv.at[video_id, 'IsTarget'] is 'Y' else 0\n        else:\n            raise Exception(\"This dataset has no reference available\")\n            \n            \n    def _generate_ffprobe_dataset_dataframe(self) -> pd.DataFrame:\n        dfs = [] #creates a new dataframe that's empty\n        for i in tqdm(os.listdir(self.basepath)):\n            if i.endswith(\"mp4\"):\n                video_path = f'{self.basepath}/{i}'\n                video_metadata = self._flatten(skvideo.io.ffprobe(video_path))\n                video_df = pd.DataFrame.from_dict(video_metadata, orient='index', columns=[i])\n                dfs.append(video_df)\n        vids_df = pd.concat(dfs, axis=1, sort=False)\n        vids_df = vids_df.transpose()\n        vids_df = vids_df.apply(pd.to_numeric, errors='ignore')\n        for col in MfcVideoProcessor.frac_list:\n            if col in vids_df.columns:\n                vids_df[col] = vids_df[col].apply(self._conv_to_float)\n        for col in MfcVideoProcessor.time_list:\n            def time_transform(x): \n                if pd.notnull(x) and type(x) is not str:\n                    return x.to_datetime64().astype(np.int64) \n                else:\n                    return np.nan\n            if col in vids_df.columns:\n                vids_df[col] = vids_df[col].apply(pd.to_datetime, errors='ignore').apply(time_transform)\n        return vids_df\n\n    def _flatten(self, d: dict, parent_key: str = '', sep: str = '_') -> dict:\n        items = []\n        for k, v in d.items():\n            new_key = '{0}{1}{2}'.format(parent_key,sep,k) if parent_key else k\n            if isinstance(v, MutableMapping):\n                items.extend(self._flatten(v, new_key, sep=sep).items())\n            elif isinstance(v, list):\n                # apply itself to each element of the list - that's it!\n                dic_list = self._flatten_tag(v, parent_key=new_key)\n                for k_2, v_2 in dic_list.items():\n                    items.append((k_2, v_2))\n            else:\n                items.append((new_key, v))\n        return dict(items)\n\n    def _flatten_tag(self, l: list, parent_key: str = '', sep: str = '_') -> dict:\n        items = []\n        for ele in l:\n            new_key = '{0}{1}{2}{3}'.format(parent_key,sep,'@',ele['@key']) \n            items.append((new_key, ele['@value']))\n        return dict(items)\n\n    def _conv_to_float(self, frac: str) -> float:\n        if pd.notnull(frac):\n            try: return float(fractions.Fraction(frac))\n            except ZeroDivisionError: return 0\n        else:\n            return frac\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import skvideo\nfrom collections.abc import MutableMapping\nimport fractions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_datasets_location = Path(\"//kaggle/input/deepfake-detection-challenge/test_videos/\")\ntest = MfcVideoProcessor(\"dfdc_vid\", test_datasets_location, ref_avail=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = gs_forest_50.predict(test.ffprobe_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub=pd.read_csv('/kaggle/input/deepfake-detection-challenge/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = '/kaggle/input/deepfake-detection-challenge/test_videos/'\nfilenames = os.listdir(test_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for pred,name in zip(pred,filenames):\n    name=name.replace('/kaggle/input/deepfake-detection-challenge/test_videos/','')\n    sub.iloc[list(sub['filename']).index(name),1]=pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}