{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport glob\nimport random\nimport math\nimport numpy as np\nrandom.seed(123)\n\nimport cv2\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom IPython.display import display\nfrom scipy.stats import norm\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data"},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"s_submission_df = pd.read_csv(\"../input/deepfake-detection-challenge/sample_submission.csv\")\nmetadata_df = pd.read_json('../input/deepfake-detection-challenge/train_sample_videos/metadata.json').transpose()\n\ntrain_embeddings = pd.read_csv(\"../input/edadfdc/train_embeddings.csv\")\ntest_embeddings = pd.read_csv(\"../input/edadfdc/test_embeddings.csv\")\n\naligned_train_embeddings = pd.read_csv(\"../input/edadfdc/aligned_train_embeddings.csv\")\naligned_test_embeddings = pd.read_csv(\"../input/edadfdc/aligned_test_embeddings.csv\")\n\ntrain_embeddings = pd.merge(left=train_embeddings, right=metadata_df[['label']], left_on='128', right_index=True)\naligned_train_embeddings = pd.merge(left=aligned_train_embeddings, right=metadata_df[['label']], left_on='128', right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metadata_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def func(pct, allvals):\n    absolute = int(pct/100.*np.sum(allvals))\n    return \"{:.1f}%\\n({:d} faces)\".format(pct, absolute)\n\nfig, axes = plt.subplots(1, 2, figsize=(10, 5))\n\naxes[0].bar(['train', 'test'], [train_embeddings.shape[0], test_embeddings.shape[0]])\naxes[0].set_ylabel('# of faces')\n\n\ndata = [train_embeddings.label.value_counts().REAL,\n        train_embeddings.label.value_counts().FAKE]\n\nwedges, texts, autotexts = axes[1].pie(data, autopct=lambda pct: func(pct, data),\n                                       textprops=dict(color=\"w\"))\naxes[1].legend(wedges, ['real', 'fake'],\n          title=\"Label\",\n          loc=\"center left\",\n          bbox_to_anchor=(1, 0, 0.5, 1))\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize Data"},{"metadata":{},"cell_type":"markdown","source":"## Embedding vectors"},{"metadata":{},"cell_type":"markdown","source":"Clean data<p>\nVideos that have been claimed to have more than 625 faces by the detector needs to be removed (we are assuming that each video can only have one or more persons which would have a maximum of 600 faces)"},{"metadata":{"trusted":true},"cell_type":"code","source":"outliers = train_embeddings.groupby('128').size()\nindex = (outliers > 625).to_numpy().nonzero()[0]\ntrain_embeddings = train_embeddings[~train_embeddings['128'].isin(outliers[index].index)]\naligned_train_embeddings = aligned_train_embeddings[~aligned_train_embeddings['128'].isin(outliers[index].index)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_embeddings.groupby('128').size())\nplt.title(\"Amount of embeddings in a video\")\nplt.xlabel('# of embeddings')\nplt.ylabel('frequency')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Visualize data points"},{"metadata":{},"cell_type":"markdown","source":"Comparing the not aligned and aligned figures  "},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.decomposition import PCA\n\ndef reduce_dims(df, video):\n    fake_emb_reduced = PCA(n_components=2).fit_transform(df.loc[(df['128']==video) & (df.label=='FAKE')][np.arange(0,128).astype(str)])\n    real_video = metadata_df.loc[metadata_df.index==video]['original'].values[0]\n    real_emb_reduced = PCA(n_components=2).fit_transform(df.loc[(df['128']==real_video) & (df.label=='REAL')][np.arange(0,128).astype(str)])\n    return fake_emb_reduced, real_emb_reduced, real_video\n    \nfake_samples = np.random.choice(metadata_df.loc[\n    (metadata_df.label=='FAKE') &\n    (metadata_df.original.isin(train_embeddings['128'])) &\n    (metadata_df.index.isin(train_embeddings['128']))].index, 4)\n\nfig, axes = plt.subplots(2, 4, figsize=(18, 8))\nfig.suptitle('Scatter plot fake/real video embeddings', fontsize=16)\nfor idx, video in enumerate(fake_samples):\n    fake_emb_reduced, real_emb_reduced, real_video = reduce_dims(train_embeddings, video)\n    axes[0, idx].scatter(fake_emb_reduced[:, 0], fake_emb_reduced[:, 1])\n    axes[0, idx].set_title(f'{video} (FAKE)')\n    axes[1, idx].scatter(real_emb_reduced[:, 0], real_emb_reduced[:, 1], color='pink')\n    axes[1, idx].set_title(f'{real_video} (REAL)')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 4, figsize=(18, 8))\nfig.suptitle('Scatter plot real/fake aligned video embeddings', fontsize=16)\nfor idx, video in enumerate(fake_samples):\n    real_video = metadata_df.loc[metadata_df.index==video]['original'].values[0]\n    fake_emb_reduced, real_emb_reduced, real_video = reduce_dims(aligned_train_embeddings, video)\n    axes[0, idx].scatter(fake_emb_reduced[:, 0], fake_emb_reduced[:, 1])\n    axes[0, idx].set_title(f'{video} (FAKE)')\n    axes[1, idx].scatter(real_emb_reduced[:, 0], real_emb_reduced[:, 1], color='pink')\n    axes[1, idx].set_title(f'{real_video} (REAL)')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"distances_videos = {}\ndistances_align_videos = {}\n\n\ndef get_mean_distance(df, video):\n    emb = df[(df['128']==video)][np.arange(0,128).astype(str)]\n    avg_point = np.mean(emb)\n    distances = np.linalg.norm(emb - avg_point, axis=1)\n    return np.mean(distances)\n    \nfor video in train_embeddings['128'].unique():\n    distances_videos[video] = get_mean_distance(train_embeddings, video)\n\nfor video in aligned_train_embeddings['128'].unique():\n    distances_align_videos[video] = get_mean_distance(aligned_train_embeddings, video)\n\ndistances_df = pd.DataFrame(distances_videos.items())\ndistances_df.columns = ['Video', 'Spread']\ndistances_df = distances_df.set_index('Video')\n\nal_distances_df = pd.DataFrame(distances_align_videos.items())\nal_distances_df.columns = ['Video', 'Spread']\nal_distances_df = al_distances_df.set_index('Video')\n\ntrain_embeddings = pd.merge(left=train_embeddings, right=distances_df['Spread'], left_on='128', right_index=True)\naligned_train_embeddings = pd.merge(left=aligned_train_embeddings, right=al_distances_df['Spread'], left_on='128', right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12, 4))\naxes[0].hist(train_embeddings.loc[train_embeddings.label=='FAKE'].groupby('128').mean()['Spread'])\naxes[1].hist(train_embeddings.loc[train_embeddings.label=='REAL'].groupby('128').mean()['Spread'], color='pink')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(12, 4))\naxes[0].hist(aligned_train_embeddings.loc[aligned_train_embeddings.label=='FAKE'].groupby('128').mean()['Spread'])\naxes[1].hist(aligned_train_embeddings.loc[aligned_train_embeddings.label=='REAL'].groupby('128').mean()['Spread'], color='pink')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = [train_embeddings.loc[train_embeddings.label=='FAKE'].groupby('128').mean()['Spread'],\n        train_embeddings.loc[train_embeddings.label=='REAL'].groupby('128').mean()['Spread']]\nal_data = [aligned_train_embeddings.loc[aligned_train_embeddings.label=='FAKE'].groupby('128').mean()['Spread'],\n        aligned_train_embeddings.loc[aligned_train_embeddings.label=='REAL'].groupby('128').mean()['Spread']]\n\nfig, axes = plt.subplots(1, 2, figsize=(12, 5))\nfig.suptitle('Boxplot average distance of points in a video')\nb1 = axes[0].boxplot(data, labels=['fake', 'real'], patch_artist=True, widths=[0.3, 0.3])\nb2 = axes[1].boxplot(al_data, labels=['fake', 'real'], patch_artist=True, widths=[0.3, 0.3])\n\naxes[0].yaxis.grid(True)\naxes[0].set_title('Not Aligned')\naxes[1].yaxis.grid(True)\naxes[1].set_title('Aligned')\n\nb1['boxes'][1].set_facecolor('pink')\nb2['boxes'][1].set_facecolor('pink')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.cluster import DBSCAN\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import log_loss\n\nfrom xgboost import XGBClassifier\nfrom sklearn.svm import SVC\n\ndef cluster_faces(embeddings_df, eps, min_samples):\n    embeddings_data = []\n    for video in metadata_df.index:\n        data = embeddings_df.loc[embeddings_df['128']==video]\n        embeddings = data[np.arange(0,128).astype(str)]\n        avg_point = np.mean(embeddings)\n        distances = np.linalg.norm(embeddings - avg_point, axis=1)\n        label = metadata_df.loc[metadata_df.index==video].label.values[0]\n        if embeddings.empty:\n            embeddings_data.append([video, 0, 0, 0, label])\n            continue\n        clf = DBSCAN(eps=eps, min_samples=min_samples, n_jobs=6).fit(embeddings)\n        classes = np.bincount(clf.labels_+1).shape[0] - 1\n        noises = np.bincount(clf.labels_+1)[0]\n        embeddings_data.append([video, classes, noises, np.mean(distances), label])\n    return embeddings_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def evaluate_cluster(embeddings_df, eps=0.5, min_samples=100):\n    embeddings_data = pd.DataFrame(cluster_faces(embeddings_df, eps, min_samples))\n    embeddings_data.columns = ['Video', 'Classes', 'Noises', 'Spread', 'Label']\n    y = embeddings_data.Label.apply(lambda x: 0 if x == \"REAL\" else 1).values\n    X = embeddings_data[['Classes', 'Noises', 'Spread']]\n#     clf = LogisticRegression(solver='lbfgs', random_state=0)\n    clf = SVC(kernel='rbf', gamma=0.7, C=1.0, probability=True)\n    score = cross_val_score(clf, X, y, cv=5, scoring='neg_log_loss')\n    return score\n\ndef evaluate_aligned_cluster(min_samples):\n    eps = np.median(aligned_train_embeddings.loc[aligned_train_embeddings.label=='REAL'].groupby('128').mean()['Spread'])\n    return evaluate_cluster(aligned_train_embeddings, eps, min_samples)\n    \ndef evaluate_not_aligned_cluster(min_samples):\n    eps = np.median(train_embeddings.loc[train_embeddings.label=='REAL'].groupby('128').mean()['Spread'])\n    return evaluate_cluster(train_embeddings, eps, min_samples)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"min_sample_values = np.arange(30, 34, 1)\nprint ('Calculando para dados não alinhados')\nscore = (list(map(evaluate_not_aligned_cluster, min_sample_values)))\nmean_score = list(map(lambda s: -s.mean(), score))\n\nprint ('Calculando para dados alinhados')\naligned_score = (list(map(evaluate_aligned_cluster, min_sample_values)))\naligned_mean_score = list(map(lambda s: -s.mean(), aligned_score))\n\nfor idx, s in enumerate(mean_score):\n    print (f'[Not aligned] Score for {min_sample_values[idx]}: {s}')\nfor idx, s in enumerate(aligned_mean_score):\n    print (f'[Aligned] Score for {min_sample_values[idx]}: {s}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are going to start the evaluation with a *min_samples* value of 300, wich means that for a point to belong to the dense region of a cluster it needs to have at least 300 points close to its neighborhood"},{"metadata":{"trusted":true},"cell_type":"code","source":"eps = np.median(train_embeddings.loc[train_embeddings.label=='REAL'].groupby('128').mean()['Spread'])\nal_eps = np.median(aligned_train_embeddings.loc[aligned_train_embeddings.label=='REAL'].groupby('128').mean()['Spread'])\n\nembeddings_data_df = pd.DataFrame(cluster_faces(train_embeddings, eps=eps, min_samples=32))\nembeddings_data_df.columns = ['Video', 'Classes', 'Noises', 'Spread', 'Label']\ndisplay(embeddings_data_df.head(5))\n\nal_embeddings_data_df = pd.DataFrame(cluster_faces(aligned_train_embeddings, eps=al_eps, min_samples=33))\nal_embeddings_data_df.columns = ['Video', 'Classes', 'Noises', 'Spread', 'Label']\ndisplay(al_embeddings_data_df.head(5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. Noises"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(14,8))\n\nlbls = ['REAL', 'FAKE']\nc = ['pink', 'C0']\nfor idx in range(2):\n    axes[0, idx].hist(embeddings_data_df.loc[embeddings_data_df.Label==lbls[idx]].Noises.values, color=c[idx])\n    axes[0, idx].set_xlabel(f'# of noise in {lbls[idx].lower()} videos (not aligned)')\n    axes[0, idx].set_ylabel('frequency')\n\nfor idx in range(2, 4):\n    axes[1, idx-2].hist(al_embeddings_data_df.loc[al_embeddings_data_df.Label==lbls[idx-2]].Noises.values, color=c[idx-2])\n    axes[1, idx-2].set_xlabel(f'# of noise in {lbls[idx-2].lower()} videos (aligned)')\n    axes[1, idx-2].set_ylabel('frequency')\n    \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(14, 4))\n\nc = list(map(lambda x: 'pink' if x == 'REAL' else 'C0', embeddings_data_df.Label.values))\naxes[0].scatter(embeddings_data_df.Noises, embeddings_data_df.Label, c=c)\naxes[0].set_xlabel('# of noises')\naxes[0].set_title('Target vs Noises in videos')\naxes[1].scatter(al_embeddings_data_df.Noises, embeddings_data_df.Label, c=c)\naxes[1].set_title('Target vs Noises in aligned videos')\naxes[1].set_xlabel('# of noises')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2. Classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(14, 8))\n\nlbls = ['REAL', 'FAKE']\nc = ['pink', 'C0']\nfor idx in range(2):\n    axes[0, idx].hist(embeddings_data_df.loc[embeddings_data_df.Label==lbls[idx]].Classes.values, color=c[idx])\n    axes[0, idx].set_xlabel(f'# of classes in {lbls[idx].lower()} videos (not aligned)')\n\nfor idx in range(2, 4):\n    axes[1, idx-2].hist(al_embeddings_data_df.loc[al_embeddings_data_df.Label==lbls[idx-2]].Classes.values, color=c[idx-2])\n    axes[1, idx-2].set_xlabel(f'# of classes in {lbls[idx-2].lower()} videos (aligned)')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3. Noise vs Classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(14, 4))\nc = list(map(lambda x: 'pink' if x == 'REAL' else 'C0', embeddings_data_df.Label.values))\nal_c = list(map(lambda x: 'pink' if x == 'REAL' else 'C0', al_embeddings_data_df.Label.values))\naxes[0].scatter(embeddings_data_df.Classes, embeddings_data_df.Noises, c=c)\naxes[1].scatter(al_embeddings_data_df.Classes, al_embeddings_data_df.Noises, c=al_c)\n\naxes[0].set_xlabel('Classes')\naxes[0].set_ylabel('Noise')\naxes[1].set_xlabel('Classes')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"3. Spread vs Noises and Classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(14, 8))\nc = list(map(lambda x: 'pink' if x == 'REAL' else 'C0', embeddings_data_df.Label.values))\nal_c = list(map(lambda x: 'pink' if x == 'REAL' else 'C0', al_embeddings_data_df.Label.values))\naxes[0, 0].scatter(embeddings_data_df.Spread, embeddings_data_df.Noises, c=c)\naxes[0, 1].scatter(al_embeddings_data_df.Spread, al_embeddings_data_df.Noises, c=al_c)\n\naxes[1, 0].scatter(embeddings_data_df.Spread, embeddings_data_df.Classes, c=c)\naxes[1, 1].scatter(al_embeddings_data_df.Spread, al_embeddings_data_df.Classes, c=al_c)\n\naxes[0, 0].set_ylabel('Noise')\naxes[1, 0].set_ylabel('Classes')\naxes[1, 0].set_xlabel('Spread')\naxes[1, 1].set_xlabel('Spread')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ensemble with deep fake score"},{"metadata":{"trusted":true},"cell_type":"code","source":"dfscore_train_result = pd.read_csv(\"../input/edadfdc/resnext_dfscore_train_result.csv\")\ndfscore_test_result = pd.read_csv(\"../input/edadfdc/resnext_dfscore_test_result.csv\")\n\ndfscore_train_result = dfscore_train_result.set_index('0')\ndfscore_train_result.columns = ['DFscore']\ndfscore_test_result = dfscore_test_result.set_index('0')\ndfscore_test_result.columns = ['DFscore']\ndfscore_train_result.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_score = dfscore_train_result.groupby(dfscore_train_result.index).mean()\nal_embeddings_data_df = pd.merge(left=al_embeddings_data_df, right=df_score['DFscore'], left_on='Video', right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Write Model\nimport pickle\n\nclf = LogisticRegression(solver='lbfgs', random_state=0)\n\ny = al_embeddings_data_df.Label.apply(lambda x: 0 if x == \"REAL\" else 1).values\nX = al_embeddings_data_df[['Classes', 'Noises', 'Spread', 'DFscore']]\n# score = cross_val_score(clf, X, y, cv=5, scoring='neg_log_loss')\nclf.fit(X, y)\n\n# print (score.mean())\npickle.dump(clf, open('lr_dfdc_resnext.dat', 'wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nsubmissions = pd.read_csv('../input/edadfdc/dfscore_test_result.csv')\nsubmissions.hist()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}