{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Proper Clustering with Facenet Embeddings\nThis kernel shows how to use facenet embeddings to cluster similar faces throughout the training data and create a safe validation strategy for trainining and validation splits. You can see below how to use PCA, T-SNE and DBSCAN to efficiently cluster high-dimensional data. The found clusters are exported and can be used to improve your training and validation split.\n","metadata":{}},{"cell_type":"code","source":"%%capture\n# Install facenet-pytorch\n!pip install /kaggle/input/facenet-pytorch-vggface2/facenet_pytorch-1.0.1-py3-none-any.whl\n\n# Copy model checkpoints to torch cache so they are loaded automatically by the package\n!mkdir -p /tmp/.cache/torch/checkpoints/\n!cp /kaggle/input/facenet-pytorch-vggface2/20180402-114759-vggface2-logits.pth /tmp/.cache/torch/checkpoints/vggface2_DG3kwML46X.pt\n!cp /kaggle/input/facenet-pytorch-vggface2/20180402-114759-vggface2-features.pth /tmp/.cache/torch/checkpoints/vggface2_G5aNV2VSMn.pt","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom pathlib import Path\nfrom facenet_pytorch import MTCNN\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport cv2\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom time import time\nimport shutil\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfrom kaggle_pytorch_utility_script import *\n\nseed_everything(42)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom facenet_pytorch import MTCNN, InceptionResnetV1, extract_face\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'Running on device: {device}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"margin = 80\nimage_size = 150\n\n# Load face detector\nmtcnn = MTCNN(keep_all=False, select_largest=False, post_process=False,\n              device=device, min_face_size=100,\n              margin=margin, image_size=image_size).eval()\n\n# Load facial recognition model\nresnet = InceptionResnetV1(pretrained='vggface2', device=device).eval()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract first face from all real videos","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0"}},{"cell_type":"code","source":"test_videos = '/kaggle/input/deepfake-detection-challenge/test_videos'\ntrain_sample_videos = '/kaggle/input/deepfake-detection-challenge/train_sample_videos'\ntrain_videos = '/kaggle/input/deepfake'\nfaces_path = '/kaggle/working/faces'\n\n!mkdir -p {faces_path}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nface_files = []\nmeta = pd.DataFrame()\nfor jsn in glob.glob(f'{train_videos}/metadata*.json'):\n    df = pd.read_json(jsn).transpose()\n    chunk = int(jsn[len(f'{train_videos}/metadata'):-len('.json')])\n    df['chunk'] = chunk\n    meta = pd.concat([meta, df])\n    path = f'{train_videos}/DeepFake{chunk:02}/DeepFake{chunk:02}'\n    print(meta.shape, jsn, path)\n    assert os.path.isdir(path)\n    # symlink all images to 'faces_path'\n    !ls {path} | xargs -IN ln -sf {path}/N {faces_path}/\n    faces = [f'{faces_path}/{vid[:-4]}.jpg' for vid in df[df.label == 'REAL'].index.tolist()]\n    face_files.extend(faces)\nprint(f'Found {len(face_files)} real videos in {len(meta.chunk.unique())} folders')\nassert len(face_files) == len(meta[meta.label == 'REAL'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get missing images from their fakes\nmissing_files, recovered_files = [], []\ndf = []\nfor idx in tqdm(meta[meta.label == 'REAL'].index,\n                total=sum(meta.label == 'REAL')):\n    real_image = f'{faces_path}/{idx[:-4]}.jpg'\n    if not os.path.isfile(real_image):\n#         print(idx, real_image)\n        for fidx in meta.loc[meta.original == idx].index:\n            fake_image = f'{faces_path}/{fidx[:-4]}.jpg'\n            if os.path.isfile(fake_image):\n#                 print(idx, fake_image)\n                # reuse the first valid fake face as the face for the real video\n                !ln -sf {fake_image} {real_image}\n                assert os.path.isfile(real_image)\n                recovered_files.append(idx)\n                break\n        if not os.path.isfile(real_image):\n            missing_files.append(idx)\nprint('Recovered', len(recovered_files), 'files, but still missing', len(missing_files),\n      'in the total of', len(face_files))\n\nface_files = [f for f in face_files if os.path.isfile(f)]\nprint('New total:', len(face_files))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def save_frame(file, folder):\n#     reader = cv2.VideoCapture(file)\n#     _, image = reader.read()\n#     image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#     pilimg = Image.fromarray(image)\n#     with torch.no_grad():\n# #         boxes, probs = mtcnn.detect(pilimg)\n#         face_image = mtcnn(pilimg)\n#         try:\n#             if face_image is not None:\n#                 pilface = Image.fromarray(face_image.byte().numpy().transpose([1,2,0]))\n#                 imgfile = f'{Path(file).stem}.jpg'\n#                 pilface.save(Path(folder)/imgfile)\n#         except Exception as e:\n#             print(e)\n#             return","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# folder = '/kaggle/working/faces'\n# Path(folder).mkdir(parents=True, exist_ok=True)\n# for file in tqdm(list_files):\n#     save_frame(file, folder)\n\n# face_files = [str(x) for x in Path(folder).glob('*')]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate embedding vectors from all images\nusing a pre-trained Facenet model, as the point of this notebook is not training/generating fake/real predictions, but group similar faces using embeddings. Facenet outputs 512-dimensional embeddings.","metadata":{}},{"cell_type":"code","source":"from torchvision.transforms import ToTensor\n\ntf_img = lambda i: ToTensor()(i).unsqueeze(0)\nembeddings = lambda input: resnet(input)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_embs = []\nwith torch.no_grad():\n    for face in tqdm(face_files):\n        t = tf_img(Image.open(face)).to(device)\n        e = embeddings(t).squeeze().cpu().tolist()\n        list_embs.append(e)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame({'face': face_files[:len(list_embs)], 'embedding': list_embs})\ndf['video'] = df.face.apply(lambda x: f'{Path(x).stem}.mp4')\ndf['chunk'] = df.video.apply(lambda x: int(meta.loc[x].chunk))\ndf = df[['video', 'face', 'chunk', 'embedding']]\ndf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Principal Component Analysis (PCA)","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\ndef scatter_thumbnails(data, images, zoom=0.12, colors=None):\n    assert len(data) == len(images)\n\n    # reduce embedding dimentions to 2\n    x = PCA(n_components=2).fit_transform(data) if len(data[0]) > 2 else data\n\n    # create a scatter plot.\n    f = plt.figure(figsize=(22, 15))\n    ax = plt.subplot(aspect='equal')\n    sc = ax.scatter(x[:,0], x[:,1], s=4)\n    _ = ax.axis('off')\n    _ = ax.axis('tight')\n\n    # add thumbnails :)\n    from matplotlib.offsetbox import OffsetImage, AnnotationBbox\n    for i in range(len(images)):\n        image = plt.imread(images[i])\n        im = OffsetImage(image, zoom=zoom)\n        bboxprops = dict(edgecolor=colors[i]) if colors is not None else None\n        ab = AnnotationBbox(im, x[i], xycoords='data',\n                            frameon=(bboxprops is not None),\n                            pad=0.02,\n                            bboxprops=bboxprops)\n        ax.add_artist(ab)\n    return ax\n\n_ = scatter_thumbnails(df.embedding.tolist(), df.face.tolist())\nplt.title('Facial Embeddings - Principal Component Analysis')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that similar images are close to each other. But it looks really hard to sepearete them in clusters/groups.","metadata":{}},{"cell_type":"markdown","source":"# t-Distributed Stochastic Neighbor Embedding (t-SNE)\n","metadata":{}},{"cell_type":"code","source":"%%time\nfrom sklearn.manifold import TSNE\n# PCA first to speed it up\nx = PCA(n_components=50).fit_transform(df['embedding'].tolist())\nx = TSNE(perplexity=50,\n         n_components=3).fit_transform(x)\n\n_ = scatter_thumbnails(x, df.face.tolist(), zoom=0.06)\nplt.title('3D t-Distributed Stochastic Neighbor Embedding')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Density Based Clustering (DBSCAN)\nDensity-Based Spatial Clustering of Applications with Noise","metadata":{}},{"cell_type":"code","source":"# !pip install -q hdbscan\n# import hdbscan\nimport sklearn.cluster as cluster","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_clusters(data, algorithm, *args, **kwds):\n    labels = algorithm(*args, **kwds).fit_predict(data)\n    palette = sns.color_palette('deep', np.max(labels) + 1)\n    colors = [palette[x] if x >= 0 else (0,0,0) for x in labels]\n    ax = scatter_thumbnails(x, df.face.tolist(), 0.06, colors)\n    plt.title(f'Clusters found by {algorithm.__name__}')\n    return labels\n\n# clusters = plot_clusters(x, hdbscan.HDBSCAN, alpha=1.0, min_cluster_size=2, min_samples=1)\nclusters = plot_clusters(x, cluster.DBSCAN, n_jobs=-1, eps=1.0, min_samples=1)\ndf['cluster'] = clusters","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clusters and the number of images on each one of them\nids, counts = np.unique(clusters, return_counts=True)\n_ = pd.DataFrame(counts, index=ids).hist(bins=len(ids), log=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get similar faces using Spotify's Annoy\nCheck assigned clusters","metadata":{}},{"cell_type":"code","source":"from annoy import AnnoyIndex\n\nf = len(df['embedding'][0])\nt = AnnoyIndex(f, metric='euclidean')\nntree = 50\n\nfor i, vector in enumerate(df['embedding']):\n    t.add_item(i, vector)\n_  = t.build(ntree)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_similar_images_annoy(img_index, n=8, max_dist=1.0):\n    vid, face  = df.iloc[img_index, [0, 1]]\n    similar_img_ids, dist = t.get_nns_by_item(img_index, n+1, include_distances=True)\n    similar_img_ids = [s for s,d in zip(similar_img_ids, dist) if d <= max_dist][1:]  # first item is always its own video\n    return vid, face, df.iloc[similar_img_ids], dist","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sample_n_similar(sample_idx):\n    vid, face, similar, distances = get_similar_images_annoy(sample_idx)\n    \n    fig = plt.figure(figsize=(15, 7))\n    gs = fig.add_gridspec(2, 6)\n    ax1 = fig.add_subplot(gs[0:2, 0:2])\n    ax2 = fig.add_subplot(gs[0, 2])\n    ax3 = fig.add_subplot(gs[0, 3])\n    ax4 = fig.add_subplot(gs[0, 4])\n    ax5 = fig.add_subplot(gs[0, 5])\n    ax6 = fig.add_subplot(gs[1, 2])\n    ax7 = fig.add_subplot(gs[1, 3])\n    ax8 = fig.add_subplot(gs[1, 4])\n    ax9 = fig.add_subplot(gs[1, 5])\n    axx = [ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8, ax9]\n    for ax in axx:\n        ax.set_axis_off()\n    list_plot = [face] + similar['face'].values.tolist()\n    list_cluster = [df.iloc[sample_idx]['cluster']] + similar['cluster'].values.tolist()\n    for ax, face, cluster, dist in zip(axx, list_plot, list_cluster, distances):\n        ax.imshow(plt.imread(face))\n        ax.set_title(f'{face.split(\"/\")[-1][:-4]} @{dist:.2f}\\ncluster:{cluster}') # show video filename and distance","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display samples and their nearest neighbors\nfor i in np.random.choice(len(df), 8, replace=False):\n    get_sample_n_similar(i)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Cluster Analysis","metadata":{}},{"cell_type":"code","source":"chunks = df.groupby('cluster').chunk.nunique().to_frame('n_chunks')\nchunks = chunks.merge(df.groupby('cluster').video.nunique().to_frame('n_videos'),\n                     left_index=True, right_index=True).sort_values(by='n_chunks')\n\nprint(f'{sum(chunks.n_chunks > 1)} clusters are spread across more than one data chunk')\nchunks","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The largest cluster is probably just whatever didn't fit anywhere else, but let's inspect the other clusters present in more than one data chunk (the division given by the hosts).\n\nIt's a long list and the clustering is not perfect, as some actors still get split in 2 different clusters and others (generally similar faces) get clustered together. However, we can see that many clusters are legit and indeed go across 2 or even several different chunks of data.\n\nTherefore, if you want to split your data by actors, do not use just the chunk number (0-49). I will export the clusters below and you can use it, or similar techniques to improve your split.\n\n","metadata":{}},{"cell_type":"code","source":"# sample 2 images of each ['cluster', 'chunk'] pair \nmixed_clusters = chunks[(chunks.n_chunks > 1) & (chunks.n_videos < 100)].index.values\nvideo_samples = df[df.cluster.isin(mixed_clusters)].groupby(['cluster', 'chunk']).face\nvideo_samples = video_samples.agg(['min', 'max']).reset_index()\n\nfor cluster in mixed_clusters:\n    chunk_samples = video_samples[video_samples.cluster == cluster]\n    fig, axes = plt.subplots(2, len(chunk_samples), figsize=(len(chunk_samples)*2, 4))\n    print(f'Cluster {cluster} with {len(chunk_samples)} chunks')\n    for i, (idx, row) in enumerate(chunk_samples.iterrows()):\n        axes[0, i].imshow(plt.imread(row['min']))\n        axes[0, i].set_axis_off()\n        axes[0, i].set_title(f\"\"\"Data chunk: {row.chunk}\n{row['min'].split('/')[-1][:-4]}.mp4\"\"\")\n        if row['max'] != row['min']:\n            axes[1, i].imshow(plt.imread(row['max']))\n            axes[1, i].set_title(f\"\"\"{row['max'].split('/')[-1][:-4]}.mp4\"\"\")\n        axes[1, i].set_axis_off()\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"chunks = df.groupby('cluster').chunk.unique().to_frame('chunks')\nchunks = chunks[chunks.chunks.apply(lambda c: len(c)) > 1] # filter non-unique clusters\nchunks_df = pd.DataFrame(range(50), columns=['chunk'])\nchunks_df['n_nonunique_clusters'] = 0\nchunks_df['nonunique_clusters'] = [[] for _ in range(50)]\nfor i in range(50):\n    chunks_df.loc[i, 'n_nonunique_clusters'] = len(chunks[chunks.chunks.apply(lambda chunks: i in chunks)])\n    chunks_df.loc[i, 'nonunique_clusters'].extend(chunks[chunks.chunks.apply(lambda chunks: i in chunks)].index.tolist())\nchunks_df.sort_values(by='n_nonunique_clusters').head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare and Export Clusters","metadata":{}},{"cell_type":"code","source":"# clean up working dir\nif not is_interactive():\n    shutil.rmtree(Path(faces_path))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# save face clusters\ndf.to_csv('face_clusters.csv.zip', index=False)\ndf[['video', 'chunk', 'cluster']].to_feather('face_clusters.feather')\ndf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta['cluster'] = -1\nmeta.loc[df.video, 'cluster'] = df.cluster.values\n\n# propagate real video clusters to their fake versions\nfor index, real in tqdm(meta[~meta.cluster.isnull()].iterrows(),\n                        total=len(meta[~meta.cluster.isnull()])):\n    meta.loc[meta.original == index, 'cluster'] = int(real.cluster)\n\nmeta.reset_index().to_feather('metadata.feather')\nmeta[~meta.cluster.isnull()]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exported files\n!ls -sh face_clusters*\n!ls -sh metadata*","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}