{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Simple function to test if Dataframe is ready for submission\n\nI trimmed down a full run to just be an outline of a run, but mostly to show the function that test submissions, there are some other helpful items here too.\n\n## After a failed Leader Board submission due to a simple error, I decided I needed a function to let me know if I was about to waste a submission with a dataframe with an easy error.\n\n## So I wrote the function below for myself, but hopefully it will help you too.  Give an upvote if you like it.\n\n- Mark"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"import pandas as pd\nimport glob\nimport os\nimport tqdm.notebook as tqdm\nimport json\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Function that should catch most submit errors and let you know.\n- if bChange is True for a Submit for Leader Board, it will show a fixed score if the database is not ready for submission (it will waste a submission, but may identify the issue for you.)  The score should be over 8, something like 9 for 0.3,10 for 0.2,12 for 0.1 for each error condition, I haven't done the math, so may be higher.  This should flag you for which type of error you have even if you don't get a print.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Need to have 2 col, filename and label, eg: \"fsjdfal.mp4\", \"0.5678\"\n# Checks total length, matching filenames, in sorted order and predictions in 0-1 range.\ndef checkSubmittable(data, bChange = True):\n    if bChange == False:\n        data_orig = data.copy()\n    nReturn = 0\n    test_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\n    test_videos = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"])\n    nVids = len(test_videos)\n    nPredVids = len(data['filename'])\n    if (nVids != nPredVids):\n        print(\"ERROR - Dataframe Size mismatch\")\n        data = data.iloc[0:0] # erase df\n        data['filename'] = test_videos\n        data['label'] = 0.3 # If in LB score will be obvious\n        nReturn = -1\n    for i in range(nVids):\n        strFn = data.at[i,'filename']\n        fPred = data.at[i,'label']\n        if (strFn != test_videos[i] ):\n            print(\"Filename mismatch, either not ordered or not matching expected!\")\n            data['filename'] = test_videos\n            data['label'] = 0.1 # If in LB score will be obvious\n            nReturn = -1\n        if (fPred < 0.0 ) or (fPred > 1.0):\n            print(\"Prediction value out of range!\")\n            data['filename'] = test_videos\n            data['label'] = 0.2 # If in LB score will be obvious\n            nReturn = -1\n    if nReturn < 0:\n        print(\"************************************************************\")\n        print(\"********** SUMBIT ERROR - Pred = .1 .2 .3 ******************\")\n        print(\"************************************************************\")\n    else:\n        print(\"Pred. Data looks OK, Files = \", nVids)\n        \n    if bChange == False:\n        return data_orig   \n    \n    return data.reset_index(drop=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames_train = glob.glob('/kaggle/input/deepfake-detection-challenge/train_sample_videos/*.mp4')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = json.load(open('/kaggle/input/deepfake-detection-challenge/train_sample_videos/metadata.json', encoding=\"utf8\"))\n\nlabels = pd.DataFrame(labels).transpose()\nlabels = labels.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dictFake = { 'REAL':0, 'FAKE':1}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nlabels['bFake'] = 0\nnCnt = 0\nfor sub in labels['label']:\n    labels.at[nCnt,'bFake'] = int(dictFake[sub])\n    nCnt += 1\n\nlabels.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_dir = \"/kaggle/input/deepfake-detection-challenge/test_videos/\"\nfilenames = sorted([x for x in os.listdir(test_dir) if x[-4:] == \".mp4\"]) # np.zeros((5,), dtype=int)\nfilenames_full = filenames.copy()\nfor i in range(len(filenames_full)):\n    filenames_full[i] = os.path.join(test_dir, filenames_full[i] )\npredictions = np.zeros(len(filenames,),dtype=float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nsubmission_df = pd.DataFrame({\"filename\": filenames, \"label\": predictions})\n\nsubmission_df.label = 0.48\n\nsubmission_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TEST CODE"},{"metadata":{"trusted":true},"cell_type":"code","source":"def isVideoFake(filename):\n    return False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# filenames_train\n# labels is the training dataframe data\nnCount1=0\n#labels['bFake']\nfor filename in tqdm.tqdm(filenames_train): ## train data\n    \n    fn = filename.split('/')[-1]\n    if (nCount1 < 999999):\n        bFakeVideo = isVideoFake(filename)\n    else:\n        bFakeVideo = False\n    \n    if bFakeVideo == True:\n        labels.loc[labels['index']==fn, 'label'] = 0.80\n    else:\n        labels.loc[labels['index']==fn, 'label'] = 0.48\n\n    nCount1 += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#labels.head(50)\nprint(labels[0:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nCount=0\n## run Test Here\nfor filename in tqdm.tqdm(filenames_full):\n    \n    fn2 = filename.split('/')[-1]\n    if (nCount < 999999):\n        bFakeVideo = isVideoFake(filename)\n    else:\n        bFakeVideo = False\n    \n    if bFakeVideo == True:\n        submission_df.at[nCount, 'label'] = 0.80\n        print(\"file Fake = \", fn)\n    else:\n        submission_df.at[nCount, 'label'] = 0.48       \n\n    nCount += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Check if submittable before exporting to file"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = checkSubmittable(submission_df, bChange=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Format for Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Print to verify it looks good"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"print(sub)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"DONE\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}