{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Validating data is always correctly arranged\nThis is a Simple notebook with an algorithm to validate that all the data contained on the dataset is organized the same.\nThis knowledge is important, since the Evaluation for the challenge reports that video frames will be passed only as rows with values `x`,`y`,`z` and no other information. \nKnowing that all data is correctly arranged, would allow us to asume data structure when pre-processing the data.\n\n\n## How does it work?\nIt loops through ALL the training data.\nOn each video it makes the following validations:\n- All frames are sorted in ASCENDING order\n- All types are sorted in the same order\n- All landmark indexes are sorted in ASCENDING order\n\nIf at any point any of the above validations fails, the process stops.","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nDATA_PATH = '/kaggle/input/asl-signs'\ntrain_df = pd.read_csv(os.path.join(DATA_PATH,'train.csv'))\n\nimport tqdm\n\ndef check_sorted():\n\ttypes = []\n\t# Loach ead parquet file\n\tfor i in tqdm.tqdm(range(0, len(train_df))):\n\t\tvideo_df = pd.read_parquet(os.path.join(DATA_PATH, train_df.iloc[i]['path']))\n\t\t# Check that all frames are sorted\n\t\tif not video_df.frame.is_monotonic_increasing:\n\t\t\tprint('Video frames not sorted for video',i)\n\t\t\treturn\n\n\t\tframes = video_df.frame.unique()\n\t\tfor j, frame in enumerate(frames):\n\t\t\tvideo_frame = video_df.loc[video_df['frame'] == frame]\n\t\t\tframe_types = video_frame.type.unique()\n\t\t\tif i == 0 and j == 0:\n\t\t\t\ttypes = frame_types\n\t\t\t\tprint('Order of types:', types)\n\t\t\telse:\n\t\t\t\ttypes_same_order = all(x == y for x, y in zip(types, frame_types))\n\n\t\t\t\tif not types_same_order:\n\t\t\t\t\tprint('Types not sorted for video',i,'on frame',frame)\n\t\t\t\t\treturn\n\n\t\t\t\tfor type in frame_types:\n\t\t\t\t\tvideo_frame_type = video_frame.loc[video_frame.type == type]\n\t\t\t\t\t\n\t\t\t\t\tif not video_frame_type.landmark_index.is_monotonic_increasing:\n\t\t\t\t\t\tprint('Video frame type', type,' for video',i,'on frame',frame,'is not sorted')\n\t\t\t\t\t\treturn\n\t\t# Check that, for each frame, the landmarks are sorted\n\ncheck_sorted()","metadata":{"execution":{"iopub.status.busy":"2023-03-05T18:42:18.137600Z","iopub.execute_input":"2023-03-05T18:42:18.138108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}