{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-08T04:43:11.159054Z","iopub.execute_input":"2023-04-08T04:43:11.159763Z","iopub.status.idle":"2023-04-08T04:43:40.678219Z","shell.execute_reply.started":"2023-04-08T04:43:11.159724Z","shell.execute_reply":"2023-04-08T04:43:40.676914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:03:05.768888Z","iopub.execute_input":"2023-04-08T05:03:05.769822Z","iopub.status.idle":"2023-04-08T05:03:05.774969Z","shell.execute_reply.started":"2023-04-08T05:03:05.769781Z","shell.execute_reply":"2023-04-08T05:03:05.773544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/asl-signs/ -GFlash --color","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:43:12.694625Z","iopub.execute_input":"2023-04-08T05:43:12.695922Z","iopub.status.idle":"2023-04-08T05:43:13.820214Z","shell.execute_reply.started":"2023-04-08T05:43:12.695854Z","shell.execute_reply":"2023-04-08T05:43:13.818907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv('/kaggle/input/asl-signs/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-08T04:58:19.231986Z","iopub.execute_input":"2023-04-08T04:58:19.232712Z","iopub.status.idle":"2023-04-08T04:58:19.460652Z","shell.execute_reply.started":"2023-04-08T04:58:19.232649Z","shell.execute_reply":"2023-04-08T04:58:19.459709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-08T04:58:19.794276Z","iopub.execute_input":"2023-04-08T04:58:19.795322Z","iopub.status.idle":"2023-04-08T04:58:19.802944Z","shell.execute_reply.started":"2023-04-08T04:58:19.795281Z","shell.execute_reply":"2023-04-08T04:58:19.801928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T04:58:41.841919Z","iopub.execute_input":"2023-04-08T04:58:41.842607Z","iopub.status.idle":"2023-04-08T04:58:41.867989Z","shell.execute_reply.started":"2023-04-08T04:58:41.842569Z","shell.execute_reply":"2023-04-08T04:58:41.867027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nplt.style.use('seaborn-colorblind')","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:03:19.916212Z","iopub.execute_input":"2023-04-08T05:03:19.916850Z","iopub.status.idle":"2023-04-08T05:03:19.922224Z","shell.execute_reply.started":"2023-04-08T05:03:19.916810Z","shell.execute_reply":"2023-04-08T05:03:19.921183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#what sign we are trying to predict?\n#250 unique sign ranging from 299 - 415 example of each\ntrain['sign'].value_counts().head(20).sort_values(ascending=True).plot(kind='barh',figsize=(10,5),xlabel='number of training example',title=\"Top 20 Signs\")\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:19:41.559998Z","iopub.execute_input":"2023-04-08T05:19:41.560318Z","iopub.status.idle":"2023-04-08T05:19:42.047436Z","shell.execute_reply.started":"2023-04-08T05:19:41.560285Z","shell.execute_reply":"2023-04-08T05:19:42.046623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Parquet landmark data","metadata":{}},{"cell_type":"code","source":"train.query('sign==\"listen\" ')","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:21:21.829107Z","iopub.execute_input":"2023-04-08T05:21:21.829529Z","iopub.status.idle":"2023-04-08T05:21:21.852710Z","shell.execute_reply.started":"2023-04-08T05:21:21.829491Z","shell.execute_reply":"2023-04-08T05:21:21.851894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bdir='../input/asl-signs/'","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:45:38.203593Z","iopub.execute_input":"2023-04-08T05:45:38.203994Z","iopub.status.idle":"2023-04-08T05:45:38.210994Z","shell.execute_reply.started":"2023-04-08T05:45:38.203961Z","shell.execute_reply":"2023-04-08T05:45:38.209072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nexample_fn=train.query('sign==\"listen\" ')['path'].values[0]\nexample_landmark=pd.read_parquet(f'{bdir}/{example_fn}')","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:46:21.225394Z","iopub.execute_input":"2023-04-08T05:46:21.225838Z","iopub.status.idle":"2023-04-08T05:46:21.374686Z","shell.execute_reply.started":"2023-04-08T05:46:21.225796Z","shell.execute_reply":"2023-04-08T05:46:21.373445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:47:01.934127Z","iopub.execute_input":"2023-04-08T05:47:01.935390Z","iopub.status.idle":"2023-04-08T05:47:01.961819Z","shell.execute_reply.started":"2023-04-08T05:47:01.935328Z","shell.execute_reply":"2023-04-08T05:47:01.959905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#finding the unique frame in file\nunique_frame=example_landmark['frame'].nunique()\nprint(f'the file has {unique_frame} - unique frames ')\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:48:45.875494Z","iopub.execute_input":"2023-04-08T05:48:45.875978Z","iopub.status.idle":"2023-04-08T05:48:45.885620Z","shell.execute_reply.started":"2023-04-08T05:48:45.875935Z","shell.execute_reply":"2023-04-08T05:48:45.884270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_type=example_landmark['type'].nunique()\nprint(f'the file has {unique_type} - unique type ')\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T07:38:04.365111Z","iopub.execute_input":"2023-04-08T07:38:04.365579Z","iopub.status.idle":"2023-04-08T07:38:04.374307Z","shell.execute_reply.started":"2023-04-08T07:38:04.365542Z","shell.execute_reply":"2023-04-08T07:38:04.372865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"comparing different for a bunch of parquet files ","metadata":{}},{"cell_type":"code","source":"listen_files=train.query('sign == \"listen\" ')['path'].values\nfor i ,f in enumerate(listen_files):\n    example_landmark=pd.read_parquet(f\"{bdir}/{f}\")\n    unique_frames=example_landmark['frame'].nunique()\n    unique_types=example_landmark['type'].nunique()\n    print(f\"The files has {unique_frames} unique frames and {unique_types} unique types\")\n    if i==20:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-04-08T07:39:02.716970Z","iopub.execute_input":"2023-04-08T07:39:02.717459Z","iopub.status.idle":"2023-04-08T07:39:03.002108Z","shell.execute_reply.started":"2023-04-08T07:39:02.717406Z","shell.execute_reply":"2023-04-08T07:39:03.000737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**create metadata for training data**","metadata":{"execution":{"iopub.status.busy":"2023-04-08T07:39:27.163446Z","iopub.execute_input":"2023-04-08T07:39:27.164078Z","iopub.status.idle":"2023-04-08T07:39:27.170997Z","shell.execute_reply.started":"2023-04-08T07:39:27.164002Z","shell.execute_reply":"2023-04-08T07:39:27.169398Z"}}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-04-08T07:40:05.257017Z","iopub.execute_input":"2023-04-08T07:40:05.258302Z","iopub.status.idle":"2023-04-08T07:40:05.274988Z","shell.execute_reply.started":"2023-04-08T07:40:05.258251Z","shell.execute_reply":"2023-04-08T07:40:05.273436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-04-08T07:40:40.142962Z","iopub.execute_input":"2023-04-08T07:40:40.143424Z","iopub.status.idle":"2023-04-08T07:40:40.155545Z","shell.execute_reply.started":"2023-04-08T07:40:40.143385Z","shell.execute_reply":"2023-04-08T07:40:40.154157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,d in tqdm(train.iterrows(),total=len(train)):\n    file_path=d['path']\n    example_landmark=pd.read_parquet(f\"{bdir}/{file_path}\")\n    break\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-04-08T07:45:19.582921Z","iopub.execute_input":"2023-04-08T07:45:19.583713Z","iopub.status.idle":"2023-04-08T07:45:19.652238Z","shell.execute_reply.started":"2023-04-08T07:45:19.583662Z","shell.execute_reply":"2023-04-08T07:45:19.651011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_landmark['frame'].nunique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}