{"cells":[{"metadata":{"_uuid":"ea72c07831285bd8776167ccffc2715807dfd7c7"},"cell_type":"markdown","source":"This notebook provides an quick solution to calculate speed from data. There is probably better way to do it with other method (some NaN and inifit values are found) but removing this values does not affect the results, e.g. concussion are occuring in the middle of the run.\nHere is a more general approach since time delta can be different. See the other proposal with [aggregation method](https://www.kaggle.com/mtodisco10/speedy-speed-mph-functions)\n\nThe simple way is to calculate this simple formula : \n$$\\|\\vec{V}\\|=\\|\\frac{\\vec{dx}}{dt}\\|=\\frac{\\sqrt{x^2+y^2}}{t_{i+1}-t_{i}}$$ \nwith $t_i$ the time at coodinate $i$"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nlistNGS_CSV = glob.glob(\"../input/how-to-import-large-csv-files-and-save-efficiently/NGS*pkl\")\nimport feather\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4661bf552d55aa3764643c5ceee28e5b8dcf79e"},"cell_type":"markdown","source":"# Defining functions"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def calculate_speeds(df, dt=None, SI=False):\n    data_selected = df[['Time', 'x','y']]\n    if SI==True:\n        data_selected.x = data_selected.x / 1.0936132983\n        data_selected.y = data_selected.y / 1.0936132983\n    # Might have used shift pd function ?\n    data_selected_diff = data_selected.diff()\n    if dt==None:\n        # Time is now a timedelta and need to be converted\n        data_selected_diff.Time = data_selected_diff.Time.apply(lambda x: (x.total_seconds()))\n        data_selected_diff['Speed'] = (data_selected_diff.x **2 + data_selected_diff.y **2).astype(np.float64).apply(np.sqrt) / data_selected_diff.Time\n    else:\n        # Need to be sure about the time step...\n        data_selected_diff['Speed'] = (data_selected_diff.x **2 + data_selected_diff.y **2).astype(np.float64).apply(np.sqrt) / dt\n    #data_selected_diff.rename(columns={'Time':'TimeDelta'}, inplace=True)\n    #return data_selected_diff\n    df['TimeDelta'] = data_selected_diff.Time\n    df['Speed'] = data_selected_diff.Speed\n    return df[1:]\n\ndef remove_wrong_values(df, tested_columns=['Season_Year', 'GameKey', 'PlayID', 'GSISID', 'TimeDelta'], cutspeed=None):\n    dump = df.copy()\n    colums = dump.columns\n    mask = []\n    for col in tested_columns:\n        dump['shift_'+col] = dump[col].shift(-1)\n        mask.append(\"( dump['shift_\"+col+\"'] == dump['\"+col+\"'])\")\n    mask =eval(\" & \".join(mask))\n    # Keep results where next rows is equally space\n    dump = dump[mask]\n    dump = dump[colums]\n    if cutspeed!=None:\n        dump = dump[dump.Speed < cutspeed]\n    return dump\n\ndef get_speed(df):\n    df_with_speed = df.copy()\n    date_format = '%Y-%m-%d %H:%M:%S.%f'\n    sortBy = ['Season_Year', 'GameKey', 'PlayID', 'GSISID', 'Time']\n    df_with_speed.Time = pd.to_datetime(df_with_speed.Time, format =date_format)\n    df_with_speed.sort_values(sortBy, inplace=True)\n    df_with_speed = calculate_speeds(df_with_speed, SI=True)\n    cut_speed=100 / 9.58 # World record 9,857232 m/s for NFL\n    df_with_speed = remove_wrong_values(df_with_speed, cutspeed=cut_speed)\n    return df_with_speed\n\ndef memory(df):\n    if isinstance(df,pd.DataFrame):\n        value = df.memory_usage(deep=True).sum() / 1024 ** 2\n    else: # we assume if not a df it's a series\n        value = df.memory_usage(deep=True) / 1024 ** 2\n    return value, \"{:03.2f} MB\".format(value)\n\ndef downCast(df, verbose=True):\n    mem, _ =  memory(df)\n    date_format = '%Y-%m-%d %H:%M:%S.%f'\n    converted_obj = df.select_dtypes(include=['int']).astype('category')\n    df[converted_obj.columns] = converted_obj\n    converted_obj = df.select_dtypes(include=['float']).apply(pd.to_numeric,downcast='float')\n    df[converted_obj.columns] = converted_obj\n    if 'Time' in df: df.Time = pd.to_datetime(df.Time,format=date_format)\n    memDown, _ = memory(df)\n    if verbose: print(\"Gain {:03.2f} %\".format(mem/memDown*100.0))\n    return df\n\ndef downCastAllCSV(listBigCSV, verbose=True):\n    for csvFile in listBigCSV:\n        dataframe = pd.read_csv(csvFile, engine='c')\n        dataframe = downCast(dataframe)\n        dataframe.to_pickle(os.path.basename(csvFile[:-4]+'.pkl'))\n        del dataframe\n        \nimport inspect\n\ndef retrieve_name(var):\n    callers_local_vars = inspect.currentframe().f_back.f_locals.items()\n    return [var_name for var_name, var_val in callers_local_vars if var_val is var][0]\n\ndef filter_expr(key,filtered):\n    return '(df[\"'+key+'\"] == '+str(filtered)+')'\n\ndef filter_from_dict(df, filters, inplace=False):\n    mask = []\n    maskList=[]\n    for key, filtered in filters.items():\n        if type(filtered) == str:\n            filtered = '\"'+filtered+'\"'\n            mask.append(filter_expr(key,filtered))\n        else:\n            if type(filtered) == list:\n                for fil in filtered:\n                    maskList.append(filter_expr(key,fil))\n                mask.append('('+'|'.join(maskList)+')')\n            else:\n                mask.append(filter_expr(key,filtered))\n    print( 'Mask generated: ',' & '.join(mask))\n    if inplace:\n        df = df[(eval('&'.join(mask)))]\n    else:\n        return df[(eval('&'.join(mask)))]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc90356e03f69b6f50a04c47219c13d49d1c3d2f"},"cell_type":"code","source":"df = pd.read_csv('../input/NFL-Punt-Analytics-Competition/NGS-2016-pre.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef125afd4b0d0568fe41f6f6c990df4bfe505b9d"},"cell_type":"markdown","source":"# Some formating and sorting is needed\nWe need to calculate the speed of each player. There is a number of rows with the same time. Sorting makes things easier."},{"metadata":{"trusted":true,"_uuid":"91573e74893063231ce1909df3b19ff75d51c29f"},"cell_type":"code","source":"date_format = '%Y-%m-%d %H:%M:%S.%f'\nsortBy = ['Season_Year', 'GameKey', 'PlayID', 'GSISID', 'Time']\ndf.Time = pd.to_datetime(df.Time, format =date_format)\ndf.sort_values(sortBy, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcbc0b85bde532b75dbe51d46fd49eea08809d39"},"cell_type":"code","source":"%timeit calculate_speeds(df[:10000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c20df073dac9a28d878092da763f27bb247f3a28"},"cell_type":"code","source":"df_with_speed = calculate_speeds(df, SI=True)\ndf_with_speed.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c073c5ad359e48c5d610ba9e6ecafd52928e2daf"},"cell_type":"markdown","source":"# Clean values and Speed above world record"},{"metadata":{"trusted":true,"_uuid":"456c30883a4c0095bc5c4ad9bf7b1f21950ddac5","_kg_hide-input":true},"cell_type":"code","source":"cut_speed=100 / 9.58 # World record 9,857232 m/s for NFL\ndf_remove_bad_values = remove_wrong_values(df, cutspeed=cut_speed)\ndf_remove_bad_values.Speed.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c84a0aeefcc917a5b8bba374d097abc57b28ccdf","_kg_hide-input":true},"cell_type":"code","source":"df_remove_bad_values.describe()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b0b464c54a2260053c86f7c1550af5be9cc03256"},"cell_type":"markdown","source":"# What is the speed of a player 19714 during the game 234 ?"},{"metadata":{"trusted":true,"_uuid":"f66018e1f716c36bdc7ef1853eaf2f9e6da53fe2"},"cell_type":"code","source":"game_play_player = df_remove_bad_values[((df_remove_bad_values.GameKey==5)&(df_remove_bad_values.PlayID==3129)&(df_remove_bad_values.GSISID==31057))]\ngame_play_player.plot(x='Time',y='Speed')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"45ba7fe1f8f01c2462d6a288dfe6aaee3646f2e1"},"cell_type":"markdown","source":"To compare with [aggregation method](https://www.kaggle.com/mtodisco10/speedy-speed-mph-functions) we can calculate maximum and mean for this play. "},{"metadata":{"trusted":true,"_uuid":"85ad4024131dce6bd4d1cdd61351e726b5f03e2f"},"cell_type":"code","source":"game_play_player.Speed.max() *2.23694","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b42937e2960041a789b4d8ae38fa74432fd59ec4"},"cell_type":"code","source":"game_play_player.Speed.mean()*2.23694","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"68008ed269fdf7c52d381d104cf9e00d73c8a917"},"cell_type":"markdown","source":"# One for all"},{"metadata":{"trusted":true,"_uuid":"641fedd9c1f62793a25ddd597fdeceeefbd95639"},"cell_type":"code","source":"def calculate_speeds2(df, dt=None, SI=False):\n    data_selected = df[['Time', 'x','y']]\n    if SI==True:\n        data_selected.x = data_selected.x.values / 1.0936132983\n        data_selected.y = data_selected.y.values / 1.0936132983\n    # Might have used shift pd function ?\n    data_selected_diff = data_selected.diff()\n    if dt==None:\n        # Time is now a timedelta and need to be converted\n        data_selected_diff.Time = data_selected_diff.Time.apply(lambda x: (x.total_seconds()))\n        data_selected_diff['Speed'] = np.sqrt(data_selected_diff.x.values **2 + data_selected_diff.y.values **2) / data_selected_diff.Time.values\n    else:\n        # Need to be sure about the time step...\n        data_selected_diff['Speed'] = (data_selected_diff.x.values **2 + data_selected_diff.y.values **2).astype(np.float64).apply(np.sqrt) / dt.values\n    #data_selected_diff.rename(columns={'Time':'TimeDelta'}, inplace=True)\n    #return data_selected_diff\n    df['TimeDelta'] = data_selected_diff.Time\n    df['Speed'] = data_selected_diff.Speed\n    return df[1:]\n\ndef get_speed(df):\n    df_with_speed = df.copy()\n    date_format = '%Y-%m-%d %H:%M:%S.%f'\n    sortBy = ['Season_Year', 'GameKey', 'PlayID', 'GSISID', 'Time']\n    df_with_speed.Time = pd.to_datetime(df_with_speed.Time, format =date_format)\n    df_with_speed.sort_values(sortBy, inplace=True)\n    df_with_speed = calculate_speeds2(df_with_speed, SI=True)\n    cut_speed=100 / 9.58 # World record 9,857232 m/s for NFL\n    df_with_speed = remove_wrong_values(df_with_speed, cutspeed=cut_speed)\n    return df_with_speed\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"354eac89ca89dbd4e008cc68dd6ca50afcef8380"},"cell_type":"code","source":"df_test = get_speed(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f44dfe69f6f6fa9d445eb0ec991803b123722c1b"},"cell_type":"code","source":"df_remove_bad_values == df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8eeb0f4e91f54e8c25769548f3d18a3522ddc402"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa631967fa930965edd73366a9fc54ecba43611e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4f7ee8e1993c0d58ffddfe6f7702a7830cac4a8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dc2dd21bc45b00a900eb7bf9d71ae1f1ee39e27"},"cell_type":"code","source":"df = get_speed(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c11ba7ba59b1ed1326bc3425a36d3724eb5ea974"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}