{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np\nimport tensorflow as tf","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":6.60333,"end_time":"2022-11-01T05:45:30.888809","exception":false,"start_time":"2022-11-01T05:45:24.285479","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T03:53:54.811230Z","iopub.execute_input":"2022-11-05T03:53:54.811809Z","iopub.status.idle":"2022-11-05T03:53:59.754237Z","shell.execute_reply.started":"2022-11-05T03:53:54.811728Z","shell.execute_reply":"2022-11-05T03:53:59.752907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_df = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/train_dtypes.csv')\ndtypes = {k: v for (k, v) in zip(dtypes_df.column, dtypes_df.dtype)}\ndfs = []\nfor i in range(10):\n    df = pd.read_csv(f'/kaggle/input/tabular-playground-series-oct-2022/train_{i}.csv', dtype=dtypes)\n    df.drop(['event_id', 'player_scoring_next', 'team_A_scoring_within_10sec', 'team_B_scoring_within_10sec'], axis=1, inplace=True)\n    dfs.append(df)\n    print(f'done {i}')\nfull_train_df = pd.concat(dfs)\nlen(full_train_df)","metadata":{"papermill":{"duration":459.542374,"end_time":"2022-11-01T05:53:10.435909","exception":false,"start_time":"2022-11-01T05:45:30.893535","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T04:00:19.347028Z","iopub.execute_input":"2022-11-05T04:00:19.347715Z","iopub.status.idle":"2022-11-05T04:02:43.686252Z","shell.execute_reply.started":"2022-11-05T04:00:19.347657Z","shell.execute_reply":"2022-11-05T04:02:43.684958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_train_df.columns","metadata":{"papermill":{"duration":0.022556,"end_time":"2022-11-01T05:53:10.463332","exception":false,"start_time":"2022-11-01T05:53:10.440776","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T04:02:43.688730Z","iopub.execute_input":"2022-11-05T04:02:43.689149Z","iopub.status.idle":"2022-11-05T04:02:43.699532Z","shell.execute_reply.started":"2022-11-05T04:02:43.689117Z","shell.execute_reply":"2022-11-05T04:02:43.697959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\nseed = hash('kaggle') % 2**32\nrandom.seed(seed)\nnp.random.seed(seed)\ntf.random.set_seed(seed)","metadata":{"papermill":{"duration":0.01664,"end_time":"2022-11-01T05:53:10.488600","exception":false,"start_time":"2022-11-01T05:53:10.471960","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T04:04:09.937020Z","iopub.execute_input":"2022-11-05T04:04:09.937477Z","iopub.status.idle":"2022-11-05T04:04:09.944314Z","shell.execute_reply.started":"2022-11-05T04:04:09.937442Z","shell.execute_reply":"2022-11-05T04:04:09.943268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nFRACTION_TAKE = 1\nFRACTION_VAL = 0.05\ngame_nums = list(full_train_df.game_num.unique())\nrandom.shuffle(game_nums)\ngame_nums = game_nums[:int(FRACTION_TAKE * len(game_nums))]\nstop_idx = int(FRACTION_VAL * len(game_nums))\ntrain_game_nums = set(game_nums[stop_idx:])\nval_game_nums = set(game_nums[:stop_idx])\ndel game_nums\nprint(len(train_game_nums))\nprint(len(val_game_nums))","metadata":{"papermill":{"duration":0.133545,"end_time":"2022-11-01T05:53:10.627113","exception":false,"start_time":"2022-11-01T05:53:10.493568","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T04:04:27.734569Z","iopub.execute_input":"2022-11-05T04:04:27.734951Z","iopub.status.idle":"2022-11-05T04:04:27.816110Z","shell.execute_reply.started":"2022-11-05T04:04:27.734912Z","shell.execute_reply":"2022-11-05T04:04:27.814659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_features(df):\n    df.drop([col for col in df.columns if col.startswith('boost')], axis=1, inplace=True)\n    \n    for p in range(6):\n        # add demoed column\n        df.insert(list(df.columns).index(f'p{p}_boost') + 1, f'p{p}_demoed', df[f'p{p}_boost'].isna())\n        \n        # Convert nan values from demoed state into dummy values, specifically sitting above the stadium, flying upwards with 0 boost\n        x = -20 + 20 * (p % 3)\n        y = -20 + 40 * (p // 3)\n        cond = df[f'p{p}_boost'].isna()\n        df.loc[cond, [f'p{p}_pos_x', f'p{p}_vel_x', f'p{p}_pos_y', f'p{p}_vel_y', f'p{p}_pos_z', f'p{p}_vel_z', f'p{p}_boost']] = \\\n            [x, x, y, y, 100, 10, 0]","metadata":{"papermill":{"duration":0.019128,"end_time":"2022-11-01T05:53:10.652400","exception":false,"start_time":"2022-11-01T05:53:10.633272","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T04:04:27.817962Z","iopub.execute_input":"2022-11-05T04:04:27.818636Z","iopub.status.idle":"2022-11-05T04:04:27.827463Z","shell.execute_reply.started":"2022-11-05T04:04:27.818601Z","shell.execute_reply":"2022-11-05T04:04:27.826242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def postprocess(df, salt='nacl'):\n    df['hash'] = [hash(f'{salt}{x}') for x in range(len(df))]\n    df.sort_values('hash', inplace=True)\n    df.drop(['game_num', 'hash'], axis=1, inplace=True)","metadata":{"papermill":{"duration":0.017769,"end_time":"2022-11-01T05:53:10.675715","exception":false,"start_time":"2022-11-01T05:53:10.657946","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npreprocess_features(full_train_df)\nfull_train_df.team_scoring_next = full_train_df.team_scoring_next.apply(lambda t: -1 if t == 'A' else 1 if t == 'B' else 0).astype('int8')\n\ntrain_df = full_train_df[full_train_df.game_num.isin(train_game_nums)].copy()\nval_df = full_train_df[full_train_df.game_num.isin(val_game_nums)].copy()\ndel full_train_df, train_game_nums, val_game_nums\npostprocess(train_df)\npostprocess(val_df)\nprint(f'{len(train_df)} train rows, {len(val_df)} val rows')","metadata":{"papermill":{"duration":76.948241,"end_time":"2022-11-01T05:54:27.629079","exception":false,"start_time":"2022-11-01T05:53:10.680838","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T04:04:27.849745Z","iopub.execute_input":"2022-11-05T04:04:27.850613Z","iopub.status.idle":"2022-11-05T04:05:15.566025Z","shell.execute_reply.started":"2022-11-05T04:04:27.850579Z","shell.execute_reply":"2022-11-05T04:05:15.564170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def write_df(df, filename_prefix):\n    filename = f'{filename_prefix}.csv'\n    df.to_csv(filename, index=False)\n    print(f'wrote {len(df)} rows to {filename}')\n    filename = f'{filename_prefix}_df.pickle'\n    df.copy().to_pickle(filename)\n    print(f'wrote {filename}')\n\nn_pages = 4\nrows_per_page = len(train_df) // n_pages\nfor i in range(n_pages):\n    write_df(train_df[i*rows_per_page:(i+1)*rows_per_page], f'train_{i}')\n\nwrite_df(val_df, 'val')","metadata":{"papermill":{"duration":885.015903,"end_time":"2022-11-01T06:09:12.680156","exception":false,"start_time":"2022-11-01T05:54:27.664253","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-11-05T04:05:15.568575Z","iopub.execute_input":"2022-11-05T04:05:15.569095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df, val_df","metadata":{"papermill":{"duration":0.0473,"end_time":"2022-11-01T06:09:12.732931","exception":false,"start_time":"2022-11-01T06:09:12.685631","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_df = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/test_dtypes.csv')\ndtypes = {k: v for (k, v) in zip(dtypes_df.column, dtypes_df.dtype)}\ntest_df = pd.read_csv('/kaggle/input/tabular-playground-series-oct-2022/test.csv', dtype=dtypes)\nwrite_df(test_df, 'test')","metadata":{"papermill":{"duration":17.859191,"end_time":"2022-11-01T06:13:36.579957","exception":false,"start_time":"2022-11-01T06:13:18.720766","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}