{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n!pip install feature_engine 2>/dev/null 1>&2\n!pip install fastparquet 2>/dev/null 1>&2\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\n\nfrom sklearn.preprocessing import StandardScaler\nfrom feature_engine.wrappers import SklearnTransformerWrapper as SKWrapper\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import optimizers\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Dropout, BatchNormalization\n\nfrom matplotlib import pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-10T07:06:59.199918Z","iopub.execute_input":"2022-10-10T07:06:59.201112Z","iopub.status.idle":"2022-10-10T07:07:32.770103Z","shell.execute_reply.started":"2022-10-10T07:06:59.200963Z","shell.execute_reply":"2022-10-10T07:07:32.768123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Downloading data**","metadata":{}},{"cell_type":"code","source":"INPUT = '../input/tabular-playground-series-oct-2022/'\n\ndf_train_dtypes = pd.read_csv(INPUT + 'train_dtypes.csv')\ndf_test_dtypes = pd.read_csv(INPUT + 'test_dtypes.csv')\ntrain_dtypes = {k: v for (k, v) in zip(df_train_dtypes.column, df_train_dtypes.dtype)}\ntest_dtypes = {k: v for (k, v) in zip(df_test_dtypes.column, df_test_dtypes.dtype)}\n\ntrain_list = []\nnum = 5\nfor i in range(num):\n#     df = pd.read_csv(INPUT + f'train_{i}.csv', dtype = train_dtypes)\n    df = pd.read_csv(INPUT + f'train_{i}.csv')\n    df.to_parquet(f'train_{i}.parquet.gzip', compression='gzip')\n    print('Done with File', i)\n    train_list.append(pd.read_parquet(f'train_{i}.parquet.gzip'))\n\n# dft = pd.read_csv(INPUT + 'test.csv', dtype = test_dtypes)\ndft = pd.read_csv(INPUT + 'test.csv')\ndft.to_parquet('test.parquet.gzip', compression='gzip')\nprint('Done with File test')\ndf_test = pd.read_parquet('test.parquet.gzip')\ndf_sample = pd.read_csv(INPUT + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-10-10T07:12:09.103670Z","iopub.execute_input":"2022-10-10T07:12:09.104397Z","iopub.status.idle":"2022-10-10T07:14:52.914947Z","shell.execute_reply.started":"2022-10-10T07:12:09.104356Z","shell.execute_reply":"2022-10-10T07:14:52.913690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Preprocessing data**","metadata":{}},{"cell_type":"markdown","source":"Thanks to @Jose Cáliz for feature engineering ideas!","metadata":{}},{"cell_type":"code","source":"for i in range(num):\n    print(train_list[i].shape)\n    games = random.sample(list(train_list[i].game_num.unique()), 150)\n    train_list[i] = train_list[i][train_list[i].game_num.isin(games)]\n    print(train_list[i].shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-10T07:14:57.704877Z","iopub.execute_input":"2022-10-10T07:14:57.705412Z","iopub.status.idle":"2022-10-10T07:14:58.050454Z","shell.execute_reply.started":"2022-10-10T07:14:57.705372Z","shell.execute_reply":"2022-10-10T07:14:58.049077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(num):\n    train_list[i]['label'] = train_list[i].team_A_scoring_within_10sec + train_list[i].team_B_scoring_within_10sec.replace(1, 2)\n    train_list[i].label.value_counts(True).to_frame(name='label proportion')","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:26.705584Z","iopub.execute_input":"2022-10-10T05:54:26.706048Z","iopub.status.idle":"2022-10-10T05:54:26.755910Z","shell.execute_reply.started":"2022-10-10T05:54:26.705995Z","shell.execute_reply":"2022-10-10T05:54:26.754643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(num):\n    train_list[i]['ball_distance_to_goal_A'] = np.sqrt(\n        (train_list[i].ball_pos_x)**2 + (train_list[i].ball_pos_y + 100)**2\n    )\n    train_list[i]['ball_distance_to_goal_B'] = np.sqrt(\n        (train_list[i].ball_pos_x)**2 + (train_list[i].ball_pos_y - 100)**2\n    )\n    \ndf_test['ball_distance_to_goal_A'] = np.sqrt(\n        (df_test.ball_pos_x)**2 + (df_test.ball_pos_y + 100)**2\n    )\ndf_test['ball_distance_to_goal_B'] = np.sqrt(\n        (df_test.ball_pos_x)**2 + (df_test.ball_pos_y - 100)**2\n    )","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:26.757498Z","iopub.execute_input":"2022-10-10T05:54:26.757939Z","iopub.status.idle":"2022-10-10T05:54:26.933194Z","shell.execute_reply.started":"2022-10-10T05:54:26.757901Z","shell.execute_reply":"2022-10-10T05:54:26.931487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(num):\n#     train_list[i] = train_list[i].dropna()","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:26.934875Z","iopub.execute_input":"2022-10-10T05:54:26.935345Z","iopub.status.idle":"2022-10-10T05:54:26.940102Z","shell.execute_reply.started":"2022-10-10T05:54:26.935306Z","shell.execute_reply":"2022-10-10T05:54:26.938782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_keep = [\n    'ball_pos_y', 'ball_pos_x', 'ball_vel_y',\n    'p0_pos_y', 'p1_pos_y', 'p2_pos_y',\n    'p3_pos_y', 'p4_pos_y', 'p5_pos_y', \n    'ball_distance_to_goal_A', 'ball_distance_to_goal_B'\n]\ntarget = []\ntrain = []\nfor i in range(num):\n    target.append(pd.get_dummies(train_list[i]['label']))\n#     target.append(train_list[i][['team_A_scoring_within_10sec','team_B_scoring_within_10sec']])\n    train.append(train_list[i].drop(['game_num', 'event_id', 'event_time', 'player_scoring_next', 'team_scoring_next', 'team_A_scoring_within_10sec', 'team_B_scoring_within_10sec', 'label'], axis = 1))\n#     train.append(train_list[i][columns_to_keep])\n\nfor i in range(num):\n    target[i].columns = ['nobody_scores', 'team_A_scores', 'team_b_scores']\n    \ntest = df_test.drop(['id'], axis = 1)\n# test = df_test[columns_to_keep]","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:26.942985Z","iopub.execute_input":"2022-10-10T05:54:26.943478Z","iopub.status.idle":"2022-10-10T05:54:29.095728Z","shell.execute_reply.started":"2022-10-10T05:54:26.943437Z","shell.execute_reply":"2022-10-10T05:54:29.093909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(num):\n#     train[i] = train[i].fillna(train[i].median())\n    train[i] = train[i].fillna(0)\n    \n# test = test.fillna(test.median())\ntest = test.fillna(0)\n","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:29.099820Z","iopub.execute_input":"2022-10-10T05:54:29.101362Z","iopub.status.idle":"2022-10-10T05:54:30.380942Z","shell.execute_reply.started":"2022-10-10T05:54:29.101245Z","shell.execute_reply":"2022-10-10T05:54:30.379169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:30.382826Z","iopub.execute_input":"2022-10-10T05:54:30.383440Z","iopub.status.idle":"2022-10-10T05:54:30.469758Z","shell.execute_reply.started":"2022-10-10T05:54:30.383362Z","shell.execute_reply":"2022-10-10T05:54:30.468278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaler = StandardScaler()\nscaler = SKWrapper(StandardScaler(), variables=train[0].columns.tolist())\nscaler.fit(train[0])\nfor i in range(num):\n    train[i] = scaler.transform(train[i])\ntest = scaler.transform(test)","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:30.474070Z","iopub.execute_input":"2022-10-10T05:54:30.474625Z","iopub.status.idle":"2022-10-10T05:54:38.143417Z","shell.execute_reply.started":"2022-10-10T05:54:30.474577Z","shell.execute_reply":"2022-10-10T05:54:38.141766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train, X_valid, y_train, y_valid = train_test_split(train[0], target[0], test_size = 0.2, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:38.145181Z","iopub.execute_input":"2022-10-10T05:54:38.145679Z","iopub.status.idle":"2022-10-10T05:54:38.152839Z","shell.execute_reply.started":"2022-10-10T05:54:38.145632Z","shell.execute_reply":"2022-10-10T05:54:38.151249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model evaluation**","metadata":{}},{"cell_type":"code","source":"modelnn = Sequential()\n# modelnn.add(Dense(512, activation=\"leaky_relu\"))\n# modelnn.add(BatchNormalization())\n# # modelnn.add(Dropout(0.2))\nmodelnn.add(Dense(256, activation=\"leaky_relu\"))\nmodelnn.add(BatchNormalization())\nmodelnn.add(Dropout(0.1))\nmodelnn.add(Dense(128, activation=\"leaky_relu\"))\nmodelnn.add(BatchNormalization())\nmodelnn.add(Dropout(0.1))\nmodelnn.add(Dense(64, activation=\"relu\"))\nmodelnn.add(BatchNormalization())\nmodelnn.add(Dropout(0.1))\nmodelnn.add(Dense(32, activation=\"relu\"))\nmodelnn.add(BatchNormalization())\nmodelnn.add(Dense(3, activation=\"softmax\"))\n\nopt = keras.optimizers.Adam(learning_rate=0.0001)\n\nloss_fn = tf.keras.losses.CategoricalCrossentropy()\nmodelnn.compile(loss=loss_fn, optimizer=opt)\n\nearly_stopping = keras.callbacks.EarlyStopping(\n    patience=30,\n    min_delta=0.001,\n    restore_best_weights=True,\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:38.155232Z","iopub.execute_input":"2022-10-10T05:54:38.155867Z","iopub.status.idle":"2022-10-10T05:54:38.530912Z","shell.execute_reply.started":"2022-10-10T05:54:38.155809Z","shell.execute_reply":"2022-10-10T05:54:38.528582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = modelnn.fit(X_train, y_train, \n#                          validation_data=(X_valid, y_valid), \n#                          batch_size = 1024, \n#                          epochs=40, \n#                          callbacks=[early_stopping])","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:38.532832Z","iopub.execute_input":"2022-10-10T05:54:38.533342Z","iopub.status.idle":"2022-10-10T05:54:38.540083Z","shell.execute_reply.started":"2022-10-10T05:54:38.533284Z","shell.execute_reply":"2022-10-10T05:54:38.538400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(8,8),dpi=200)\n# plt.plot(history.history['loss'])\n# plt.plot(history.history['val_loss'])\n# plt.title('model train vs validation loss')\n# plt.ylabel('loss')\n# plt.xlabel('epoch')\n# plt.legend(['train','validation'], loc='upper right')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:38.542543Z","iopub.execute_input":"2022-10-10T05:54:38.543398Z","iopub.status.idle":"2022-10-10T05:54:38.554777Z","shell.execute_reply.started":"2022-10-10T05:54:38.543334Z","shell.execute_reply":"2022-10-10T05:54:38.552277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = modelnn.fit(train[i], target[i], \n# #                               validation_data=(train[num-1], target[num-1]), \n#                               batch_size = 512, \n#                               epochs=8, \n#                               class_weight = {\n#                                         0: 0.5,\n#                                         1: 2,\n#                                         2: 2\n#                                         },\n#                               callbacks=[early_stopping])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in range(8):\n    for i in range(num):\n        history = modelnn.fit(train[i], target[i], \n#                               validation_data=(train[num-1], target[num-1]), \n                              batch_size = 512, \n                              epochs=1, \n                              class_weight = {\n                                        0: 0.5,\n                                        1: 2,\n                                        2: 2\n                                        },\n                              callbacks=[early_stopping])\n","metadata":{"execution":{"iopub.status.busy":"2022-10-10T05:54:38.556995Z","iopub.execute_input":"2022-10-10T05:54:38.557753Z","iopub.status.idle":"2022-10-10T06:11:01.243720Z","shell.execute_reply.started":"2022-10-10T05:54:38.557681Z","shell.execute_reply":"2022-10-10T06:11:01.242643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Making prediction**","metadata":{}},{"cell_type":"code","source":"preds = pd.DataFrame(modelnn.predict(test), \n                    columns=['nobody_scores', 'team_A_scoring_within_10sec', 'team_B_scoring_within_10sec'],\n                    index=test.index)\n# preds = np.round(preds).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-10-10T06:11:01.245464Z","iopub.execute_input":"2022-10-10T06:11:01.246831Z","iopub.status.idle":"2022-10-10T06:11:45.141143Z","shell.execute_reply.started":"2022-10-10T06:11:01.246784Z","shell.execute_reply":"2022-10-10T06:11:45.139745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"execution":{"iopub.status.busy":"2022-10-10T06:11:45.143498Z","iopub.execute_input":"2022-10-10T06:11:45.144730Z","iopub.status.idle":"2022-10-10T06:11:45.163266Z","shell.execute_reply.started":"2022-10-10T06:11:45.144673Z","shell.execute_reply":"2022-10-10T06:11:45.161648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample[['team_A_scoring_within_10sec', 'team_B_scoring_within_10sec']] = preds[['team_A_scoring_within_10sec', 'team_B_scoring_within_10sec']]","metadata":{"execution":{"iopub.status.busy":"2022-10-10T06:11:45.165534Z","iopub.execute_input":"2022-10-10T06:11:45.166508Z","iopub.status.idle":"2022-10-10T06:11:45.194442Z","shell.execute_reply.started":"2022-10-10T06:11:45.166446Z","shell.execute_reply":"2022-10-10T06:11:45.192766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample.to_csv('submission.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-10-10T06:11:45.196507Z","iopub.execute_input":"2022-10-10T06:11:45.197149Z","iopub.status.idle":"2022-10-10T06:11:47.290071Z","shell.execute_reply.started":"2022-10-10T06:11:45.197101Z","shell.execute_reply":"2022-10-10T06:11:47.288760Z"},"trusted":true},"execution_count":null,"outputs":[]}]}