{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook do Relatório de Projetos em Ciência de Dados\n\n**Grupo 5: Carlos Fonseca, João Lucas Duim, Juliana Carvalho**\n","metadata":{"execution":{"iopub.execute_input":"2021-06-04T23:07:54.351665Z","iopub.status.busy":"2021-06-04T23:07:54.351302Z","iopub.status.idle":"2021-06-04T23:07:54.356206Z","shell.execute_reply":"2021-06-04T23:07:54.355039Z","shell.execute_reply.started":"2021-06-04T23:07:54.351637Z"}}},{"cell_type":"markdown","source":"# Notebook Setup (Este Notebook está preparado para rodar no Kaggle apenas)","metadata":{}},{"cell_type":"code","source":"#### Import Python Libraries and Set Script Options ####\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import mean_absolute_error\nfrom datetime import datetime, timezone\n\n# Plotly libraries\nimport plotly as pl\nimport plotly.express as px\nimport plotly.offline as pyo\nimport plotly.graph_objs as go\n\n# Library for interactive Python widgets\nimport ipywidgets as widgets\n\n# Utility libraries\nimport gc\nfrom pathlib import Path\n\n# Set notebook mode to make plotly graphics offline\npyo.init_notebook_mode()\n\n# Expand max column width when displaying data frames \npd.set_option('display.max_colwidth', 100)\n\n# Lists all input data files from \"../input/\" directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-21T03:54:50.615429Z","iopub.execute_input":"2023-04-21T03:54:50.615847Z","iopub.status.idle":"2023-04-21T03:54:50.795851Z","shell.execute_reply.started":"2023-04-21T03:54:50.615810Z","shell.execute_reply":"2023-04-21T03:54:50.794990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read in Kaggle Data Files","metadata":{}},{"cell_type":"markdown","source":"#### Read in MLB Player Digital Engagement Forecasting Data from CSVs into pandas DFs","metadata":{}},{"cell_type":"code","source":"# Start with input file path\ninput_file_path = Path('/kaggle/input/mlb-player-digital-engagement-forecasting/')\n\n# Create table with list of CSV files to be read in, w/ corresponding df name\n# This does include large 'train' data set (read in separately)\ncsv_and_df_names = pd.DataFrame(data = {\n  'csv_name': ['seasons', 'teams', 'players', 'awards',\n    'example_test', 'example_sample_submission'],\n  'df_name': ['seasons', 'teams', 'players', 'awards_pre2018',\n    'example_test', 'example_sample_submission'] \n  })\n\n# Set up for tabbed output\nkaggle_data_tabs = widgets.Tab()\n\n# Add Output widgets for each (eventual) DF as tabs' children\nkaggle_data_tabs.children = list([widgets.Output() for df_name \n  in csv_and_df_names['df_name']])\n\nfor index, row in csv_and_df_names.iterrows():\n    \n    csv_name = row['csv_name']\n    df_name = row['df_name']\n    \n    # Read from CSV and create df with specified name in environment\n    globals()[df_name] = pd.read_csv(input_file_path / f\"{csv_name}.csv\")\n\n    # Set tab title to df name\n    kaggle_data_tabs.set_title(index, df_name)\n    \n    # Display corresponding table output for this tab name\n    with kaggle_data_tabs.children[index]:\n        display(eval(df_name))\n\ndisplay(kaggle_data_tabs)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-21T03:54:50.797302Z","iopub.execute_input":"2023-04-21T03:54:50.797801Z","iopub.status.idle":"2023-04-21T03:54:52.481596Z","shell.execute_reply.started":"2023-04-21T03:54:50.797765Z","shell.execute_reply":"2023-04-21T03:54:52.480646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define os dados como vindos do train_update","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv(input_file_path / 'train_updated.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-21T03:54:52.483157Z","iopub.execute_input":"2023-04-21T03:54:52.483673Z","iopub.status.idle":"2023-04-21T03:55:56.738445Z","shell.execute_reply.started":"2023-04-21T03:54:52.483635Z","shell.execute_reply":"2023-04-21T03:55:56.737389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [Para ser usado posteriormente]: Substitui os dados 'NaN' do twitter pelo dado anterior (metodo ffiill)","metadata":{}},{"cell_type":"code","source":"### adicionar os dados do twitter\ndata[\"playerTwitterFollowers\"].fillna( method ='ffill', inplace = True)","metadata":{"execution":{"iopub.status.busy":"2023-04-21T03:55:56.740058Z","iopub.execute_input":"2023-04-21T03:55:56.740389Z","iopub.status.idle":"2023-04-21T03:55:56.745741Z","shell.execute_reply.started":"2023-04-21T03:55:56.740357Z","shell.execute_reply":"2023-04-21T03:55:56.744974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Unnest and look at data from each of the nested data frames within the daily data","metadata":{}},{"cell_type":"code","source":"\ndaily_data_nested_df_names = data.drop('date', axis = 1).columns.values.tolist()\n\nfor df_name in daily_data_nested_df_names:\n    date_nested_table = data[['date', df_name]]\n\n    date_nested_table = (date_nested_table[\n      ~pd.isna(date_nested_table[df_name])\n      ].\n      reset_index(drop = True)\n      )\n\n    daily_dfs_collection = []\n\n    for date_index, date_row in date_nested_table.iterrows():\n        daily_df = pd.read_json(date_row[df_name])\n\n        daily_df['dailyDataDate'] = date_row['date']\n\n        daily_dfs_collection = daily_dfs_collection + [daily_df]\n\n    # Concatenate all daily dfs into single df for each row\n    unnested_table = (pd.concat(daily_dfs_collection,\n      ignore_index = True).\n      # Set and reset index to move 'dailyDataDate' to front of df\n      set_index('dailyDataDate').\n      reset_index()\n      )\n\n    # Creates 1 pandas df per unnested df from daily data read in, with same name\n    globals()[df_name] = unnested_table    \n\n    # Clean up tables and collection of daily data frames for this df\n    del(date_nested_table, daily_dfs_collection, unnested_table)\n\n# Set up for tabbed output\ndaily_data_unnested_tabs = widgets.Tab()\n\n# Add Output widgets for each (eventual) DF as tabs' children\ndaily_data_unnested_tabs.children = list([widgets.Output() \n  for df_name in daily_data_nested_df_names])\n\nfor index in range(0, len(daily_data_nested_df_names)):\n    df_name = daily_data_nested_df_names[index]\n\n    # Rename tab bar titles to df names\n    daily_data_unnested_tabs.set_title(index, df_name)\n\n    # Display corresponding table output for this tab name\n    with daily_data_unnested_tabs.children[index]:\n        display(eval(df_name))\n\ndisplay(daily_data_unnested_tabs)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-21T03:55:56.747171Z","iopub.execute_input":"2023-04-21T03:55:56.747750Z","iopub.status.idle":"2023-04-21T04:01:04.292584Z","shell.execute_reply.started":"2023-04-21T03:55:56.747710Z","shell.execute_reply":"2023-04-21T04:01:04.291422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Delete to free up memory","metadata":{}},{"cell_type":"code","source":"del(data)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:01:04.294188Z","iopub.execute_input":"2023-04-21T04:01:04.294625Z","iopub.status.idle":"2023-04-21T04:01:04.431175Z","shell.execute_reply.started":"2023-04-21T04:01:04.294581Z","shell.execute_reply":"2023-04-21T04:01:04.430115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split em train, test, validation","metadata":{}},{"cell_type":"code","source":"# Define the borderline dates\ntrain_start_date = 20180101 \ntrain_end_date = 20210131\nval_start_date = 20210201 \nval_end_date = 20210430\ntest_start_date = 20210501\ntest_end_date = 20210731\n\n# Split the data \nnextDayPlayerEngagement_train = nextDayPlayerEngagement[(nextDayPlayerEngagement.dailyDataDate <= train_end_date) & (nextDayPlayerEngagement.dailyDataDate >= train_start_date)] \nnextDayPlayerEngagement_val = nextDayPlayerEngagement[(nextDayPlayerEngagement.dailyDataDate <= val_end_date) & (nextDayPlayerEngagement.dailyDataDate >= val_start_date)] \nnextDayPlayerEngagement_test = nextDayPlayerEngagement[(nextDayPlayerEngagement.dailyDataDate <= test_end_date) & (nextDayPlayerEngagement.dailyDataDate >= test_start_date)]\n\n# nextDayPlayerEngagement_train_and_val = nextDayPlayerEngagement_train + nextDayPlayerEngagement_val\nnextDayPlayerEngagement_train_and_val = nextDayPlayerEngagement[(nextDayPlayerEngagement.dailyDataDate >= train_start_date) & (nextDayPlayerEngagement.dailyDataDate <= val_end_date)]","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:01:04.433760Z","iopub.execute_input":"2023-04-21T04:01:04.434082Z","iopub.status.idle":"2023-04-21T04:01:04.893711Z","shell.execute_reply.started":"2023-04-21T04:01:04.434045Z","shell.execute_reply":"2023-04-21T04:01:04.892825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Baselines","metadata":{}},{"cell_type":"code","source":"targets = ['target1', 'target2', 'target3', 'target4']\n\nmean_total = nextDayPlayerEngagement_train_and_val[targets].mean()\nmean_per_player = pd.DataFrame(nextDayPlayerEngagement_train_and_val.groupby('playerId')[targets].mean())\nmean_per_player.index.name = 'playerId'\n\nmedian_total = nextDayPlayerEngagement_train_and_val[targets].median()\nmedian_per_player = pd.DataFrame(nextDayPlayerEngagement_train_and_val.groupby('playerId')[targets].median())\nmedian_per_player.index.name = 'playerId'\n\nnaive = pd.DataFrame(nextDayPlayerEngagement_train_and_val.groupby('playerId').last()[targets])\nnaive.index.name = 'playerId'\n\nmaes = pd.DataFrame(columns=targets, index=['MEAN', 'MEAN_PER_PLAYER', 'MEDIAN', 'MEDIAN_PER_PLAYER', 'NAIVE'])\n\nfor target in targets:\n    y_true = nextDayPlayerEngagement_test[target]\n    \n    mean_player_pred = [mean_per_player.loc[p_id, target] for p_id in nextDayPlayerEngagement_test['playerId']]\n    median_player_pred = [median_per_player.loc[p_id, target] for p_id in nextDayPlayerEngagement_test['playerId']]\n    naive_pred = [naive.loc[p_id, target] for p_id in nextDayPlayerEngagement_test['playerId']]\n    median_all_pred = [median_total[target] for i in nextDayPlayerEngagement_test.index]\n    mean_all_pred = [mean_total[target] for i in nextDayPlayerEngagement_test.index]\n\n    maes.loc['MEAN', target] = mean_absolute_error(y_true, mean_all_pred)\n    maes.loc['MEAN_PER_PLAYER', target] = mean_absolute_error(y_true, mean_player_pred)\n    maes.loc['MEDIAN', target] = mean_absolute_error(y_true, median_all_pred)\n    maes.loc['MEDIAN_PER_PLAYER', target] = mean_absolute_error(y_true, median_player_pred)\n    maes.loc['NAIVE', target] = mean_absolute_error(y_true, naive_pred)\n    \nmaes['Final Scores'] = maes.mean(axis=1)\ndisplay(maes)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:01:04.895784Z","iopub.execute_input":"2023-04-21T04:01:04.896120Z","iopub.status.idle":"2023-04-21T04:01:37.175336Z","shell.execute_reply.started":"2023-04-21T04:01:04.896087Z","shell.execute_reply":"2023-04-21T04:01:37.174272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lasso","metadata":{}},{"cell_type":"markdown","source":"## Define as variáveis a serem utilizadas pelo Lasso","metadata":{}},{"cell_type":"markdown","source":"### Trata as variáveis","metadata":{}},{"cell_type":"code","source":"# COM DADOS DE JOGO\n## Targets+PlayersBoxScores\nmergeBS = pd.merge(nextDayPlayerEngagement, playerBoxScores)\nmergeBS = mergeBS.fillna(0.0) # Preencher os dados NaN que vieram do PBS com 0\nmergeBS = mergeBS.drop_duplicates(subset=['dailyDataDate','playerId'], keep='last') #eliminar chaves duplicadas \n\n\n## Awards -> Se o jogador ganhou o premio no dia anterior ele ganha 1, c.c. ganha 0\nawards1=awards[['dailyDataDate','playerId']]\nawards1['premio']=1\nawards1 = awards1.drop_duplicates().reset_index().drop('index',axis=1)\nmergeBS = pd.merge(mergeBS,awards1, how='left').fillna(0)\n\n## Se o jogador se lesionou no jogo, ele recebe um valor de dias em que ficará de recuperação\nrosters1 = rosters.copy()\nrosters1['diasLesionado']=0\nrosters1.loc[rosters.statusCode=='D10','diasLesionado']=10\nrosters1.loc[rosters.statusCode=='D60','diasLesionado']=60\nrosters1.loc[rosters.statusCode=='D7','diasLesionado']=7\nrosters1=rosters1.drop_duplicates(subset=['dailyDataDate', 'playerId'], keep='last')\n\nmergeBS = pd.merge(mergeBS,rosters1[['dailyDataDate','playerId','diasLesionado']],how='left').fillna(0)\n\n## Resultados do Jogo\nhome = games[['gameType','dailyDataDate','homeId',\n       'homeWins', 'homeLosses',\n       'homeWinner', 'homeScore',]].rename(columns={'homeId':'teamId',\n       'homeWins':'teamWins', 'homeLosses':'teamLosses',\n       'homeWinner':'teamWinner', 'homeScore':'teamScore'})\naway = games[['gameType','dailyDataDate', 'awayId', \n              'awayWins', 'awayLosses', 'awayWinner', 'awayScore']].rename(\n    columns={'awayId':'teamId', 'awayWins':'teamWins', 'awayLosses':'teamLosses',\n             'awayWinner':'teamWinner', 'awayScore':'teamScore'}\n)\n\njogos = pd.concat([home,away],ignore_index=True)\n\njogos.teamWinner=jogos.teamWinner*1\n\nmapa = [{\"id\":\"S\",\"description\":\"Spring Training\"},{\"id\":\"R\",\"description\":\"Regular Season\"},{\"id\":\"F\",\"description\":\"Wild Card Game\"},{\"id\":\"D\",\"description\":\"Division Series\"},{\"id\":\"L\",\"description\":\"League Championship Series\"},{\"id\":\"W\",\"description\":\"World Series\"},{\"id\":\"C\",\"description\":\"Championship\"},{\"id\":\"N\",\"description\":\"Nineteenth Century Series\"},{\"id\":\"P\",\"description\":\"Playoffs\"},{\"id\":\"A\",\"description\":\"All-Star Game\"},{\"id\":\"I\",\"description\":\"Intrasquad\"},{\"id\":\"E\",\"description\":\"Exhibition\"}]\ndic={}\nfor a in mapa:\n    dic[a['id']]=a['description'].replace(' ','')\n\njogos = jogos.replace(dic)\n\n\njogos[['All-StarGame', 'DivisionSeries', 'Exhibition',\n       'LeagueChampionshipSeries', 'RegularSeason', 'SpringTraining',\n       'WildCardGame', 'WorldSeries']]=pd.get_dummies(jogos['gameType'])\n\njogos=jogos.drop(['gameType'],axis=1)\njogos=jogos.drop_duplicates(subset=['dailyDataDate', 'teamId'], keep='last')\n\nmergeBS = pd.merge(mergeBS,jogos,how='left')\n\n## Número de Seguidores no TWITTER\n### caso o número seja NaN será prenchido com 0\n### no inicio foi feito uma função degrau que preenche os vazios com os dados do dia anterior\nmergeBS = pd.merge(mergeBS,playerTwitterFollowers[['dailyDataDate','playerId','numberOfFollowers']],how='left').fillna(0)\n\n## Drop columns categoricas\nmergeBS = mergeBS.drop(columns=['engagementMetricsDate','gamePk', 'gameDate', 'gameTimeUTC', 'teamId',\n                                'teamName', 'playerName', 'jerseyNum', 'positionCode',\n                                'positionName', 'positionType'])\n\n## Dados de Baseline (Não colocamos apenas os dados que envolvia o resultado do jogador, pois as medidas gerais são apenas constantes)\nmergeBS = pd.merge(mergeBS,mean_per_player.rename(columns={a:a.replace('target','mean') for a in mean_per_player.columns}),on='playerId')\nmergeBS = pd.merge(mergeBS,median_per_player.rename(columns={a:a.replace('target','median') for a in median_per_player.columns}),on='playerId')\nmergeBS = pd.merge(mergeBS,naive.rename(columns={a:a.replace('target','naive') for a in naive.columns}),on='playerId')\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:01:37.176639Z","iopub.execute_input":"2023-04-21T04:01:37.176943Z","iopub.status.idle":"2023-04-21T04:01:45.044901Z","shell.execute_reply.started":"2023-04-21T04:01:37.176913Z","shell.execute_reply":"2023-04-21T04:01:45.043834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Sem dadoos de jogo","metadata":{}},{"cell_type":"code","source":"# SEM DADOS DE JOGO\n## TARGET's quando não tem dados de jogo\nmergeSem = pd.merge(nextDayPlayerEngagement,playerBoxScores,how='left')\nmergeSem = mergeSem[mergeSem.home.isna()]\nmergeSem = mergeSem[nextDayPlayerEngagement.columns]\n\n## Seguidores do TWITTER\nmergeSem = mergeSem.merge(playerTwitterFollowers[['dailyDataDate','playerId','numberOfFollowers']],how='left')\n\n## Premio\nmergeSem = mergeSem.merge(awards1,how='left').fillna(0)\n\nmergeSem = mergeSem.drop(columns=['engagementMetricsDate'])\n\n## Dados de Baseline (Não colocamos apenas os dados que envolvia o resultado do jogador, pois as medidas gerais são apenas constantes)\nmergeSem = pd.merge(mergeSem,mean_per_player.rename(columns={a:a.replace('target','mean') for a in mean_per_player.columns}),on='playerId')\nmergeSem = pd.merge(mergeSem,median_per_player.rename(columns={a:a.replace('target','median') for a in median_per_player.columns}),on='playerId')\nmergeSem = pd.merge(mergeSem,naive.rename(columns={a:a.replace('target','naive') for a in naive.columns}),on='playerId')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Com DADOS DE JOGO\n","metadata":{}},{"cell_type":"code","source":"# Com DADOS DE JOGO\n\n# Split the data \nmergeBS_train = mergeBS[(mergeBS.dailyDataDate <= train_end_date) & (mergeBS.dailyDataDate >= train_start_date)].fillna(0)\nmergeBS_val = mergeBS[(mergeBS.dailyDataDate <= val_end_date) & (mergeBS.dailyDataDate >= val_start_date)].fillna(0)\nmergeBS_test = mergeBS[(mergeBS.dailyDataDate <= test_end_date) & (mergeBS.dailyDataDate >= test_start_date)].fillna(0)\n\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Calculo","metadata":{}},{"cell_type":"code","source":"XBS_train = mergeBS_train.iloc[:,6:].to_numpy()\n\n## Normalização\nMEAN_BS = XBS_train.mean(axis=0)\nSTD_BS = XBS_train.std(axis=0)+0.00000000000000000000001\n\nXBS_train=(XBS_train-MEAN_BS)/STD_BS\n\nXBS_val = (mergeBS_val.iloc[:,6:].to_numpy()-MEAN_BS)/STD_BS\nXBS_test = (mergeBS_test.iloc[:,6:].to_numpy()-MEAN_BS)/STD_BS\n\nyBS_train1=mergeBS_train.iloc[:,2].to_numpy().reshape(-1,1)\nyBS_train2=mergeBS_train.iloc[:,3].to_numpy().reshape(-1,1)\nyBS_train3=mergeBS_train.iloc[:,4].to_numpy().reshape(-1)\nyBS_train4=mergeBS_train.iloc[:,5].to_numpy().reshape(-1)\nyBS_val1=mergeBS_val.iloc[:,2].to_numpy().reshape(-1,1)\nyBS_val2=mergeBS_val.iloc[:,3].to_numpy().reshape(-1)\nyBS_val3=mergeBS_val.iloc[:,4].to_numpy().reshape(-1)\nyBS_val4=mergeBS_val.iloc[:,5].to_numpy().reshape(-1)\nyBS_test1=mergeBS_test.iloc[:,2].to_numpy().reshape(-1,1)\nyBS_test2=mergeBS_test.iloc[:,3].to_numpy().reshape(-1)\nyBS_test3=mergeBS_test.iloc[:,4].to_numpy().reshape(-1)\nyBS_test4=mergeBS_test.iloc[:,5].to_numpy().reshape(-1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Sem DADOS DE JOGO\n","metadata":{}},{"cell_type":"code","source":"# Split the data \nmergeSem_train = mergeSem[(mergeSem.dailyDataDate <= train_end_date) & (mergeSem.dailyDataDate >= train_start_date)] \nmergeSem_val = mergeSem[(mergeSem.dailyDataDate <= val_end_date) & (mergeSem.dailyDataDate >= val_start_date)] \nmergeSem_test = mergeSem[(mergeSem.dailyDataDate <= test_end_date) & (mergeSem.dailyDataDate >= test_start_date)]\n\nXSem_train = mergeSem_train.iloc[:,6:].to_numpy()\n\n## Normalização\nMEAN_Sem = XSem_train.mean(axis=0)\nSTD_Sem = XSem_train.std(axis=0)+0.00000000000000000000001\n\nXSem_train=(XSem_train-MEAN_Sem)/STD_Sem\nXSem_val = (mergeSem_val.iloc[:,6:].to_numpy()-MEAN_Sem)/STD_Sem\nXSem_test = (mergeSem_test.iloc[:,6:].to_numpy()-MEAN_Sem)/STD_Sem\n\nySem_train1=mergeSem_train.iloc[:,2].to_numpy().reshape(-1,1)\nySem_train2=mergeSem_train.iloc[:,3].to_numpy().reshape(-1,1)\nySem_train3=mergeSem_train.iloc[:,4].to_numpy().reshape(-1)\nySem_train4=mergeSem_train.iloc[:,5].to_numpy().reshape(-1)\nySem_val1=mergeSem_val.iloc[:,2].to_numpy().reshape(-1,1)\nySem_val2=mergeSem_val.iloc[:,3].to_numpy().reshape(-1)\nySem_val3=mergeSem_val.iloc[:,4].to_numpy().reshape(-1)\nySem_val4=mergeSem_val.iloc[:,5].to_numpy().reshape(-1)\nySem_test1=mergeSem_test.iloc[:,2].to_numpy().reshape(-1,1)\nySem_test2=mergeSem_test.iloc[:,3].to_numpy().reshape(-1)\nySem_test3=mergeSem_test.iloc[:,4].to_numpy().reshape(-1)\nySem_test4=mergeSem_test.iloc[:,5].to_numpy().reshape(-1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Variaveis que estamos utilizando","metadata":{}},{"cell_type":"code","source":"### Variaveis que estamos utilizando\nvariaveisBS = mergeBS.columns[6:]\nvariaveisSem = mergeSem.columns[6:]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Lasso: seleção hiperparametros","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Lasso\nfrom sklearn.metrics import make_scorer, mean_absolute_error\nimport numpy as np\n\ndef grid_search(X_train, y_train, X_val, y_val, alphas, fit_intercepts):\n    best_alpha = None\n    best_intercept = None\n    best_score = float('inf')\n    \n    for alpha in alphas:\n        for fit_intercept in fit_intercepts:\n            model = Lasso(alpha=alpha, fit_intercept=fit_intercept)\n            model.fit(X_train, y_train)\n            y_pred = np.clip(model.predict(X_val), 0., 100.)\n            score = mean_absolute_error(y_val, y_pred)\n            \n            if score < best_score:\n                best_score = score\n                best_alpha = alpha\n                best_intercept = fit_intercept\n    \n            print('♠',end='')\n    print('')\n    return {'best_params':{'alpha':best_alpha, 'intercept':best_intercept}, 'best_score':best_score}\n\n# Definir os valores para o parâmetro alpha que deseja testar\nalphas = np.array([0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.7, 1, 1.5, 2, 3, 5])\n\n# Definir os valores para o parâmetro fit_intercept que deseja testar\nfit_intercepts = [True, False]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Grid search","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Lasso\nfrom sklearn.metrics import make_scorer, mean_absolute_error\nimport numpy as np\n\ndef grid_search(X_train, y_train, X_val, y_val, alphas, fit_intercepts):\n    best_alpha = None\n    best_intercept = None\n    best_score = float('inf')\n    \n    for alpha in alphas:\n        for fit_intercept in fit_intercepts:\n            model = Lasso(alpha=alpha, fit_intercept=fit_intercept)\n            model.fit(X_train, y_train)\n            y_pred = np.clip(model.predict(X_val), 0., 100.)\n            score = mean_absolute_error(y_val, y_pred)\n            \n            if score < best_score:\n                best_score = score\n                best_alpha = alpha\n                best_intercept = fit_intercept\n    \n            print('♠',end='')\n    print('')\n    return {'best_params':{'alpha':best_alpha, 'intercept':best_intercept}, 'best_score':best_score}\n\n# Definir os valores para o parâmetro alpha que deseja testar\nalphas = np.array([0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.7, 1, 1.5, 2, 3, 5])\n\n# Definir os valores para o parâmetro fit_intercept que deseja testar\nfit_intercepts = [True, False]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LASSO: Calculo de MAE","metadata":{}},{"cell_type":"code","source":"def mae(gridBS,gridSem, \n        XBS_train,XSem_train,yBS_train,ySem_train,\n        XBS_test,XSem_test,yBS_test, ySem_test):\n    mBS = Lasso(alpha = gridBS['best_params']['alpha'],\n                          fit_intercept=gridBS['best_params']['intercept']).fit(XBS_train, yBS_train)\n    mSem = Lasso(alpha = gridSem['best_params']['alpha'],\n                 fit_intercept=gridSem['best_params']['intercept']).fit(XSem_train, ySem_train)\n    y = np.concatenate((yBS_test,ySem_test))\n    y_pred = np.clip(np.concatenate((mBS.predict(XBS_test),mSem.predict(XSem_test))),0,100)\n    return {'MAE': mean_absolute_error(y,y_pred),'param':{'BS':mBS.coef_,'Sem':mSem.coef_}}\n\nmae1 = mae(gridBS1,gridSem1,\n           XBS_train,XSem_train,yBS_train1,ySem_train1,\n           XBS_test,XSem_test ,yBS_test1, ySem_test1)\n\nmae2 = mae(gridBS2,gridSem2,\n           XBS_train,XSem_train,yBS_train2,ySem_train2,\n           XBS_test,XSem_test ,yBS_test2, ySem_test2)\n\nmae3 = mae(gridBS3,gridSem3,\n           XBS_train,XSem_train,yBS_train3,ySem_train3,\n           XBS_test,XSem_test ,yBS_test3, ySem_test3)\n\nmae4 = mae(gridBS4,gridSem4,\n           XBS_train,XSem_train,yBS_train4,ySem_train4,\n           XBS_test,XSem_test ,yBS_test4, ySem_test4)\n\nprint('Mae1 :', mae1)\nprint('Mae2 :', mae2)\nprint('Mae3 :', mae3)\nprint('Mae4 :', mae4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Realiza o merge de todas as variáveis a serem utilizadas","metadata":{}},{"cell_type":"code","source":"# Box Scores + Targets\nmerge1 = pd.merge(nextDayPlayerEngagement, playerBoxScores)\nmerge1 = merge1.fillna(0.0)\n\n# Merge + Awards\nmerge1 = pd.merge(merge1,awards1, how='left').fillna(0)\n\n# Situação do Jogador\nmerge1 = pd.merge(merge1,rosters1[['dailyDataDate','playerId','diasLesionado']],how='left').fillna(0)\n\n# Resultado do jogo\nmerge1 = pd.merge(merge1,jogos,how='left')\n\n# Seguidores no TWITTER\nmerge1 = pd.merge(merge1,playerTwitterFollowers[['dailyDataDate','playerId','numberOfFollowers']],how='left').fillna(0)\nmerge1.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:01:45.046347Z","iopub.execute_input":"2023-04-21T04:01:45.046705Z","iopub.status.idle":"2023-04-21T04:01:50.398314Z","shell.execute_reply.started":"2023-04-21T04:01:45.046661Z","shell.execute_reply":"2023-04-21T04:01:50.397411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Gradient Boosting","metadata":{}},{"cell_type":"markdown","source":"# Trata as variáveis - aproveitando as do Lasso, mas sem os targets","metadata":{}},{"cell_type":"code","source":"# Box Scores + Targets\nmerge_gb = pd.merge(nextDayPlayerEngagement, playerBoxScores)\nmerge_gb = merge_gb.fillna(0.0)\n\n# playerBoxScores + Awards\nmerge_gb = pd.merge(merge_gb,awards1, how='left').fillna(0)\n\n# Situação do Jogador\nmerge_gb = pd.merge(merge_gb,rosters1[['dailyDataDate','playerId','diasLesionado']],how='left').fillna(0)\n\n# Resultado do jogo\nmerge_gb = pd.merge(merge_gb,jogos,how='left')\n\n# Seguidores no TWITTER\nmerge_gb = pd.merge(merge_gb,playerTwitterFollowers[['dailyDataDate','playerId','numberOfFollowers']],how='left').fillna(0)\nmerge_gb.info()\nmerge_gb","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:01:50.399408Z","iopub.execute_input":"2023-04-21T04:01:50.399839Z","iopub.status.idle":"2023-04-21T04:01:55.944106Z","shell.execute_reply.started":"2023-04-21T04:01:50.399807Z","shell.execute_reply":"2023-04-21T04:01:55.943021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.experimental import enable_hist_gradient_boosting\nfrom sklearn.ensemble import HistGradientBoostingRegressor\ngb = HistGradientBoostingRegressor()\n\n# Split the data \nmerge_gb_train = merge_gb[(merge_gb.dailyDataDate <= train_end_date) & (merge_gb.dailyDataDate >= train_start_date)] \nmerge_gb_val = merge_gb[(merge_gb.dailyDataDate <= val_end_date) & (merge_gb.dailyDataDate >= val_start_date)] \nmerge_gb_test = merge_gb[(merge_gb.dailyDataDate <= test_end_date) & (merge_gb.dailyDataDate >= test_start_date)]\n\n# merge_gb_train_and_val = merge_gb_train + merge_gb_val\nmerge_gb_train_and_val = merge_gb[(merge_gb.dailyDataDate >= train_start_date) & (merge_gb.dailyDataDate <= val_end_date)]\n\nmerge_gb_tv_features = merge_gb_train_and_val.drop(columns=targets)\nmerge_gb_tv_targets = merge_gb_train_and_val.loc[:, targets]\n\nmerge_gb_test_features = merge_gb_test.drop(columns=targets)\nmerge_gb_test_targets = merge_gb_test.loc[:, targets]\n\n\ndef drop_categorical_columns(df):\n    categorical_columns = df.select_dtypes(include=['object', 'category']).columns.tolist()\n    df = df.drop(categorical_columns, axis=1)\n    # drop all columns except 'plateAppearances', 'totalBases', 'rbi'\n    # df = df.drop(['dailyDataDate', 'playerId', 'target1', 'target2', 'target3', 'target4'], axis=1)\n    # get columns 'plateAppearances', 'totalBases', 'rbi' and merge them into a new dataframe\n    # df = pd.concat([df['plateAppearances']], axis=1)\n    return df\n\ndef GB(target):\n    X_train = merge_gb_tv_features\n    y_train = merge_gb_tv_targets[f'{target}']\n\n    X_test = merge_gb_test_features\n    y_test = merge_gb_test_targets[f'{target}']\n\n    X_train = drop_categorical_columns(X_train)\n    X_test = drop_categorical_columns(X_test)\n\n    gb.fit(X_train, y_train)\n\n    # Realizando as previsões com o conjunto de teste\n    y_pred = gb.predict(X_test)\n\n    mae_target = mean_absolute_error(y_test, y_pred)\n    print(f\"MAE {target}: {mae_target}\")\n    return mae_target\n    \nmean = 0\nfor target in targets:\n    mean += GB(target)\nmean /= len(targets)\nmean","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:01:55.945730Z","iopub.execute_input":"2023-04-21T04:01:55.946101Z","iopub.status.idle":"2023-04-21T04:02:21.492652Z","shell.execute_reply.started":"2023-04-21T04:01:55.946047Z","shell.execute_reply":"2023-04-21T04:02:21.491759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test different models with lazy predict\nO Lazy Predict ajuda a construir muitos modelos básicos sem muito código e ajuda a entender quais modelos funcionam melhor sem nenhum ajuste de parâmetro.","metadata":{}},{"cell_type":"code","source":"os.system('pip install lazypredict')\nimport lazypredict\nfrom lazypredict.Supervised import LazyRegressor\n#Print complete list of regressor\nlazypredict.Supervised.REGRESSORS\n\n# exclude regressors that takes long to run\ndoesnt_work = ['GammaRegressor', 'GaussianProcessRegressor', 'QuantileRegressor', ]\n# high error: RANSACRegressor\n#takes toto long: ExtraTreesRegressor\nexclude = ['KernelRidge', 'ExtraTreesRegressor', 'NuSVR', 'MLPRegressor', 'RANSACRegressor']\nexclude.append(doesnt_work)\nlazypredict.Supervised.REGRESSORS = [reg for reg in lazypredict.Supervised.REGRESSORS if reg[0] not in exclude]\nreg = LazyRegressor(verbose=2, ignore_warnings=False, custom_metric=mean_absolute_error)\n\nX_train = merge_gb_tv_features\ny_train = merge_gb_tv_targets['target1']\n\nX_test = merge_gb_test_features\ny_test = merge_gb_test_targets['target1']\n\nX_train = drop_categorical_columns(X_train)\nX_test = drop_categorical_columns(X_test)\n\nmodels, predictions = reg.fit(X_train, X_test, y_train, y_test)\n\nprint(models)","metadata":{"execution":{"iopub.status.busy":"2023-04-21T04:02:21.496331Z","iopub.execute_input":"2023-04-21T04:02:21.498248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}