{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# train.csvの特徴量エンジニアリング","metadata":{}},{"cell_type":"markdown","source":"このノートブックでは、train.csvのJSONファイルを見て、<br>どういった特徴があるのか理解することを目的としています。<br>\n([English ver is here](https://www.kaggle.com/fumiyakomatsu/explanation-of-train-csv-each-variable))","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-20T06:50:49.094893Z","iopub.execute_input":"2021-06-20T06:50:49.095281Z","iopub.status.idle":"2021-06-20T06:50:49.110565Z","shell.execute_reply.started":"2021-06-20T06:50:49.095246Z","shell.execute_reply":"2021-06-20T06:50:49.109479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport gc\nimport sys\nimport ipywidgets as widgets\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:50:49.473402Z","iopub.execute_input":"2021-06-20T06:50:49.473768Z","iopub.status.idle":"2021-06-20T06:50:50.658657Z","shell.execute_reply.started":"2021-06-20T06:50:49.473739Z","shell.execute_reply":"2021-06-20T06:50:50.657965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train.csvの読み込み","metadata":{}},{"cell_type":"code","source":"data_dir = Path('../input/mlb-player-digital-engagement-forecasting/')\ntraining = pd.read_csv(data_dir / 'train.csv')\n\n# Convert training data date field to datetime type\ntraining['date'] = pd.to_datetime(training['date'], format=\"%Y%m%d\")\n\ndisplay(training.info())","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:50:51.377866Z","iopub.execute_input":"2021-06-20T06:50:51.378390Z","iopub.status.idle":"2021-06-20T06:51:57.994375Z","shell.execute_reply.started":"2021-06-20T06:50:51.378360Z","shell.execute_reply":"2021-06-20T06:51:57.993456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"indexごとに日付が進んでいく<br>\niloc[行数]を使うことで指定した日付のJSONデータを取得できる。","metadata":{}},{"cell_type":"code","source":"training","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:51:57.995746Z","iopub.execute_input":"2021-06-20T06:51:57.996176Z","iopub.status.idle":"2021-06-20T06:51:58.182200Z","shell.execute_reply.started":"2021-06-20T06:51:57.996146Z","shell.execute_reply":"2021-06-20T06:51:58.181178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:52:06.052066Z","iopub.execute_input":"2021-06-20T06:52:06.052408Z","iopub.status.idle":"2021-06-20T06:52:06.058921Z","shell.execute_reply.started":"2021-06-20T06:52:06.052379Z","shell.execute_reply":"2021-06-20T06:52:06.058081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# JSONファイルの展開用関数\nJSONファイルを展開するための関数<br>\n公式の[スターターノート](https://www.kaggle.com/ryanholbrook/getting-started-with-mlb-player-digital-engagement)から引用","metadata":{}},{"cell_type":"code","source":"# Helper function to unpack json found in daily data\ndef unpack_json(json_str):\n    return pd.DataFrame() if pd.isna(json_str) else pd.read_json(json_str)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:52:18.343440Z","iopub.execute_input":"2021-06-20T06:52:18.343847Z","iopub.status.idle":"2021-06-20T06:52:18.348635Z","shell.execute_reply.started":"2021-06-20T06:52:18.343817Z","shell.execute_reply":"2021-06-20T06:52:18.347612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# nextDayPlayerEngagement\n目的変数を含んだ重要なデータになります。<br>\nこのtarget1~4を予測します","metadata":{}},{"cell_type":"code","source":"nextDayPlayerEngagement = unpack_json(training['nextDayPlayerEngagement'].iloc[0])\nnextDayPlayerEngagement","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:52:45.462844Z","iopub.execute_input":"2021-06-20T06:52:45.463325Z","iopub.status.idle":"2021-06-20T06:52:45.504490Z","shell.execute_reply.started":"2021-06-20T06:52:45.463285Z","shell.execute_reply":"2021-06-20T06:52:45.503058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nextDayPlayerEngagement.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:53:01.999721Z","iopub.execute_input":"2021-06-20T06:53:02.000075Z","iopub.status.idle":"2021-06-20T06:53:02.006110Z","shell.execute_reply.started":"2021-06-20T06:53:02.000047Z","shell.execute_reply":"2021-06-20T06:53:02.004748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# games\n試合情報<br>\n初めのデータが2018-2-23でこの日から'gameType'がSであることから、<br>\n春季トレーニングのデータが入っているとわかる。<br>\nその後レギュラーシーズンなどの情報がここには入っている。","metadata":{}},{"cell_type":"code","source":"games = unpack_json(training['games'].iloc[53])\ngames","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:53:44.444446Z","iopub.execute_input":"2021-06-20T06:53:44.444734Z","iopub.status.idle":"2021-06-20T06:53:44.496756Z","shell.execute_reply.started":"2021-06-20T06:53:44.444709Z","shell.execute_reply":"2021-06-20T06:53:44.495598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"games.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:53:45.029463Z","iopub.execute_input":"2021-06-20T06:53:45.029777Z","iopub.status.idle":"2021-06-20T06:53:45.034073Z","shell.execute_reply.started":"2021-06-20T06:53:45.029744Z","shell.execute_reply":"2021-06-20T06:53:45.033604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# rosters\nチーム名簿情報<br>\nケガやマイナーへの降格情報など確認できる","metadata":{}},{"cell_type":"code","source":"unpack_json(training['rosters'].iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.038876Z","iopub.execute_input":"2021-06-20T05:41:23.039312Z","iopub.status.idle":"2021-06-20T05:41:23.066227Z","shell.execute_reply.started":"2021-06-20T05:41:23.039269Z","shell.execute_reply":"2021-06-20T05:41:23.065205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# playerBoxScores\n選手の試合成績<br>\n試合ごとに集計されている<br>\n2018-3-29が初めてのデータ<br>\nこの日からレギュラーシーズンが始まったとわかる。<br>","metadata":{}},{"cell_type":"code","source":"playerBoxScores = unpack_json(training['playerBoxScores'].iloc[87])\nplayerBoxScores","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:54:12.357605Z","iopub.execute_input":"2021-06-20T06:54:12.357950Z","iopub.status.idle":"2021-06-20T06:54:12.470172Z","shell.execute_reply.started":"2021-06-20T06:54:12.357920Z","shell.execute_reply":"2021-06-20T06:54:12.468415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"playerBoxScores.columns","metadata":{"execution":{"iopub.status.busy":"2021-06-20T06:54:12.937928Z","iopub.execute_input":"2021-06-20T06:54:12.938290Z","iopub.status.idle":"2021-06-20T06:54:12.944810Z","shell.execute_reply.started":"2021-06-20T06:54:12.938255Z","shell.execute_reply":"2021-06-20T06:54:12.943872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# teamBoxScores\nチームごとの試合情報<br>\nその日に行われる試合数は異なるため、行数が日によって違う","metadata":{}},{"cell_type":"code","source":"unpack_json(training['teamBoxScores'].iloc[87])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.171296Z","iopub.execute_input":"2021-06-20T05:41:23.171603Z","iopub.status.idle":"2021-06-20T05:41:23.230057Z","shell.execute_reply.started":"2021-06-20T05:41:23.171574Z","shell.execute_reply":"2021-06-20T05:41:23.229019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# transactions\n選手やチームのトランザクション<br>\n一行目を見てみると、選手のトレード情報だとわかる。","metadata":{}},{"cell_type":"code","source":"unpack_json(training['transactions'].iloc[1])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.231372Z","iopub.execute_input":"2021-06-20T05:41:23.23168Z","iopub.status.idle":"2021-06-20T05:41:23.263537Z","shell.execute_reply.started":"2021-06-20T05:41:23.23165Z","shell.execute_reply":"2021-06-20T05:41:23.262607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# standings\nチームの順位情報<br>\n全30チーム分のデータがある","metadata":{}},{"cell_type":"code","source":"unpack_json(training['standings'].iloc[87])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.264873Z","iopub.execute_input":"2021-06-20T05:41:23.265169Z","iopub.status.idle":"2021-06-20T05:41:23.33392Z","shell.execute_reply.started":"2021-06-20T05:41:23.26514Z","shell.execute_reply":"2021-06-20T05:41:23.332956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# awards\n選手の表彰情報<br>\nこの日の情報を見てみると、awardSeasonが2017であることから<br>\n2017年のシーズンについての表彰だとわかる","metadata":{}},{"cell_type":"code","source":"unpack_json(training['awards'].iloc[14])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.335293Z","iopub.execute_input":"2021-06-20T05:41:23.335594Z","iopub.status.idle":"2021-06-20T05:41:23.356671Z","shell.execute_reply.started":"2021-06-20T05:41:23.335564Z","shell.execute_reply":"2021-06-20T05:41:23.355704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# events\nフィールド上で起きた出来事のデータ<br>\nボールの座標データなどもあるためかなり次元が多い","metadata":{}},{"cell_type":"code","source":"unpack_json(training['events'].iloc[87])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.357837Z","iopub.execute_input":"2021-06-20T05:41:23.358108Z","iopub.status.idle":"2021-06-20T05:41:23.713592Z","shell.execute_reply.started":"2021-06-20T05:41:23.358083Z","shell.execute_reply":"2021-06-20T05:41:23.712614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# playerTwitterFollowers\n選手のTwitterフォロワー数<br>\n毎月一日に更新されている","metadata":{}},{"cell_type":"code","source":"unpack_json(training['playerTwitterFollowers'].iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.714852Z","iopub.execute_input":"2021-06-20T05:41:23.71515Z","iopub.status.idle":"2021-06-20T05:41:23.748639Z","shell.execute_reply.started":"2021-06-20T05:41:23.715121Z","shell.execute_reply":"2021-06-20T05:41:23.747925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# teamTwitterFollowers\n全３０チーム公式Twitterアカウントのフォロワー数\n<br>先ほど同様、毎月一日に更新される","metadata":{}},{"cell_type":"code","source":"unpack_json(training['teamTwitterFollowers'].iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T05:41:23.749725Z","iopub.execute_input":"2021-06-20T05:41:23.750172Z","iopub.status.idle":"2021-06-20T05:41:23.776748Z","shell.execute_reply.started":"2021-06-20T05:41:23.750127Z","shell.execute_reply":"2021-06-20T05:41:23.775632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"補足情報などありましたらコメントでご指摘お願いします。","metadata":{}}]}