{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import json\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport mlb\n\n%matplotlib inline\n\ntrain = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/train_updated.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-28T13:33:09.926566Z","iopub.execute_input":"2021-07-28T13:33:09.926954Z","iopub.status.idle":"2021-07-28T13:34:23.369845Z","shell.execute_reply.started":"2021-07-28T13:33:09.926868Z","shell.execute_reply":"2021-07-28T13:34:23.369103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:34:37.102615Z","iopub.execute_input":"2021-07-28T13:34:37.102969Z","iopub.status.idle":"2021-07-28T13:34:37.231709Z","shell.execute_reply.started":"2021-07-28T13:34:37.102928Z","shell.execute_reply":"2021-07-28T13:34:37.230765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nengagements = []\nfor i, row in train.iterrows():\n    try:\n        parsed = json.loads(row['nextDayPlayerEngagement'])\n        engagements.extend(parsed)\n    except:\n        pass\n\nengagement_df = pd.DataFrame(engagements)\nengagement_df['cumcount'] = engagement_df.groupby('engagementMetricsDate').cumcount()\nsampled = engagement_df.sample(500000)","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:34:43.547052Z","iopub.execute_input":"2021-07-28T13:34:43.547395Z","iopub.status.idle":"2021-07-28T13:34:55.396424Z","shell.execute_reply.started":"2021-07-28T13:34:43.547366Z","shell.execute_reply":"2021-07-28T13:34:55.395487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cumcount VS Target","metadata":{}},{"cell_type":"code","source":"plt.scatter(sampled['cumcount'], sampled['target1'])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:36:11.598316Z","iopub.execute_input":"2021-07-28T13:36:11.598862Z","iopub.status.idle":"2021-07-28T13:36:13.457448Z","shell.execute_reply.started":"2021-07-28T13:36:11.598812Z","shell.execute_reply":"2021-07-28T13:36:13.456427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(sampled['cumcount'], sampled['target2'])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:36:21.039394Z","iopub.execute_input":"2021-07-28T13:36:21.039740Z","iopub.status.idle":"2021-07-28T13:36:22.873649Z","shell.execute_reply.started":"2021-07-28T13:36:21.039706Z","shell.execute_reply":"2021-07-28T13:36:22.872704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(sampled['cumcount'], sampled['target3'])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:36:26.880015Z","iopub.execute_input":"2021-07-28T13:36:26.880362Z","iopub.status.idle":"2021-07-28T13:36:28.717437Z","shell.execute_reply.started":"2021-07-28T13:36:26.880328Z","shell.execute_reply":"2021-07-28T13:36:28.716502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(sampled['cumcount'], sampled['target4'])","metadata":{"execution":{"iopub.status.busy":"2021-07-28T13:36:32.721172Z","iopub.execute_input":"2021-07-28T13:36:32.721513Z","iopub.status.idle":"2021-07-28T13:36:34.576044Z","shell.execute_reply.started":"2021-07-28T13:36:32.721480Z","shell.execute_reply":"2021-07-28T13:36:34.573785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The order of the data within the same day has a strong relationship with the target.\nI make sure that the data returned from the mlb module as well as train_updated.csv are in the same order. By including this leak in the features, we can improve the CV/public LB.\n\nThe host can easily eliminate this leak by shuffling the rows in the test data since August. I hope that this leak will be eliminated to make the winning model worthwhile.\n","metadata":{}}]}