{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Dataset Link: https://www.kaggle.com/mrutyunjaybiswal/mlbunpackedtrainjson","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\nimport matplotlib.pyplot as plt\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-08T16:15:57.909831Z","iopub.execute_input":"2021-07-08T16:15:57.911383Z","iopub.status.idle":"2021-07-08T16:15:58.122512Z","shell.execute_reply.started":"2021-07-08T16:15:57.911162Z","shell.execute_reply":"2021-07-08T16:15:58.121207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# paths\ninp_folder = \"../input/mlb-player-digital-engagement-forecasting\"\n\ntrain_ = os.path.join(inp_folder, \"train.csv\")\ntest_ = os.path.join(inp_folder, \"example_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:15:58.124745Z","iopub.execute_input":"2021-07-08T16:15:58.125190Z","iopub.status.idle":"2021-07-08T16:15:58.132986Z","shell.execute_reply.started":"2021-07-08T16:15:58.125151Z","shell.execute_reply":"2021-07-08T16:15:58.131385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test = pd.read_csv(test_)\nsample_test","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:15:58.135060Z","iopub.execute_input":"2021-07-08T16:15:58.135400Z","iopub.status.idle":"2021-07-08T16:15:59.136396Z","shell.execute_reply.started":"2021-07-08T16:15:58.135368Z","shell.execute_reply":"2021-07-08T16:15:59.135174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_test.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:15:59.138024Z","iopub.execute_input":"2021-07-08T16:15:59.138367Z","iopub.status.idle":"2021-07-08T16:15:59.163963Z","shell.execute_reply.started":"2021-07-08T16:15:59.138335Z","shell.execute_reply":"2021-07-08T16:15:59.162476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_cols_ = sample_test.set_index('date').columns.to_list()\ntrain_cols_ = ['nextDayPlayerEngagement'] + test_cols_","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:15:59.166842Z","iopub.execute_input":"2021-07-08T16:15:59.167455Z","iopub.status.idle":"2021-07-08T16:15:59.181529Z","shell.execute_reply.started":"2021-07-08T16:15:59.167401Z","shell.execute_reply":"2021-07-08T16:15:59.180280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unpack_json(json_str, date):\n    res_df = pd.DataFrame()\n    \n    if pd.isna(json_str):\n        return res_df\n    else:\n        res_df = pd.read_json(json_str)\n        \n    res_df['pk_date'] = date\n    \n    return res_df\n\n\ndef unpack_data(data, dfs=None, n_jobs=-1):\n    if dfs is not None:\n        data = data.loc[:, dfs]\n    unnested_dfs = {}\n    for name, column in tqdm(data.iteritems(),total=len(data.columns)):\n        daily_dfs = Parallel(n_jobs=n_jobs)(\n            delayed(unpack_json)(item, date) for date, item in column.iteritems())\n        df = pd.concat(daily_dfs)\n        unnested_dfs[name] = df.set_index(\"pk_date\")\n        del data[name], df\n        _ = gc.collect()\n        \n    return unnested_dfs","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:15:59.183416Z","iopub.execute_input":"2021-07-08T16:15:59.183918Z","iopub.status.idle":"2021-07-08T16:15:59.200900Z","shell.execute_reply.started":"2021-07-08T16:15:59.183842Z","shell.execute_reply":"2021-07-08T16:15:59.198635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(train_)\nstorage_dict = unpack_data(train.set_index('date'), train_cols_)\n\nif not os.path.exists(\"./unpacked_train\"):\n    os.makedirs(\"./unpacked_train\")\n    \n\nfor col in train_cols_:\n    storage_dict[col].to_csv(f\"./unpacked_train/train_{col}.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:15:59.202318Z","iopub.execute_input":"2021-07-08T16:15:59.202653Z","iopub.status.idle":"2021-07-08T16:22:40.829170Z","shell.execute_reply.started":"2021-07-08T16:15:59.202623Z","shell.execute_reply":"2021-07-08T16:22:40.827545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp -r -v ../input/mlb-player-digital-engagement-forecasting/* ./","metadata":{"execution":{"iopub.status.busy":"2021-07-08T16:24:05.806446Z","iopub.execute_input":"2021-07-08T16:24:05.806854Z"},"trusted":true},"execution_count":null,"outputs":[]}]}