{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is my preprocessing pipeline to create unnested and seperated train dataframes which I can build models off. This notebook has nothing original it it. All the relevant code is taken from the starter notebook: https://www.kaggle.com/alokpattani/mlb-player-digital-engagement-data-exploration\n\nThe only reason I am releasing this notebook is that other people can run the PyTorch Baseline I published without problems. You can check it out here: https://www.kaggle.com/nicohrubec/mlb-pytorch-dnn-loop-feature-engineering/notebook","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unnest_data(df, mode):\n    # Get names of all \"nested\" data frames in daily training set\n    daily_data_nested_df_names = df.drop('date', axis = 1).columns.values.tolist()\n\n    for df_name in daily_data_nested_df_names:\n        print(\"Processing \", df_name, \" ...\")\n        date_nested_table = df[['date', df_name]]\n\n        date_nested_table = (date_nested_table[\n          ~pd.isna(date_nested_table[df_name])\n          ].\n          reset_index(drop = True)\n          )\n\n        daily_dfs_collection = []\n\n        for date_index, date_row in date_nested_table.iterrows():\n            daily_df = pd.read_json(date_row[df_name])\n\n            daily_df['dailyDataDate'] = date_row['date']\n\n            daily_dfs_collection = daily_dfs_collection + [daily_df]\n\n        # Concatenate all daily dfs into single df for each row\n        unnested_table = (pd.concat(daily_dfs_collection,\n          ignore_index = True).\n          # Set and reset index to move 'dailyDataDate' to front of df\n          set_index('dailyDataDate').\n          reset_index()\n          )\n        \n        path = mode + '_' + df_name + '.csv'\n        unnested_table.to_csv(path, index=False)\n\n        # Clean up tables and collection of daily data frames for this df\n        del(date_nested_table, daily_dfs_collection, unnested_table)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unnest_data(train, 'train')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}