{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport json\nfrom pandas import json_normalize\nfrom scipy.stats import norm\nimport datetime\n\nimport gc\ngc.enable()\n\ndir = '../input/ga-customer-revenue-prediction/'\nfor _, _, filenames in os.walk(dir):\n    for filename in filenames:\n        print(filename)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T05:42:27.253264Z","iopub.execute_input":"2022-07-15T05:42:27.253747Z","iopub.status.idle":"2022-07-15T05:42:28.322007Z","shell.execute_reply.started":"2022-07-15T05:42:27.253656Z","shell.execute_reply":"2022-07-15T05:42:28.320978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nJSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\ndfs = pd.read_csv(dir + 'train_v2.csv', sep=',',\n                 converters={column: json.loads for column in JSON_COLUMNS}, \n                 dtype={'fullVisitorId': 'str'}, # Important!!\n                chunksize = 100000)\n\ncount = 0\n\nfor df in dfs:\n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df.reset_index(inplace=True, drop=True)\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n        \n    df.index = df.index + count*100000\n    count += 1\n    print(f\"Shape: {df.shape}\")\n\n    for col in df.columns:\n        if col != 'hits':\n            df[col].to_csv('train_' + col + '.csv', mode='a', header=False, index=True)\n\n    del df\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T05:48:35.455688Z","iopub.execute_input":"2022-07-15T05:48:35.456080Z","iopub.status.idle":"2022-07-15T05:50:39.690957Z","shell.execute_reply.started":"2022-07-15T05:48:35.456050Z","shell.execute_reply":"2022-07-15T05:50:39.689956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nJSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\ndfs = pd.read_csv(dir + 'test_v2.csv', sep=',',\n                 converters={column: json.loads for column in JSON_COLUMNS}, \n                 dtype={'fullVisitorId': 'str'}, # Important!!\n                chunksize = 100000)\n\ncount = 0\n\nfor df in dfs:\n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df.reset_index(inplace=True, drop=True)\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n    df.index = df.index + count*100000\n    count += 1\n    print(f\"Shape: {df.shape}\")\n\n    for col in df.columns:\n        if col != 'hits':\n            df[col].to_csv('test_' + col + '.csv', mode='a', header=False, index=True)\n\n    del df\n    gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls -lh ","metadata":{"execution":{"iopub.status.busy":"2022-06-30T04:18:42.108464Z","iopub.execute_input":"2022-06-30T04:18:42.108865Z","iopub.status.idle":"2022-06-30T04:18:42.986058Z","shell.execute_reply.started":"2022-06-30T04:18:42.108832Z","shell.execute_reply":"2022-06-30T04:18:42.984744Z"},"trusted":true},"execution_count":null,"outputs":[]}]}