{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Put Validation and Test Data to list of group by Session","metadata":{}},{"cell_type":"code","source":"VER = 1\nimport pandas as pd, numpy as np\nimport pickle, glob, gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-01T09:32:31.324700Z","iopub.execute_input":"2022-12-01T09:32:31.325424Z","iopub.status.idle":"2022-12-01T09:32:31.472945Z","shell.execute_reply.started":"2022-12-01T09:32:31.325327Z","shell.execute_reply":"2022-12-01T09:32:31.471854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation data","metadata":{}},{"cell_type":"code","source":"%%time\ntype_labels = {'clicks':0, 'carts':1, 'orders':2}\n\ndef load_test(files):    \n    dfs = []\n    for e, chunk_file in enumerate(glob.glob(files)):\n        chunk = pd.read_parquet(chunk_file)\n        chunk.ts = (chunk.ts/1000).astype('int32')\n        chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n        dfs.append(chunk)\n    return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})\n\nvalid = load_test('../input/otto-validation/test_parquet/*')\nprint('Valid data has shape',valid.shape)\nvalid.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T09:32:31.474921Z","iopub.execute_input":"2022-12-01T09:32:31.475327Z","iopub.status.idle":"2022-12-01T09:32:34.956379Z","shell.execute_reply.started":"2022-12-01T09:32:31.475291Z","shell.execute_reply":"2022-12-01T09:32:34.955149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntemp = [group for name, group in valid.sort_values([\"session\", \"ts\"]).groupby([\"session\"])]\nprint(len(temp))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T09:32:34.958188Z","iopub.execute_input":"2022-12-01T09:32:34.959427Z","iopub.status.idle":"2022-12-01T09:33:21.722567Z","shell.execute_reply.started":"2022-12-01T09:32:34.959376Z","shell.execute_reply":"2022-12-01T09:33:21.721086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nPIECES = 5\n\nfor PART in range(PIECES):\n#for PART in range(1):\n    \n    print('### PART {} - from {} to {}'.format(PART + 1, PART * len(temp)//PIECES, (PART+1) * len(temp) // PIECES))\n    \n    mylist = [[h.session.iloc[0], h.aid.to_list(), h.type.to_list()] for h in temp[PART * len(temp)//PIECES:(PART+1) * len(temp) // PIECES]]\n    \n    with open(f'valid_group_tolist_{PART}_{VER}.pkl', 'wb') as f:\n        pickle.dump(mylist, f) ","metadata":{"execution":{"iopub.status.busy":"2022-12-01T09:41:24.684653Z","iopub.execute_input":"2022-12-01T09:41:24.685055Z","iopub.status.idle":"2022-12-01T09:47:25.417072Z","shell.execute_reply.started":"2022-12-01T09:41:24.685012Z","shell.execute_reply":"2022-12-01T09:47:25.415974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del valid, temp, mylist\n_ = gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T09:48:21.094435Z","iopub.execute_input":"2022-12-01T09:48:21.094855Z","iopub.status.idle":"2022-12-01T09:48:21.120793Z","shell.execute_reply.started":"2022-12-01T09:48:21.094823Z","shell.execute_reply":"2022-12-01T09:48:21.118937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmylist = []\nfor PART in range(PIECES):\n    with open(f'valid_group_tolist_{PART}_{VER}.pkl', 'rb') as f:\n        mylist.extend(pickle.load(f))\nprint(len(mylist))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T09:50:02.874951Z","iopub.execute_input":"2022-12-01T09:50:02.876188Z","iopub.status.idle":"2022-12-01T09:50:13.808630Z","shell.execute_reply.started":"2022-12-01T09:50:02.876143Z","shell.execute_reply":"2022-12-01T09:50:13.807192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test data","metadata":{}},{"cell_type":"code","source":"%%time\ntest = load_test('../input/otto-chunk-data-inparquet-format/test_parquet/*')\nprint('Test data has shape',test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T09:51:39.949336Z","iopub.execute_input":"2022-12-01T09:51:39.949777Z","iopub.status.idle":"2022-12-01T09:51:43.021685Z","shell.execute_reply.started":"2022-12-01T09:51:39.949740Z","shell.execute_reply":"2022-12-01T09:51:43.020345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntemp = [group for name, group in test.sort_values([\"session\", \"ts\"]).groupby([\"session\"])]\nprint(len(temp))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T09:52:15.615455Z","iopub.execute_input":"2022-12-01T09:52:15.615842Z","iopub.status.idle":"2022-12-01T09:53:05.379999Z","shell.execute_reply.started":"2022-12-01T09:52:15.615811Z","shell.execute_reply":"2022-12-01T09:53:05.378630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nPIECES = 5\n\nfor PART in range(PIECES):\n#for PART in range(1):\n    \n    print('### PART {} - from {} to {}'.format(PART + 1, PART * len(temp)//PIECES, (PART+1) * len(temp) // PIECES))\n    \n    mylist = [[h.session.iloc[0], h.aid.to_list(), h.type.to_list()] for h in temp[PART * len(temp)//PIECES:(PART+1) * len(temp) // PIECES]]\n    \n    with open(f'test_group_tolist_{PART}_{VER}.pkl', 'wb') as f:\n        pickle.dump(mylist, f) ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test, temp, mylist\n_ = gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmylist = []\nfor PART in range(PIECES):\n    with open(f'test_group_tolist_{PART}_{VER}.pkl', 'rb') as f:\n        mylist.extend(pickle.load(f))\nprint(len(mylist))","metadata":{},"execution_count":null,"outputs":[]}]}