{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":38760,"databundleVersionId":4493939,"sourceType":"competition"},{"sourceId":4474043,"sourceType":"datasetVersion","datasetId":2601572}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport cudf\nimport cupy as cp  # CuPy를 사용하기 위해 import\nimport os","metadata":{"execution":{"iopub.status.busy":"2024-09-13T07:45:17.51787Z","iopub.execute_input":"2024-09-13T07:45:17.518273Z","iopub.status.idle":"2024-09-13T07:45:21.470512Z","shell.execute_reply.started":"2024-09-13T07:45:17.518229Z","shell.execute_reply":"2024-09-13T07:45:21.469555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2type = ['clicks', 'carts', 'orders'] # I have analyzed the data\n                                          # and so I know we can expect these event types\ntype2id = {a: i for i, a in enumerate(id2type)}\n\nid2type, type2id","metadata":{"execution":{"iopub.status.busy":"2024-09-13T07:45:21.472271Z","iopub.execute_input":"2024-09-13T07:45:21.473168Z","iopub.status.idle":"2024-09-13T07:45:21.480938Z","shell.execute_reply.started":"2024-09-13T07:45:21.473121Z","shell.execute_reply":"2024-09-13T07:45:21.480068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.to_pickle(id2type, '/kaggle/working/id2type.pkl')\npd.to_pickle(type2id, '/kaggle/working/type2id.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-09-13T07:45:21.949917Z","iopub.execute_input":"2024-09-13T07:45:21.950532Z","iopub.status.idle":"2024-09-13T07:45:21.955917Z","shell.execute_reply.started":"2024-09-13T07:45:21.950489Z","shell.execute_reply":"2024-09-13T07:45:21.954781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jsonl_to_df(fn, output_dir='/kaggle/working/output_chunks', chunk_size=100_000, split_size=1_000_000):\n    sessions = []\n    aids = []\n    tss = []\n    types = []\n    \n    # 각 파일이 저장될 chunk 번호를 저장하는 변수\n    chunk_num = 0\n\n    # 저장할 디렉토리가 없으면 생성\n    if not os.path.exists(output_dir):\n        os.makedirs(output_dir)\n\n    chunks = pd.read_json(fn, lines=True, chunksize=chunk_size)\n\n    # row count tracking\n    total_rows = 0\n    \n    for chunk in chunks:\n        for row_idx, session_data in chunk.iterrows():\n            num_events = len(session_data.events)\n            # NumPy 대신 CuPy를 사용하여 배열을 처리\n            sessions += cp.asnumpy(cp.repeat(cp.array(session_data.session), num_events)).tolist()\n            for event in session_data.events:\n                aids.append(event['aid'])\n                tss.append(event['ts'])\n                types.append(type2id[event['type']])\n                \n            total_rows += num_events\n\n            # 일정 크기만큼 데이터가 모이면 Parquet로 저장\n            if total_rows >= split_size:\n                # CuPy 배열을 사용하여 DataFrame 생성\n                df = pd.DataFrame(data={\n                    'session': sessions,\n                    'aid': aids,\n                    'ts': tss,\n                    'type': cp.asnumpy(cp.array(types)).tolist()  # CuPy 배열을 다시 NumPy로 변환하여 pandas에서 처리\n                })\n                \n                # 메모리 절약을 위해 데이터 타입을 변환\n                df.type = df.type.astype(cp.uint8)  # CuPy의 uint8을 사용\n                \n                # Parquet 파일로 저장\n                parquet_file = os.path.join(output_dir, f'chunk_{chunk_num}.parquet')\n                df.to_parquet(parquet_file, index=False)\n                \n                # 다음 chunk 파일을 위해 초기화\n                sessions = []\n                aids = []\n                tss = []\n                types = []\n                total_rows = 0\n                chunk_num += 1\n    \n    # 마지막으로 남은 데이터 저장\n    if sessions:\n        df = pd.DataFrame(data={\n            'session': sessions,\n            'aid': aids,\n            'ts': tss,\n            'type': cp.asnumpy(cp.array(types)).tolist()\n        })\n        df.type = df.type.astype(cp.uint8)\n        parquet_file = os.path.join(output_dir, f'chunk_{chunk_num}.parquet')\n        df.to_parquet(parquet_file, index=False)\n\n    print(f\"Data saved in {output_dir} in {chunk_num + 1} files.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-13T07:45:24.787245Z","iopub.execute_input":"2024-09-13T07:45:24.788155Z","iopub.status.idle":"2024-09-13T07:45:24.802271Z","shell.execute_reply.started":"2024-09-13T07:45:24.788112Z","shell.execute_reply":"2024-09-13T07:45:24.801206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time jsonl_to_df('/kaggle/input/otto-recommender-system/train.jsonl', output_dir='train_chunks', chunk_size=300_000, split_size=3_000_000)","metadata":{"execution":{"iopub.status.busy":"2024-09-13T07:45:26.756115Z","iopub.execute_input":"2024-09-13T07:45:26.756514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_df = cudf.read_parquet('/kaggle/working/train_chunks/*.parquet')\ntotal_df.to_parquet('/kaggle/working/train.parquet')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}