{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### This Notebook is simple demonsration of Polars Library to convert JSONL file data with around 12GB data to 1.5GB parquest data in kaggle cpu. ","metadata":{}},{"cell_type":"code","source":"#Install Polars with all optional dependencies.\n!pip install 'polars[all]'","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:48:59.055789Z","iopub.execute_input":"2022-12-24T14:48:59.056307Z","iopub.status.idle":"2022-12-24T14:49:15.677253Z","shell.execute_reply.started":"2022-12-24T14:48:59.056212Z","shell.execute_reply":"2022-12-24T14:49:15.676080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#importing Libraries\nimport polars as pl\nfrom io import StringIO\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:49:15.680023Z","iopub.execute_input":"2022-12-24T14:49:15.680874Z","iopub.status.idle":"2022-12-24T14:49:15.741516Z","shell.execute_reply.started":"2022-12-24T14:49:15.680822Z","shell.execute_reply":"2022-12-24T14:49:15.740585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking the number of session data avilable in training data\nwith open(\"/kaggle/input/otto-recommender-system/train.jsonl\") as json_file:\n    file_lines = json_file.readlines()\n\nprint(f'Training data consists of {len(file_lines)} sessions')","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:49:15.742911Z","iopub.execute_input":"2022-12-24T14:49:15.743281Z","iopub.status.idle":"2022-12-24T14:51:23.019193Z","shell.execute_reply.started":"2022-12-24T14:49:15.743251Z","shell.execute_reply":"2022-12-24T14:51:23.016826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating a polar dataframe\nslice_size = 1_000_000\ndf = pl.concat(\n    [\n        pl.read_json(\n            StringIO(\"\".join(file_lines[offset: (offset + slice_size)])),\n            json_lines=True,\n        )\n        for offset in range(0, len(file_lines), slice_size)\n    ]\n)\n#converting struct of events in 3 columns(aid,ts,type) \ndf_final=df.explode('events').unnest('events')","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:51:23.024306Z","iopub.execute_input":"2022-12-24T14:51:23.025391Z","iopub.status.idle":"2022-12-24T14:55:36.274336Z","shell.execute_reply.started":"2022-12-24T14:51:23.025329Z","shell.execute_reply":"2022-12-24T14:55:36.272773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:55:36.276095Z","iopub.execute_input":"2022-12-24T14:55:36.276475Z","iopub.status.idle":"2022-12-24T14:55:37.166772Z","shell.execute_reply.started":"2022-12-24T14:55:36.276433Z","shell.execute_reply":"2022-12-24T14:55:37.165583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.select(pl.col(\"type\").unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:55:37.168788Z","iopub.execute_input":"2022-12-24T14:55:37.169150Z","iopub.status.idle":"2022-12-24T14:55:40.810890Z","shell.execute_reply.started":"2022-12-24T14:55:37.169117Z","shell.execute_reply":"2022-12-24T14:55:40.810099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from_ = [\"clicks\",\"orders\",\"carts\"]\nto_ = [0,1,2]\n\n\ndef replace(column, from_, to_):\n    # initiate the expression with `pl.when`\n    branch =  pl.when(pl.col(column) == from_[0]).then(to_[0])\n\n    \n    # for every value add a `when.then`\n    for (from_value, to_value) in zip(from_, to_):\n        branch = branch.when(pl.col(column) == from_value).then(to_value)\n\n    # finish with an `otherwise`\n    return branch.otherwise(pl.col(column)).alias(column)\n    \n\n\ndf_final=df_final.with_column(replace(\"type\", from_, to_))","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:55:40.812304Z","iopub.execute_input":"2022-12-24T14:55:40.813280Z","iopub.status.idle":"2022-12-24T14:56:23.699238Z","shell.execute_reply.started":"2022-12-24T14:55:40.813245Z","shell.execute_reply":"2022-12-24T14:56:23.698268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:56:23.700667Z","iopub.execute_input":"2022-12-24T14:56:23.701162Z","iopub.status.idle":"2022-12-24T14:56:23.711279Z","shell.execute_reply.started":"2022-12-24T14:56:23.701117Z","shell.execute_reply":"2022-12-24T14:56:23.710269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final=df_final.with_column(pl.col(\"type\").cast(pl.Int32, strict=False).alias(\"type\"))\n","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:56:23.713550Z","iopub.execute_input":"2022-12-24T14:56:23.714057Z","iopub.status.idle":"2022-12-24T14:56:26.393550Z","shell.execute_reply.started":"2022-12-24T14:56:23.714015Z","shell.execute_reply":"2022-12-24T14:56:26.392665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:56:26.396582Z","iopub.execute_input":"2022-12-24T14:56:26.397129Z","iopub.status.idle":"2022-12-24T14:56:26.404602Z","shell.execute_reply.started":"2022-12-24T14:56:26.397093Z","shell.execute_reply":"2022-12-24T14:56:26.403661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_final.write_parquet('otto-train.parquet', use_pyarrow=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-24T14:56:26.406042Z","iopub.execute_input":"2022-12-24T14:56:26.406633Z","iopub.status.idle":"2022-12-24T14:57:01.297022Z","shell.execute_reply.started":"2022-12-24T14:56:26.406599Z","shell.execute_reply":"2022-12-24T14:57:01.296051Z"},"trusted":true},"execution_count":null,"outputs":[]}]}