{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:19.664704Z","iopub.execute_input":"2023-01-12T15:53:19.666120Z","iopub.status.idle":"2023-01-12T15:53:32.012790Z","shell.execute_reply.started":"2023-01-12T15:53:19.665985Z","shell.execute_reply":"2023-01-12T15:53:32.011048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nimport random\nfrom polars.testing import assert_frame_equal","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.015797Z","iopub.execute_input":"2023-01-12T15:53:32.016735Z","iopub.status.idle":"2023-01-12T15:53:32.066750Z","shell.execute_reply.started":"2023-01-12T15:53:32.016679Z","shell.execute_reply":"2023-01-12T15:53:32.065302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"\npd.set_option('display.max_colwidth', None)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.068979Z","iopub.execute_input":"2023-01-12T15:53:32.069508Z","iopub.status.idle":"2023-01-12T15:53:32.078852Z","shell.execute_reply.started":"2023-01-12T15:53:32.069466Z","shell.execute_reply":"2023-01-12T15:53:32.077013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Radek's validation set is using the older version of organizer's script (there is a little cold start problem on aid)\ntrain_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/train.parquet')\ntest_v = pl.scan_parquet('/kaggle/input/otto-train-and-test-data-for-local-validation/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.080523Z","iopub.execute_input":"2023-01-12T15:53:32.080888Z","iopub.status.idle":"2023-01-12T15:53:32.110949Z","shell.execute_reply.started":"2023-01-12T15:53:32.080856Z","shell.execute_reply":"2023-01-12T15:53:32.109560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I created this validation set on paperspace and optimized on Kaggle\ntrain_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/train_sessions.parquet')\ntest_v7 = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions.parquet')\ntest_v7_full = pl.scan_parquet('/kaggle/input/ottovalidation7days/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.115336Z","iopub.execute_input":"2023-01-12T15:53:32.115720Z","iopub.status.idle":"2023-01-12T15:53:32.138432Z","shell.execute_reply.started":"2023-01-12T15:53:32.115689Z","shell.execute_reply":"2023-01-12T15:53:32.137392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I created this validation set on paperspace and optimized on Kaggle too\ntrain_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/train_sessions.parquet')\ntest_v7_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions.parquet')\ntest_v7_full_2nd = pl.scan_parquet('/kaggle/input/ottovalidation7days2nd/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.139711Z","iopub.execute_input":"2023-01-12T15:53:32.140626Z","iopub.status.idle":"2023-01-12T15:53:32.162278Z","shell.execute_reply.started":"2023-01-12T15:53:32.140593Z","shell.execute_reply":"2023-01-12T15:53:32.161304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I created this validation set on Kaggle and optimized on Kaggle too\ntrain_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/train_sessions.parquet')\ntest_v7_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions.parquet')\ntest_v7_full_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_sessions_full.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.163762Z","iopub.execute_input":"2023-01-12T15:53:32.165729Z","iopub.status.idle":"2023-01-12T15:53:32.189053Z","shell.execute_reply.started":"2023-01-12T15:53:32.165691Z","shell.execute_reply":"2023-01-12T15:53:32.187932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create test_sessions_full","metadata":{}},{"cell_type":"code","source":"# train_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/train_ms.parquet')\n# test_ms = pl.scan_parquet('/kaggle/input/otto-radek-style-polars/test_ms.parquet')\n\n# split_ts_ms = train_ms.select([\n#     (pl.col('ts').max() - 7*24*60*60*1000).alias('split_ts')\n# ]).collect().to_series().to_list()[0]\n\n# # building `train_items` as it is in organizer's script https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L100\n# unique_aids_train_valid = (\n#     train_ms\n#     .filter(~(pl.col('ts').first() > split_ts_ms).over('session'))\n#     .filter(pl.col('ts') < split_ts_ms)\n#     .filter((pl.col('aid').count()>=2).over('session'))\n#     .select([\n#         pl.col('aid').unique().alias('unique_aids')\n#     ]).collect().to_series().to_list()\n# )\n# len(unique_aids_train_valid) # 1825325\n\n# test_sessions_full = (\n#     train_ms\n#     .filter((pl.col('ts').first() > split_ts_ms).over('session')).collect() # stop being lazy early can avoid error and wrong answers\n#     .filter(pl.col('aid').is_in(unique_aids_train_valid)) # step 2-2: https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L75\n#     .filter((pl.col('aid').count()>=2).over('session')) # step 3: https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py#L76\n# )\n\n# test_sessions_full.select([\n#         pl.col('session').n_unique().alias('total_sessions'),\n#         pl.col('aid').n_unique().alias('total_aid'),    \n#         pl.col('session').count().alias('total_rows'),       \n#     ])","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.190218Z","iopub.execute_input":"2023-01-12T15:53:32.190500Z","iopub.status.idle":"2023-01-12T15:53:32.196178Z","shell.execute_reply.started":"2023-01-12T15:53:32.190475Z","shell.execute_reply":"2023-01-12T15:53:32.194947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## use `frame_equal` to check whether `test_sessions_full` is correct","metadata":{}},{"cell_type":"code","source":"# assert_frame_equal(test_sessions_full, test_v7_full_2nd.collect())\n\n# assert_frame_equal(test_sessions_full, test_v7_full_3rd.collect())","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.197734Z","iopub.execute_input":"2023-01-12T15:53:32.198176Z","iopub.status.idle":"2023-01-12T15:53:32.210202Z","shell.execute_reply.started":"2023-01-12T15:53:32.198130Z","shell.execute_reply":"2023-01-12T15:53:32.209026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## In this notebook I would like to reimplement `test_sessions` and `test_labels` as it is in the organizer's [scripts](https://github.com/otto-de/recsys-dataset/blob/main/src/testset.py).\n\nstep 1: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/testset.py#L38-L40\n\n```python\n    last_labels = []\n    splitted_sessions = []\n\n    for _, session in tqdm(sessions.iterrows(), desc=\"Creating trimmed testset\", total=len(sessions)):\n        # take all rows of each session from `test_sessions_full`, and give them to `split_events` function\n        session = session.to_dict()\n        splitted_events, labels = split_events(session['events'])\n```\n\nstep 2: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/testset.py#L22-L23\n\n```python\ndef split_events(events: list[dict], split_idx=None):\n    # make a deep copy of the entire session \n    # and give it to function `ground_truth` to generate `test_events` which include labels or ground_truth\n    test_events = ground_truth(deepcopy(events)) \n```\n\nstep 3: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/testset.py#L24-L25\n\n```python\n    if not split_idx:\n        split_idx = random.randint(1, len(test_events)) # create a random split number `split_idx`\n```\n\nstep 4: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/testset.py#L26\n\n```python\n    test_events = test_events[:split_idx] # use `split_idx` to get `test_events`\n```    \n\nstep 5: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/testset.py#L27\n\n```python\n    labels = test_events[-1]['labels'] # get the ground truth from the last event's labels\n```\n\nstep 6: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/testset.py#L28-L29\n\n```python\n    for event in test_events:\n        del event['labels'] # remove `labels` from every event of the `test_events` of the session\n    return test_events, labels        \n```\n","metadata":{}},{"cell_type":"markdown","source":"## How to convert a session into a `test_events` with ground truth? [script](https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/labels.py)\n\nstep 1: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/labels.py#L6\n\n```python\ndef ground_truth(events: list[dict]):\n    prev_labels = {\"clicks\": None, \"carts\": set(), \"orders\": set()} # to store previous labels for previous events\n``` \n\nstep 2: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/labels.py#L8-L9\n\n```python\n    for event in reversed(events): # loop from the last event to the first event (Important! so that only the first click-aid will be kept)\n            event[\"labels\"] = {} # add an empty dict named 'labels' to each event\n```\n\nstep 3: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/labels.py#L11-L16\n\n```python\n        for label in ['clicks', 'carts', 'orders']:\n            if prev_labels[label]: # when `prev_labels[label]` is not empty or none\n                if label != 'clicks':\n                    # give `prev_labels['carts']` to `event['labels']['carts']`\n                    # give `prev_labels['orders']` to `event['labels']['orders']`                    \n                    event[\"labels\"][label] = prev_labels[label].copy() \n                else:\n                    event[\"labels\"][label] = prev_labels[label] # give `prev_labels['clicks']` to `event['labels']['clicks']`                    \n```\n\nstep 4: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/labels.py#L18-L23\n\n```python\n        if event[\"type\"] == \"clicks\":\n            prev_labels['clicks'] = event[\"aid\"] # update/replace `prev_labels['clicks]` with the current event's click-aid \n        if event[\"type\"] == \"carts\":\n            prev_labels['carts'].add(event[\"aid\"]) # add current event's cart-aid to the set of `prev_labels['carts']`\n        elif event[\"type\"] == \"orders\":\n            prev_labels['orders'].add(event[\"aid\"]) # add current event's order-aid to the set of `prev_labels['orders']`\n```","metadata":{}},{"cell_type":"markdown","source":"step 5: https://github.com/otto-de/recsys-dataset/blob/0aa8346e0caec260ebd1cb47f556147cda5f770d/src/labels.py#L25\n\n```python\n    return events[:-1] # return `events` without the last event\n    # why organizer does it? \n    # 1. because by now, the test_labels have been created and added to each event, and test_sessions won't include the last event anyway, so removing it won't affect anything. \n    # 2. more importantly later on the codes `split_idx = random.randint(1, len(test_events))`  and `test_events[:split_idx]` could include the last event of each session into `test_events` or `test_sessions`, which is not what we want. So, this removing the last event can ensure `test_events` or `test_sessions` get its number or split right.\n```","metadata":{}},{"cell_type":"code","source":"[1,2,3][:-1]","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.211723Z","iopub.execute_input":"2023-01-12T15:53:32.212122Z","iopub.status.idle":"2023-01-12T15:53:32.225501Z","shell.execute_reply.started":"2023-01-12T15:53:32.212086Z","shell.execute_reply":"2023-01-12T15:53:32.224343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create test_sessions","metadata":{}},{"cell_type":"code","source":"test_sessions_full = test_v7_full_3rd.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:32.226775Z","iopub.execute_input":"2023-01-12T15:53:32.227328Z","iopub.status.idle":"2023-01-12T15:53:33.658402Z","shell.execute_reply.started":"2023-01-12T15:53:32.227292Z","shell.execute_reply":"2023-01-12T15:53:33.657498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_full.select([\n    pl.col('aid').count().alias('total_rows'),\n    pl.col('session').n_unique().alias('total_sess')\n])","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:33.659752Z","iopub.execute_input":"2023-01-12T15:53:33.660235Z","iopub.status.idle":"2023-01-12T15:53:34.106267Z","shell.execute_reply.started":"2023-01-12T15:53:33.660203Z","shell.execute_reply":"2023-01-12T15:53:34.105097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_full.groupby('session').agg([\n    pl.col('aid').count().alias('sess_length')\n]).select('sess_length').describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:34.107564Z","iopub.execute_input":"2023-01-12T15:53:34.107865Z","iopub.status.idle":"2023-01-12T15:53:34.929635Z","shell.execute_reply.started":"2023-01-12T15:53:34.107838Z","shell.execute_reply":"2023-01-12T15:53:34.928887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"13376874 + 1783737","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:34.933528Z","iopub.execute_input":"2023-01-12T15:53:34.934453Z","iopub.status.idle":"2023-01-12T15:53:34.942021Z","shell.execute_reply.started":"2023-01-12T15:53:34.934418Z","shell.execute_reply":"2023-01-12T15:53:34.940397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_full = (\n    test_sessions_full\n    .select([\n        pl.all().shift(1).over('session') # remove the last event of each session, see code in step 5 of ground_truth\n    ])\n    .drop_nulls()\n)\ntest_sessions_full","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:34.944293Z","iopub.execute_input":"2023-01-12T15:53:34.944718Z","iopub.status.idle":"2023-01-12T15:53:44.283127Z","shell.execute_reply.started":"2023-01-12T15:53:34.944635Z","shell.execute_reply":"2023-01-12T15:53:44.282387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_sessions_full\n    .groupby('session').agg([\n        pl.col('aid').count().alias('sess_length'),\n    ])\n    .sort('session')\n\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:44.284389Z","iopub.execute_input":"2023-01-12T15:53:44.284873Z","iopub.status.idle":"2023-01-12T15:53:45.088878Z","shell.execute_reply.started":"2023-01-12T15:53:44.284843Z","shell.execute_reply":"2023-01-12T15:53:45.087490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sess_length = (\n    test_sessions_full\n    .groupby('session').agg([\n        pl.col('aid').count().alias('sess_length'),\n    ])\n    .sort('session')\n    .select('sess_length')\n    .to_series().to_list()\n)\nlen(sess_length)\nsess_length[:10]\nmin(sess_length)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:45.090186Z","iopub.execute_input":"2023-01-12T15:53:45.090954Z","iopub.status.idle":"2023-01-12T15:53:45.986212Z","shell.execute_reply.started":"2023-01-12T15:53:45.090921Z","shell.execute_reply":"2023-01-12T15:53:45.985291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# help(random.randint)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:45.987365Z","iopub.execute_input":"2023-01-12T15:53:45.988674Z","iopub.status.idle":"2023-01-12T15:53:45.992774Z","shell.execute_reply.started":"2023-01-12T15:53:45.988638Z","shell.execute_reply":"2023-01-12T15:53:45.991923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.randint(1, 1)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:45.994053Z","iopub.execute_input":"2023-01-12T15:53:45.994528Z","iopub.status.idle":"2023-01-12T15:53:46.014665Z","shell.execute_reply.started":"2023-01-12T15:53:45.994496Z","shell.execute_reply":"2023-01-12T15:53:46.013400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# help(np.random.randint)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:46.015912Z","iopub.execute_input":"2023-01-12T15:53:46.016257Z","iopub.status.idle":"2023-01-12T15:53:46.024375Z","shell.execute_reply.started":"2023-01-12T15:53:46.016231Z","shell.execute_reply":"2023-01-12T15:53:46.023090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    np.random.randint(1,1)\nexcept ValueError as e: \n    print(e)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:46.025805Z","iopub.execute_input":"2023-01-12T15:53:46.027273Z","iopub.status.idle":"2023-01-12T15:53:46.037814Z","shell.execute_reply.started":"2023-01-12T15:53:46.027224Z","shell.execute_reply":"2023-01-12T15:53:46.036312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.seed(42)\nsplit_idx_sess = [random.randint(1, x) for x in sess_length] # random seed will work here\nlen(split_idx_sess)\nsplit_idx_sess[:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:46.040158Z","iopub.execute_input":"2023-01-12T15:53:46.040607Z","iopub.status.idle":"2023-01-12T15:53:48.582413Z","shell.execute_reply.started":"2023-01-12T15:53:46.040574Z","shell.execute_reply":"2023-01-12T15:53:48.581155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_sessions_full\n    .groupby('session').agg([\n        pl.col('aid').count().alias('sess_length'),\n    ])\n    .sort('session')\n    .with_columns([\n        pl.Series(name='split_idx', values=split_idx_sess)\n    ])\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:48.584164Z","iopub.execute_input":"2023-01-12T15:53:48.584509Z","iopub.status.idle":"2023-01-12T15:53:49.357247Z","shell.execute_reply.started":"2023-01-12T15:53:48.584483Z","shell.execute_reply":"2023-01-12T15:53:49.356019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"split_df = (\n    np.random.seed(42),\n    test_sessions_full\n    .groupby('session').agg([\n        pl.col('aid').count().alias('sess_length'),\n        pl.col('aid').count().apply(lambda x: random.randint(1, x)).alias('split_idx') # random seed won't work here\n    ])\n    .sort('session')\n)\nsplit_df","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:49.358555Z","iopub.execute_input":"2023-01-12T15:53:49.358862Z","iopub.status.idle":"2023-01-12T15:53:52.707879Z","shell.execute_reply.started":"2023-01-12T15:53:49.358832Z","shell.execute_reply":"2023-01-12T15:53:52.706693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_split = (\n    test_sessions_full\n    .groupby('session').agg([\n        pl.col('aid').count().alias('sess_length'),\n    ])\n    .sort('session')\n    .with_columns([\n        pl.Series(name='split_idx', values=split_idx_sess)\n    ])\n)\ntest_sessions_split","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:52.710199Z","iopub.execute_input":"2023-01-12T15:53:52.710634Z","iopub.status.idle":"2023-01-12T15:53:53.487170Z","shell.execute_reply.started":"2023-01-12T15:53:52.710593Z","shell.execute_reply":"2023-01-12T15:53:53.485425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_sessions_full\n    .with_columns([\n        pl.col('aid').cumcount().over('session').alias('event_idx_each_session'),\n        pl.col('aid').count().over('session').alias('session_length'),\n    ])\n\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:53.489084Z","iopub.execute_input":"2023-01-12T15:53:53.489496Z","iopub.status.idle":"2023-01-12T15:53:55.589399Z","shell.execute_reply.started":"2023-01-12T15:53:53.489463Z","shell.execute_reply":"2023-01-12T15:53:55.588495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_sessions_full\n    .with_columns([\n        pl.col('aid').cumcount().over('session').alias('event_idx_each_session'),\n        pl.col('aid').count().over('session').alias('session_length'),\n    ])\n    .join(test_sessions_split, on='session')\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:55.592681Z","iopub.execute_input":"2023-01-12T15:53:55.594438Z","iopub.status.idle":"2023-01-12T15:53:58.192722Z","shell.execute_reply.started":"2023-01-12T15:53:55.594397Z","shell.execute_reply":"2023-01-12T15:53:58.192044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_sessions_full\n    .with_columns([\n        pl.col('aid').cumcount().over('session').alias('event_idx_each_session'),\n        pl.col('aid').count().over('session').alias('session_length'),\n    ])\n    .join(test_sessions_split, on='session')\n    .filter(pl.col('event_idx_each_session') < pl.col('split_idx'))\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:53:58.193787Z","iopub.execute_input":"2023-01-12T15:53:58.194505Z","iopub.status.idle":"2023-01-12T15:54:01.018616Z","shell.execute_reply.started":"2023-01-12T15:53:58.194476Z","shell.execute_reply":"2023-01-12T15:54:01.017189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_handmade = (\n    test_sessions_full\n    .with_columns([\n        pl.col('aid').cumcount().over('session').alias('event_idx_each_session'),\n        pl.col('aid').count().over('session').alias('session_length'),\n    ])\n    .join(test_sessions_split, on='session')\n    .filter(pl.col('event_idx_each_session') < pl.col('split_idx'))\n    .select([\n        'session',\n        'aid',\n        'ts',\n        'type'\n    ])\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:01.019842Z","iopub.execute_input":"2023-01-12T15:54:01.020185Z","iopub.status.idle":"2023-01-12T15:54:03.880232Z","shell.execute_reply.started":"2023-01-12T15:54:01.020157Z","shell.execute_reply":"2023-01-12T15:54:03.878520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    assert_frame_equal(test_sessions_handmade, test_v7.collect())\nexcept AssertionError as e:\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:03.881990Z","iopub.execute_input":"2023-01-12T15:54:03.882441Z","iopub.status.idle":"2023-01-12T15:54:04.962665Z","shell.execute_reply.started":"2023-01-12T15:54:03.882409Z","shell.execute_reply":"2023-01-12T15:54:04.961764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    assert_frame_equal(test_sessions_handmade, test_v7_2nd.collect())\nexcept AssertionError as e:\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:04.963890Z","iopub.execute_input":"2023-01-12T15:54:04.964190Z","iopub.status.idle":"2023-01-12T15:54:06.023548Z","shell.execute_reply.started":"2023-01-12T15:54:04.964163Z","shell.execute_reply":"2023-01-12T15:54:06.022556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    assert_frame_equal(test_sessions_handmade, test_v7_3rd.collect())\nexcept AssertionError as e:\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:06.024781Z","iopub.execute_input":"2023-01-12T15:54:06.025659Z","iopub.status.idle":"2023-01-12T15:54:07.011494Z","shell.execute_reply.started":"2023-01-12T15:54:06.025623Z","shell.execute_reply":"2023-01-12T15:54:07.010372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create test_labels","metadata":{}},{"cell_type":"code","source":"test_sessions_full = test_v7_full_3rd","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:07.013422Z","iopub.execute_input":"2023-01-12T15:54:07.013883Z","iopub.status.idle":"2023-01-12T15:54:07.019644Z","shell.execute_reply.started":"2023-01-12T15:54:07.013838Z","shell.execute_reply":"2023-01-12T15:54:07.017662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_sessions = (\n    test_sessions_full\n    .collect()\n    .with_columns([\n        pl.col('aid').cumcount().over('session').alias('event_idx_each_session'),\n        pl.col('aid').count().over('session').alias('session_length'),\n    ])\n    .join(test_sessions_split, on='session') # include the column 'split_idx' which take into account that the last event removed\n    .filter(~(pl.col('event_idx_each_session') < pl.col('split_idx'))) # the last event of each session is preserved here.\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:07.021557Z","iopub.execute_input":"2023-01-12T15:54:07.021959Z","iopub.status.idle":"2023-01-12T15:54:10.506695Z","shell.execute_reply.started":"2023-01-12T15:54:07.021922Z","shell.execute_reply":"2023-01-12T15:54:10.505548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cfg = pl.Config.restore_defaults()  \npl.Config.set_tbl_rows(50)  \npl.Config.set_fmt_str_lengths(1000)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:10.508292Z","iopub.execute_input":"2023-01-12T15:54:10.508707Z","iopub.status.idle":"2023-01-12T15:54:10.519780Z","shell.execute_reply.started":"2023-01-12T15:54:10.508668Z","shell.execute_reply":"2023-01-12T15:54:10.518467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels = (\n    test_labels_sessions\n    .with_columns([\n        pl.col('aid').filter(pl.col('type') == 0).first().over('session').alias('label_clicks'),# won't turned into a list\n        pl.col('aid').filter(pl.col('type') == 1).unique().list().over('session').alias('label_carts'),\n        pl.col('aid').filter(pl.col('type') == 2).unique().list().over('session').alias('label_orders'),        \n    ])\n    .with_columns([\n        pl.concat_list(['label_clicks']),# make it a list\n        pl.col('label_orders').arr.eval(pl.element().cast(pl.Utf8)).alias(\"str\"), \n    ])\n    .select([\n        pl.all().exclude(['str', 'event_idx_each_session', 'session_length', 'sess_length', 'split_idx']),\n        pl.col('str').arr.join(\" \").alias('label_orders_str')\n    ])\n)\ntest_labels","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:10.521386Z","iopub.execute_input":"2023-01-12T15:54:10.521716Z","iopub.status.idle":"2023-01-12T15:54:20.470641Z","shell.execute_reply.started":"2023-01-12T15:54:10.521687Z","shell.execute_reply":"2023-01-12T15:54:20.469746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels = (\n    test_labels_sessions\n    .with_columns([\n        pl.col('aid').filter(pl.col('type') == 0).first().over('session').alias('label_clicks'),# won't turned into a list\n        pl.col('aid').filter(pl.col('type') == 1).unique().list().over('session').alias('label_carts'),\n        pl.col('aid').filter(pl.col('type') == 2).unique().list().over('session').alias('label_orders'),        \n    ])\n    .with_columns([\n        pl.concat_list(['label_clicks']),# make it a list\n        pl.col('label_orders').arr.eval(pl.element().cast(pl.Utf8)).alias(\"str\"), \n    ])\n    .select([\n        pl.all().exclude(['str', 'event_idx_each_session', 'session_length', 'sess_length', 'split_idx']),\n        pl.col('str').arr.join(\" \").alias('label_orders_str')\n    ])\n    .groupby('session').agg([\n        pl.all().first()\n    ])\n    .sort('session')\n)\ntest_labels","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:20.471920Z","iopub.execute_input":"2023-01-12T15:54:20.473073Z","iopub.status.idle":"2023-01-12T15:54:31.125843Z","shell.execute_reply.started":"2023-01-12T15:54:20.473032Z","shell.execute_reply":"2023-01-12T15:54:31.125173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_clicks = test_labels.select([\n    pl.col('session'),\n    pl.lit('clicks').alias('type'),\n    pl.col('label_clicks').alias('ground_truth'),\n])\nlabel_clicks","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.126762Z","iopub.execute_input":"2023-01-12T15:54:31.127286Z","iopub.status.idle":"2023-01-12T15:54:31.167551Z","shell.execute_reply.started":"2023-01-12T15:54:31.127260Z","shell.execute_reply":"2023-01-12T15:54:31.166427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_clicks = test_labels.select([\n    pl.col('session'),\n    pl.lit('clicks').alias('type'),\n    pl.col('label_clicks').alias('ground_truth'),\n]).filter(pl.col('ground_truth').arr.first().is_not_null())\nlabel_clicks","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.168695Z","iopub.execute_input":"2023-01-12T15:54:31.169497Z","iopub.status.idle":"2023-01-12T15:54:31.251474Z","shell.execute_reply.started":"2023-01-12T15:54:31.169461Z","shell.execute_reply":"2023-01-12T15:54:31.250625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_carts = test_labels.select([\n    pl.col('session'),\n    pl.lit('carts').alias('type'),\n    pl.col('label_carts').alias('ground_truth'),\n])\nlabel_carts","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.252531Z","iopub.execute_input":"2023-01-12T15:54:31.252795Z","iopub.status.idle":"2023-01-12T15:54:31.295801Z","shell.execute_reply.started":"2023-01-12T15:54:31.252771Z","shell.execute_reply":"2023-01-12T15:54:31.294148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len([])","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.302509Z","iopub.execute_input":"2023-01-12T15:54:31.302882Z","iopub.status.idle":"2023-01-12T15:54:31.311200Z","shell.execute_reply.started":"2023-01-12T15:54:31.302846Z","shell.execute_reply":"2023-01-12T15:54:31.309465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_carts = test_labels.select([\n    pl.col('session'),\n    pl.lit('carts').alias('type'),\n    pl.col('label_carts').alias('ground_truth'),\n]).filter(pl.col('ground_truth').arr.lengths()>0)\nlabel_carts","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.312678Z","iopub.execute_input":"2023-01-12T15:54:31.313161Z","iopub.status.idle":"2023-01-12T15:54:31.377889Z","shell.execute_reply.started":"2023-01-12T15:54:31.313121Z","shell.execute_reply":"2023-01-12T15:54:31.376437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_orders = test_labels.select([\n    pl.col('session'),\n    pl.lit('orders').alias('type'),\n    pl.col('label_orders').alias('ground_truth'),\n])\nlabel_orders","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.379633Z","iopub.execute_input":"2023-01-12T15:54:31.380077Z","iopub.status.idle":"2023-01-12T15:54:31.422366Z","shell.execute_reply.started":"2023-01-12T15:54:31.380038Z","shell.execute_reply":"2023-01-12T15:54:31.420958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_orders = test_labels.select([\n    pl.col('session'),\n    pl.lit('orders').alias('type'),\n    pl.col('label_orders').alias('ground_truth'),\n]).filter(pl.col('ground_truth').arr.lengths()>0)\nlabel_orders","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.424183Z","iopub.execute_input":"2023-01-12T15:54:31.424594Z","iopub.status.idle":"2023-01-12T15:54:31.481510Z","shell.execute_reply.started":"2023-01-12T15:54:31.424556Z","shell.execute_reply":"2023-01-12T15:54:31.480298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade = pl.concat([label_clicks, label_carts, label_orders]).sort('session').select([\n    pl.col('session').cast(pl.Int32),\n    pl.col('type'),\n    pl.col('ground_truth').arr.eval(pl.element().cast(pl.Int32)) # cast into Int32 for each element of a list in a column\n])    ","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:31.482951Z","iopub.execute_input":"2023-01-12T15:54:31.483404Z","iopub.status.idle":"2023-01-12T15:54:32.886040Z","shell.execute_reply.started":"2023-01-12T15:54:31.483365Z","shell.execute_reply":"2023-01-12T15:54:32.884788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_3rd = pl.scan_parquet('/kaggle/input/otto-validation-optimized-parquet/test_labels.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:32.887339Z","iopub.execute_input":"2023-01-12T15:54:32.887753Z","iopub.status.idle":"2023-01-12T15:54:32.901923Z","shell.execute_reply.started":"2023-01-12T15:54:32.887720Z","shell.execute_reply":"2023-01-12T15:54:32.900286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_labels_3rd\n    .collect()\n    .select([\n        pl.all().head().suffix(\"_head\"),\n        pl.all().tail().suffix(\"_tail\")\n    ])\n    \n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:32.903206Z","iopub.execute_input":"2023-01-12T15:54:32.904168Z","iopub.status.idle":"2023-01-12T15:54:33.387277Z","shell.execute_reply.started":"2023-01-12T15:54:32.904133Z","shell.execute_reply":"2023-01-12T15:54:33.386579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_labels_handmade\n    .select([\n        pl.all().head().suffix(\"_head\"),\n        pl.all().tail().suffix(\"_tail\")\n    ])\n    \n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:33.388365Z","iopub.execute_input":"2023-01-12T15:54:33.389514Z","iopub.status.idle":"2023-01-12T15:54:33.400085Z","shell.execute_reply.started":"2023-01-12T15:54:33.389449Z","shell.execute_reply":"2023-01-12T15:54:33.398237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade.frame_equal(test_labels_3rd.collect())","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:33.401599Z","iopub.execute_input":"2023-01-12T15:54:33.403204Z","iopub.status.idle":"2023-01-12T15:54:34.860656Z","shell.execute_reply.started":"2023-01-12T15:54:33.403152Z","shell.execute_reply":"2023-01-12T15:54:34.860038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Will they be the same when list turned into a long string? (no really)","metadata":{}},{"cell_type":"code","source":"# assert_frame_equal(test_labels_handmade, test_labels_3rd.collect()) # cause crash","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:34.861660Z","iopub.execute_input":"2023-01-12T15:54:34.862036Z","iopub.status.idle":"2023-01-12T15:54:34.865480Z","shell.execute_reply.started":"2023-01-12T15:54:34.862010Z","shell.execute_reply":"2023-01-12T15:54:34.864658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_str = (\n    test_labels_handmade\n    .select([\n        pl.all().exclude('ground_truth'),\n        pl.col('ground_truth').arr.eval(pl.element().cast(pl.Utf8)).alias('str')\n    ])\n    .select([\n        'session',\n        'type',\n        pl.col('str').arr.join(' ').alias('ground_truth_str')\n    ])\n    \n)\ntest_labels_handmade_str","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:34.866403Z","iopub.execute_input":"2023-01-12T15:54:34.866630Z","iopub.status.idle":"2023-01-12T15:54:37.036863Z","shell.execute_reply.started":"2023-01-12T15:54:34.866607Z","shell.execute_reply":"2023-01-12T15:54:37.035362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_3rd_str = (\n    test_labels_3rd\n    .collect()\n    .select([\n        pl.all().exclude('ground_truth'),\n        pl.col('ground_truth').arr.eval(pl.element().cast(pl.Utf8)).alias('str')\n    ])\n    .select([\n        'session',\n        'type',\n        pl.col('str').arr.join(' ').alias('ground_truth_str')\n    ])\n    \n)\ntest_labels_3rd_str","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:37.038529Z","iopub.execute_input":"2023-01-12T15:54:37.039220Z","iopub.status.idle":"2023-01-12T15:54:39.453289Z","shell.execute_reply.started":"2023-01-12T15:54:37.039173Z","shell.execute_reply":"2023-01-12T15:54:39.452155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assert_frame_equal(test_labels_handmade_str, test_labels_3rd_str)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.455224Z","iopub.execute_input":"2023-01-12T15:54:39.456363Z","iopub.status.idle":"2023-01-12T15:54:39.459831Z","shell.execute_reply.started":"2023-01-12T15:54:39.456312Z","shell.execute_reply":"2023-01-12T15:54:39.459111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_str.frame_equal(test_labels_3rd_str)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.460924Z","iopub.execute_input":"2023-01-12T15:54:39.462092Z","iopub.status.idle":"2023-01-12T15:54:39.497799Z","shell.execute_reply.started":"2023-01-12T15:54:39.462058Z","shell.execute_reply":"2023-01-12T15:54:39.497046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_str.select('session').to_series().series_equal(test_labels_3rd_str.select('session').to_series())\ntest_labels_handmade_str.select('type').to_series().series_equal(test_labels_3rd_str.select('type').to_series())","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.498933Z","iopub.execute_input":"2023-01-12T15:54:39.499916Z","iopub.status.idle":"2023-01-12T15:54:39.529331Z","shell.execute_reply.started":"2023-01-12T15:54:39.499885Z","shell.execute_reply":"2023-01-12T15:54:39.528501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_str.select('ground_truth_str').to_series().series_equal(test_labels_3rd_str.select('ground_truth_str').to_series())","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.530884Z","iopub.execute_input":"2023-01-12T15:54:39.531241Z","iopub.status.idle":"2023-01-12T15:54:39.553207Z","shell.execute_reply.started":"2023-01-12T15:54:39.531212Z","shell.execute_reply":"2023-01-12T15:54:39.551250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare the length of list of column 'ground_truth'","metadata":{}},{"cell_type":"code","source":"# test_labels_handmade.frame_equal(test_labels_3rd.collect())","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.554788Z","iopub.execute_input":"2023-01-12T15:54:39.555169Z","iopub.status.idle":"2023-01-12T15:54:39.560848Z","shell.execute_reply.started":"2023-01-12T15:54:39.555136Z","shell.execute_reply":"2023-01-12T15:54:39.559041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_lstlen = (\n    test_labels_handmade\n    .with_columns([\n        pl.col('ground_truth').arr.lengths().alias('lst_len')\n    ])\n    \n)\ntest_labels_handmade_lstlen","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.562582Z","iopub.execute_input":"2023-01-12T15:54:39.562976Z","iopub.status.idle":"2023-01-12T15:54:39.588993Z","shell.execute_reply.started":"2023-01-12T15:54:39.562934Z","shell.execute_reply":"2023-01-12T15:54:39.587553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_3rd_lstlen = (\n    test_labels_3rd\n    .collect()\n    .with_columns([\n        pl.col('ground_truth').arr.lengths().alias('lst_len')\n    ])\n    \n)\ntest_labels_3rd_lstlen","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.590450Z","iopub.execute_input":"2023-01-12T15:54:39.590765Z","iopub.status.idle":"2023-01-12T15:54:39.832491Z","shell.execute_reply.started":"2023-01-12T15:54:39.590739Z","shell.execute_reply":"2023-01-12T15:54:39.831641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_lstlen.select('lst_len').to_series().series_equal(test_labels_3rd_lstlen.select('lst_len').to_series())","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.833504Z","iopub.execute_input":"2023-01-12T15:54:39.834329Z","iopub.status.idle":"2023-01-12T15:54:39.843333Z","shell.execute_reply.started":"2023-01-12T15:54:39.834299Z","shell.execute_reply":"2023-01-12T15:54:39.842096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_lstlen.select('ground_truth').to_series().series_equal(test_labels_3rd_lstlen.select('ground_truth').to_series())","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:39.845400Z","iopub.execute_input":"2023-01-12T15:54:39.845817Z","iopub.status.idle":"2023-01-12T15:54:41.083459Z","shell.execute_reply.started":"2023-01-12T15:54:39.845785Z","shell.execute_reply":"2023-01-12T15:54:41.082144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assert_frame_equal(test_labels_handmade_lstlen, test_labels_3rd_lstlen) # won't work","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:41.084817Z","iopub.execute_input":"2023-01-12T15:54:41.085522Z","iopub.status.idle":"2023-01-12T15:54:41.092771Z","shell.execute_reply.started":"2023-01-12T15:54:41.085486Z","shell.execute_reply":"2023-01-12T15:54:41.091539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compare the sum of `ground_truth` column of two dataframe","metadata":{"execution":{"iopub.status.busy":"2022-12-31T06:36:54.161692Z","iopub.execute_input":"2022-12-31T06:36:54.162015Z","iopub.status.idle":"2022-12-31T06:36:54.177939Z","shell.execute_reply.started":"2022-12-31T06:36:54.161990Z","shell.execute_reply":"2022-12-31T06:36:54.177178Z"}}},{"cell_type":"code","source":"test_labels_handmade_lstsum = (\n    test_labels_3rd\n    .collect()\n    .with_columns([\n        pl.col('ground_truth').arr.sum().alias('lst_sum')\n    ])\n    \n)\ntest_labels_handmade_lstsum","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:41.094902Z","iopub.execute_input":"2023-01-12T15:54:41.096024Z","iopub.status.idle":"2023-01-12T15:54:42.384739Z","shell.execute_reply.started":"2023-01-12T15:54:41.095956Z","shell.execute_reply":"2023-01-12T15:54:42.383692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_3rd_lstsum = (\n    test_labels_3rd\n    .collect()\n    .with_columns([\n        pl.col('ground_truth').arr.sum().alias('lst_sum')\n    ])\n    \n)\ntest_labels_3rd_lstsum","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:42.386191Z","iopub.execute_input":"2023-01-12T15:54:42.386718Z","iopub.status.idle":"2023-01-12T15:54:43.645541Z","shell.execute_reply.started":"2023-01-12T15:54:42.386684Z","shell.execute_reply":"2023-01-12T15:54:43.644111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_labels_handmade_lstsum\n    .with_columns([\n        (pl.col('lst_sum') - test_labels_3rd_lstsum.select('lst_sum').to_series()).alias('res')\n    ])\n    \n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:43.646918Z","iopub.execute_input":"2023-01-12T15:54:43.647272Z","iopub.status.idle":"2023-01-12T15:54:43.665974Z","shell.execute_reply.started":"2023-01-12T15:54:43.647235Z","shell.execute_reply":"2023-01-12T15:54:43.664581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n    test_labels_handmade_lstsum\n    .with_columns([\n        (pl.col('lst_sum') - test_labels_3rd_lstsum.select('lst_sum').to_series()).alias('res')\n    ])\n    .filter(pl.col('res') != 0)\n    \n)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:43.667547Z","iopub.execute_input":"2023-01-12T15:54:43.668182Z","iopub.status.idle":"2023-01-12T15:54:43.685943Z","shell.execute_reply.started":"2023-01-12T15:54:43.668143Z","shell.execute_reply":"2023-01-12T15:54:43.685134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# so, they are the same, the only difference could be the order of the lists in 'ground_truth' column","metadata":{}},{"cell_type":"code","source":"test_labels_3rd_sort = (\n    test_labels_3rd\n    .collect()\n    .with_columns([\n        pl.col('ground_truth').arr.sort()\n    ])\n    \n)\ntest_labels_3rd_sort","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:54:43.687092Z","iopub.execute_input":"2023-01-12T15:54:43.687688Z","iopub.status.idle":"2023-01-12T15:54:44.716550Z","shell.execute_reply.started":"2023-01-12T15:54:43.687650Z","shell.execute_reply":"2023-01-12T15:54:44.715313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_handmade_sort = (\n    test_labels_handmade\n    .with_columns([\n        pl.col('ground_truth').arr.sort()\n    ])\n    \n)\ntest_labels_handmade_sort","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:59:27.806555Z","iopub.execute_input":"2023-01-12T15:59:27.806971Z","iopub.status.idle":"2023-01-12T15:59:28.638938Z","shell.execute_reply.started":"2023-01-12T15:59:27.806936Z","shell.execute_reply":"2023-01-12T15:59:28.638244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels_3rd_sort.frame_equal(test_labels_handmade_sort)","metadata":{"execution":{"iopub.status.busy":"2023-01-12T15:59:33.524835Z","iopub.execute_input":"2023-01-12T15:59:33.525234Z","iopub.status.idle":"2023-01-12T15:59:34.781220Z","shell.execute_reply.started":"2023-01-12T15:59:33.525192Z","shell.execute_reply":"2023-01-12T15:59:34.779391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Yes, after sorting, they are the same now!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}