{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Preprocessing","metadata":{}},{"cell_type":"code","source":"!pip install polars","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:45:44.077064Z","iopub.execute_input":"2023-08-03T19:45:44.078503Z","iopub.status.idle":"2023-08-03T19:45:53.762021Z","shell.execute_reply.started":"2023-08-03T19:45:44.078450Z","shell.execute_reply":"2023-08-03T19:45:53.760719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\n\ntrain = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/train.parquet')\nvalidA = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test.parquet')\nvalidB = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test_labels.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:45:53.765347Z","iopub.execute_input":"2023-08-03T19:45:53.765663Z","iopub.status.idle":"2023-08-03T19:45:58.926398Z","shell.execute_reply.started":"2023-08-03T19:45:53.765627Z","shell.execute_reply":"2023-08-03T19:45:58.925182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.dtypes)\nprint(validA.dtypes)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:45:58.927590Z","iopub.execute_input":"2023-08-03T19:45:58.929117Z","iopub.status.idle":"2023-08-03T19:45:58.936650Z","shell.execute_reply.started":"2023-08-03T19:45:58.929060Z","shell.execute_reply":"2023-08-03T19:45:58.935262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n# Get subset of data \nfraction_of_sessions = 0.2\n\ntrain_sessions = train['session'].sample(fraction=fraction_of_sessions, seed=42)\ntrain = train.filter(pl.col(\"session\").is_in(train_sessions))\ntrain = train.sort(\"session\")\n\nvalidation_sessions = validA['session'].sample(fraction=fraction_of_sessions, seed=42)\nvalidA = validA.filter(pl.col(\"session\").is_in(validation_sessions))\nvalidA = validA.sort(\"session\")\n\nvalidB = validB.filter(pl.col(\"session\").is_in(validation_sessions))\nvalidB = validB.sort(\"session\")\n\nmax_ts, min_ts = max(train['ts']), min(train['ts'])\n\nprint(train.shape[0], validA.shape[0], validB.shape[0])\n\nprint(train, validA, validB)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:45:58.940174Z","iopub.execute_input":"2023-08-03T19:45:58.940868Z","iopub.status.idle":"2023-08-03T19:46:53.714412Z","shell.execute_reply.started":"2023-08-03T19:45:58.940824Z","shell.execute_reply":"2023-08-03T19:46:53.712679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Candidate and Feature Generation","metadata":{}},{"cell_type":"code","source":"# Simple candidate generator - used to benchmark results\ndef generate_candidates_simple(df, dict_format=False):\n    df = df.groupby(['session', 'aid']).agg(pl.count())\n    df = df.sort('session', 'count', descending=[False, True]).groupby('session').head(50)\n    if dict_format:\n        df = df.groupby('session').agg(pl.col(\"aid\"))\n        result = {}\n        for session, aid_list in zip(df['session'].to_list(), df['aid'].to_list()):\n            result[session] = aid_list \n        return result\n    \n    return df.select('session', 'aid').sort(['session', 'aid'])","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:46:53.716536Z","iopub.execute_input":"2023-08-03T19:46:53.717026Z","iopub.status.idle":"2023-08-03T19:46:53.725526Z","shell.execute_reply.started":"2023-08-03T19:46:53.716988Z","shell.execute_reply":"2023-08-03T19:46:53.723672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_weight = {0: 1, 1: 6, 2: 3}\n\ndef generate_covisit(df, event_type):\n    # look at carts and orders only when generating buy2buy covisit matrix\n    if event_type == 2:\n        df = df.filter(pl.col('type').is_in([1, 2]))\n    # sort by session_id and recency\n    df = df.sort(['session', 'ts'], descending=[False, True])\n    # get most recent events for each session, otherwise merge is too costly\n    df = df.groupby('session').tail(10)\n    # find pairs of items that were accessed within a day of each other\n    df = df.join(df, on='session', suffix=\"_next\")\n    df = df.filter((pl.col(\"aid\") != pl.col(\"aid_next\")) & \n                   (pl.col(\"type\") != pl.col(\"type_next\")) &\n                   (pl.col(\"ts\") != pl.col(\"ts_next\")))\n    df = df.with_columns(((pl.col(\"ts_next\") - pl.col(\"ts\")) / (24 * 60 * 60 * 1000)).alias(\"days_elapsed\"))\n    df = df.filter((pl.col(\"days_elapsed\") >= 0) & (pl.col(\"days_elapsed\") <= 1))\n    \n    # used for predicting clicks given past clicks/carts/orders\n    if event_type == 0:\n        # weight is combination of count and recency (represented by timestamp with max-min scaling)\n        df = df.with_columns(weight = (1 + 3*(pl.col(\"ts\") - min_ts)/(max_ts - min_ts)))\n        df = df.groupby(['aid', 'aid_next']).agg(pl.sum(\"weight\"))\n    # used for predicting future cart/orders given past carts/orders\n    elif event_type == 1:\n        # pairs are weighted according to event type\n        df = df.with_columns(pl.col(\"type\").map_dict(type_weight).alias(\"weight\"))\n        df = df.groupby(['aid', 'aid_next']).agg(pl.sum(\"weight\"))\n    elif event_type == 2:\n        # count of number of occurences of each pair\n        df = df.groupby(['aid', 'aid_next']).agg(pl.count().alias('weight'))\n    \n    df = df.sort(['aid', 'aid_next', 'weight'], descending=[False, False, True]).groupby('aid').head(40).select(['aid', 'aid_next'])\n    df = df.groupby('aid').agg(pl.col('aid_next'))\n    covisit = {}\n    for aid, aid_list in zip(df[\"aid\"], df[\"aid_next\"]):\n        covisit[aid] = aid_list.to_list()\n    \n    return covisit","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:46:53.728133Z","iopub.execute_input":"2023-08-03T19:46:53.728596Z","iopub.status.idle":"2023-08-03T19:46:53.746027Z","shell.execute_reply.started":"2023-08-03T19:46:53.728565Z","shell.execute_reply":"2023-08-03T19:46:53.744338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\n\ntype_weight = {0: 1, 1: 6, 2: 3}\n\ndef generate_candidates(df, covisit):\n    aids = df[\"aid\"].to_list()\n    types = df[\"type\"].to_list()\n    unique_aids = list(dict.fromkeys(aids[::-1]))\n\n    time_weights = np.logspace(0.1,1,len(aids),base=2, endpoint=True)-1\n    aids_counter = {}\n    for aid, w, t in zip(aids, time_weights, types):\n        aids_counter[aid] = aids_counter.get(aid, 0) + w * type_weight[t]\n\n    aids_counter_sorted = sorted(aids_counter.items(), key=lambda x: x[1])\n    candidates = [k for k, v in aids_counter_sorted]\n\n    if len(candidates) <= 20:\n        secondary_candidates_counter = Counter()\n        for candidate in candidates:\n            secondary_candidates_counter.update(covisit.get(aid, []))  \n        secondary_candidates = [k for k, v in secondary_candidates_counter.most_common(40)]\n        return candidates[:40] + secondary_candidates[:(40 - len(candidates))]\n\n    return candidates[:40]","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:46:53.748579Z","iopub.execute_input":"2023-08-03T19:46:53.749400Z","iopub.status.idle":"2023-08-03T19:46:53.768412Z","shell.execute_reply.started":"2023-08-03T19:46:53.749359Z","shell.execute_reply":"2023-08-03T19:46:53.766828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate Session Features \ndef generate_session_features(train_df_user):\n    df_session_grouped = train_df_user.sort('session', 'ts', descending=[False, True]).groupby('session')\n    df_session_grouped_clicks = train_df_user.filter(train_df_user['type'] == 0).sort('session', 'ts', descending=[False, True]).groupby('session')\n    df_session_grouped_carts = train_df_user.filter(train_df_user['type'] == 1).sort('session', 'ts', descending=[False, True]).groupby('session')\n    df_session_grouped_orders = train_df_user.filter(train_df_user['type'] == 2).sort('session', 'ts', descending=[False, True]).groupby('session')\n\n    # last event type and aid\n    last_event_type = df_session_grouped.agg(last_type=pl.col('type').first())\n\n    # length of session\n    session_length = df_session_grouped.agg(session_length=pl.col('ts').first() - pl.col('ts').last())\n\n    # duplication rate\n\n    # number of clicks, carts, orders, events that session\n    clicks_ratio = df_session_grouped_clicks.agg(clicks_ratio=pl.col('type').count())\n    carts_ratio = df_session_grouped_carts.agg(carts_ratio=pl.col('type').count())\n    orders_ratio = df_session_grouped_orders.agg(orders_ratio=pl.col('type').count())\n    events_ratio = df_session_grouped.agg(events_ratio=pl.col('type').count())\n\n    return [last_event_type, session_length, events_ratio, clicks_ratio, carts_ratio, orders_ratio]","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:46:53.770260Z","iopub.execute_input":"2023-08-03T19:46:53.770675Z","iopub.status.idle":"2023-08-03T19:46:53.783453Z","shell.execute_reply.started":"2023-08-03T19:46:53.770641Z","shell.execute_reply":"2023-08-03T19:46:53.781601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# AID FEATURES\ntrain_df_item = pl.concat([train, validA])\n\n# total interaction for aid\ninteraction_counts = train_df_item['aid'].value_counts().sort(by='counts').rename({\"counts\":\"interaction_count\"})\n\n# click for aid\nclick_counts = train_df_item.filter(train_df_item['type'] == 0).groupby('aid').agg(click_count=pl.count('type'))\n\n# cart for aid\ncart_counts = train_df_item.filter(train_df_item['type'] == 1).groupby('aid').agg(cart_count=pl.count('type'))\n\n# buy for aid\nbuy_counts = train_df_item.filter(train_df_item['type'] == 2).groupby('aid').agg(buy_count=pl.count('type'))\n\n# last ts for aid\nlast_ts = train_df_item.groupby('aid').agg(last_ts=pl.col('ts').max())\n\n# click, cart, buy ratios\nglobal_events_ratio = click_counts.join(cart_counts, on='aid').join(buy_counts, on='aid')\nglobal_events_ratio = global_events_ratio.with_columns(click2cart = pl.col('click_count') / pl.col('cart_count'))\nglobal_events_ratio = global_events_ratio.with_columns(cart2buy = pl.col('cart_count') / pl.col('buy_count'))\nglobal_events_ratio = global_events_ratio.with_columns(click2buy = pl.col('click_count') / pl.col('buy_count'))\nglobal_events_ratio = global_events_ratio.drop('click_count').drop('cart_count').drop('buy_count')\n\n# interaction rate in last 7 days\nlast_ts_int = train_df_item['ts'].max()\nlast_7_days_count = train_df_item.filter(train_df_item['ts'] >= last_ts_int - (7 * 24 * 60 * 60))['aid'].value_counts().sort(by='counts').rename({\"counts\":\"last7days_count\"})\nmerged = interaction_counts.join(last_7_days_count, on='aid')\nmerged = merged.with_columns(last7days_interaction_rate = pl.col('last7days_count') / pl.col('interaction_count'))\nlast7days_interaction_count = merged.drop('interaction_count').drop('last7days_count')\n\n# inclusion rate in all sessions\n# sessions_with_aid / total_sessions\ntotal_sessions = train_df_item['aid'].n_unique()\nunique_sessions = train_df_item.groupby('aid').agg(unique_sessions=pl.n_unique('session')).sort(by='unique_sessions')\ninclusion_rate = unique_sessions.with_columns(inclusion_rate = pl.col('unique_sessions') * 1000 / total_sessions).drop('unique_sessions')\n\n# average interactions per hour over all sessions\nnum_hours = train_df_item.groupby('aid').agg(num_hours = (pl.max('ts') - pl.min('ts')) / (60 * 60)).sort(by='num_hours')\nnum_hours = num_hours.with_columns(num_hours = pl.when(num_hours['num_hours']==0).then(1).otherwise(num_hours['num_hours']))\nmerged = num_hours.join(interaction_counts, on='aid')\naverage_interactions_ph = merged.with_columns(average_interactions_ph = pl.col('interaction_count') / pl.col('num_hours'))\naverage_interactions_ph = average_interactions_ph.drop('interaction_count').drop('num_hours')\n\n# average num clicks before buy\n# this bit too hard\nxd = train_df_item.sort(by=['session','ts'])\naid_features = [interaction_counts, click_counts, buy_counts, last_ts, global_events_ratio, last7days_interaction_count, inclusion_rate, average_interactions_ph]","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:46:53.786376Z","iopub.execute_input":"2023-08-03T19:46:53.786692Z","iopub.status.idle":"2023-08-03T19:47:47.922922Z","shell.execute_reply.started":"2023-08-03T19:46:53.786668Z","shell.execute_reply":"2023-08-03T19:47:47.921384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nDISK_PIECES = 4\n\n# Improved speed for 2X using polars. \ndef pqt_to_dict(path):\n    return pl.read_parquet(path).groupby('aid_x').agg(pl.col('aid_y')).to_pandas().set_index('aid_x').aid_y.apply(list).to_dict()\n\n# LOAD THREE CO-VISITATION MATRICES\ncovisit_clicks = pqt_to_dict(f'/kaggle/input/otto-covisitation-matrix-parquet-files/top_20_clicks_v7_0.pqt')\n\nfor k in range(1,DISK_PIECES): \n    covisit_clicks.update(pd.read_parquet(f'/kaggle/input/otto-covisitation-matrix-parquet-files/top_20_clicks_v7_{k}.pqt') )\n","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:47:47.927962Z","iopub.execute_input":"2023-08-03T19:47:47.928337Z","iopub.status.idle":"2023-08-03T19:47:53.842891Z","shell.execute_reply.started":"2023-08-03T19:47:47.928308Z","shell.execute_reply":"2023-08-03T19:47:53.842124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# simple candidate generator\n# candidates_df = generate_candidates_simple(pl.concat([train, validA]))\n# train_df = validA.unique(subset=['session'])\n# train_df = candidates_df.join(train_df, on=\"session\", how=\"inner\").select(['session', 'aid'])\n\nEVENT_TYPE = 0\n# Add candidates + features to df\ntrain_df = validA.to_pandas()\n# covisit loaded above \n# covisit = generate_covisit(pl.concat([train, validA]), EVENT_TYPE)\ntrain_df = train_df.sort_values(['session', 'ts'], ascending=[True, False]).groupby('session').apply(lambda x: generate_candidates(x, covisit_clicks)).reset_index(name=\"aid\")\ntrain_df = pl.from_pandas(train_df)\ntrain_df.columns = ['session', 'aid']\ntrain_df = train_df.explode('aid')\ntrain_df = train_df.select([\n    pl.col('session').cast(pl.datatypes.Int32).alias('session'), \n    pl.col('aid').cast(pl.datatypes.Int32).alias('aid')\n])\n\nsession_features = generate_session_features(validA)\nfor session in session_features:\n    train_df = train_df.join(session, on='session', how='left')\n\nfor aid in aid_features:\n    train_df = train_df.join(aid, on='aid', how='left')","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:47:53.844552Z","iopub.execute_input":"2023-08-03T19:47:53.845217Z","iopub.status.idle":"2023-08-03T19:50:10.347756Z","shell.execute_reply.started":"2023-08-03T19:47:53.845184Z","shell.execute_reply":"2023-08-03T19:50:10.346506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# add gts\ntype2id = {\"clicks\": 0, \"carts\": 1, \"orders\": 2}\nvalidB_long = validB.explode(\"ground_truth\").select([\n    pl.col(\"session\").cast(pl.datatypes.Int32),\n    pl.col(\"ground_truth\").cast(pl.datatypes.Int32).alias(\"aid\"),\n    pl.col(\"type\").map_dict(type2id).cast(pl.datatypes.Int8),\n]).with_columns(pl.lit(1).alias('gt'))\n\nvalidB_click = validB_long.filter(pl.col(\"type\") == 0)\nvalidB_cart = validB_long.filter(pl.col(\"type\") == 1)\nvalidB_order = validB_long.filter(pl.col(\"type\") == 2)\n\ntrain_df_click = train_df.join(validB_click, on=[\"session\", \"aid\"], how=\"left\").with_columns(pl.all().fill_null(0)).drop(\"type\")\ntrain_df_cart = train_df.join(validB_cart, on=[\"session\", \"aid\"], how=\"left\").with_columns(pl.all().fill_null(0)).drop(\"type\")\ntrain_df_order = train_df.join(validB_order, on=[\"session\", \"aid\"], how=\"left\").with_columns(pl.all().fill_null(0)).drop(\"type\")\n\nprint(train_df_click, train_df_cart, train_df_order)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:51:10.317609Z","iopub.execute_input":"2023-08-03T19:51:10.317987Z","iopub.status.idle":"2023-08-03T19:51:12.174298Z","shell.execute_reply.started":"2023-08-03T19:51:10.317962Z","shell.execute_reply":"2023-08-03T19:51:12.173173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom collections import defaultdict\n\nvalidA = validA.sort(['session', 'ts'], descending=[False, True])\n\ncount_in_session = defaultdict(lambda: 0)\nfor row in validA.iter_rows():\n    count_in_session[(row[0], row[1])] += 1\n\ndef session_item_features(df):\n    df = df.with_columns(\n        pl.struct(pl.col(['session', 'aid'])).apply(lambda x: count_in_session[x['aid'], x['session']]).alias(\"count_within_session\"),\n    )\n    return df\n\ntrain_df_click = session_item_features(train_df_click).select([pl.all().exclude('gt'), pl.col('gt')])\ntrain_df_cart = session_item_features(train_df_cart).select([pl.all().exclude('gt'), pl.col('gt')])\ntrain_df_order = session_item_features(train_df_order).select([pl.all().exclude('gt'), pl.col('gt')])","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:02:12.115724Z","iopub.execute_input":"2023-08-03T20:02:12.116164Z","iopub.status.idle":"2023-08-03T20:03:14.246588Z","shell.execute_reply.started":"2023-08-03T20:02:12.116137Z","shell.execute_reply":"2023-08-03T20:03:14.245801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import polars as pl\ndata = {'a': [1, 2, 3],\n       'b': [2, 3, 4]}\n\ndef f(a, b):\n    return a + b\ndf = pl.DataFrame(data)\n\ndf.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:51:12.176060Z","iopub.execute_input":"2023-08-03T19:51:12.176480Z","iopub.status.idle":"2023-08-03T19:51:12.189048Z","shell.execute_reply.started":"2023-08-03T19:51:12.176453Z","shell.execute_reply":"2023-08-03T19:51:12.186529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# output\nimport pathlib\n\npath_click: pathlib.Path = '/kaggle/working/train_df_click.parquet'\npath_cart: pathlib.Path = '/kaggle/working/train_df_cart.parquet'\npath_order: pathlib.Path = '/kaggle/working/train_df_order.parquet'\n\ntrain_df_click.write_parquet(path_click)\ntrain_df_cart.write_parquet(path_cart)\ntrain_df_order.write_parquet(path_order)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:51:12.190157Z","iopub.execute_input":"2023-08-03T19:51:12.190584Z","iopub.status.idle":"2023-08-03T19:51:28.285146Z","shell.execute_reply.started":"2023-08-03T19:51:12.190547Z","shell.execute_reply":"2023-08-03T19:51:28.283594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Test Data","metadata":{}},{"cell_type":"code","source":"# Get test data\nimport numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\n\ndata_path = Path('/kaggle/input/recsys-dataset/')\n\ntest_sessions = pd.DataFrame()\nchunks = pd.read_json(data_path / 'otto-recsys-test.jsonl', lines=True, chunksize=100_000)\n\nfor e, chunk in enumerate(chunks):\n    event_dict = {\n        'session': [],\n        'aid': [],\n        'ts': [],\n        'type': [],\n    }\n    if e < 2:\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(event['type'])\n        chunk_session = pd.DataFrame(event_dict)\n        test_sessions = pd.concat([test_sessions, chunk_session])\n    else:\n        break\n        \n\ntest_sessions = pl.from_pandas(test_sessions.reset_index(drop=True))\ntest_sessions = test_sessions.groupby('session').agg(pl.all()).sort(by='session')","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:51:28.286747Z","iopub.execute_input":"2023-08-03T19:51:28.287121Z","iopub.status.idle":"2023-08-03T19:51:39.127471Z","shell.execute_reply.started":"2023-08-03T19:51:28.287091Z","shell.execute_reply":"2023-08-03T19:51:39.126062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split test data into testA (session up to certain point) and testB (prediction)\ndictA = {'session': [], 'aid': [], 'ts': [], 'type': []}\ndictB = {'session': [], 'aid': [], 'ts': [], 'type': []}\n\nfor row in test_sessions.iter_rows():\n    split_idx = np.random.randint(1,len(row[1]))\n    dictA['session'].append(row[0])\n    dictA['aid'].append(row[1][:split_idx])\n    dictA['ts'].append(row[2][:split_idx])\n    dictA['type'].append(row[3][:split_idx])\n\n    dictB['session'].append(row[0])\n    dictB['aid'].append(row[1][split_idx:])\n    dictB['ts'].append(row[2][split_idx:])\n    dictB['type'].append(row[3][split_idx:])\n    \ntestA = pl.DataFrame(data=dictA).explode(['aid', 'ts', 'type'])\ntestA = testA.select([pl.all().exclude('type'), pl.col('type').map_dict(type2id).alias('type')])\ntestB = pl.DataFrame(data=dictB)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:51:39.129292Z","iopub.execute_input":"2023-08-03T19:51:39.129736Z","iopub.status.idle":"2023-08-03T19:51:52.370125Z","shell.execute_reply.started":"2023-08-03T19:51:39.129696Z","shell.execute_reply":"2023-08-03T19:51:52.368070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate test candidates\n# Actual Candidates Generator\ntest_df = testA.to_pandas()\n# covisit loaded above \n# covisit = generate_covisit(pl.concat([train, validA]), EVENT_TYPE)\ntest_df = test_df.sort_values(['session', 'ts'], ascending=[True, False]).groupby('session').apply(lambda x: generate_candidates(x, covisit_clicks)).reset_index(name=\"aid\")\ntest_df = pl.from_pandas(test_df)\ntest_df.columns = ['session', 'aid']\ntest_df = test_df.explode('aid')\ntest_df = test_df.select([\n    pl.col('session').cast(pl.datatypes.Int32).alias('session'), \n    pl.col('aid').cast(pl.datatypes.Int32).alias('aid')\n])\n\n# Using simple candidates generator\n# test_df = generate_candidates_simple(testA).select([pl.all().cast(pl.datatypes.Int32)])    \n# session_features = generate_session_features(testA.select([\n#     pl.col('session').cast(pl.datatypes.Int32),\n#     pl.col('aid').cast(pl.datatypes.Int32),\n#     pl.col('ts').apply(lambda x: x / 1000).cast(pl.datatypes.Int32),\n#     pl.col(\"type\").map_dict(type2id).cast(pl.datatypes.Int8)\n# ]))\n\nfor session in session_features:\n    test_df = test_df.join(session, on='session', how='left')\n\nfor aid in aid_features:\n    test_df = test_df.join(aid, on='aid', how='left')\n    \ntest_df = test_df.with_columns(pl.all().fill_null(0))\nprint(test_df)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:51:52.371949Z","iopub.execute_input":"2023-08-03T19:51:52.372423Z","iopub.status.idle":"2023-08-03T19:52:28.541663Z","shell.execute_reply.started":"2023-08-03T19:51:52.372383Z","shell.execute_reply":"2023-08-03T19:52:28.540592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfrom collections import defaultdict\n\ntestA = testA.sort(['session', 'ts'], descending=[False, True])\n\ncount_in_session = defaultdict(lambda: 0)\nfor row in testA.iter_rows():\n    count_in_session[(row[0], row[1])] += 1\n\ndef session_item_features(df):\n    df = df.with_columns(\n        pl.struct(pl.col(['session', 'aid'])).apply(lambda x: count_in_session[x['aid'], x['session']]).alias(\"count_within_session\"),\n    )\n    return df\n\ntest_df = session_item_features(test_df)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:52:28.543293Z","iopub.execute_input":"2023-08-03T19:52:28.543956Z","iopub.status.idle":"2023-08-03T19:52:35.683031Z","shell.execute_reply.started":"2023-08-03T19:52:28.543922Z","shell.execute_reply":"2023-08-03T19:52:35.681919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_df)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:52:35.684377Z","iopub.execute_input":"2023-08-03T19:52:35.684662Z","iopub.status.idle":"2023-08-03T19:52:35.691290Z","shell.execute_reply.started":"2023-08-03T19:52:35.684639Z","shell.execute_reply":"2023-08-03T19:52:35.689924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"actual_events = testB.explode(['aid', 'ts', 'type']).select([\n    pl.col('session').cast(pl.datatypes.Int32),\n    pl.col('aid').cast(pl.datatypes.Int32),\n    pl.col('ts').apply(lambda x: x / 1000).cast(pl.datatypes.Int32),\n    pl.col('type').map_dict(type2id).cast(pl.datatypes.Int8)\n])","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:52:35.692486Z","iopub.execute_input":"2023-08-03T19:52:35.692883Z","iopub.status.idle":"2023-08-03T19:52:35.966065Z","shell.execute_reply.started":"2023-08-03T19:52:35.692847Z","shell.execute_reply":"2023-08-03T19:52:35.964004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# output\nimport pathlib\n\npath_test: pathlib.Path = '/kaggle/working/test_df.parquet'\npath_actual_events: pathlib.Path = '/kaggle/working/actual_events.parquet'\n\ntest_df.write_parquet(path_test)\nactual_events.write_parquet(path_actual_events)","metadata":{"execution":{"iopub.status.busy":"2023-08-03T19:52:35.967896Z","iopub.execute_input":"2023-08-03T19:52:35.968271Z","iopub.status.idle":"2023-08-03T19:52:37.155490Z","shell.execute_reply.started":"2023-08-03T19:52:35.968226Z","shell.execute_reply":"2023-08-03T19:52:37.154406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}