{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Create Submission File","metadata":{}},{"cell_type":"code","source":"import polars as pl\n\ntest_df = pl.read_parquet('../input/comp9417-preprocessing-and-test-data/test_df.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:40:04.069166Z","iopub.execute_input":"2023-08-03T20:40:04.069739Z","iopub.status.idle":"2023-08-03T20:40:04.519396Z","shell.execute_reply.started":"2023-08-03T20:40:04.069699Z","shell.execute_reply":"2023-08-03T20:40:04.517604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id2type = {0: 'clicks', 1: 'carts', 2: 'orders'}\ntype2id = {'clicks': 0, 'carts': 1, 'orders': 2}","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:40:04.521547Z","iopub.execute_input":"2023-08-03T20:40:04.522487Z","iopub.status.idle":"2023-08-03T20:40:04.546103Z","shell.execute_reply.started":"2023-08-03T20:40:04.522337Z","shell.execute_reply":"2023-08-03T20:40:04.544872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get test data\nimport polars as pl\nimport numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\n\ndata_path = Path('/kaggle/input/otto-recommender-system/')\n\ntest_sessions = pd.DataFrame()\nchunks = pd.read_json(data_path / 'test.jsonl', lines=True, chunksize=100_000)\n\nfor e, chunk in enumerate(chunks):\n    event_dict = {\n        'session': [],\n        'aid': [],\n        'ts': [],\n        'type': [],\n    }\n    if e < 2:\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(event['type'])\n        chunk_session = pd.DataFrame(event_dict)\n        test_sessions = pd.concat([test_sessions, chunk_session])\n    else:\n        break\n        \n\ntest_sessions = pl.from_pandas(test_sessions.reset_index(drop=True))\ntest_sessions = test_sessions.groupby('session').agg(pl.all()).sort(by='session')","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:40:04.548267Z","iopub.execute_input":"2023-08-03T20:40:04.548772Z","iopub.status.idle":"2023-08-03T20:40:16.215607Z","shell.execute_reply.started":"2023-08-03T20:40:04.548730Z","shell.execute_reply":"2023-08-03T20:40:16.214345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\ndef load_model(name):\n    with open(name, 'rb') as file:  \n        model = pickle.load(file)\n        \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:40:16.223448Z","iopub.execute_input":"2023-08-03T20:40:16.223978Z","iopub.status.idle":"2023-08-03T20:40:16.232469Z","shell.execute_reply.started":"2023-08-03T20:40:16.223938Z","shell.execute_reply":"2023-08-03T20:40:16.230596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_df(model_long, model_short):\n    event_types = ['click', 'cart', 'order']\n    models = []\n    for event in event_types:\n        models.append(load_model(f'../input/comp9417-training-{model_long}/{model_short}_{event}.pkl'))\n\n    model_click = models[0]\n    model_cart = models[1]\n    model_order = models[2]\n\n    click_scores = model_click.predict(test_df.to_pandas())\n    cart_scores = model_cart.predict(test_df.to_pandas())\n    order_scores = model_order.predict(test_df.to_pandas())\n\n    pred_df = test_df.with_columns(click_scores = pl.lit(click_scores), cart_scores = pl.lit(cart_scores), order_scores = pl.lit(order_scores))\n    \n    click_preds = pred_df.groupby('session').agg(pl.col('aid').sort_by('click_scores', descending=True).slice(0,20))\n    cart_preds = pred_df.groupby('session').agg(pl.col('aid').sort_by('cart_scores', descending=True).slice(0,20))\n    order_preds = pred_df.groupby('session').agg(pl.col('aid').sort_by('order_scores', descending=True).slice(0,20))\n\n    click_new = click_preds.with_columns(\n        pl.lit('clicks').alias('type'),\n        (pl.col('session').cast(pl.Utf8) + '_clicks').alias('session_type'),\n        pl.col('aid').cast(pl.List(pl.Utf8))\n    ).rename({'aid':'labels'}).drop('session').drop('type')\n    cart_new = cart_preds.with_columns(\n        pl.lit('carts').alias('type'),\n        (pl.col('session').cast(pl.Utf8) + '_carts').alias('session_type'),\n        pl.col('aid').cast(pl.List(pl.Utf8))\n    ).rename({'aid':'labels'}).drop('session').drop('type')\n    order_new = order_preds.with_columns(\n        pl.lit('orders').alias('type'),\n        (pl.col('session').cast(pl.Utf8) + '_orders').alias('session_type'),\n        pl.col('aid').cast(pl.List(pl.Utf8))\n    ).rename({'aid':'labels'}).drop('session').drop('type')\n\n    preds = pl.concat([click_new, cart_new, order_new]).to_pandas()\n    print(preds)\n    preds['labels'] = preds['labels'].apply(lambda x: ' '.join(x))\n    preds = preds[['session_type', 'labels']]\n    preds = preds.sort_values(by='session_type')\n\n    print(preds)\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:40:16.235157Z","iopub.execute_input":"2023-08-03T20:40:16.236602Z","iopub.status.idle":"2023-08-03T20:40:16.260097Z","shell.execute_reply.started":"2023-08-03T20:40:16.236551Z","shell.execute_reply":"2023-08-03T20:40:16.258747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# output\npreds = get_df('lightgbm', 'lgbm')\npreds.to_csv('/kaggle/working/lgbm_submission.csv', index=False)  ","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:40:16.261812Z","iopub.execute_input":"2023-08-03T20:40:16.262155Z","iopub.status.idle":"2023-08-03T20:40:31.073241Z","shell.execute_reply.started":"2023-08-03T20:40:16.262126Z","shell.execute_reply":"2023-08-03T20:40:31.071913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = get_df('xgboost', 'xgb')\npreds.to_csv('/kaggle/working/xgb_submission.csv', index=False)  ","metadata":{"execution":{"iopub.status.busy":"2023-08-03T20:40:31.075839Z","iopub.execute_input":"2023-08-03T20:40:31.076222Z","iopub.status.idle":"2023-08-03T20:40:54.813734Z","shell.execute_reply.started":"2023-08-03T20:40:31.076190Z","shell.execute_reply":"2023-08-03T20:40:54.812218Z"},"trusted":true},"execution_count":null,"outputs":[]}]}