{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-23T11:08:04.592031Z","iopub.execute_input":"2022-12-23T11:08:04.592588Z","iopub.status.idle":"2022-12-23T11:08:04.640547Z","shell.execute_reply.started":"2022-12-23T11:08:04.592477Z","shell.execute_reply":"2022-12-23T11:08:04.639134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thanks to @radek1 for his excellent notebook!\n(https://www.kaggle.com/code/radek1/word2vec-how-to-training-and-submission)\nLots of great ideas came from there.","metadata":{}},{"cell_type":"code","source":"!pip install polars\n\nimport multiprocessing\n\nimport polars as pl\nfrom gensim.test.utils import common_texts\nfrom gensim.models import Word2Vec\n\ntrain = pl.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest = pl.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:08:04.643007Z","iopub.execute_input":"2022-12-23T11:08:04.643503Z","iopub.status.idle":"2022-12-23T11:08:34.775252Z","shell.execute_reply.started":"2022-12-23T11:08:04.643456Z","shell.execute_reply":"2022-12-23T11:08:34.774130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data to train word2vec model**","metadata":{}},{"cell_type":"code","source":"sentences_df = pl.concat([train, test]).groupby('session').agg(\n    pl.col('aid').alias('sentence')\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:08:34.776837Z","iopub.execute_input":"2022-12-23T11:08:34.777511Z","iopub.status.idle":"2022-12-23T11:08:47.767571Z","shell.execute_reply.started":"2022-12-23T11:08:34.777477Z","shell.execute_reply":"2022-12-23T11:08:47.766352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_test_df = pl.concat([test]).groupby('session').agg(\n    pl.col('aid').alias('sentence')\n)\npred_test_df = pred_test_df.to_pandas().rename(columns={'sentence':'labels'})\n\nsentences_df_clicks = pl.concat([test]).filter(pl.col('type') == 0)\nsentences_df_carts = pl.concat([test]).filter(pl.col('type') == 1)\nsentences_df_orders = pl.concat([test]).filter(pl.col('type') == 2)\n\nsentences_df_clicks = sentences_df_clicks.groupby('session').agg(pl.col('aid').alias('sentence'))\nsentences_df_carts = sentences_df_carts.groupby('session').agg(pl.col('aid').alias('sentence'))\nsentences_df_orders = sentences_df_orders.groupby('session').agg(pl.col('aid').alias('sentence'))\n\nsentences_df_clicks = sentences_df_clicks.to_pandas().rename(columns={'sentence':'labels_clicks'})\nsentences_df_carts = sentences_df_carts.to_pandas().rename(columns={'sentence':'labels_carts'})\nsentences_df_orders = sentences_df_orders.to_pandas().rename(columns={'sentence':'labels_orders'})\n\npred_test_df = pred_test_df.merge(sentences_df_clicks, how='left', on='session') \\\n                           .merge(sentences_df_carts, how='left', on='session') \\\n                           .merge(sentences_df_orders, how='left', on='session') \npred_test_df","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:08:47.769939Z","iopub.execute_input":"2022-12-23T11:08:47.770267Z","iopub.status.idle":"2022-12-23T11:08:53.560205Z","shell.execute_reply.started":"2022-12-23T11:08:47.770238Z","shell.execute_reply":"2022-12-23T11:08:53.558537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_df.labels.apply(lambda x: len(x)).hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:08:53.561794Z","iopub.execute_input":"2022-12-23T11:08:53.562158Z","iopub.status.idle":"2022-12-23T11:08:53.568341Z","shell.execute_reply.started":"2022-12-23T11:08:53.562126Z","shell.execute_reply":"2022-12-23T11:08:53.566745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = sentences_df['sentence'].to_list()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:08:53.570077Z","iopub.execute_input":"2022-12-23T11:08:53.570496Z","iopub.status.idle":"2022-12-23T11:09:33.660155Z","shell.execute_reply.started":"2022-12-23T11:08:53.570460Z","shell.execute_reply":"2022-12-23T11:09:33.658874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sentences = sentences_df.to_pandas().sentence.apply(lambda x: list(x[:100])).to_list()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:09:33.661432Z","iopub.execute_input":"2022-12-23T11:09:33.662545Z","iopub.status.idle":"2022-12-23T11:09:33.684982Z","shell.execute_reply.started":"2022-12-23T11:09:33.662504Z","shell.execute_reply":"2022-12-23T11:09:33.683576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Training word2vec model**","metadata":{}},{"cell_type":"code","source":"%%time\n\n# w2vec = Word2Vec(sentences=sentences, vector_size=32, min_count=1, workers=4)\n\ncores = multiprocessing.cpu_count() # Count the number of cores in a computer\n\nw2vec = Word2Vec(\n                 sentences=sentences, \n                 window = 5, # 5: 0.509, 10: 0.508\n#                  sg = 0,  # 1 -> skip-gram, 0 -> CBOW    1: 0.509, 0: 0.508\n                 vector_size=100,  # 200: 0.509, 150: 0.509, 100: 0.510, 50: 0.509\n#                  sample=6e-5,\n                 alpha=0.04,       # 0.1: 0.494, 0.05: 0.507, 0.04: 0.509, 0.03: 0.509, 0.01: 0.503 <-200\n                                   # 0.04: 0.510, 0.02: 0.509 <-100\n                 min_alpha=0.01,  # 0.0001: 0.510, 0.001: 0.509, 0.01: 0.510\n                 min_count=1, \n                 negative=20,  # 5: 0.510, 15: 0.511 (0.04), 18: 0.512 (0.04), 19: 0.512, 20: 0.513 (alpha=0.02), 21: 0.511 (0.04), 22: 0.512 (0.04)\n                               # 25:0.512 (0.04), 30: 0.510 (alpha=0.04)\n                 workers=cores-1\n                )\n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T11:09:33.686791Z","iopub.execute_input":"2022-12-23T11:09:33.688202Z","iopub.status.idle":"2022-12-23T12:45:17.057807Z","shell.execute_reply.started":"2022-12-23T11:09:33.688155Z","shell.execute_reply":"2022-12-23T12:45:17.056222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w2vec.save(\"word2vec.model\")","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:45:17.060671Z","iopub.execute_input":"2022-12-23T12:45:17.061020Z","iopub.status.idle":"2022-12-23T12:45:19.416266Z","shell.execute_reply.started":"2022-12-23T12:45:17.060988Z","shell.execute_reply":"2022-12-23T12:45:19.415418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = Word2Vec.load(\"word2vec.model\")\nmodel = Word2Vec.load(\"/kaggle/input/word2vec-model-best/word2vec.model\")","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:45:19.422786Z","iopub.execute_input":"2022-12-23T12:45:19.423617Z","iopub.status.idle":"2022-12-23T12:45:41.213055Z","shell.execute_reply.started":"2022-12-23T12:45:19.423553Z","shell.execute_reply":"2022-12-23T12:45:41.211861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"(Function for adding most similar products using word2vec functional)","metadata":{}},{"cell_type":"code","source":"def add_most_similar_to_20(l):\n    l = l[:20]\n    sims = model.wv.most_similar(l, topn=20-len(l))\n    sim_list = [x[0] for x in sims]\n    l = list(l) + list(sim_list)\n    return l","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:45:41.214676Z","iopub.execute_input":"2022-12-23T12:45:41.215054Z","iopub.status.idle":"2022-12-23T12:45:41.221545Z","shell.execute_reply.started":"2022-12-23T12:45:41.215021Z","shell.execute_reply":"2022-12-23T12:45:41.220730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Information from test dataset that we can use","metadata":{}},{"cell_type":"code","source":"# sentences_test_df = pl.concat([test]).groupby('session').agg(\n#     pl.col('aid').alias('sentence')\n# )\n# sentences_test_df = sentences_test_df.to_pandas()\n# sentences_test_df","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:45:41.222589Z","iopub.execute_input":"2022-12-23T12:45:41.223579Z","iopub.status.idle":"2022-12-23T12:45:41.234548Z","shell.execute_reply.started":"2022-12-23T12:45:41.223536Z","shell.execute_reply":"2022-12-23T12:45:41.233312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Importing Annoy that will help us to find most similar products","metadata":{}},{"cell_type":"code","source":"%%time\n\nfrom annoy import AnnoyIndex\n\naid2idx = {aid: i for i, aid in enumerate(model.wv.index_to_key)}\n# index = AnnoyIndex(32, 'euclidean')\nindex = AnnoyIndex(100, 'angular')\n\nfor aid, idx in aid2idx.items():\n    index.add_item(aid, model.wv.vectors[idx])\n#     index.add_item(aid, model.wv.vectors[aid])\n    \nindex.build(50)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:45:41.236540Z","iopub.execute_input":"2022-12-23T12:45:41.237257Z","iopub.status.idle":"2022-12-23T12:48:10.488337Z","shell.execute_reply.started":"2022-12-23T12:45:41.237206Z","shell.execute_reply":"2022-12-23T12:48:10.487071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def to_aid(x):\n#     return list(aid2idx.keys())[list(aid2idx.values()).index(x)]\n\n# def list_to_aid(l):\n#     return [to_aid(x) for x in l]","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:10.489600Z","iopub.execute_input":"2022-12-23T12:48:10.489931Z","iopub.status.idle":"2022-12-23T12:48:10.494343Z","shell.execute_reply.started":"2022-12-23T12:48:10.489902Z","shell.execute_reply":"2022-12-23T12:48:10.493211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"(Function for adding most similar products using annoy - much faster than previous function)","metadata":{}},{"cell_type":"code","source":"def make_20(x):\n    x = x[-20:]\n#     y = list(index.get_nns_by_item(aid2idx[x[-1]], 21 - len(x)))\n#     x = list(x) + list_to_aid(y)[1:]\n    x = list(x) + list(index.get_nns_by_item(x[-1], 21 - len(x))[1:])\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:10.495570Z","iopub.execute_input":"2022-12-23T12:48:10.496050Z","iopub.status.idle":"2022-12-23T12:48:10.512248Z","shell.execute_reply.started":"2022-12-23T12:48:10.496020Z","shell.execute_reply":"2022-12-23T12:48:10.511362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_20_avg(x):\n    x = x[-20:]\n#     y = [aid2idx[i] for i in x]\n    z = 0\n    for j in x[-3:]:\n        z += model.wv.vectors[j]\n    index.add_item(2000000, z / len(x))\n    y = index.get_nns_by_item(2000000, 20 - len(x))[:]\n    x = list(x) + list(y)\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:10.513663Z","iopub.execute_input":"2022-12-23T12:48:10.514848Z","iopub.status.idle":"2022-12-23T12:48:10.530140Z","shell.execute_reply.started":"2022-12-23T12:48:10.514796Z","shell.execute_reply":"2022-12-23T12:48:10.528802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import collections\n\ndef make_20_mult(x, n):\n    x = x[-20:]\n    if n > 19:\n        n = 19\n    if len(x) >= n:\n        y = []\n        y_sum = list(np.zeros(21-len(x)))\n        for i in range(n):\n            y.append(list(index.get_nns_by_item(x[-(i+1)], 21 - len(x))[1:]))\n            y_sum += y[-1]\n#         y1 = list(index.get_nns_by_item(x[-1], 21 - len(x))[1:])\n#         y2 = list(index.get_nns_by_item(x[-2], 21 - len(x))[1:]) \n#         y = y1 + y2\n        counter = dict(collections.Counter(y_sum))\n        res = sorted(list(set(y_sum)), key = lambda d: counter[d], reverse=True)\n    else:\n        res = list(index.get_nns_by_item(x[-1], 21 - len(x))[1:])\n        \n#     x = list(x) + list(res)[:20-len(x)] # prev var\n    x = list(x) + list(res)[1:21-len(x)] # to try\n#     x = list(res)[1:21]\n    \n    return x\n\ndef make_20_mult_many(labs, x1, y1, z1):\n    \n    x = x1[-20:]\n    y = y1[-20:]\n    z = z1[-20:]\n#     x_sum = list(set(list(x) + list(y) + list(z)))\n#     x_sum = list(x) + list(y) + list(z)\n#     n = len(x_sum)\n\n#     x_sum = list(set(labs))\n    x_sum = labs\n    n = len(x_sum)\n    \n    if n > 19:\n        n = 19\n    \n    f = []\n    f_sum = list(np.zeros(21-n))\n    \n    for item in x:\n        f.append(list(index.get_nns_by_item(item, 21 - n)[1:]))\n        f_sum += f[-1]\n    for item in y:\n        f.append(list(index.get_nns_by_item(item, 21 - n)[1:]))\n        f_sum += f[-1] \n        f.append(list(index.get_nns_by_item(item, 21 - n)[1:]))\n        f_sum += f[-1] \n    for item in z:\n        f.append(list(index.get_nns_by_item(item, 21 - n)[1:]))\n        f_sum += f[-1] \n        f.append(list(index.get_nns_by_item(item, 21 - n)[1:]))\n        f_sum += f[-1] \n        f.append(list(index.get_nns_by_item(item, 21 - n)[1:]))\n        f_sum += f[-1] \n                 \n    counter = dict(collections.Counter(f_sum))\n    res = sorted(list(set(f_sum)), key = lambda d: counter[d], reverse=True)\n        \n    x_sum = list(x_sum) + list(res)[1:20-n]\n    \n    return x_sum\n\ndef test_func(labs, x, y, z):\n    \n    x = list(x[-20:])\n    y = list(y[-20:])\n    z = list(z[-20:])\n#     x_sum = list(set(list(x) + list(y) + list(z)))\n#     x_sum = list(x) + list(y) + list(z)\n    x_sum = labs\n    n = len(x_sum)\n    \n    return x_sum[:20]\n\ndef short_list(x):\n    counter = dict(collections.Counter(x))\n    res = sorted(list(set(x)), key = lambda d: counter[d], reverse=True)\n    return res","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:10.532107Z","iopub.execute_input":"2022-12-23T12:48:10.532844Z","iopub.status.idle":"2022-12-23T12:48:10.688290Z","shell.execute_reply.started":"2022-12-23T12:48:10.532797Z","shell.execute_reply":"2022-12-23T12:48:10.687067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_test_df['labels_carts'] = pred_test_df['labels_carts'].fillna(pred_test_df['labels'])\npred_test_df['labels_orders'] = pred_test_df['labels_orders'].fillna(pred_test_df['labels'])\npred_test_df['labels_clicks'] = pred_test_df['labels_clicks'].fillna(pred_test_df['labels'])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:10.690085Z","iopub.execute_input":"2022-12-23T12:48:10.690440Z","iopub.status.idle":"2022-12-23T12:48:12.664656Z","shell.execute_reply.started":"2022-12-23T12:48:10.690409Z","shell.execute_reply":"2022-12-23T12:48:12.663400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.667234Z","iopub.execute_input":"2022-12-23T12:48:12.668238Z","iopub.status.idle":"2022-12-23T12:48:12.672433Z","shell.execute_reply.started":"2022-12-23T12:48:12.668198Z","shell.execute_reply":"2022-12-23T12:48:12.671438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make_20_mult_many(pred_test_df.labels_clicks.iloc[1],pred_test_df.labels_carts.iloc[1],pred_test_df.labels_orders.iloc[1])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.674465Z","iopub.execute_input":"2022-12-23T12:48:12.674846Z","iopub.status.idle":"2022-12-23T12:48:12.685392Z","shell.execute_reply.started":"2022-12-23T12:48:12.674814Z","shell.execute_reply":"2022-12-23T12:48:12.684030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Getting 20 values for each category in each session","metadata":{}},{"cell_type":"code","source":"# test_df = test.to_pandas()\n\n# test_df_clicks = test_df[test_df.type == 0]\n# test_df_clicks.session = test_df_clicks.session.apply(lambda x: str(x) + '_clicks')\n# test_df_carts = test_df[test_df.type == 1]\n# test_df_carts.session = test_df_carts.session.apply(lambda x: str(x) + '_carts')\n# test_df_orders = test_df[test_df.type == 2]\n# test_df_orders.session = test_df_orders.session.apply(lambda x: str(x) + '_orders')\n\n# test_df_clicks = pd.DataFrame(test_df_clicks.groupby('session').aid.apply(list)).reset_index().rename(columns={'aid':'labels'})\n# test_df_carts = pd.DataFrame(test_df_carts.groupby('session').aid.apply(list)).reset_index().rename(columns={'aid':'labels'})\n# test_df_orders = pd.DataFrame(test_df_orders.groupby('session').aid.apply(list)).reset_index().rename(columns={'aid':'labels'})","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.687035Z","iopub.execute_input":"2022-12-23T12:48:12.688154Z","iopub.status.idle":"2022-12-23T12:48:12.698034Z","shell.execute_reply.started":"2022-12-23T12:48:12.688102Z","shell.execute_reply":"2022-12-23T12:48:12.696368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_df = sentences_test_df.rename(columns = {'sentence':'labels'})\n\n# clicks_pred_df = pred_test_df.copy()\n# clicks_pred_df.session = clicks_pred_df.session.apply(lambda x: str(x) + '_clicks')\n# orders_pred_df = pred_test_df.copy()\n# orders_pred_df.session = orders_pred_df.session.apply(lambda x: str(x) + '_orders')\n# carts_pred_df = pred_test_df.copy()\n# carts_pred_df.session = carts_pred_df.session.apply(lambda x: str(x) + '_carts')\n\n# orders_pred_df = orders_pred_df.merge(test_df_orders, how='left', on='session')\n# orders_pred_df['labels_y'] = orders_pred_df['labels_y'].fillna(orders_pred_df['labels_x'])\n# clicks_pred_df = clicks_pred_df.merge(test_df_clicks, how='left', on='session')\n# clicks_pred_df['labels_y'] = clicks_pred_df['labels_y'].fillna(clicks_pred_df['labels_x'])\n# carts_pred_df = carts_pred_df.merge(test_df_carts, how='left', on='session')\n# carts_pred_df['labels_y'] = carts_pred_df['labels_y'].fillna(carts_pred_df['labels_x'])\n\n# clicks_pred_df = clicks_pred_df.drop('labels_x', axis=1)\n# carts_pred_df = carts_pred_df.drop('labels_x', axis=1)\n# orders_pred_df = orders_pred_df.drop('labels_x', axis=1)\n\n# clicks_pred_df = clicks_pred_df.rename(columns={'labels_y':'labels'})\n# carts_pred_df = carts_pred_df.rename(columns={'labels_y':'labels'})\n# orders_pred_df = orders_pred_df.rename(columns={'labels_y':'labels'})","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.699522Z","iopub.execute_input":"2022-12-23T12:48:12.699990Z","iopub.status.idle":"2022-12-23T12:48:12.717153Z","shell.execute_reply.started":"2022-12-23T12:48:12.699955Z","shell.execute_reply":"2022-12-23T12:48:12.715768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_df = pd.concat(\n#     [clicks_pred_df, orders_pred_df, carts_pred_df]\n# )\n# pred_test_df.columns = ['session_type', 'labels']","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.718365Z","iopub.execute_input":"2022-12-23T12:48:12.719397Z","iopub.status.idle":"2022-12-23T12:48:12.734610Z","shell.execute_reply.started":"2022-12-23T12:48:12.719358Z","shell.execute_reply":"2022-12-23T12:48:12.733246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clicks_pred_df.sort_values(by='session')","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.736161Z","iopub.execute_input":"2022-12-23T12:48:12.736984Z","iopub.status.idle":"2022-12-23T12:48:12.747820Z","shell.execute_reply.started":"2022-12-23T12:48:12.736938Z","shell.execute_reply":"2022-12-23T12:48:12.746744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_df = sentences_test_df.rename(columns = {'sentence':'labels'})\n# pred_test_df = sentences_test_df\n# pred_test_df = pred_test_df[:100]","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.749427Z","iopub.execute_input":"2022-12-23T12:48:12.749798Z","iopub.status.idle":"2022-12-23T12:48:12.761003Z","shell.execute_reply.started":"2022-12-23T12:48:12.749763Z","shell.execute_reply":"2022-12-23T12:48:12.759967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\npred_test_df['labels'] = pred_test_df.labels.apply(lambda x: list(set(x)))\n# pred_test_df['labels_clicks'] = pred_test_df.labels_clicks.apply(lambda x: list(set(x)))\n# pred_test_df['labels_carts'] = pred_test_df.labels_carts.apply(lambda x: list(set(x)))\n# pred_test_df['labels_orders'] = pred_test_df.labels_orders.apply(lambda x: list(set(x)))","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:12.762730Z","iopub.execute_input":"2022-12-23T12:48:12.763122Z","iopub.status.idle":"2022-12-23T12:48:21.570470Z","shell.execute_reply.started":"2022-12-23T12:48:12.763089Z","shell.execute_reply":"2022-12-23T12:48:21.569287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# stop","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:21.571919Z","iopub.execute_input":"2022-12-23T12:48:21.572263Z","iopub.status.idle":"2022-12-23T12:48:21.577355Z","shell.execute_reply.started":"2022-12-23T12:48:21.572231Z","shell.execute_reply":"2022-12-23T12:48:21.576072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_df.iloc[10]","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:21.582855Z","iopub.execute_input":"2022-12-23T12:48:21.583231Z","iopub.status.idle":"2022-12-23T12:48:21.590218Z","shell.execute_reply.started":"2022-12-23T12:48:21.583200Z","shell.execute_reply":"2022-12-23T12:48:21.589055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make_20_mult_many(pred_test_df.labels.iloc[10], \n#                   pred_test_df.labels_clicks.iloc[10],\n#                   pred_test_df.labels_carts.iloc[10], \n#                   pred_test_df.labels_orders.iloc[10])","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:21.592188Z","iopub.execute_input":"2022-12-23T12:48:21.592696Z","iopub.status.idle":"2022-12-23T12:48:21.603276Z","shell.execute_reply.started":"2022-12-23T12:48:21.592651Z","shell.execute_reply":"2022-12-23T12:48:21.601985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_test_df = pred_test_df[:100]","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:21.604852Z","iopub.execute_input":"2022-12-23T12:48:21.605285Z","iopub.status.idle":"2022-12-23T12:48:21.614035Z","shell.execute_reply.started":"2022-12-23T12:48:21.605181Z","shell.execute_reply":"2022-12-23T12:48:21.612852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# from tqdm import tqdm\n\npred_test_df['labels'] = pred_test_df.labels.apply(lambda x: make_20_mult(x, len(x)))\n\n# pred_test_df['labels'] = pred_test_df.labels_orders.apply(lambda x: make_20_mult(x, len(x))) + pred_test_df.labels_carts.apply(lambda x: make_20_mult(x, len(x))) + pred_test_df.labels_clicks.apply(lambda x: make_20_mult(x, len(x)))\n        \n# pred_test_df['labels'] = pred_test_df['labels'].apply(lambda x: x[:20])\n\n# pred_test_df['labels'] = make_20_mult_many(pred_test_df.labels, \n#                                            pred_test_df.labels_clicks,\n#                                            pred_test_df.labels_carts, \n#                                            pred_test_df.labels_orders)\n\n# pred_test_df['labels'] = test_func(pred_test_df.labels_clicks, pred_test_df.labels_carts, pred_test_df.labels_orders)\n\n# for i in tqdm(range(pred_test_df.shape[0])):\n#     pred_test_df['labels'].iloc[i] = make_20_mult_many(pred_test_df.labels_clicks.iloc[i], pred_test_df.labels_carts.iloc[i], pred_test_df.labels_orders.iloc[i])\n\npred_test_df['labels'] = pred_test_df.labels.apply(lambda x: \" \".join(map(str,x)))\npred_test_df = pred_test_df.drop(['labels_clicks', 'labels_carts', 'labels_orders'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T12:48:21.615263Z","iopub.execute_input":"2022-12-23T12:48:21.615692Z","iopub.status.idle":"2022-12-23T13:08:13.239944Z","shell.execute_reply.started":"2022-12-23T12:48:21.615659Z","shell.execute_reply":"2022-12-23T13:08:13.238673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clicks_pred_df = pred_test_df.copy()\nclicks_pred_df.session = clicks_pred_df.session.apply(lambda x: str(x) + '_clicks')\norders_pred_df = pred_test_df.copy()\norders_pred_df.session = orders_pred_df.session.apply(lambda x: str(x) + '_orders')\ncarts_pred_df = pred_test_df.copy()\ncarts_pred_df.session = carts_pred_df.session.apply(lambda x: str(x) + '_carts')\n","metadata":{"execution":{"iopub.status.busy":"2022-12-23T13:08:13.241427Z","iopub.execute_input":"2022-12-23T13:08:13.241743Z","iopub.status.idle":"2022-12-23T13:08:15.437247Z","shell.execute_reply.started":"2022-12-23T13:08:13.241715Z","shell.execute_reply":"2022-12-23T13:08:15.436109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n\n# clicks_pred_df['labels'] = clicks_pred_df.labels.apply(lambda x: make_20(x))\n# clicks_pred_df['labels'] = clicks_pred_df.labels.apply(lambda x: \" \".join(map(str,x)))\n\n# orders_pred_df['labels'] = orders_pred_df.labels.apply(lambda x: make_20(x))\n# orders_pred_df['labels'] = orders_pred_df.labels.apply(lambda x: \" \".join(map(str,x)))\n\n# carts_pred_df['labels'] = carts_pred_df.labels.apply(lambda x: make_20(x))\n# carts_pred_df['labels'] = carts_pred_df.labels.apply(lambda x: \" \".join(map(str,x)))","metadata":{"execution":{"iopub.status.busy":"2022-12-23T13:08:15.438800Z","iopub.execute_input":"2022-12-23T13:08:15.439130Z","iopub.status.idle":"2022-12-23T13:08:15.443861Z","shell.execute_reply.started":"2022-12-23T13:08:15.439100Z","shell.execute_reply":"2022-12-23T13:08:15.442635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.concat(\n    [clicks_pred_df, orders_pred_df, carts_pred_df]\n)\npred_df.columns = ['session_type', 'labels']","metadata":{"execution":{"iopub.status.busy":"2022-12-23T13:08:15.445321Z","iopub.execute_input":"2022-12-23T13:08:15.445695Z","iopub.status.idle":"2022-12-23T13:08:15.749078Z","shell.execute_reply.started":"2022-12-23T13:08:15.445663Z","shell.execute_reply":"2022-12-23T13:08:15.747748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_df = pred_test_df","metadata":{"execution":{"iopub.status.busy":"2022-12-23T13:08:15.750879Z","iopub.execute_input":"2022-12-23T13:08:15.751242Z","iopub.status.idle":"2022-12-23T13:08:15.755975Z","shell.execute_reply.started":"2022-12-23T13:08:15.751210Z","shell.execute_reply":"2022-12-23T13:08:15.754790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pred_df.sort_values(by='session_type').reset_index()\npred_df = pred_df.drop('index', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T13:08:15.757794Z","iopub.execute_input":"2022-12-23T13:08:15.758244Z","iopub.status.idle":"2022-12-23T13:08:35.529667Z","shell.execute_reply.started":"2022-12-23T13:08:15.758200Z","shell.execute_reply":"2022-12-23T13:08:35.528592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df","metadata":{"execution":{"iopub.status.busy":"2022-12-23T13:08:35.531057Z","iopub.execute_input":"2022-12-23T13:08:35.531537Z","iopub.status.idle":"2022-12-23T13:08:35.545339Z","shell.execute_reply.started":"2022-12-23T13:08:35.531480Z","shell.execute_reply":"2022-12-23T13:08:35.544077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Submitting our prediction","metadata":{}},{"cell_type":"code","source":"pred_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-23T13:08:35.547064Z","iopub.execute_input":"2022-12-23T13:08:35.547538Z","iopub.status.idle":"2022-12-23T13:08:59.519754Z","shell.execute_reply.started":"2022-12-23T13:08:35.547491Z","shell.execute_reply":"2022-12-23T13:08:59.518624Z"},"trusted":true},"execution_count":null,"outputs":[]}]}