{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom tqdm.notebook import tqdm\nimport os, sys, pickle, glob, gc\nfrom collections import Counter, defaultdict\nimport itertools","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_file(f):\n    df = pd.read_parquet(f)\n    df.ts = (df.ts/1000).astype('int32')\n    df['type'] = df.type.map(type_labels).astype('int8')\n    return df\n\n\nREAD_CT = 5\nfiles = glob.glob('../input/otto-chunk-data-inparquet-format/*_parquet/*')\ntype_labels = {'clicks':0, 'carts':1, 'orders':2}\ntype_weight = {0:1, 1:8, 2:1} \nCHUNK = int( np.ceil( len(files)/6 ))\nCHUNK","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_test():    \n    dfs = []\n    for e, chunk_file in enumerate(glob.glob('../input/otto-chunk-data-inparquet-format/test_parquet/*')):\n        chunk = pd.read_parquet(chunk_file)\n        chunk.ts = (chunk.ts/1000).astype('int32')\n        chunk['type'] = chunk['type'].map(type_labels).astype('int8')\n        dfs.append(chunk)\n    return pd.concat(dfs).reset_index(drop=True) #.astype({\"ts\": \"datetime64[ms]\"})\n\ntest_df = load_test()\nprint('Test data has shape',test_df.shape)\nprint(test_df.shape);test_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndef pqt_to_dict(df):\n    return df.groupby('aid_x').aid_y.apply(list).to_dict()\n# LOAD THREE CO-VISITATION MATRICES\nx=20\nroot = \"/kaggle/input/ottorec-sys/\"\ntop_20_clicks = pqt_to_dict( pd.read_parquet(f'{root}top_20_clicks_0.pqt') )\nfor k in range(1,8): \n    top_20_clicks.update( pqt_to_dict( pd.read_parquet(f'{root}top_20_clicks_{k}.pqt') ) )\n    \ntop_20_buys = pqt_to_dict( pd.read_parquet(f'{root}top_{x}_carts_orders_0.pqt') )\nfor k in range(1,8): \n    top_20_buys.update( pqt_to_dict( pd.read_parquet(f'{root}top_{x}_carts_orders_{k}.pqt') ) )\n\n    \ntop_20_buy2buy = pqt_to_dict( pd.read_parquet(f'{root}top_{x}_buy2buy_0.pqt') )\nfor k in range(1,4): \n    top_20_buy2buy.update( pqt_to_dict( pd.read_parquet(f'{root}top_{x}_buy2buy_{k}.pqt') ) )\n    \ntop_clicks = test_df.loc[test_df['type']==0,'aid'].value_counts().index.values[:20]\ntop_orders = test_df.loc[test_df['type']==2,'aid'].value_counts().index.values[:20]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import datetime\n\n# for timestamp in test_df.ts[:5]:\n#     value = datetime.datetime.fromtimestamp(timestamp)\n#     print(f\"{value:%Y-%m-%d %H:%M:%S}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def suggest_clicks(df):\n    # USER HISTORY AIDS AND TYPES\n    aids=df.aid.tolist()\n    types = df.type.tolist()\n    unique_aids = list(dict.fromkeys(aids[::-1] ))\n    # RERANK CANDIDATES USING WEIGHTS\n    if len(unique_aids)>=20:\n        weights=np.logspace(0.1,1,len(aids),base=2, endpoint=True)-1\n        aids_temp = Counter() \n        # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n        for aid,w,t in zip(aids,weights,types): \n            aids_temp[aid] += w * type_weight[t]\n        sorted_aids = [k for k,v in aids_temp.most_common(20)]\n        return sorted_aids\n    # USE \"CLICKS\" CO-VISITATION MATRIX\n    aids2 = list(itertools.chain(*[top_20_clicks[aid] for aid in unique_aids if aid in top_20_clicks]))\n    # RERANK CANDIDATES\n    top_aids2 = [aid2 for aid2, cnt in Counter(aids2).most_common(30) if aid2 not in unique_aids]    \n    result = list(unique_aids + top_aids2[:20 - len(unique_aids)])\n    # USE TOP20 TEST CLICKS\n    return result + [i for i in top_clicks if i not in result][:20-len(result)]\n\ndef suggest_buys(df):\n    # USER HISTORY AIDS AND TYPES\n    aids=df.aid.tolist()\n    types = df.type.tolist()\n    # UNIQUE AIDS AND UNIQUE BUYS\n    unique_aids = list(dict.fromkeys(aids[::-1] ))\n    df = df.loc[(df['type']==1)|(df['type']==2)]\n    unique_buys = list(dict.fromkeys( df.aid.tolist()[::-1] ))\n    # RERANK CANDIDATES USING WEIGHTS\n    if len(unique_aids)>=20:\n        weights=np.logspace(0.5,1.5,len(aids),base=2, endpoint=True)-1\n        aids_temp = Counter() \n        # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n        for aid,w,t in zip(aids,weights,types): \n            aids_temp[aid] += w * type_weight[t]\n            \n        # RERANK CANDIDATES USING \"BUY2BUY\" CO-VISITATION MATRIX\n        aids2 = list(itertools.chain(*[top_20_buys[aid] for aid in unique_aids if aid in top_20_buys]))\n        if len(aids2)>=340:\n            aids2 = list(itertools.chain(*[top_20_buys[aid][:5] for aid in unique_aids if aid in top_20_buys]))\n        elif len(aids2)>=140 and len(aids2)<340:\n            aids2 = list(itertools.chain(*[top_20_buys[aid][:7] for aid in unique_aids if aid in top_20_buys]))\n        elif len(aids2)>=80 and len(aids2)<140:\n            aids2 = list(itertools.chain(*[top_20_buys[aid][:10] for aid in unique_aids if aid in top_20_buys]))\n        elif len(aids2)>=60 and len(aids2)<80:\n            aids2 = list(itertools.chain(*[top_20_buys[aid][:15] for aid in unique_aids if aid in top_20_buys])) \n            \n        aids3 = list(itertools.chain(*[top_20_buy2buy[aid] for aid in unique_buys if aid in top_20_buy2buy]))\n\n        for aid in aids2: aids_temp[aid] += 0.1\n        for aid in aids3: aids_temp[aid] += 0.1\n        sorted_aids = [k for k,v in aids_temp.most_common(20)]\n#         print(f\"a2: {len(aids2)} a3:{len(aids3)}\")\n        return sorted_aids\n\n    # USE \"CART ORDER\" CO-VISITATION MATRIX\n    aids2 = list(itertools.chain(*[top_20_buys[aid][:2] for aid in unique_aids if aid in top_20_buys]))\n    if len(aids2)>=40:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:1] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)<40 and len(aids2)>=20:\n        aids2 = aids2\n    elif len(aids2)<20 and len(aids2)>=14:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:3] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)<14 and len(aids2)>10:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:4] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)<=10 and len(aids2)>8:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:5] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)<=8 and len(aids2)>5:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:7] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)<=5 and len(aids2)>2:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:10] for aid in unique_aids if aid in top_20_buys]))\n    else:\n        aids2 = list(itertools.chain(*[top_20_buys[aid] for aid in unique_aids if aid in top_20_buys]))\n        \n    # USE \"BUY2BUY\" CO-VISITATION MATRIX\n    aids3 = list(itertools.chain(*[top_20_buy2buy[aid] for aid in unique_buys if aid in top_20_buy2buy]))\n    if len(aids3)<=10 and len(aids3)>5:\n        aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:2] for aid in unique_aids if aid in top_20_buy2buy]))\n        if len(aids3)<=10:\n            aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:4] for aid in unique_aids if aid in top_20_buy2buy]))\n            if len(aids3)<=15:\n                aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:6] for aid in unique_aids if aid in top_20_buy2buy]))\n        aids3 = aids3[:20]\n    elif len(aids3)<=5:\n        aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:4] for aid in unique_aids if aid in top_20_buy2buy]))\n        if len(aids3)<=8:\n            aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:8] for aid in unique_aids if aid in top_20_buy2buy]))\n            if len(aids3)<=10:\n                aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:16] for aid in unique_aids if aid in top_20_buy2buy]))\n        aids3 = aids3[:20]\n#     print(\"order: \",len(aids3))\n    # RERANK CANDIDATES\n    top_aids = [aid for aid, cnt in Counter(aids2+aids3).most_common(40) if aid not in unique_aids] \n#     print(f\"a2:{len(set(aids2))} a3:{len(set(aids3))}, order_top_aids: {len(top_aids)}\" )\n    result = list(unique_aids + top_aids[:20 - len(unique_aids)])\n    if len(result)<20:\n        aids2 = list(itertools.chain(*[top_20_buys[aid] for aid in unique_aids if aid in top_20_buys]))\n        top_aids = [aid for aid, cnt in Counter(aids2+aids3).most_common(20) if aid not in unique_aids] \n        result = list(unique_aids + top_aids[:20 - len(unique_aids)])\n    # USE TOP20 TEST ORDERS\n    return result + [i for i in top_orders if i not in result][:20-len(result)]\n\ndef suggest_carts(df):\n    # USER HISTORY AIDS AND TYPES\n    aids=df.aid.tolist()\n    types = df.type.tolist()\n    # UNIQUE AIDS AND UNIQUE BUYS\n    unique_aids = list(dict.fromkeys(aids[::-1] ))\n    df = df.loc[(df['type']==1)|(df['type']==2)]\n    unique_buys = list(dict.fromkeys( df.aid.tolist()[::-1] ))\n    # RERANK CANDIDATES USING WEIGHTS\n    if len(unique_aids)>=20:\n        weights=np.logspace(0.5,1.5,len(aids),base=2, endpoint=True)-1\n        aids_temp = Counter() \n        # RERANK BASED ON REPEAT ITEMS AND TYPE OF ITEMS\n        for aid,w,t in zip(aids,weights,types): \n            aids_temp[aid] += w * type_weight[t]\n        # RERANK CANDIDATES USING \"BUY2BUY\" CO-VISITATION MATRIX\n        aids2 = list(itertools.chain(*[top_20_buys[aid] for aid in unique_aids if aid in top_20_buys]))\n        aids3 = list(itertools.chain(*[top_20_buy2buy[aid] for aid in unique_buys if aid in top_20_buy2buy]))\n        for aid in aids2: aids_temp[aid] += 0.1\n        for aid in aids3: aids_temp[aid] += 0.1\n        sorted_aids = [k for k,v in aids_temp.most_common(20)]\n        return sorted_aids\n    # USE \"CART ORDER\" CO-VISITATION MATRIX\n    aids2 = list(itertools.chain(*[top_20_buys[aid] for aid in unique_aids if aid in top_20_buys]))\n    if len(aids2)>=340:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:5] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)>=140 and len(aids2)<340:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:7] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)>=80 and len(aids2)<140:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:10] for aid in unique_aids if aid in top_20_buys]))\n    elif len(aids2)>=60 and len(aids2)<80:\n        aids2 = list(itertools.chain(*[top_20_buys[aid][:15] for aid in unique_aids if aid in top_20_buys]))    \n        \n    # USE \"BUY2BUY\" CO-VISITATION MATRIX\n    aids3 = list(itertools.chain(*[top_20_buy2buy[aid] for aid in unique_buys if aid in top_20_buy2buy]))\n    if len(aids3)<=10 and len(aids3)>5:\n        aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:2] for aid in unique_aids if aid in top_20_buy2buy]))\n        if len(aids3)<=10:\n            aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:4] for aid in unique_aids if aid in top_20_buy2buy]))\n            if len(aids3)<=15:\n                aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:6] for aid in unique_aids if aid in top_20_buy2buy]))\n        aids3 = aids3[:20]\n    elif len(aids3)<=5:\n        aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:4] for aid in unique_aids if aid in top_20_buy2buy]))\n        if len(aids3)<=8:\n            aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:8] for aid in unique_aids if aid in top_20_buy2buy]))\n            if len(aids3)<=10:\n                aids3 = list(itertools.chain(*[top_20_buy2buy[aid][:16] for aid in unique_aids if aid in top_20_buy2buy]))\n        aids3 = aids3[:20]\n#     print(\"cart: \",len(aids3))\n    # RERANK CANDIDATES\n    top_aids = [aid for aid, cnt in Counter(aids2+aids3).most_common(40) if aid not in unique_aids] \n#     print(f\"a2:{len(set(aids2))} a3:{len(set(aids3))}, cart_top_aids: {len(top_aids)}\" )\n    result = list(unique_aids + top_aids[:20 - len(unique_aids)])\n    if len(result)<20:\n        aids2 = list(itertools.chain(*[top_20_buys[aid] for aid in unique_aids if aid in top_20_buys]))\n        top_aids = [aid for aid, cnt in Counter(aids2+aids3).most_common(20) if aid not in unique_aids] \n        result = list(unique_aids + top_aids[:20 - len(unique_aids)])\n    # USE TOP20 TEST ORDERS\n    return result + [i for i in top_orders if i not in result][:20-len(result)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# test_df = test_df[:200]\npred_df_clicks = test_df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n    lambda x: suggest_clicks(x)\n)\n\npred_df_buys = test_df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n    lambda x: suggest_buys(x)\n)\n\npred_df_carts = test_df.sort_values([\"session\", \"ts\"]).groupby([\"session\"]).apply(\n    lambda x: suggest_carts(x)\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clicks_pred_df = pd.DataFrame(pred_df_clicks.add_suffix(\"_clicks\"), columns=[\"labels\"]).reset_index()\norders_pred_df = pd.DataFrame(pred_df_buys.add_suffix(\"_orders\"), columns=[\"labels\"]).reset_index()\ncarts_pred_df = pd.DataFrame(pred_df_carts.add_suffix(\"_carts\"), columns=[\"labels\"]).reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df = pd.concat([clicks_pred_df, orders_pred_df, carts_pred_df])\npred_df.columns = [\"session_type\", \"labels\"]\npred_df[\"labels\"] = pred_df.labels.apply(lambda x: \" \".join(map(str,x)))\npred_df.to_csv(\"submission.csv\", index=False)\npred_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}