{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nfrom scipy.stats import pearsonr, linregress\n\nfrom math import exp\n\nfrom scipy.optimize import curve_fit\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-20T22:48:26.814006Z","iopub.execute_input":"2022-12-20T22:48:26.815428Z","iopub.status.idle":"2022-12-20T22:48:26.825889Z","shell.execute_reply.started":"2022-12-20T22:48:26.815344Z","shell.execute_reply":"2022-12-20T22:48:26.824399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reading the data: thanks to https://www.kaggle.com/code/columbia2131/otto-read-a-chunk-of-jsonl-to-manageable-df\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom pathlib import Path\n\ndata_path = Path('/kaggle/input/otto-recommender-system/')","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:48:27.052489Z","iopub.execute_input":"2022-12-20T22:48:27.052936Z","iopub.status.idle":"2022-12-20T22:48:27.059425Z","shell.execute_reply.started":"2022-12-20T22:48:27.052889Z","shell.execute_reply":"2022-12-20T22:48:27.058533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntrain_sessions = pd.DataFrame()\nchunks = pd.read_json(data_path / 'train.jsonl', lines=True, chunksize=100_000)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:48:27.061327Z","iopub.execute_input":"2022-12-20T22:48:27.061772Z","iopub.status.idle":"2022-12-20T22:48:27.078238Z","shell.execute_reply.started":"2022-12-20T22:48:27.061731Z","shell.execute_reply":"2022-12-20T22:48:27.076606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor e, chunk in enumerate(chunks):\n    event_dict = {\n        'session': [],\n        'aid': [],\n        'ts': [],\n        'type': [],\n    }\n    if e < 2:\n        # train_sessions = pd.concat([train_sessions, chunk])\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(event['type'])\n        chunk_session = pd.DataFrame(event_dict)\n        train_sessions = pd.concat([train_sessions, chunk_session])\n    else:\n        break\n        \ntrain_sessions = train_sessions.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:48:27.206624Z","iopub.execute_input":"2022-12-20T22:48:27.207493Z","iopub.status.idle":"2022-12-20T22:49:17.543502Z","shell.execute_reply.started":"2022-12-20T22:48:27.207436Z","shell.execute_reply":"2022-12-20T22:49:17.542251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.float_format', '{:.6f}'.format)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:17.546408Z","iopub.execute_input":"2022-12-20T22:49:17.547166Z","iopub.status.idle":"2022-12-20T22:49:17.553081Z","shell.execute_reply.started":"2022-12-20T22:49:17.547112Z","shell.execute_reply":"2022-12-20T22:49:17.551475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_sessions)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:17.554615Z","iopub.execute_input":"2022-12-20T22:49:17.555006Z","iopub.status.idle":"2022-12-20T22:49:17.577565Z","shell.execute_reply.started":"2022-12-20T22:49:17.554946Z","shell.execute_reply":"2022-12-20T22:49:17.576391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions[train_sessions['type']!='clicks']","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:17.580948Z","iopub.execute_input":"2022-12-20T22:49:17.581552Z","iopub.status.idle":"2022-12-20T22:49:18.910792Z","shell.execute_reply.started":"2022-12-20T22:49:17.581499Z","shell.execute_reply":"2022-12-20T22:49:18.909976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_mean=np.mean(train_sessions['aid'])\nnorm_mean","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:18.912256Z","iopub.execute_input":"2022-12-20T22:49:18.913395Z","iopub.status.idle":"2022-12-20T22:49:18.939279Z","shell.execute_reply.started":"2022-12-20T22:49:18.913356Z","shell.execute_reply":"2022-12-20T22:49:18.938028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_std = np.std(train_sessions['aid'])\nnorm_std","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:18.940596Z","iopub.execute_input":"2022-12-20T22:49:18.940940Z","iopub.status.idle":"2022-12-20T22:49:19.022237Z","shell.execute_reply.started":"2022-12-20T22:49:18.940909Z","shell.execute_reply":"2022-12-20T22:49:19.021108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.min(train_sessions['aid'])","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:19.023626Z","iopub.execute_input":"2022-12-20T22:49:19.023972Z","iopub.status.idle":"2022-12-20T22:49:19.047047Z","shell.execute_reply.started":"2022-12-20T22:49:19.023929Z","shell.execute_reply":"2022-12-20T22:49:19.046066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int(np.random.normal(norm_mean, norm_std))","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:19.048338Z","iopub.execute_input":"2022-12-20T22:49:19.048850Z","iopub.status.idle":"2022-12-20T22:49:19.056085Z","shell.execute_reply.started":"2022-12-20T22:49:19.048817Z","shell.execute_reply":"2022-12-20T22:49:19.054881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks=train_sessions[train_sessions['type']=='clicks']\ntrain_sessions_carts=train_sessions[train_sessions['type']=='carts']\ntrain_sessions_orders=train_sessions[train_sessions['type']=='orders']","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:19.057823Z","iopub.execute_input":"2022-12-20T22:49:19.058208Z","iopub.status.idle":"2022-12-20T22:49:23.358562Z","shell.execute_reply.started":"2022-12-20T22:49:19.058175Z","shell.execute_reply":"2022-12-20T22:49:23.357291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in [train_sessions_clicks, train_sessions_carts, train_sessions_orders]:\n    df.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:23.364505Z","iopub.execute_input":"2022-12-20T22:49:23.365517Z","iopub.status.idle":"2022-12-20T22:49:23.371812Z","shell.execute_reply.started":"2022-12-20T22:49:23.365472Z","shell.execute_reply":"2022-12-20T22:49:23.370542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l_1 = len(train_sessions_clicks)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:23.373422Z","iopub.execute_input":"2022-12-20T22:49:23.374714Z","iopub.status.idle":"2022-12-20T22:49:23.384357Z","shell.execute_reply.started":"2022-12-20T22:49:23.374662Z","shell.execute_reply":"2022-12-20T22:49:23.383098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l_2 = len(train_sessions_carts)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:23.386159Z","iopub.execute_input":"2022-12-20T22:49:23.386666Z","iopub.status.idle":"2022-12-20T22:49:23.398147Z","shell.execute_reply.started":"2022-12-20T22:49:23.386629Z","shell.execute_reply":"2022-12-20T22:49:23.396814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l_3 = len(train_sessions_orders)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:23.399762Z","iopub.execute_input":"2022-12-20T22:49:23.401049Z","iopub.status.idle":"2022-12-20T22:49:23.409490Z","shell.execute_reply.started":"2022-12-20T22:49:23.401002Z","shell.execute_reply":"2022-12-20T22:49:23.408158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l_1 + l_2 + l_3","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:23.411083Z","iopub.execute_input":"2022-12-20T22:49:23.412121Z","iopub.status.idle":"2022-12-20T22:49:23.425212Z","shell.execute_reply.started":"2022-12-20T22:49:23.412077Z","shell.execute_reply":"2022-12-20T22:49:23.423769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:23.427219Z","iopub.execute_input":"2022-12-20T22:49:23.427728Z","iopub.status.idle":"2022-12-20T22:49:23.434772Z","shell.execute_reply.started":"2022-12-20T22:49:23.427678Z","shell.execute_reply":"2022-12-20T22:49:23.433722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_aid_clicks=Counter(train_sessions_clicks['aid']).most_common()\nmost_aid_carts=Counter(train_sessions_carts['aid']).most_common()\nmost_aid_orders=Counter(train_sessions_orders['aid']).most_common()","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:23.439210Z","iopub.execute_input":"2022-12-20T22:49:23.440027Z","iopub.status.idle":"2022-12-20T22:49:28.464774Z","shell.execute_reply.started":"2022-12-20T22:49:23.439951Z","shell.execute_reply":"2022-12-20T22:49:28.463714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_aid_clicks_aid=[x[0] for x in most_aid_clicks]\nmost_aid_carts_aid=[x[0] for x in most_aid_carts]\nmost_aid_orders_aid=[x[0] for x in most_aid_orders]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:28.466656Z","iopub.execute_input":"2022-12-20T22:49:28.467272Z","iopub.status.idle":"2022-12-20T22:49:28.748814Z","shell.execute_reply.started":"2022-12-20T22:49:28.467233Z","shell.execute_reply":"2022-12-20T22:49:28.747838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"most_aid_clicks_dict={x[0]:x[1] for x in most_aid_clicks}\nmost_aid_carts_dict={x[0]:x[1] for x in most_aid_carts}\nmost_aid_orders_dict={x[0]:x[1] for x in most_aid_orders}","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:28.750261Z","iopub.execute_input":"2022-12-20T22:49:28.750619Z","iopub.status.idle":"2022-12-20T22:49:29.430513Z","shell.execute_reply.started":"2022-12-20T22:49:28.750585Z","shell.execute_reply":"2022-12-20T22:49:29.429232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_aid=set(most_aid_orders_aid) & set(most_aid_carts_aid) & set(most_aid_clicks_aid)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:29.432465Z","iopub.execute_input":"2022-12-20T22:49:29.433243Z","iopub.status.idle":"2022-12-20T22:49:29.748481Z","shell.execute_reply.started":"2022-12-20T22:49:29.433186Z","shell.execute_reply":"2022-12-20T22:49:29.747250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"common_aid=list(common_aid)\nlen(common_aid)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:29.750264Z","iopub.execute_input":"2022-12-20T22:49:29.751027Z","iopub.status.idle":"2022-12-20T22:49:29.764667Z","shell.execute_reply.started":"2022-12-20T22:49:29.750982Z","shell.execute_reply":"2022-12-20T22:49:29.763629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions.sort_values(by='session')","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:29.766372Z","iopub.execute_input":"2022-12-20T22:49:29.767225Z","iopub.status.idle":"2022-12-20T22:49:31.155241Z","shell.execute_reply.started":"2022-12-20T22:49:29.767179Z","shell.execute_reply":"2022-12-20T22:49:31.154012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_sorted=train_sessions.sort_values(by=['session', 'ts'])","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:31.156945Z","iopub.execute_input":"2022-12-20T22:49:31.157593Z","iopub.status.idle":"2022-12-20T22:49:46.258601Z","shell.execute_reply.started":"2022-12-20T22:49:31.157552Z","shell.execute_reply":"2022-12-20T22:49:46.257492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_sorted","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:46.260242Z","iopub.execute_input":"2022-12-20T22:49:46.260613Z","iopub.status.idle":"2022-12-20T22:49:46.276438Z","shell.execute_reply.started":"2022-12-20T22:49:46.260581Z","shell.execute_reply":"2022-12-20T22:49:46.274934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_sorted['ts_num']=train_sessions_sorted.groupby('session')['ts'].diff().fillna(0)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:49:46.277924Z","iopub.execute_input":"2022-12-20T22:49:46.278651Z","iopub.status.idle":"2022-12-20T22:50:28.020822Z","shell.execute_reply.started":"2022-12-20T22:49:46.278610Z","shell.execute_reply":"2022-12-20T22:50:28.019559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_sorted","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:28.022169Z","iopub.execute_input":"2022-12-20T22:50:28.022553Z","iopub.status.idle":"2022-12-20T22:50:28.039192Z","shell.execute_reply.started":"2022-12-20T22:50:28.022520Z","shell.execute_reply":"2022-12-20T22:50:28.037764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 24 hours is 86400 sconds, so let's create new colomn of ts expressed in days unit\n\ntrain_sessions_sorted['ts_days']=train_sessions_sorted['ts_num']/86400\ntrain_sessions_sorted[(train_sessions_sorted['session']==0)]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:28.040727Z","iopub.execute_input":"2022-12-20T22:50:28.041115Z","iopub.status.idle":"2022-12-20T22:50:28.360335Z","shell.execute_reply.started":"2022-12-20T22:50:28.041081Z","shell.execute_reply":"2022-12-20T22:50:28.359171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_sorted[(train_sessions_sorted['type']=='orders') & (train_sessions_sorted['session']==199997)]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:28.362119Z","iopub.execute_input":"2022-12-20T22:50:28.362865Z","iopub.status.idle":"2022-12-20T22:50:29.570287Z","shell.execute_reply.started":"2022-12-20T22:50:28.362811Z","shell.execute_reply":"2022-12-20T22:50:29.569330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_sorted[train_sessions_sorted['type']=='carts']['ts_days'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:29.578606Z","iopub.execute_input":"2022-12-20T22:50:29.579156Z","iopub.status.idle":"2022-12-20T22:50:30.959148Z","shell.execute_reply.started":"2022-12-20T22:50:29.579121Z","shell.execute_reply":"2022-12-20T22:50:30.958110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Number of aids per session**","metadata":{}},{"cell_type":"code","source":"train_sessions_sorted['ts_day_mean'] = train_sessions_sorted.groupby(['session', 'type'])['ts_days'].transform('mean')","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:30.961948Z","iopub.execute_input":"2022-12-20T22:50:30.962330Z","iopub.status.idle":"2022-12-20T22:50:33.769168Z","shell.execute_reply.started":"2022-12-20T22:50:30.962299Z","shell.execute_reply":"2022-12-20T22:50:33.767943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_sorted","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:33.770844Z","iopub.execute_input":"2022-12-20T22:50:33.771233Z","iopub.status.idle":"2022-12-20T22:50:33.791099Z","shell.execute_reply.started":"2022-12-20T22:50:33.771199Z","shell.execute_reply":"2022-12-20T22:50:33.789914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions=train_sessions_sorted","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:33.795129Z","iopub.execute_input":"2022-12-20T22:50:33.795701Z","iopub.status.idle":"2022-12-20T22:50:33.951554Z","shell.execute_reply.started":"2022-12-20T22:50:33.795667Z","shell.execute_reply":"2022-12-20T22:50:33.950488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions.drop_duplicates('session', inplace=True)\ntrain_sessions","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:33.952999Z","iopub.execute_input":"2022-12-20T22:50:33.953935Z","iopub.status.idle":"2022-12-20T22:50:34.724387Z","shell.execute_reply.started":"2022-12-20T22:50:33.953893Z","shell.execute_reply":"2022-12-20T22:50:34.723204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_ts_days=train_sessions_sorted.groupby('session').agg({'ts_days': 'mean',\n                                                                     \n                                                                    })","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:34.726059Z","iopub.execute_input":"2022-12-20T22:50:34.727204Z","iopub.status.idle":"2022-12-20T22:50:34.755711Z","shell.execute_reply.started":"2022-12-20T22:50:34.727157Z","shell.execute_reply":"2022-12-20T22:50:34.754531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_ts_days","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:34.757224Z","iopub.execute_input":"2022-12-20T22:50:34.757671Z","iopub.status.idle":"2022-12-20T22:50:34.770778Z","shell.execute_reply.started":"2022-12-20T22:50:34.757638Z","shell.execute_reply":"2022-12-20T22:50:34.769526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:34.772898Z","iopub.execute_input":"2022-12-20T22:50:34.773783Z","iopub.status.idle":"2022-12-20T22:50:34.793161Z","shell.execute_reply.started":"2022-12-20T22:50:34.773730Z","shell.execute_reply":"2022-12-20T22:50:34.791907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sessions_by_aid_number(df):\n    new_df=df\n    new_df['ts_num']=new_df.groupby('session')['ts'].diff().fillna(0)\n    new_df['ts_days']=new_df['ts_num']/86400\n    new_df['ts_days_mean'] = new_df.groupby(['session'])['ts_days'].transform('mean')\n    new_df['aid_N']=new_df.groupby(['session'])['aid'].transform('count')\n    \n    new_df.drop_duplicates('session', inplace=True)\n    \n    new_df=new_df[['session', 'aid_N', 'ts_days_mean']]\n    \n    return new_df","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:34.795932Z","iopub.execute_input":"2022-12-20T22:50:34.797182Z","iopub.status.idle":"2022-12-20T22:50:34.805009Z","shell.execute_reply.started":"2022-12-20T22:50:34.797135Z","shell.execute_reply":"2022-12-20T22:50:34.803652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks_N=get_sessions_by_aid_number(train_sessions_clicks)\ntrain_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:50:34.806512Z","iopub.execute_input":"2022-12-20T22:50:34.806900Z","iopub.status.idle":"2022-12-20T22:51:15.260941Z","shell.execute_reply.started":"2022-12-20T22:50:34.806865Z","shell.execute_reply":"2022-12-20T22:51:15.260030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_sessions_clicks_N.to_csv('train_sessions_clicks_N.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.262431Z","iopub.execute_input":"2022-12-20T22:51:15.262839Z","iopub.status.idle":"2022-12-20T22:51:15.269185Z","shell.execute_reply.started":"2022-12-20T22:51:15.262803Z","shell.execute_reply":"2022-12-20T22:51:15.267631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correlation between number of 'clicked aids' and nb days between visits\ncorrelation=pearsonr(train_sessions_clicks_N['ts_days_mean'], train_sessions_clicks_N['aid_N'])\ncorrelation","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.270824Z","iopub.execute_input":"2022-12-20T22:51:15.271665Z","iopub.status.idle":"2022-12-20T22:51:15.291296Z","shell.execute_reply.started":"2022-12-20T22:51:15.271621Z","shell.execute_reply":"2022-12-20T22:51:15.289159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.293337Z","iopub.execute_input":"2022-12-20T22:51:15.294780Z","iopub.status.idle":"2022-12-20T22:51:15.322268Z","shell.execute_reply.started":"2022-12-20T22:51:15.294719Z","shell.execute_reply":"2022-12-20T22:51:15.320465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def expo(x):\n    return exp(x)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.324767Z","iopub.execute_input":"2022-12-20T22:51:15.326019Z","iopub.status.idle":"2022-12-20T22:51:15.333268Z","shell.execute_reply.started":"2022-12-20T22:51:15.325945Z","shell.execute_reply":"2022-12-20T22:51:15.331264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks_N['exp_ts_days']=np.exp(train_sessions_clicks_N['ts_days_mean'])","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.335321Z","iopub.execute_input":"2022-12-20T22:51:15.336488Z","iopub.status.idle":"2022-12-20T22:51:15.357378Z","shell.execute_reply.started":"2022-12-20T22:51:15.336432Z","shell.execute_reply":"2022-12-20T22:51:15.355925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.359335Z","iopub.execute_input":"2022-12-20T22:51:15.360741Z","iopub.status.idle":"2022-12-20T22:51:15.388629Z","shell.execute_reply.started":"2022-12-20T22:51:15.360687Z","shell.execute_reply":"2022-12-20T22:51:15.387246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks_N['log_aid_N']=np.log(train_sessions_clicks_N['aid_N'])","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.390829Z","iopub.execute_input":"2022-12-20T22:51:15.391705Z","iopub.status.idle":"2022-12-20T22:51:15.407451Z","shell.execute_reply.started":"2022-12-20T22:51:15.391654Z","shell.execute_reply":"2022-12-20T22:51:15.405912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.409039Z","iopub.execute_input":"2022-12-20T22:51:15.409489Z","iopub.status.idle":"2022-12-20T22:51:15.427562Z","shell.execute_reply.started":"2022-12-20T22:51:15.409443Z","shell.execute_reply":"2022-12-20T22:51:15.426376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"correlation=pearsonr(train_sessions_clicks_N['log_aid_N'], train_sessions_clicks_N['ts_days_mean'])\ncorrelation","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.429737Z","iopub.execute_input":"2022-12-20T22:51:15.430666Z","iopub.status.idle":"2022-12-20T22:51:15.450891Z","shell.execute_reply.started":"2022-12-20T22:51:15.430610Z","shell.execute_reply":"2022-12-20T22:51:15.448554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def func(x, a, b):\n    return np.exp(a * x + b)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.454040Z","iopub.execute_input":"2022-12-20T22:51:15.457674Z","iopub.status.idle":"2022-12-20T22:51:15.468189Z","shell.execute_reply.started":"2022-12-20T22:51:15.457581Z","shell.execute_reply":"2022-12-20T22:51:15.465644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=train_sessions_clicks_N['log_aid_N']\nx=train_sessions_clicks_N['ts_days_mean']\npopt= curve_fit(func, y, x, maxfev=1000000)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.471411Z","iopub.execute_input":"2022-12-20T22:51:15.472741Z","iopub.status.idle":"2022-12-20T22:51:15.664949Z","shell.execute_reply.started":"2022-12-20T22:51:15.472617Z","shell.execute_reply":"2022-12-20T22:51:15.663928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"slope = linregress(x, y)[0]\nslope","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.666310Z","iopub.execute_input":"2022-12-20T22:51:15.667478Z","iopub.status.idle":"2022-12-20T22:51:15.680326Z","shell.execute_reply.started":"2022-12-20T22:51:15.667430Z","shell.execute_reply":"2022-12-20T22:51:15.678998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intercept = linregress(x, y)[1]\nintercept","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.682209Z","iopub.execute_input":"2022-12-20T22:51:15.683094Z","iopub.status.idle":"2022-12-20T22:51:15.696918Z","shell.execute_reply.started":"2022-12-20T22:51:15.683041Z","shell.execute_reply":"2022-12-20T22:51:15.695708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"linregress(x, y)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.698458Z","iopub.execute_input":"2022-12-20T22:51:15.699596Z","iopub.status.idle":"2022-12-20T22:51:15.710709Z","shell.execute_reply.started":"2022-12-20T22:51:15.699540Z","shell.execute_reply":"2022-12-20T22:51:15.709700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict number aid clicked for a time = 10 days\npred= func(1461, slope, intercept)\npred","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.712078Z","iopub.execute_input":"2022-12-20T22:51:15.713178Z","iopub.status.idle":"2022-12-20T22:51:15.721152Z","shell.execute_reply.started":"2022-12-20T22:51:15.713118Z","shell.execute_reply":"2022-12-20T22:51:15.719911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading test data","metadata":{}},{"cell_type":"code","source":"%%time\n\ntest_sessions = pd.DataFrame()\nchunks = pd.read_json(data_path / 'test.jsonl', lines=True, chunksize=100_000)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.723479Z","iopub.execute_input":"2022-12-20T22:51:15.723990Z","iopub.status.idle":"2022-12-20T22:51:15.738584Z","shell.execute_reply.started":"2022-12-20T22:51:15.723920Z","shell.execute_reply":"2022-12-20T22:51:15.737421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nfor e, chunk in enumerate(chunks):\n    event_dict = {\n        'session': [],\n        'aid': [],\n        'ts': [],\n        'type': [],\n    }\n    if e < 2:\n        # train_sessions = pd.concat([train_sessions, chunk])\n        for session, events in zip(chunk['session'].tolist(), chunk['events'].tolist()):\n            for event in events:\n                event_dict['session'].append(session)\n                event_dict['aid'].append(event['aid'])\n                event_dict['ts'].append(event['ts'])\n                event_dict['type'].append(event['type'])\n        chunk_session = pd.DataFrame(event_dict)\n        test_sessions = pd.concat([test_sessions, chunk_session])\n    else:\n        break\n        \ntest_sessions = test_sessions.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:15.739907Z","iopub.execute_input":"2022-12-20T22:51:15.740390Z","iopub.status.idle":"2022-12-20T22:51:28.183927Z","shell.execute_reply.started":"2022-12-20T22:51:15.740352Z","shell.execute_reply":"2022-12-20T22:51:28.182587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:28.185268Z","iopub.execute_input":"2022-12-20T22:51:28.185609Z","iopub.status.idle":"2022-12-20T22:51:28.202433Z","shell.execute_reply.started":"2022-12-20T22:51:28.185578Z","shell.execute_reply":"2022-12-20T22:51:28.201021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks=test_sessions[test_sessions['type']=='clicks']\ntest_sessions_carts=test_sessions[test_sessions['type']=='carts']\ntest_sessions_orders=test_sessions[test_sessions['type']=='orders']\n\nfor df in [test_sessions_clicks, test_sessions_carts, test_sessions_orders]:\n    df.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:28.204692Z","iopub.execute_input":"2022-12-20T22:51:28.205336Z","iopub.status.idle":"2022-12-20T22:51:28.743519Z","shell.execute_reply.started":"2022-12-20T22:51:28.205287Z","shell.execute_reply":"2022-12-20T22:51:28.742548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(test_sessions_clicks))\nprint(len(test_sessions_carts))\nprint(len(test_sessions_orders))","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:28.744948Z","iopub.execute_input":"2022-12-20T22:51:28.745556Z","iopub.status.idle":"2022-12-20T22:51:28.751098Z","shell.execute_reply.started":"2022-12-20T22:51:28.745518Z","shell.execute_reply":"2022-12-20T22:51:28.749935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(len(test_sessions_clicks) + len(test_sessions_carts) + len(test_sessions_orders)) * 3","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:28.752678Z","iopub.execute_input":"2022-12-20T22:51:28.753201Z","iopub.status.idle":"2022-12-20T22:51:28.766299Z","shell.execute_reply.started":"2022-12-20T22:51:28.753167Z","shell.execute_reply":"2022-12-20T22:51:28.764769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_length = 5015409","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:28.767571Z","iopub.execute_input":"2022-12-20T22:51:28.767912Z","iopub.status.idle":"2022-12-20T22:51:28.776306Z","shell.execute_reply.started":"2022-12-20T22:51:28.767881Z","shell.execute_reply":"2022-12-20T22:51:28.775301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:28.777541Z","iopub.execute_input":"2022-12-20T22:51:28.778908Z","iopub.status.idle":"2022-12-20T22:51:28.797495Z","shell.execute_reply.started":"2022-12-20T22:51:28.778866Z","shell.execute_reply":"2022-12-20T22:51:28.796582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N = get_sessions_by_aid_number(test_sessions_clicks)\ntest_sessions_carts_N = get_sessions_by_aid_number(test_sessions_carts)\ntest_sessions_orders_N = get_sessions_by_aid_number(test_sessions_orders)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:51:28.798929Z","iopub.execute_input":"2022-12-20T22:51:28.800337Z","iopub.status.idle":"2022-12-20T22:52:13.751261Z","shell.execute_reply.started":"2022-12-20T22:51:28.800296Z","shell.execute_reply":"2022-12-20T22:52:13.749902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.753516Z","iopub.execute_input":"2022-12-20T22:52:13.754129Z","iopub.status.idle":"2022-12-20T22:52:13.768464Z","shell.execute_reply.started":"2022-12-20T22:52:13.754049Z","shell.execute_reply":"2022-12-20T22:52:13.767387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['pred_aid_N']=func(test_sessions_clicks_N['ts_days_mean'], slope, intercept)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.770108Z","iopub.execute_input":"2022-12-20T22:52:13.771375Z","iopub.status.idle":"2022-12-20T22:52:13.786074Z","shell.execute_reply.started":"2022-12-20T22:52:13.771314Z","shell.execute_reply":"2022-12-20T22:52:13.784834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.787522Z","iopub.execute_input":"2022-12-20T22:52:13.788612Z","iopub.status.idle":"2022-12-20T22:52:13.804553Z","shell.execute_reply.started":"2022-12-20T22:52:13.788567Z","shell.execute_reply":"2022-12-20T22:52:13.803112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['pred_aid_N']=np.clip(test_sessions_clicks_N['pred_aid_N'], 1, 20)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.806450Z","iopub.execute_input":"2022-12-20T22:52:13.807459Z","iopub.status.idle":"2022-12-20T22:52:13.821173Z","shell.execute_reply.started":"2022-12-20T22:52:13.807408Z","shell.execute_reply":"2022-12-20T22:52:13.819883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['pred_aid_N']=test_sessions_clicks_N['pred_aid_N'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.823568Z","iopub.execute_input":"2022-12-20T22:52:13.824146Z","iopub.status.idle":"2022-12-20T22:52:13.835204Z","shell.execute_reply.started":"2022-12-20T22:52:13.824089Z","shell.execute_reply":"2022-12-20T22:52:13.833574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N.reset_index(drop=True, inplace=True)\ntest_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.838025Z","iopub.execute_input":"2022-12-20T22:52:13.838699Z","iopub.status.idle":"2022-12-20T22:52:13.855054Z","shell.execute_reply.started":"2022-12-20T22:52:13.838644Z","shell.execute_reply":"2022-12-20T22:52:13.853645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sessions_clicks","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.856944Z","iopub.execute_input":"2022-12-20T22:52:13.858211Z","iopub.status.idle":"2022-12-20T22:52:13.878608Z","shell.execute_reply.started":"2022-12-20T22:52:13.858146Z","shell.execute_reply":"2022-12-20T22:52:13.877535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_sessions_clicks_N['pred_aids']=[np.random.normal(norm_mean, norm_std, size=test_sessions_clicks_N['pred_aid_N'][i]) for i in range(len(test_sessions_clicks_N))]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:13.879766Z","iopub.execute_input":"2022-12-20T22:52:13.880126Z","iopub.status.idle":"2022-12-20T22:52:16.523792Z","shell.execute_reply.started":"2022-12-20T22:52:13.880095Z","shell.execute_reply":"2022-12-20T22:52:16.522481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:16.525616Z","iopub.execute_input":"2022-12-20T22:52:16.526559Z","iopub.status.idle":"2022-12-20T22:52:16.550163Z","shell.execute_reply.started":"2022-12-20T22:52:16.526517Z","shell.execute_reply":"2022-12-20T22:52:16.548804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['pred_aids']=[np.clip(x, 1, max(x)) for x in test_sessions_clicks_N['pred_aids']]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:16.551767Z","iopub.execute_input":"2022-12-20T22:52:16.552247Z","iopub.status.idle":"2022-12-20T22:52:20.601064Z","shell.execute_reply.started":"2022-12-20T22:52:16.552202Z","shell.execute_reply":"2022-12-20T22:52:20.599630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:20.602549Z","iopub.execute_input":"2022-12-20T22:52:20.602907Z","iopub.status.idle":"2022-12-20T22:52:20.624900Z","shell.execute_reply.started":"2022-12-20T22:52:20.602873Z","shell.execute_reply":"2022-12-20T22:52:20.623780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['pred_aids']=[[int(y) for y in test_sessions_clicks_N['pred_aids'][i]] for i in range(len(test_sessions_clicks_N))]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:20.626574Z","iopub.execute_input":"2022-12-20T22:52:20.626915Z","iopub.status.idle":"2022-12-20T22:52:24.325433Z","shell.execute_reply.started":"2022-12-20T22:52:20.626885Z","shell.execute_reply":"2022-12-20T22:52:24.324138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.328420Z","iopub.execute_input":"2022-12-20T22:52:24.328938Z","iopub.status.idle":"2022-12-20T22:52:24.349447Z","shell.execute_reply.started":"2022-12-20T22:52:24.328887Z","shell.execute_reply":"2022-12-20T22:52:24.348608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['session_clicks']=test_sessions_clicks_N['session'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.350816Z","iopub.execute_input":"2022-12-20T22:52:24.351365Z","iopub.status.idle":"2022-12-20T22:52:24.491031Z","shell.execute_reply.started":"2022-12-20T22:52:24.351331Z","shell.execute_reply":"2022-12-20T22:52:24.489669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['session_clicks']=[str(x) + '_clicks' for x in test_sessions_clicks_N['session']]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.492798Z","iopub.execute_input":"2022-12-20T22:52:24.493219Z","iopub.status.idle":"2022-12-20T22:52:24.621509Z","shell.execute_reply.started":"2022-12-20T22:52:24.493182Z","shell.execute_reply":"2022-12-20T22:52:24.620199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.625099Z","iopub.execute_input":"2022-12-20T22:52:24.625571Z","iopub.status.idle":"2022-12-20T22:52:24.650300Z","shell.execute_reply.started":"2022-12-20T22:52:24.625528Z","shell.execute_reply":"2022-12-20T22:52:24.649036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_list=[1256, 12, 4558]\nkk=[*my_list]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.651837Z","iopub.execute_input":"2022-12-20T22:52:24.652254Z","iopub.status.idle":"2022-12-20T22:52:24.658253Z","shell.execute_reply.started":"2022-12-20T22:52:24.652216Z","shell.execute_reply":"2022-12-20T22:52:24.657002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk=''\nfor num in my_list:\n    kk = kk + str(num) + ' '","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.660308Z","iopub.execute_input":"2022-12-20T22:52:24.660818Z","iopub.status.idle":"2022-12-20T22:52:24.671660Z","shell.execute_reply.started":"2022-12-20T22:52:24.660766Z","shell.execute_reply":"2022-12-20T22:52:24.670296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kk.strip()","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.673457Z","iopub.execute_input":"2022-12-20T22:52:24.673903Z","iopub.status.idle":"2022-12-20T22:52:24.686432Z","shell.execute_reply.started":"2022-12-20T22:52:24.673867Z","shell.execute_reply":"2022-12-20T22:52:24.685451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['session_type']=test_sessions_clicks_N['session_clicks']","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.688162Z","iopub.execute_input":"2022-12-20T22:52:24.689589Z","iopub.status.idle":"2022-12-20T22:52:24.702773Z","shell.execute_reply.started":"2022-12-20T22:52:24.689532Z","shell.execute_reply":"2022-12-20T22:52:24.701365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.718122Z","iopub.execute_input":"2022-12-20T22:52:24.718553Z","iopub.status.idle":"2022-12-20T22:52:24.743248Z","shell.execute_reply.started":"2022-12-20T22:52:24.718518Z","shell.execute_reply":"2022-12-20T22:52:24.742074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_sessions_clicks_N['session_clicks']","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.744853Z","iopub.execute_input":"2022-12-20T22:52:24.745646Z","iopub.status.idle":"2022-12-20T22:52:24.758317Z","shell.execute_reply.started":"2022-12-20T22:52:24.745604Z","shell.execute_reply":"2022-12-20T22:52:24.756678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['pred_aids']=[[str(y) for y in x] for x in test_sessions_clicks_N['pred_aids']]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:24.760144Z","iopub.execute_input":"2022-12-20T22:52:24.760659Z","iopub.status.idle":"2022-12-20T22:52:27.493464Z","shell.execute_reply.started":"2022-12-20T22:52:24.760610Z","shell.execute_reply":"2022-12-20T22:52:27.492058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N['labels']=[' '.join(x) for x in test_sessions_clicks_N['pred_aids']]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.495228Z","iopub.execute_input":"2022-12-20T22:52:27.496056Z","iopub.status.idle":"2022-12-20T22:52:27.718521Z","shell.execute_reply.started":"2022-12-20T22:52:27.496003Z","shell.execute_reply":"2022-12-20T22:52:27.717055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.720038Z","iopub.execute_input":"2022-12-20T22:52:27.720439Z","iopub.status.idle":"2022-12-20T22:52:27.745293Z","shell.execute_reply.started":"2022-12-20T22:52:27.720402Z","shell.execute_reply":"2022-12-20T22:52:27.743742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission=test_sessions_clicks_N[['session_type', 'labels']]\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.746766Z","iopub.execute_input":"2022-12-20T22:52:27.747158Z","iopub.status.idle":"2022-12-20T22:52:27.812818Z","shell.execute_reply.started":"2022-12-20T22:52:27.747122Z","shell.execute_reply":"2022-12-20T22:52:27.811527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.814502Z","iopub.execute_input":"2022-12-20T22:52:27.815165Z","iopub.status.idle":"2022-12-20T22:52:27.821721Z","shell.execute_reply.started":"2022-12-20T22:52:27.815125Z","shell.execute_reply":"2022-12-20T22:52:27.820393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_carts_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.823517Z","iopub.execute_input":"2022-12-20T22:52:27.823880Z","iopub.status.idle":"2022-12-20T22:52:27.843550Z","shell.execute_reply.started":"2022-12-20T22:52:27.823848Z","shell.execute_reply":"2022-12-20T22:52:27.842144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_orders_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.845542Z","iopub.execute_input":"2022-12-20T22:52:27.846375Z","iopub.status.idle":"2022-12-20T22:52:27.862838Z","shell.execute_reply.started":"2022-12-20T22:52:27.846328Z","shell.execute_reply":"2022-12-20T22:52:27.861660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in [test_sessions_clicks_N, test_sessions_carts_N, test_sessions_orders_N]:\n    df.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.864306Z","iopub.execute_input":"2022-12-20T22:52:27.864677Z","iopub.status.idle":"2022-12-20T22:52:27.874989Z","shell.execute_reply.started":"2022-12-20T22:52:27.864644Z","shell.execute_reply":"2022-12-20T22:52:27.873888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_carts_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.876302Z","iopub.execute_input":"2022-12-20T22:52:27.877296Z","iopub.status.idle":"2022-12-20T22:52:27.897811Z","shell.execute_reply.started":"2022-12-20T22:52:27.877256Z","shell.execute_reply":"2022-12-20T22:52:27.896707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df = test_sessions_clicks_N.append(test_sessions_carts_N)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.899563Z","iopub.execute_input":"2022-12-20T22:52:27.900331Z","iopub.status.idle":"2022-12-20T22:52:27.941916Z","shell.execute_reply.started":"2022-12-20T22:52:27.900289Z","shell.execute_reply":"2022-12-20T22:52:27.940829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_sessions_clicks_N","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.943257Z","iopub.execute_input":"2022-12-20T22:52:27.943842Z","iopub.status.idle":"2022-12-20T22:52:27.968183Z","shell.execute_reply.started":"2022-12-20T22:52:27.943807Z","shell.execute_reply":"2022-12-20T22:52:27.966406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lab = test_sessions_clicks_N['labels']","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:27.969934Z","iopub.execute_input":"2022-12-20T22:52:27.970423Z","iopub.status.idle":"2022-12-20T22:52:28.054732Z","shell.execute_reply.started":"2022-12-20T22:52:27.970381Z","shell.execute_reply":"2022-12-20T22:52:28.053456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(lab)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:28.056635Z","iopub.execute_input":"2022-12-20T22:52:28.057087Z","iopub.status.idle":"2022-12-20T22:52:28.070702Z","shell.execute_reply.started":"2022-12-20T22:52:28.057005Z","shell.execute_reply":"2022-12-20T22:52:28.069789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lab = lab.append(lab)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:28.072223Z","iopub.execute_input":"2022-12-20T22:52:28.072609Z","iopub.status.idle":"2022-12-20T22:52:28.096548Z","shell.execute_reply.started":"2022-12-20T22:52:28.072575Z","shell.execute_reply":"2022-12-20T22:52:28.095122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(lab)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:28.098220Z","iopub.execute_input":"2022-12-20T22:52:28.098590Z","iopub.status.idle":"2022-12-20T22:52:28.107062Z","shell.execute_reply.started":"2022-12-20T22:52:28.098553Z","shell.execute_reply":"2022-12-20T22:52:28.105725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lab = lab[0:5015409]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:28.108916Z","iopub.execute_input":"2022-12-20T22:52:28.110144Z","iopub.status.idle":"2022-12-20T22:52:28.120634Z","shell.execute_reply.started":"2022-12-20T22:52:28.110081Z","shell.execute_reply":"2022-12-20T22:52:28.119046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(lab)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:28.122105Z","iopub.execute_input":"2022-12-20T22:52:28.122937Z","iopub.status.idle":"2022-12-20T22:52:28.138208Z","shell.execute_reply.started":"2022-12-20T22:52:28.122899Z","shell.execute_reply":"2022-12-20T22:52:28.137050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv(\"/kaggle/input/otto-recommender-system/sample_submission.csv\")\nsample","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:28.139875Z","iopub.execute_input":"2022-12-20T22:52:28.140404Z","iopub.status.idle":"2022-12-20T22:52:32.984082Z","shell.execute_reply.started":"2022-12-20T22:52:28.140359Z","shell.execute_reply":"2022-12-20T22:52:32.982735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:32.986042Z","iopub.execute_input":"2022-12-20T22:52:32.986457Z","iopub.status.idle":"2022-12-20T22:52:32.998474Z","shell.execute_reply.started":"2022-12-20T22:52:32.986409Z","shell.execute_reply":"2022-12-20T22:52:32.997074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(lab)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:32.999945Z","iopub.execute_input":"2022-12-20T22:52:33.000551Z","iopub.status.idle":"2022-12-20T22:52:33.012170Z","shell.execute_reply.started":"2022-12-20T22:52:33.000509Z","shell.execute_reply":"2022-12-20T22:52:33.011309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(sample)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:33.013714Z","iopub.execute_input":"2022-12-20T22:52:33.014399Z","iopub.status.idle":"2022-12-20T22:52:33.025552Z","shell.execute_reply.started":"2022-12-20T22:52:33.014361Z","shell.execute_reply":"2022-12-20T22:52:33.024673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lab = pd.DataFrame(lab)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:33.027201Z","iopub.execute_input":"2022-12-20T22:52:33.027907Z","iopub.status.idle":"2022-12-20T22:52:33.050997Z","shell.execute_reply.started":"2022-12-20T22:52:33.027853Z","shell.execute_reply":"2022-12-20T22:52:33.049899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = sample.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:33.052540Z","iopub.execute_input":"2022-12-20T22:52:33.053189Z","iopub.status.idle":"2022-12-20T22:52:33.261564Z","shell.execute_reply.started":"2022-12-20T22:52:33.053151Z","shell.execute_reply":"2022-12-20T22:52:33.260344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['lab']=[1 for i in range(len(sample))]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:33.263119Z","iopub.execute_input":"2022-12-20T22:52:33.263493Z","iopub.status.idle":"2022-12-20T22:52:34.804864Z","shell.execute_reply.started":"2022-12-20T22:52:33.263456Z","shell.execute_reply":"2022-12-20T22:52:34.803593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[sample.index.duplicated()]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:34.806587Z","iopub.execute_input":"2022-12-20T22:52:34.810456Z","iopub.status.idle":"2022-12-20T22:52:34.898655Z","shell.execute_reply.started":"2022-12-20T22:52:34.810399Z","shell.execute_reply":"2022-12-20T22:52:34.897391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sample['labels'][0])","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:34.902107Z","iopub.execute_input":"2022-12-20T22:52:34.903315Z","iopub.status.idle":"2022-12-20T22:52:34.910554Z","shell.execute_reply.started":"2022-12-20T22:52:34.903261Z","shell.execute_reply":"2022-12-20T22:52:34.909180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:34.912291Z","iopub.execute_input":"2022-12-20T22:52:34.912654Z","iopub.status.idle":"2022-12-20T22:52:34.929617Z","shell.execute_reply.started":"2022-12-20T22:52:34.912622Z","shell.execute_reply":"2022-12-20T22:52:34.928367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['labels']=[np.random.normal(norm_mean, norm_std, size = 20) for i in range(len(sample))]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:52:34.931107Z","iopub.execute_input":"2022-12-20T22:52:34.931486Z","iopub.status.idle":"2022-12-20T22:53:01.065663Z","shell.execute_reply.started":"2022-12-20T22:52:34.931451Z","shell.execute_reply":"2022-12-20T22:53:01.064247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:53:01.067258Z","iopub.execute_input":"2022-12-20T22:53:01.067782Z","iopub.status.idle":"2022-12-20T22:53:01.088740Z","shell.execute_reply.started":"2022-12-20T22:53:01.067742Z","shell.execute_reply":"2022-12-20T22:53:01.087727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['lab'] = [[int(x) for x in sample['labels'][i]] for i in range(len(sample))]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:53:01.090625Z","iopub.execute_input":"2022-12-20T22:53:01.091791Z","iopub.status.idle":"2022-12-20T22:54:22.266031Z","shell.execute_reply.started":"2022-12-20T22:53:01.091737Z","shell.execute_reply":"2022-12-20T22:54:22.264614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:54:22.267820Z","iopub.execute_input":"2022-12-20T22:54:22.268768Z","iopub.status.idle":"2022-12-20T22:54:22.294836Z","shell.execute_reply.started":"2022-12-20T22:54:22.268703Z","shell.execute_reply":"2022-12-20T22:54:22.293456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['lab'] = [[abs(x) for x in sample['lab'][i]] for i in range(len(sample))]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:54:22.296523Z","iopub.execute_input":"2022-12-20T22:54:22.296995Z","iopub.status.idle":"2022-12-20T22:55:30.505472Z","shell.execute_reply.started":"2022-12-20T22:54:22.296943Z","shell.execute_reply":"2022-12-20T22:55:30.504218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['lab'] = [[str(x) for x in sample['lab'][i]] for i in range(len(sample))]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:55:30.507289Z","iopub.execute_input":"2022-12-20T22:55:30.507680Z","iopub.status.idle":"2022-12-20T22:56:58.586027Z","shell.execute_reply.started":"2022-12-20T22:55:30.507645Z","shell.execute_reply":"2022-12-20T22:56:58.584709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:56:58.587881Z","iopub.execute_input":"2022-12-20T22:56:58.588427Z","iopub.status.idle":"2022-12-20T22:56:58.614695Z","shell.execute_reply.started":"2022-12-20T22:56:58.588367Z","shell.execute_reply":"2022-12-20T22:56:58.612999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['lab'] = [' '.join(x) for x in sample['lab']]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:56:58.616589Z","iopub.execute_input":"2022-12-20T22:56:58.617020Z","iopub.status.idle":"2022-12-20T22:57:17.523796Z","shell.execute_reply.started":"2022-12-20T22:56:58.616981Z","shell.execute_reply":"2022-12-20T22:57:17.522355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:57:17.525486Z","iopub.execute_input":"2022-12-20T22:57:17.525865Z","iopub.status.idle":"2022-12-20T22:57:17.546577Z","shell.execute_reply.started":"2022-12-20T22:57:17.525832Z","shell.execute_reply":"2022-12-20T22:57:17.545025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = sample[['session_type', 'lab']]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:57:17.548542Z","iopub.execute_input":"2022-12-20T22:57:17.548901Z","iopub.status.idle":"2022-12-20T22:57:19.050935Z","shell.execute_reply.started":"2022-12-20T22:57:17.548869Z","shell.execute_reply":"2022-12-20T22:57:19.049529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.columns = ['session_type', 'labels']","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:57:19.052539Z","iopub.execute_input":"2022-12-20T22:57:19.052939Z","iopub.status.idle":"2022-12-20T22:57:19.059451Z","shell.execute_reply.started":"2022-12-20T22:57:19.052902Z","shell.execute_reply":"2022-12-20T22:57:19.058150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-20T22:57:19.061324Z","iopub.execute_input":"2022-12-20T22:57:19.061838Z","iopub.status.idle":"2022-12-20T22:57:19.081309Z","shell.execute_reply.started":"2022-12-20T22:57:19.061785Z","shell.execute_reply":"2022-12-20T22:57:19.080026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T23:20:26.120861Z","iopub.execute_input":"2022-12-20T23:20:26.121437Z","iopub.status.idle":"2022-12-20T23:20:47.509121Z","shell.execute_reply.started":"2022-12-20T23:20:26.121395Z","shell.execute_reply":"2022-12-20T23:20:47.507524Z"},"trusted":true},"execution_count":null,"outputs":[]}]}