{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport vaex\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-19T15:13:29.244507Z","iopub.execute_input":"2022-12-19T15:13:29.245319Z","iopub.status.idle":"2022-12-19T15:13:29.269261Z","shell.execute_reply.started":"2022-12-19T15:13:29.245280Z","shell.execute_reply":"2022-12-19T15:13:29.267970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import vaex","metadata":{"execution":{"iopub.status.busy":"2022-12-19T15:13:29.271803Z","iopub.execute_input":"2022-12-19T15:13:29.272506Z","iopub.status.idle":"2022-12-19T15:13:29.278258Z","shell.execute_reply.started":"2022-12-19T15:13:29.272454Z","shell.execute_reply":"2022-12-19T15:13:29.276709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file = vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000200000_000300000.parquet')\nfile2= vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000400000_000500000.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T15:37:20.613726Z","iopub.execute_input":"2022-12-19T15:37:20.614640Z","iopub.status.idle":"2022-12-19T15:37:20.634964Z","shell.execute_reply.started":"2022-12-19T15:37:20.614601Z","shell.execute_reply":"2022-12-19T15:37:20.633715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file7=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000600000_000700000.parquet')\nfile8=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000900000_001000000.parquet')\nfile9=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000900000_001000000.parquet')\nfile10=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000100000_000200000.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:14.902825Z","iopub.execute_input":"2022-12-19T16:04:14.903232Z","iopub.status.idle":"2022-12-19T16:04:14.947401Z","shell.execute_reply.started":"2022-12-19T16:04:14.903201Z","shell.execute_reply":"2022-12-19T16:04:14.946260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file3=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/001300000_001400000.parquet')\nfile4=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000000000_000100000.parquet')\nfile5=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000800000_000900000.parquet')\nfile6=vaex.open('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/000500000_000600000.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:17.550262Z","iopub.execute_input":"2022-12-19T16:04:17.550646Z","iopub.status.idle":"2022-12-19T16:04:17.570851Z","shell.execute_reply.started":"2022-12-19T16:04:17.550614Z","shell.execute_reply":"2022-12-19T16:04:17.569986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TAB = vaex.concat([file,file2,file3,file4,file5,file6,file7,file8,file9,file10])","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:21.056235Z","iopub.execute_input":"2022-12-19T16:04:21.057064Z","iopub.status.idle":"2022-12-19T16:04:21.065710Z","shell.execute_reply.started":"2022-12-19T16:04:21.057023Z","shell.execute_reply":"2022-12-19T16:04:21.064266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TAB.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:23.986897Z","iopub.execute_input":"2022-12-19T16:04:23.987593Z","iopub.status.idle":"2022-12-19T16:04:24.267361Z","shell.execute_reply.started":"2022-12-19T16:04:23.987557Z","shell.execute_reply":"2022-12-19T16:04:24.266147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tab_order=TAB[TAB['type']=='orders']\ntab_clicks=TAB[TAB['type']=='clicks']\ntab_carts=TAB[TAB['type']=='carts']\ndf=tab_order.groupby('aid').agg({'session':'nunique'})\n","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:04:54.437453Z","iopub.execute_input":"2022-12-19T17:04:54.438222Z","iopub.status.idle":"2022-12-19T17:04:55.175292Z","shell.execute_reply.started":"2022-12-19T17:04:54.438174Z","shell.execute_reply":"2022-12-19T17:04:55.174009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:34.330342Z","iopub.execute_input":"2022-12-19T16:04:34.331234Z","iopub.status.idle":"2022-12-19T16:04:34.342086Z","shell.execute_reply.started":"2022-12-19T16:04:34.331195Z","shell.execute_reply":"2022-12-19T16:04:34.340885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"taborder=tab_order.groupby(by='session').agg({'aid':'nunique'})","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:10:00.869787Z","iopub.execute_input":"2022-12-19T17:10:00.870320Z","iopub.status.idle":"2022-12-19T17:10:01.694734Z","shell.execute_reply.started":"2022-12-19T17:10:00.870276Z","shell.execute_reply":"2022-12-19T17:10:01.692964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(taborder['aid'].values)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:10:14.200027Z","iopub.execute_input":"2022-12-19T17:10:14.200531Z","iopub.status.idle":"2022-12-19T17:10:14.503525Z","shell.execute_reply.started":"2022-12-19T17:10:14.200492Z","shell.execute_reply":"2022-12-19T17:10:14.501821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tabclicks=tab_clicks.groupby(by='session').agg({'aid':'nunique'})\nplt.hist(tabclicks['aid'].values)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:11:41.969557Z","iopub.execute_input":"2022-12-19T17:11:41.970104Z","iopub.status.idle":"2022-12-19T17:11:44.784488Z","shell.execute_reply.started":"2022-12-19T17:11:41.970060Z","shell.execute_reply":"2022-12-19T17:11:44.783111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tabcarts=tab_carts.groupby(by='session').agg({'aid':'nunique'})\nplt.hist(tabcarts['aid'].values)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:12:45.564185Z","iopub.execute_input":"2022-12-19T17:12:45.565110Z","iopub.status.idle":"2022-12-19T17:12:46.807996Z","shell.execute_reply.started":"2022-12-19T17:12:45.565060Z","shell.execute_reply":"2022-12-19T17:12:46.806628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_refined=df[df['session']>150]","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:41.981864Z","iopub.execute_input":"2022-12-19T16:04:41.982266Z","iopub.status.idle":"2022-12-19T16:04:41.988991Z","shell.execute_reply.started":"2022-12-19T16:04:41.982235Z","shell.execute_reply":"2022-12-19T16:04:41.987948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nx=df_refined.aid.tolist()\ny=df_refined.session.tolist()\nplt.scatter(x,y)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:45.158809Z","iopub.execute_input":"2022-12-19T16:04:45.159206Z","iopub.status.idle":"2022-12-19T16:04:45.850411Z","shell.execute_reply.started":"2022-12-19T16:04:45.159176Z","shell.execute_reply":"2022-12-19T16:04:45.849236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_refined.sort(by=\"session\",ascending=False)[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:04:50.144644Z","iopub.execute_input":"2022-12-19T16:04:50.145067Z","iopub.status.idle":"2022-12-19T16:04:50.190020Z","shell.execute_reply.started":"2022-12-19T16:04:50.145034Z","shell.execute_reply":"2022-12-19T16:04:50.188815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_aid2=TAB.groupby(\"session\").agg({'aid':'nunique'})","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:54:19.369866Z","iopub.execute_input":"2022-12-19T16:54:19.371789Z","iopub.status.idle":"2022-12-19T16:54:22.125280Z","shell.execute_reply.started":"2022-12-19T16:54:19.371694Z","shell.execute_reply":"2022-12-19T16:54:22.124084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_aid2.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:54:39.873306Z","iopub.execute_input":"2022-12-19T16:54:39.874251Z","iopub.status.idle":"2022-12-19T16:54:40.249327Z","shell.execute_reply.started":"2022-12-19T16:54:39.874177Z","shell.execute_reply":"2022-12-19T16:54:40.247972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x1=df_aid2.session.tolist()\ny1=df_aid2.aid.tolist()\nplt.scatter(x1,y1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:55:29.604781Z","iopub.execute_input":"2022-12-19T16:55:29.605413Z","iopub.status.idle":"2022-12-19T16:55:39.554516Z","shell.execute_reply.started":"2022-12-19T16:55:29.605358Z","shell.execute_reply":"2022-12-19T16:55:39.553236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_aid=TAB.groupby(\"session\").agg({'aid':'list'})","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:54:24.995685Z","iopub.execute_input":"2022-12-19T16:54:24.996763Z","iopub.status.idle":"2022-12-19T16:54:26.054071Z","shell.execute_reply.started":"2022-12-19T16:54:24.996670Z","shell.execute_reply":"2022-12-19T16:54:26.052533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_aid['aid']=df_aid['aid'].apply(set)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:54:28.238466Z","iopub.execute_input":"2022-12-19T16:54:28.238983Z","iopub.status.idle":"2022-12-19T16:54:28.247354Z","shell.execute_reply.started":"2022-12-19T16:54:28.238938Z","shell.execute_reply":"2022-12-19T16:54:28.245376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"13318382 pour avoir une session de 100 articles\n14215550 pour avoir une session de 50 articles","metadata":{}},{"cell_type":"code","source":"df_aid[df_aid['session']==14215550]['aid'].values","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:57:11.654471Z","iopub.execute_input":"2022-12-19T16:57:11.656127Z","iopub.status.idle":"2022-12-19T16:57:11.708892Z","shell.execute_reply.started":"2022-12-19T16:57:11.656063Z","shell.execute_reply":"2022-12-19T16:57:11.707338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_aid[-1]","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:27:31.768881Z","iopub.execute_input":"2022-12-19T16:27:31.769360Z","iopub.status.idle":"2022-12-19T16:27:31.799972Z","shell.execute_reply.started":"2022-12-19T16:27:31.769319Z","shell.execute_reply":"2022-12-19T16:27:31.798622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(L)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:48:31.750073Z","iopub.execute_input":"2022-12-19T16:48:31.750650Z","iopub.status.idle":"2022-12-19T16:48:31.760037Z","shell.execute_reply.started":"2022-12-19T16:48:31.750604Z","shell.execute_reply":"2022-12-19T16:48:31.758727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"longueur=[]\nfor el in L:\n    if len(el)!=50:\n        longueur.append(len(el))\n\nprint(longueur)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:03:34.843685Z","iopub.execute_input":"2022-12-19T17:03:34.845012Z","iopub.status.idle":"2022-12-19T17:03:34.853135Z","shell.execute_reply.started":"2022-12-19T17:03:34.844951Z","shell.execute_reply":"2022-12-19T17:03:34.851717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(longueur)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:03:37.628504Z","iopub.execute_input":"2022-12-19T17:03:37.629553Z","iopub.status.idle":"2022-12-19T17:03:37.886990Z","shell.execute_reply.started":"2022-12-19T17:03:37.629498Z","shell.execute_reply":"2022-12-19T17:03:37.885558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c={1341188, 906757, 1501574, 1596040, 1827344, 1222035, 628372, 603542, 355863, 1411864, 1789086, 347294, 869536, 924062, 176034, 545573, 963115, 938926, 1198385, 472245, 1399225, 1838010, 631617, 599876, 1037380, 1564746, 1505615, 336720, 439121, 1200722, 1063764, 63957, 993877, 1127511, 1025622, 1382233, 1648986, 1787994, 808927, 756065, 672482, 1041122, 1378662, 646503, 1466856, 604656, 1506804, 1653110, 761721, 864507}\nL=[]\nfor el in df_aid['aid'].values:\n    inter=c.intersection(el)\n    if len(inter)>1:\n        L.append(inter)\n        \nprint(L)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T17:00:33.765943Z","iopub.execute_input":"2022-12-19T17:00:33.766570Z","iopub.status.idle":"2022-12-19T17:00:46.441509Z","shell.execute_reply.started":"2022-12-19T17:00:33.766515Z","shell.execute_reply":"2022-12-19T17:00:46.439860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c.intersection({684424, 1172873, 1527732})","metadata":{"execution":{"iopub.status.busy":"2022-12-19T16:24:22.926213Z","iopub.execute_input":"2022-12-19T16:24:22.926638Z","iopub.status.idle":"2022-12-19T16:24:22.934752Z","shell.execute_reply.started":"2022-12-19T16:24:22.926602Z","shell.execute_reply":"2022-12-19T16:24:22.933183Z"},"trusted":true},"execution_count":null,"outputs":[]}]}