{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-08T06:36:26.30399Z","iopub.execute_input":"2023-06-08T06:36:26.304391Z","iopub.status.idle":"2023-06-08T06:36:26.355323Z","shell.execute_reply.started":"2023-06-08T06:36:26.304358Z","shell.execute_reply":"2023-06-08T06:36:26.354192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install nvtabular==1.3.3 merlin-models polars merlin-core==v0.4.0 dask_cuda","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:36:26.357463Z","iopub.execute_input":"2023-06-08T06:36:26.357898Z","iopub.status.idle":"2023-06-08T06:40:29.168582Z","shell.execute_reply.started":"2023-06-08T06:36:26.35786Z","shell.execute_reply":"2023-06-08T06:40:29.16718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport pandas as pd\nfrom pathlib import Path\nimport os\nimport random\nimport numpy as np\nimport json\nfrom datetime import timedelta\nfrom collections import Counter\nfrom tqdm.notebook import tqdm\nfrom heapq import nlargest\n\nimport polars as pl\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_theme()\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:29.170623Z","iopub.execute_input":"2023-06-08T06:40:29.171093Z","iopub.status.idle":"2023-06-08T06:40:30.345999Z","shell.execute_reply.started":"2023-06-08T06:40:29.171055Z","shell.execute_reply":"2023-06-08T06:40:30.344774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test.parquet')\ntrain_labels = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test_labels.parquet')\ntrain_XG = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test.parquet')\ntrain_labels_XG = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test_labels.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-06-08T07:05:16.458126Z","iopub.execute_input":"2023-06-08T07:05:16.458534Z","iopub.status.idle":"2023-06-08T07:05:17.663851Z","shell.execute_reply.started":"2023-06-08T07:05:16.458504Z","shell.execute_reply":"2023-06-08T07:05:17.663021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()\n#В polars более удобные данные и весит меньше.\n# session - сессия пользователя\n# aid - article id\n# ts - unix time\n# type 0-click 1-cart 2-buy","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:31.574277Z","iopub.execute_input":"2023-06-08T06:40:31.574726Z","iopub.status.idle":"2023-06-08T06:40:31.602043Z","shell.execute_reply.started":"2023-06-08T06:40:31.574685Z","shell.execute_reply":"2023-06-08T06:40:31.600981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:31.603437Z","iopub.execute_input":"2023-06-08T06:40:31.604491Z","iopub.status.idle":"2023-06-08T06:40:31.619158Z","shell.execute_reply.started":"2023-06-08T06:40:31.60444Z","shell.execute_reply":"2023-06-08T06:40:31.617885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_action_num_reverse_chrono(df):\n    return df.select([\n        pl.col('*'),\n        pl.col('session').cumcount().reverse().over('session').alias('action_num_reverse_chrono')\n    ])\n\ndef add_session_length(df):\n    return df.select([\n        pl.col('*'),\n        pl.col('session').count().over('session').alias('session_length')\n    ])\n\ndef add_log_recency_score(df):\n    linear_interpolation = 0.1 + ((1-0.1) / (df['session_length']-1)) * (df['session_length']-df['action_num_reverse_chrono']-1)\n    return df.with_columns(pl.Series(2**linear_interpolation - 1).alias('log_recency_score')).fill_nan(1)\n\ndef add_type_weighted_log_recency_score(df):\n    type_weights = {0:1, 1:6, 2:3}\n    type_weighted_log_recency_score = pl.Series(df['log_recency_score'] / df['type'].apply(lambda x: type_weights[x]))\n    return df.with_columns(type_weighted_log_recency_score.alias('type_weighted_log_recency_score'))\n\ndef apply(df, pipeline):\n    for f in pipeline:\n        df = f(df)\n    return df\n\npipeline = [add_action_num_reverse_chrono, add_session_length, add_log_recency_score, add_type_weighted_log_recency_score]\ntrain = apply(train, pipeline)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:31.620743Z","iopub.execute_input":"2023-06-08T06:40:31.621898Z","iopub.status.idle":"2023-06-08T06:40:39.701162Z","shell.execute_reply.started":"2023-06-08T06:40:31.621856Z","shell.execute_reply":"2023-06-08T06:40:39.69998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type2id = {\"clicks\": 0, \"carts\": 1, \"orders\": 2}\ntrain_labels = train_labels.explode('ground_truth').with_columns([\n    pl.col('ground_truth').alias('aid'),\n    pl.col('type').apply(lambda x: type2id[x])\n])[['session', 'type', 'aid']]","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:39.702817Z","iopub.execute_input":"2023-06-08T06:40:39.703309Z","iopub.status.idle":"2023-06-08T06:40:41.314461Z","shell.execute_reply.started":"2023-06-08T06:40:39.703266Z","shell.execute_reply":"2023-06-08T06:40:41.313295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = train_labels.with_columns([\n    pl.col('session').cast(pl.datatypes.Int32),\n    pl.col('type').cast(pl.datatypes.UInt8),\n    pl.col('aid').cast(pl.datatypes.Int32)\n])\ntrain_labels = train_labels.with_columns(pl.lit(1).alias('gt'))\ntrain = train.join(train_labels, how='left', on=['session', 'type', 'aid']).with_columns(pl.col('gt').fill_null(0))","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:41.315935Z","iopub.execute_input":"2023-06-08T06:40:41.316413Z","iopub.status.idle":"2023-06-08T06:40:42.322894Z","shell.execute_reply.started":"2023-06-08T06:40:41.316372Z","shell.execute_reply":"2023-06-08T06:40:42.321915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:42.326683Z","iopub.execute_input":"2023-06-08T06:40:42.327025Z","iopub.status.idle":"2023-06-08T06:40:42.338266Z","shell.execute_reply.started":"2023-06-08T06:40:42.326996Z","shell.execute_reply":"2023-06-08T06:40:42.337096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_session_lenghts(df):\n    return df.groupby('session').agg([\n        pl.col('session').count().alias('session_length')\n    ])['session_length'].to_numpy()\nsession_lengths_train = get_session_lenghts(train)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:42.340035Z","iopub.execute_input":"2023-06-08T06:40:42.341482Z","iopub.status.idle":"2023-06-08T06:40:42.693186Z","shell.execute_reply.started":"2023-06-08T06:40:42.341433Z","shell.execute_reply":"2023-06-08T06:40:42.692295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm.sklearn import LGBMRanker","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:42.697173Z","iopub.execute_input":"2023-06-08T06:40:42.697546Z","iopub.status.idle":"2023-06-08T06:40:43.751839Z","shell.execute_reply.started":"2023-06-08T06:40:42.697515Z","shell.execute_reply":"2023-06-08T06:40:43.750568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranker = LGBMRanker(\n    objective=\"lambdarank\",\n    metric=\"ndcg\",\n    boosting_type=\"dart\",\n    n_estimators=20,\n    importance_type='gain',\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:43.753186Z","iopub.execute_input":"2023-06-08T06:40:43.753582Z","iopub.status.idle":"2023-06-08T06:40:43.760329Z","shell.execute_reply.started":"2023-06-08T06:40:43.75355Z","shell.execute_reply":"2023-06-08T06:40:43.758659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:43.762622Z","iopub.execute_input":"2023-06-08T06:40:43.763307Z","iopub.status.idle":"2023-06-08T06:40:43.777954Z","shell.execute_reply.started":"2023-06-08T06:40:43.763235Z","shell.execute_reply":"2023-06-08T06:40:43.776756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = ['aid', 'type', 'action_num_reverse_chrono', 'session_length', 'log_recency_score', 'type_weighted_log_recency_score']\ntarget = 'gt'","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:43.780082Z","iopub.execute_input":"2023-06-08T06:40:43.780566Z","iopub.status.idle":"2023-06-08T06:40:43.788759Z","shell.execute_reply.started":"2023-06-08T06:40:43.780523Z","shell.execute_reply":"2023-06-08T06:40:43.787328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ranker = ranker.fit(\n    train[feature_cols].to_pandas(),\n    train[target].to_pandas(),\n    group=session_lengths_train,\n)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:40:43.790034Z","iopub.execute_input":"2023-06-08T06:40:43.790516Z","iopub.status.idle":"2023-06-08T06:41:02.291753Z","shell.execute_reply.started":"2023-06-08T06:40:43.790467Z","shell.execute_reply":"2023-06-08T06:41:02.290884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pl.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\ntest = apply(test, pipeline)\nscores = ranker.predict(test[feature_cols].to_pandas())\n","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:41:02.292998Z","iopub.execute_input":"2023-06-08T06:41:02.29364Z","iopub.status.idle":"2023-06-08T06:41:13.561084Z","shell.execute_reply.started":"2023-06-08T06:41:02.293607Z","shell.execute_reply":"2023-06-08T06:41:13.560127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(scores)","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:41:13.562553Z","iopub.execute_input":"2023-06-08T06:41:13.563147Z","iopub.status.idle":"2023-06-08T06:41:13.570135Z","shell.execute_reply.started":"2023-06-08T06:41:13.563114Z","shell.execute_reply":"2023-06-08T06:41:13.569063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.with_columns(pl.Series(name='score', values=scores))\ntest_predictions = test.sort(['session', 'score'], descending =True).groupby('session').agg([\n    pl.col(\"aid\").limit(20)\n])","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:41:13.571535Z","iopub.execute_input":"2023-06-08T06:41:13.57228Z","iopub.status.idle":"2023-06-08T06:41:14.250528Z","shell.execute_reply.started":"2023-06-08T06:41:13.572243Z","shell.execute_reply":"2023-06-08T06:41:14.249412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_types = []\nlabels = []\n\nfor session, preds in zip(test_predictions['session'].to_numpy(), test_predictions['aid'].to_numpy()):\n    l = ' '.join(str(p) for p in preds)\n    for session_type in ['clicks', 'carts', 'orders']:\n        labels.append(l)\n        session_types.append(f'{session}_{session_type}')","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:41:14.252027Z","iopub.execute_input":"2023-06-08T06:41:14.252375Z","iopub.status.idle":"2023-06-08T06:41:32.903668Z","shell.execute_reply.started":"2023-06-08T06:41:14.252346Z","shell.execute_reply":"2023-06-08T06:41:32.902343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pl.DataFrame({'session_type': session_types, 'labels': labels})\nsubmission.write_csv('submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-06-08T06:41:32.905355Z","iopub.execute_input":"2023-06-08T06:41:32.905726Z","iopub.status.idle":"2023-06-08T06:41:36.895301Z","shell.execute_reply.started":"2023-06-08T06:41:32.905696Z","shell.execute_reply":"2023-06-08T06:41:36.8943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_XG = apply(train_XG, pipeline)\ntrain_labels_XG = train_labels_XG.explode('ground_truth').with_columns([\n    pl.col('ground_truth').alias('aid'),\n    pl.col('type').apply(lambda x: type2id[x])\n])[['session', 'type', 'aid']]\ntrain_labels = train_labels.with_columns([\n    pl.col('session').cast(pl.datatypes.Int32),\n    pl.col('type').cast(pl.datatypes.UInt8),\n    pl.col('aid').cast(pl.datatypes.Int32)\n])\n\ntrain_labels = train_labels.with_column(pl.lit(1).alias('gt'))\n\ntrain = train.join(train_labels, how='left', on=['session', 'type', 'aid']).with_column(pl.col('gt').fill_null(0))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}