{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install polars\n\nimport polars as pl\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:48:18.415947Z","iopub.execute_input":"2023-01-13T06:48:18.416677Z","iopub.status.idle":"2023-01-13T06:48:34.722611Z","shell.execute_reply.started":"2023-01-13T06:48:18.416531Z","shell.execute_reply":"2023-01-13T06:48:34.721092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test.parquet')\ntrain_labels = pl.read_parquet('../input/otto-train-and-test-data-for-local-validation/test_labels.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:48:34.725715Z","iopub.execute_input":"2023-01-13T06:48:34.726236Z","iopub.status.idle":"2023-01-13T06:48:35.804800Z","shell.execute_reply.started":"2023-01-13T06:48:34.726187Z","shell.execute_reply":"2023-01-13T06:48:35.803612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_action_num_reverse_chrono(df):\n    return df.select([\n        pl.col('*'),\n        pl.col('session').cumcount().reverse().over('session').alias('action_num_reverse_chrono')\n    ])\n\ndef add_session_length(df):\n    return df.select([\n        pl.col('*'),\n        pl.col('session').count().over('session').alias('session_length')\n    ])\n\ndef add_log_recency_score(df):\n    linear_interpolation = 0.1 + ((1-0.1) / (df['session_length']-1)) * (df['session_length']-df['action_num_reverse_chrono']-1)\n    return df.with_columns(pl.Series(2**linear_interpolation - 1).alias('log_recency_score')).fill_nan(1)\n\ndef add_type_weighted_log_recency_score(df):\n    type_weights = {0:1, 1:6, 2:3}\n    type_weighted_log_recency_score = pl.Series(df['log_recency_score'] / df['type'].apply(lambda x: type_weights[x]))\n    return df.with_column(type_weighted_log_recency_score.alias('type_weighted_log_recency_score'))\n\ndef apply(df, pipeline):\n    for f in pipeline:\n        df = f(df)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:48:54.047246Z","iopub.execute_input":"2023-01-13T06:48:54.047731Z","iopub.status.idle":"2023-01-13T06:48:54.059708Z","shell.execute_reply.started":"2023-01-13T06:48:54.047693Z","shell.execute_reply":"2023-01-13T06:48:54.057918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline = [add_action_num_reverse_chrono, add_session_length, add_log_recency_score, add_type_weighted_log_recency_score]\n\ntrain = apply(train, pipeline)","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:49:15.038003Z","iopub.execute_input":"2023-01-13T06:49:15.038555Z","iopub.status.idle":"2023-01-13T06:49:22.858448Z","shell.execute_reply.started":"2023-01-13T06:49:15.038507Z","shell.execute_reply":"2023-01-13T06:49:22.857141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:49:49.072585Z","iopub.execute_input":"2023-01-13T06:49:49.073124Z","iopub.status.idle":"2023-01-13T06:49:49.085086Z","shell.execute_reply.started":"2023-01-13T06:49:49.073080Z","shell.execute_reply":"2023-01-13T06:49:49.083686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type2id = {\"clicks\": 0, \"carts\": 1, \"orders\": 2}\n\ntrain_labels = train_labels.explode('ground_truth').with_columns([\n    pl.col('ground_truth').alias('aid'),\n    pl.col('type').apply(lambda x: type2id[x])])[['session', 'type', 'aid']]","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:51:40.733306Z","iopub.execute_input":"2023-01-13T06:51:40.733788Z","iopub.status.idle":"2023-01-13T06:51:42.287430Z","shell.execute_reply.started":"2023-01-13T06:51:40.733754Z","shell.execute_reply":"2023-01-13T06:51:42.286312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = train_labels.with_columns([\n    pl.col('session').cast(pl.datatypes.Int32),\n    pl.col('type').cast(pl.datatypes.UInt8),\n    pl.col('aid').cast(pl.datatypes.Int32)\n])\n\ntrain_labels = train_labels.with_column(pl.lit(1).alias('gt'))","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:52:12.841144Z","iopub.execute_input":"2023-01-13T06:52:12.841624Z","iopub.status.idle":"2023-01-13T06:52:12.886001Z","shell.execute_reply.started":"2023-01-13T06:52:12.841586Z","shell.execute_reply":"2023-01-13T06:52:12.884602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:52:24.478340Z","iopub.execute_input":"2023-01-13T06:52:24.478761Z","iopub.status.idle":"2023-01-13T06:52:24.488081Z","shell.execute_reply.started":"2023-01-13T06:52:24.478730Z","shell.execute_reply":"2023-01-13T06:52:24.486453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.join(train_labels, how='left', on=['session', 'type', 'aid']).with_column(pl.col('gt').fill_null(0))\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:52:37.896421Z","iopub.execute_input":"2023-01-13T06:52:37.896853Z","iopub.status.idle":"2023-01-13T06:52:38.984426Z","shell.execute_reply.started":"2023-01-13T06:52:37.896820Z","shell.execute_reply":"2023-01-13T06:52:38.983499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_session_lenghts(df):\n    return df.groupby('session').agg([\n        pl.col('session').count().alias('session_length')\n    ])['session_length'].to_numpy()\n\nsession_lengths_train = get_session_lenghts(train)","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:53:13.176037Z","iopub.execute_input":"2023-01-13T06:53:13.176642Z","iopub.status.idle":"2023-01-13T06:53:14.504162Z","shell.execute_reply.started":"2023-01-13T06:53:13.176566Z","shell.execute_reply":"2023-01-13T06:53:14.503014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm.sklearn import LGBMRanker\n\nranker = LGBMRanker(\n    objective=\"lambdarank\",\n    metric=\"ndcg\",\n    boosting_type=\"dart\",\n    n_estimators=20,\n    importance_type='gain',\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:53:36.433248Z","iopub.execute_input":"2023-01-13T06:53:36.433789Z","iopub.status.idle":"2023-01-13T06:53:38.105487Z","shell.execute_reply.started":"2023-01-13T06:53:36.433748Z","shell.execute_reply":"2023-01-13T06:53:38.104011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = ['aid', 'type', 'action_num_reverse_chrono', 'session_length', 'log_recency_score', 'type_weighted_log_recency_score']\ntarget = 'gt'\n\nmodel = ranker.fit(\n    train[feature_cols].to_pandas(),\n    train[target].to_pandas(),\n    group=session_lengths_train,\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:54:02.832121Z","iopub.execute_input":"2023-01-13T06:54:02.832751Z","iopub.status.idle":"2023-01-13T06:54:22.127137Z","shell.execute_reply.started":"2023-01-13T06:54:02.832706Z","shell.execute_reply":"2023-01-13T06:54:22.125965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pl.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\ntest = apply(test, pipeline)\n\nscores = model.predict(test[feature_cols].to_pandas())\ntest = test.with_columns(pl.Series(name='score', values=scores))\ntest_predictions = test.sort(['session', 'score'], reverse=True).groupby('session').agg([\n    pl.col('aid').limit(20).list()\n])","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:54:32.178693Z","iopub.execute_input":"2023-01-13T06:54:32.179207Z","iopub.status.idle":"2023-01-13T06:54:43.874074Z","shell.execute_reply.started":"2023-01-13T06:54:32.179161Z","shell.execute_reply":"2023-01-13T06:54:43.872954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"session_types = []\nlabels = []\n\nfor session, preds in zip(test_predictions['session'].to_numpy(), test_predictions['aid'].to_numpy()):\n    l = ' '.join(str(p) for p in preds)\n    for session_type in ['clicks', 'carts', 'orders']:\n        labels.append(l)\n        session_types.append(f'{session}_{session_type}')\n        \nsubmission = pl.DataFrame({'session_type': session_types, 'labels': labels})\nsubmission.write_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-13T06:55:08.357098Z","iopub.execute_input":"2023-01-13T06:55:08.357672Z","iopub.status.idle":"2023-01-13T06:55:22.623458Z","shell.execute_reply.started":"2023-01-13T06:55:08.357621Z","shell.execute_reply":"2023-01-13T06:55:22.622482Z"},"trusted":true},"execution_count":null,"outputs":[]}]}