{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About\n\n### What is done in this notebook:\n  * Creating xgboost ranker model to rank candidates gotten from covisitation matrices\n  * Calculating score on validation set\n  * Creating submission for public test set\n  \n### About data\n  * For each session in train,val,test sets, 40 candidate items (more is better) are generated using handcrafted rules and covisitation matrices \n  * For generated candidates various features are added and train, val, test sets prepared\n  * This data is saved in /kaggle/input/prepared-candidates-from-covisitation-mat-3 dataset.\n  * Train set negatives are sampled so only 0.2 of data is kept\n  * Val and test data is not downsampled and is split into 10 files because of memory consumption\n\n### Thanks\n* Thanks to <a href='https://www.kaggle.com/cdeotte'>Chris</a>, <a href='https://www.kaggle.com/radek1'>Radek</a>, and others that supplied knowledge, code, comments","metadata":{}},{"cell_type":"markdown","source":"# Import ","metadata":{"papermill":{"duration":0.022544,"end_time":"2023-01-31T14:31:47.841746","exception":false,"start_time":"2023-01-31T14:31:47.819202","status":"completed"},"tags":[]}},{"cell_type":"code","source":"VER = 1","metadata":{"papermill":{"duration":0.034236,"end_time":"2023-01-31T14:31:47.896784","exception":false,"start_time":"2023-01-31T14:31:47.862548","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:41.851999Z","iopub.execute_input":"2023-02-02T16:27:41.853078Z","iopub.status.idle":"2023-02-02T16:27:41.880896Z","shell.execute_reply.started":"2023-02-02T16:27:41.852967Z","shell.execute_reply":"2023-02-02T16:27:41.880012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\nimport datetime\nimport cudf\nimport glob\nimport sys\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport subprocess\nimport os\nimport psutil\nimport numpy as np\nimport gc\nimport time\nimport random\nfrom time import sleep","metadata":{"papermill":{"duration":3.488868,"end_time":"2023-01-31T14:31:51.406379","exception":false,"start_time":"2023-01-31T14:31:47.917511","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:41.891140Z","iopub.execute_input":"2023-02-02T16:27:41.891468Z","iopub.status.idle":"2023-02-02T16:27:45.522191Z","shell.execute_reply.started":"2023-02-02T16:27:41.891440Z","shell.execute_reply":"2023-02-02T16:27:45.521099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"class cfg:\n    local         = False # on local machine only some testing is done (low memory)\n    seed          = 101\n    folds_num     = 5\n    # train_folds   = [0] # it is faster to check model improvement of only one fold, when possible\n    train_folds   = list(range(folds_num))\n    val_fold      = 0 # validate only on this fold\n    test_folds    = train_folds # predict using all train folds","metadata":{"papermill":{"duration":0.029597,"end_time":"2023-01-31T14:31:51.456918","exception":false,"start_time":"2023-01-31T14:31:51.427321","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.528581Z","iopub.execute_input":"2023-02-02T16:27:45.530854Z","iopub.status.idle":"2023-02-02T16:27:45.538436Z","shell.execute_reply.started":"2023-02-02T16:27:45.530815Z","shell.execute_reply":"2023-02-02T16:27:45.537438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# On my local machine, I found it good to set resource limit to number a bit above available RAM \n\nif cfg.local:\n    import resource\n    resource.setrlimit(resource.RLIMIT_AS, (int(21*(10**9)), -1))\n    print(resource.getrlimit(resource.RLIMIT_AS)[0] / 1e9, 'GB')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T16:27:45.543177Z","iopub.execute_input":"2023-02-02T16:27:45.545832Z","iopub.status.idle":"2023-02-02T16:27:45.553769Z","shell.execute_reply.started":"2023-02-02T16:27:45.545794Z","shell.execute_reply":"2023-02-02T16:27:45.552784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Seed","metadata":{}},{"cell_type":"code","source":"# Sets the seed of the entire notebook so results are the same every time we run. This is for REPRODUCIBILITY\ndef set_seed(seed = 42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n\n    cudf.cupy.random.seed(seed)\n\n    print('Setting seed done')\n    \nset_seed(cfg.seed)","metadata":{"papermill":{"duration":0.237995,"end_time":"2023-01-31T14:31:51.716295","exception":false,"start_time":"2023-01-31T14:31:51.478300","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.556651Z","iopub.execute_input":"2023-02-02T16:27:45.557267Z","iopub.status.idle":"2023-02-02T16:27:45.850412Z","shell.execute_reply.started":"2023-02-02T16:27:45.557230Z","shell.execute_reply":"2023-02-02T16:27:45.849403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### File paths:","metadata":{}},{"cell_type":"code","source":"print('Train:')\ntrain_candidates_files = {}\nfor ev in ['clicks', 'carts', 'orders']:\n    if cfg.local:\n        train_candidates_files[ev] = glob.glob(f'./train_candidates_{ev}*')\n    else:\n        train_candidates_files[ev] = glob.glob(f'/kaggle/input/prepared-candidates-from-covisitation-mat-3/train_candidates_{ev}*')\n    print(len(train_candidates_files[ev]))\n    train_candidates_files[ev] = train_candidates_files[ev][0]\n    \nprint('Val:')\nval_candidates_files = {}\nfor ev in ['clicks', 'carts', 'orders']:\n    if cfg.local:\n        val_candidates_files[ev] = glob.glob(f'./val_{ev}_candidates_*')\n    else:\n        val_candidates_files[ev] = glob.glob(f'/kaggle/input/prepared-candidates-from-covisitation-mat-3/val_{ev}_candidates_*')\n    print(len(val_candidates_files[ev]))\n\nprint('Test:')\ntest_candidates_files = {}\nfor ev in ['clicks', 'carts', 'orders']:\n    if cfg.local:\n        test_candidates_files[ev] = glob.glob(f'./test_{ev}_candidates_*')\n    else:\n        test_candidates_files[ev] = glob.glob(f'/kaggle/input/prepared-candidates-from-covisitation-mat-3/test_{ev}_candidates_df_*')\n    print(len(test_candidates_files[ev]))\n","metadata":{"papermill":{"duration":0.051536,"end_time":"2023-01-31T14:31:51.789325","exception":false,"start_time":"2023-01-31T14:31:51.737789","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.854108Z","iopub.execute_input":"2023-02-02T16:27:45.854832Z","iopub.status.idle":"2023-02-02T16:27:45.902831Z","shell.execute_reply.started":"2023-02-02T16:27:45.854793Z","shell.execute_reply":"2023-02-02T16:27:45.901789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if cfg.local:\n    folds_path = glob.glob('./df_fold.pqt')\nelse:\n    folds_path = glob.glob('/kaggle/input/prepared-candidates-from-covisitation-mat-3/df_fold.pqt')\n\nif len(folds_path) == 1:\n    folds_path = folds_path[0]\n    print('Folds df present')\nelse:\n#     print('Folds missing')\n    assert False, 'Folds missing'","metadata":{"papermill":{"duration":0.031497,"end_time":"2023-01-31T14:31:51.841661","exception":false,"start_time":"2023-01-31T14:31:51.810164","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.907242Z","iopub.execute_input":"2023-02-02T16:27:45.909957Z","iopub.status.idle":"2023-02-02T16:27:45.921119Z","shell.execute_reply.started":"2023-02-02T16:27:45.909913Z","shell.execute_reply":"2023-02-02T16:27:45.919949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Last week of train set is used for training here\n\nif cfg.local:\n    train_files = glob.glob('../../../downloaded_data/split_data_local_validation/test_parquet/*')\n    train_labels_path = '../../../downloaded_data/split_data_local_validation/test_labels.parquet'\n    \n    test_files = glob.glob('../../../downloaded_data/split_data/test_parquet/*')\nelse:\n    train_files = glob.glob('/kaggle/input/otto-validation/test_parquet/*')\n    train_labels_path = '/kaggle/input/otto-validation/test_labels.parquet'\n\n    test_files = glob.glob('/kaggle/input/otto-chunk-data-inparquet-format/test_parquet/*')","metadata":{"papermill":{"duration":0.031754,"end_time":"2023-01-31T14:31:51.894908","exception":false,"start_time":"2023-01-31T14:31:51.863154","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.926324Z","iopub.execute_input":"2023-02-02T16:27:45.928868Z","iopub.status.idle":"2023-02-02T16:27:45.943680Z","shell.execute_reply.started":"2023-02-02T16:27:45.928827Z","shell.execute_reply":"2023-02-02T16:27:45.942386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{"papermill":{"duration":0.02047,"end_time":"2023-01-31T14:31:51.936034","exception":false,"start_time":"2023-01-31T14:31:51.915564","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def ts_format(ts):\n    return datetime.datetime.fromtimestamp(ts).strftime(\"%b %d %Y  %H:%M:%S\")\ndef print_date_ts(ts, end='\\n'):\n    print(ts_format(ts), end=end)\ndef print_date(d, end='\\n'):\n    print(d.strftime(\"%b %d %Y  %H:%M:%S\"), end=end)\ndef mils_format(x):\n    return f'{x:,}'\ndef len_mils(x):\n    return mils_format(len(x))\ndef load_df_from_files(files_list):\n    df = cudf.DataFrame()\n    for f in files_list:\n        df = cudf.concat([df, cudf.read_parquet(f)], ignore_index=True)\n    df = df.reset_index(drop=True)\n    return df\ndef preview_df(df, num_to_show=1):\n    print(len_mils(df))\n    display(df.head(num_to_show))","metadata":{"papermill":{"duration":0.029028,"end_time":"2023-01-31T14:31:51.985917","exception":false,"start_time":"2023-01-31T14:31:51.956889","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.948390Z","iopub.execute_input":"2023-02-02T16:27:45.951070Z","iopub.status.idle":"2023-02-02T16:27:45.962378Z","shell.execute_reply.started":"2023-02-02T16:27:45.951029Z","shell.execute_reply":"2023-02-02T16:27:45.961420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Track gpu memory","metadata":{"papermill":{"duration":0.020668,"end_time":"2023-01-31T14:31:52.176981","exception":false,"start_time":"2023-01-31T14:31:52.156313","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"* This part of notebook is used only for plotting GPU memory consumption over time\n* It is not neccessary, but exploratory :)","metadata":{"papermill":{"duration":0.020641,"end_time":"2023-01-31T14:31:52.218426","exception":false,"start_time":"2023-01-31T14:31:52.197785","status":"completed"},"tags":[]}},{"cell_type":"code","source":"track_gpu_mem = True","metadata":{"papermill":{"duration":0.027241,"end_time":"2023-01-31T14:31:52.267410","exception":false,"start_time":"2023-01-31T14:31:52.240169","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.967214Z","iopub.execute_input":"2023-02-02T16:27:45.968962Z","iopub.status.idle":"2023-02-02T16:27:45.976196Z","shell.execute_reply.started":"2023-02-02T16:27:45.968921Z","shell.execute_reply":"2023-02-02T16:27:45.975177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpu_available = None\n\ntry:\n    subprocess.check_output('nvidia-smi')\n    gpu_available = True\n\n    print('Nvidia GPU detected!')\nexcept Exception:\n    gpu_available = False\n    track_gpu_mem = False\n\n    print('No Nvidia GPU in system!')","metadata":{"papermill":{"duration":0.095222,"end_time":"2023-01-31T14:31:52.383734","exception":false,"start_time":"2023-01-31T14:31:52.288512","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:45.985093Z","iopub.execute_input":"2023-02-02T16:27:45.987310Z","iopub.status.idle":"2023-02-02T16:27:46.076271Z","shell.execute_reply.started":"2023-02-02T16:27:45.987273Z","shell.execute_reply":"2023-02-02T16:27:46.074919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if gpu_available:\n    dev0 = cudf.cupy.cuda.Device(0)\n    def print_gpu_mem_info():\n        free_memory, total_memory = dev0.mem_info\n        print('Free -', free_memory // 1024**2, 'MiB', '\\tTotal -', total_memory // 1024**2, 'MiB')\n    print_gpu_mem_info()","metadata":{"papermill":{"duration":1.806188,"end_time":"2023-01-31T14:31:54.211565","exception":false,"start_time":"2023-01-31T14:31:52.405377","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:46.079600Z","iopub.execute_input":"2023-02-02T16:27:46.080018Z","iopub.status.idle":"2023-02-02T16:27:48.600691Z","shell.execute_reply.started":"2023-02-02T16:27:46.079966Z","shell.execute_reply":"2023-02-02T16:27:48.599552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clear_gpu_log():\n    with open('./gpu-report.csv', 'w') as file:\n        cols = [\n            'date', 'used_MiB'\n        ]\n        line = ','.join(cols) + '\\n'\n        file.write(line)\n\ndef start_logging_gpu_info(every='2'):\n    log_gpu_subprocess = subprocess.Popen(\n        [\"watch\",\"-n\",every, \"./track-gpu.sh\"],\n        stdout=subprocess.PIPE\n    )\n    \n    return log_gpu_subprocess\n\ndef stop_logging_gpu_info():\n    for proc in psutil.process_iter():\n        if 'watch' in proc.name() and './track-gpu.sh' in proc.cmdline():\n            proc.kill()\n\ndef plot_gpu_usage(mins_to_plot=2, interval=2):\n    if not os.path.exists('./gpu-report.csv'):\n        return\n        \n    df = pd.read_csv('./gpu-report.csv')\n\n    entries_num = int(mins_to_plot * 60 // interval)\n    df = df[-entries_num:]\n    df['date'] = df.date.apply(lambda d: datetime.datetime.fromisoformat(d))\n\n    plt.figure(figsize=(12,4))\n    plt.title(f'used GPU Memory')\n    plt.plot(df.date, df['used_MiB'], color='blue')\n    plt.show()\n\ndef create_file_with_logging_command():\n    with open('./track-gpu.sh', 'w') as file:\n        file.write(\n            'values=$(nvidia-smi | sed -n \\'10p\\' | awk \\'{print $9}\\' | sed -nz \\'s/MiB\\\\n//gp\\'); echo $(date +%Y-%m-%d#%H:%M:%S),$(($values)) >> ' + f'{os.getcwd()}/gpu-report.csv'\n        )\n\ndef run_logging_file():\n    subprocess.run(\n        [f'{os.getcwd()}/track-gpu.sh'],\n        shell=True\n    )\n\ndef allow_logging_file_execution():\n    os.system('chmod a+x ./track-gpu.sh')","metadata":{"papermill":{"duration":0.034778,"end_time":"2023-01-31T14:31:54.267402","exception":false,"start_time":"2023-01-31T14:31:54.232624","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:48.605677Z","iopub.execute_input":"2023-02-02T16:27:48.606612Z","iopub.status.idle":"2023-02-02T16:27:48.623772Z","shell.execute_reply.started":"2023-02-02T16:27:48.606573Z","shell.execute_reply":"2023-02-02T16:27:48.622549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if track_gpu_mem:\n    create_file_with_logging_command()\n    allow_logging_file_execution()\n\n    stop_logging_gpu_info()\n    clear_gpu_log()\n    run_logging_file() # to add one memory value\n    \n    watch_process = start_logging_gpu_info()\n    \n    df = pd.read_csv('./gpu-report.csv')\n    df.tail()","metadata":{"papermill":{"duration":0.150998,"end_time":"2023-01-31T14:31:54.439093","exception":false,"start_time":"2023-01-31T14:31:54.288095","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:48.627599Z","iopub.execute_input":"2023-02-02T16:27:48.628007Z","iopub.status.idle":"2023-02-02T16:27:48.795909Z","shell.execute_reply.started":"2023-02-02T16:27:48.627969Z","shell.execute_reply":"2023-02-02T16:27:48.792577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if track_gpu_mem:\n    sleep(2)\n    plot_gpu_usage()","metadata":{"papermill":{"duration":0.304555,"end_time":"2023-01-31T14:31:54.779678","exception":false,"start_time":"2023-01-31T14:31:54.475123","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:48.798426Z","iopub.execute_input":"2023-02-02T16:27:48.798829Z","iopub.status.idle":"2023-02-02T16:27:51.103246Z","shell.execute_reply.started":"2023-02-02T16:27:48.798787Z","shell.execute_reply":"2023-02-02T16:27:51.102269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{"papermill":{"duration":0.021451,"end_time":"2023-01-31T14:31:54.822503","exception":false,"start_time":"2023-01-31T14:31:54.801052","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Load train data","metadata":{"papermill":{"duration":0.020742,"end_time":"2023-01-31T14:31:54.864180","exception":false,"start_time":"2023-01-31T14:31:54.843438","status":"completed"},"tags":[]}},{"cell_type":"code","source":"type2id_df = cudf.DataFrame({'type': ['clicks', 'carts', 'orders'], 'type_id': [0,1,2]})\ntype2id_df.type_id = type2id_df.type_id.astype('int8')\n\ndef read_transform_file(f):\n    df = cudf.read_parquet(f)\n    df.ts = (df.ts/1000).astype('int32')\n    df = cudf.merge(df, type2id_df, on='type', how='inner').drop('type', axis=1).rename(columns={'type_id': 'type'})\n    return df\ndef load_train_week():\n    train = cudf.DataFrame()\n    for f in train_files:\n        train = cudf.concat([train, read_transform_file(f)], ignore_index=True)\n\n    train = train.reset_index(drop=True)\n    return train","metadata":{"papermill":{"duration":0.029866,"end_time":"2023-01-31T14:31:55.020494","exception":false,"start_time":"2023-01-31T14:31:54.990628","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:51.106201Z","iopub.execute_input":"2023-02-02T16:27:51.106509Z","iopub.status.idle":"2023-02-02T16:27:51.129252Z","shell.execute_reply.started":"2023-02-02T16:27:51.106480Z","shell.execute_reply":"2023-02-02T16:27:51.128401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = load_train_week()\npreview_df(train)","metadata":{"papermill":{"duration":3.026073,"end_time":"2023-01-31T14:31:58.117490","exception":false,"start_time":"2023-01-31T14:31:55.091417","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:51.130779Z","iopub.execute_input":"2023-02-02T16:27:51.131125Z","iopub.status.idle":"2023-02-02T16:27:54.824251Z","shell.execute_reply.started":"2023-02-02T16:27:51.131089Z","shell.execute_reply":"2023-02-02T16:27:54.823401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_date_ts(train['ts'].min())\nprint_date_ts(train['ts'].max())","metadata":{"papermill":{"duration":0.034593,"end_time":"2023-01-31T14:31:58.174184","exception":false,"start_time":"2023-01-31T14:31:58.139591","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:54.825492Z","iopub.execute_input":"2023-02-02T16:27:54.826078Z","iopub.status.idle":"2023-02-02T16:27:54.837155Z","shell.execute_reply.started":"2023-02-02T16:27:54.826041Z","shell.execute_reply":"2023-02-02T16:27:54.836222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load train labels","metadata":{"papermill":{"duration":0.020902,"end_time":"2023-01-31T14:31:58.216726","exception":false,"start_time":"2023-01-31T14:31:58.195824","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"* Train labels consist of (session,aid_list) pairs that did happen in session continuation","metadata":{"papermill":{"duration":0.020765,"end_time":"2023-01-31T14:31:58.258728","exception":false,"start_time":"2023-01-31T14:31:58.237963","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def load_train_week_labels(do_print=False):\n    train_labels = cudf.read_parquet(train_labels_path)\n\n    if do_print:\n        print(train_labels.shape)\n        preview_df(train_labels)\n\n    # Change ground_truth from \"aid in list\" to \"one aid per row\" -> (session, type) pairs will now be repeated where len(ground_truth) was >=2\n    train_labels = cudf.merge(train_labels, type2id_df, on='type', how='inner').drop('type', axis=1).rename(columns={'type_id': 'type'})\n    train_labels = train_labels.explode('ground_truth').rename(columns={'ground_truth': 'aid'})[['session', 'type', 'aid']]\n    train_labels = train_labels.astype({\n        'session': 'int32',\n        'aid': 'int32',\n        'type': 'int8',\n    })\n\n    if do_print:\n        print(train_labels.shape)\n        preview_df(train_labels)\n\n    # * Add columns ground_truth - gt that marks all pairs as 1, that is they happened\n    train_labels['gt'] = 1\n    train_labels['gt'] = train_labels['gt'].astype('int8')\n    train_labels.dtypes\n\n    return train_labels","metadata":{"papermill":{"duration":0.031393,"end_time":"2023-01-31T14:31:58.311003","exception":false,"start_time":"2023-01-31T14:31:58.279610","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:54.838543Z","iopub.execute_input":"2023-02-02T16:27:54.838856Z","iopub.status.idle":"2023-02-02T16:27:54.855856Z","shell.execute_reply.started":"2023-02-02T16:27:54.838826Z","shell.execute_reply":"2023-02-02T16:27:54.854986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = load_train_week_labels(do_print=True)","metadata":{"papermill":{"duration":1.787202,"end_time":"2023-01-31T14:32:00.119276","exception":false,"start_time":"2023-01-31T14:31:58.332074","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:54.856838Z","iopub.execute_input":"2023-02-02T16:27:54.857757Z","iopub.status.idle":"2023-02-02T16:27:56.679176Z","shell.execute_reply.started":"2023-02-02T16:27:54.857720Z","shell.execute_reply":"2023-02-02T16:27:56.678230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load train candidates function","metadata":{"papermill":{"duration":0.02276,"end_time":"2023-01-31T14:32:00.164257","exception":false,"start_time":"2023-01-31T14:32:00.141497","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_train_df(event):\n    candidates_df = cudf.read_parquet(train_candidates_files[event])\n    \n    return candidates_df","metadata":{"papermill":{"duration":0.030032,"end_time":"2023-01-31T14:32:00.265545","exception":false,"start_time":"2023-01-31T14:32:00.235513","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:56.680835Z","iopub.execute_input":"2023-02-02T16:27:56.681243Z","iopub.status.idle":"2023-02-02T16:27:56.687115Z","shell.execute_reply.started":"2023-02-02T16:27:56.681201Z","shell.execute_reply":"2023-02-02T16:27:56.685413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load folds","metadata":{}},{"cell_type":"code","source":"df_fold = cudf.read_parquet(folds_path)\ndf_fold.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T16:27:56.689484Z","iopub.execute_input":"2023-02-02T16:27:56.690108Z","iopub.status.idle":"2023-02-02T16:27:56.815405Z","shell.execute_reply.started":"2023-02-02T16:27:56.690071Z","shell.execute_reply":"2023-02-02T16:27:56.814469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calc model val score","metadata":{"papermill":{"duration":0.022361,"end_time":"2023-01-31T14:32:00.309457","exception":false,"start_time":"2023-01-31T14:32:00.287096","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_val_data(event):\n    val_data = cudf.DataFrame()\n    for f in val_candidates_files[event]:\n        val_data = cudf.concat([val_data, cudf.read_parquet(f).sort_values('session')], ignore_index=True)\n    val_data = val_data.reset_index(drop=True)\n    val_data['session'] = val_data['session'].astype('int32')\n    val_data['aid'] = val_data['aid'].astype('int32')\n    return val_data","metadata":{"papermill":{"duration":0.03089,"end_time":"2023-01-31T14:32:00.361663","exception":false,"start_time":"2023-01-31T14:32:00.330773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:56.816947Z","iopub.execute_input":"2023-02-02T16:27:56.817316Z","iopub.status.idle":"2023-02-02T16:27:56.825270Z","shell.execute_reply.started":"2023-02-02T16:27:56.817280Z","shell.execute_reply":"2023-02-02T16:27:56.824232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_event_recall(ev_preds_df, event_name, clip_amount=20):\n    sub = ev_preds_df\n    sub['session'] = sub.session_type.apply(lambda x: int(x.split('_')[0]))\n    # sub.labels = sub.labels.apply(lambda x: x[:clip_amount])\n    sub.labels = sub.labels.apply(lambda x: x[:clip_amount])\n\n    val_labels = pd.read_parquet(train_labels_path)\n    val_labels = val_labels.loc[val_labels['type']==event_name]\n\n    val_labels = pd.merge(val_labels, df_fold.to_pandas(), on='session')\n    val_labels = val_labels.loc[val_labels.fold == cfg.val_fold]\n    del val_labels['fold']\n\n    val_labels = val_labels.merge(sub, how='left', on=['session'])\n\n    val_labels['hits'] = val_labels.apply(lambda df: len(set(df.ground_truth).intersection(set(df.labels))), axis=1)\n\n    val_labels['gt_count'] = val_labels.ground_truth.str.len().clip(0, clip_amount)\n\n    recall = val_labels['hits'].sum() / val_labels['gt_count'].sum()\n    print(f'{event_name} recall =',recall)\n\n    return recall\ndef calc_val_score(event_name, clip_amount=20):\n    # events_names = ['clicks', 'carts', 'orders']\n    events_names = [event_name]\n\n    preds = {}\n    for event_name in events_names:\n        print(event_name)\n        \n        model = xgb.Booster()\n        model.load_model(f'XGB_fold{valid_fold_num}_{event_name}.xgb')\n        model.set_param({'predictor': 'gpu_predictor'})\n        \n        preds[event_name] = []\n        for i,f in enumerate(val_candidates_files[event_name]):\n            print(i, end=' ')\n\n            val_candidates = cudf.read_parquet(f)\n            val_candidates = val_candidates.sort_values('session')\n            \n            groups = val_candidates.groupby('session').size().to_frame('size')['size']\n            dtest = xgb.DMatrix(data=val_candidates[feature_cols[event_name]], group=groups)\n            \n            preds[event_name].append(model.predict(dtest))\n            del dtest, val_candidates\n        \n        preds[event_name] = np.concatenate(preds[event_name], axis=None)\n        np.save(f'val_preds_{event_name}.npy', preds[event_name])\n        print()\n\n    predictions = {}\n    sub = {}\n    predictions[event_name] = get_val_data(event_name)[['session','aid']]\n\n    predictions[event_name]['pred'] = preds[event_name]\n\n    # Pick top 20 predictions:\n    predictions[event_name] = predictions[event_name].sort_values(['session','pred'], ascending=[True,False]).reset_index(drop=True)\n    predictions[event_name]['n'] = predictions[event_name].groupby('session').aid.cumcount().astype('int8')\n    predictions[event_name] = predictions[event_name].loc[predictions[event_name].n<clip_amount]\n    del predictions[event_name]['pred']\n    del predictions[event_name]['n']\n\n    # Transform (session,aid) pairs to (session,string of all aids) pairs\n    predictions[event_name] = predictions[event_name].to_pandas() # it was faster to do next functions in pandas then in cudf\n    sub[event_name] = predictions[event_name].groupby('session').aid.apply(list)\n    del predictions[event_name]\n\n    sub[event_name] = sub[event_name].to_frame().reset_index()\n    to_add_str = '_' + event_name\n    sub[event_name]['session'] = sub[event_name].session.astype('str') + to_add_str\n\n    sub[event_name].columns = ['session_type', 'labels']\n\n    return calc_event_recall(sub[event_name], event_name, clip_amount)","metadata":{"papermill":{"duration":0.042835,"end_time":"2023-01-31T14:32:00.425905","exception":false,"start_time":"2023-01-31T14:32:00.383070","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:56.827826Z","iopub.execute_input":"2023-02-02T16:27:56.828261Z","iopub.status.idle":"2023-02-02T16:27:56.846810Z","shell.execute_reply.started":"2023-02-02T16:27:56.828232Z","shell.execute_reply":"2023-02-02T16:27:56.845693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model config","metadata":{"papermill":{"duration":0.031643,"end_time":"2023-01-31T14:32:00.788967","exception":false,"start_time":"2023-01-31T14:32:00.757324","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_feature_cols(df, event_name):\n    feature_cols = list(df.columns)\n    \n    remove_cols = [\n        'session', 'aid', 'gt', 'fold',\n    ]\n    \n    for col in remove_cols:\n        if col in feature_cols:\n            feature_cols.remove(col)\n    \n    return feature_cols\ndef get_target_cols():\n    return ['gt']\ndef get_qid_cols():\n    return ['session']","metadata":{"papermill":{"duration":0.046191,"end_time":"2023-01-31T14:32:00.867442","exception":false,"start_time":"2023-01-31T14:32:00.821251","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:56.848509Z","iopub.execute_input":"2023-02-02T16:27:56.848987Z","iopub.status.idle":"2023-02-02T16:27:56.860080Z","shell.execute_reply.started":"2023-02-02T16:27:56.848945Z","shell.execute_reply":"2023-02-02T16:27:56.859119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_cols = {}","metadata":{"papermill":{"duration":0.041954,"end_time":"2023-01-31T14:32:01.016567","exception":false,"start_time":"2023-01-31T14:32:00.974613","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:56.861533Z","iopub.execute_input":"2023-02-02T16:27:56.862024Z","iopub.status.idle":"2023-02-02T16:27:56.870148Z","shell.execute_reply.started":"2023-02-02T16:27:56.861987Z","shell.execute_reply":"2023-02-02T16:27:56.869153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_parms = {\n    'objective':'rank:pairwise', \n    'tree_method':'gpu_hist', \n    'seed': cfg.seed, \n    'random_state': cfg.seed,\n    'max_depth': 6,\n    'min_child_weight': 100,\n    'eta': 0.1,\n    'gamma': 25.0,\n    'learning_rate': 0.2,\n    'colsample_bytree': 0.75,\n    'subsample': 0.9,\n    'eval_metric': 'ndcg',\n}","metadata":{"papermill":{"duration":0.044154,"end_time":"2023-01-31T14:32:00.943177","exception":false,"start_time":"2023-01-31T14:32:00.899023","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:36:23.860835Z","iopub.execute_input":"2023-02-02T16:36:23.861216Z","iopub.status.idle":"2023-02-02T16:36:23.866864Z","shell.execute_reply.started":"2023-02-02T16:36:23.861180Z","shell.execute_reply":"2023-02-02T16:36:23.865786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clicks","metadata":{"papermill":{"duration":0.036297,"end_time":"2023-01-31T14:32:01.226025","exception":false,"start_time":"2023-01-31T14:32:01.189728","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print_gpu_mem_info()","metadata":{"papermill":{"duration":0.04763,"end_time":"2023-01-31T14:32:01.307173","exception":false,"start_time":"2023-01-31T14:32:01.259543","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:27:56.885181Z","iopub.execute_input":"2023-02-02T16:27:56.885729Z","iopub.status.idle":"2023-02-02T16:27:56.896471Z","shell.execute_reply.started":"2023-02-02T16:27:56.885694Z","shell.execute_reply":"2023-02-02T16:27:56.895540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_clicks = get_train_df('clicks')\n\ntrain_clicks = cudf.merge(\n    train_clicks, train_labels.loc[train_labels.type==0][['session', 'aid', 'gt']], \n    how='left', on=['session', 'aid'])\n\ntrain_clicks['gt'] = train_clicks['gt'].fillna(0)\nprint(train_clicks.shape)\n\ntrain_clicks = cudf.merge(train_clicks, df_fold, on='session', how='left')\n\n# Delete all cols that do not improve score\ndel_cols = [\n#         'type', \n    'ts_max', \n    'ts_min', \n    'ts_mean', \n    'ts_med',\n    'ts_duration', \n#     'session_clicks',\n    'session_carts', \n    'session_orders', \n#     'session_length',\n    'repetitions',\n    'aid_num_log', \n    'aid_clicks_num_log', \n    'aid_carts_num_log', \n    'aid_orders_num_log', \n    'aid_clicks_num_ps_log', \n    'aid_carts_num_ps_log',\n    'aid_orders_num_ps_log',\n#     'num_clicks_last_2_hours', \n#     'num_carts_last_2_hours',\n#     'num_orders_last_2_hours', \n    'cart_order_diff_last_2_hours',\n    'clicks_in_day_1',\n    'clicks_in_day_2',\n    'clicks_in_day_3',\n    'type_mean'\n]\nfor col in del_cols:\n    if col in train_clicks.columns:\n        del train_clicks[col]\n\nfeature_cols['clicks'] = get_feature_cols(train_clicks, 'clicks')\nprint(len(feature_cols['clicks']))\nprint(feature_cols['clicks'])","metadata":{"papermill":{"duration":6.027795,"end_time":"2023-01-31T14:32:07.369730","exception":false,"start_time":"2023-01-31T14:32:01.341935","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:36:26.073390Z","iopub.execute_input":"2023-02-02T16:36:26.073748Z","iopub.status.idle":"2023-02-02T16:36:26.783798Z","shell.execute_reply.started":"2023-02-02T16:36:26.073717Z","shell.execute_reply":"2023-02-02T16:36:26.781919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_parms['learning_rate'] = 0.1\nxgb_parms['max_depth'] = 8\nxgb_parms['subsample'] = 0.9\nxgb_parms['colsample_bytree'] = 0.8\nxgb_parms['min_child_weight'] = 200\n\nfor valid_fold_num in cfg.train_folds:\n    print('Fold', valid_fold_num)\n\n    # X_train = train_clicks\n    X_train = train_clicks.loc[train_clicks.fold != valid_fold_num]\n#     X_train = X_train.sort_values('session')\n    X_train = X_train.sort_values(['session', 'clicks_final_wgt'], ascending=[True, False])\n    \n    y_train = X_train['gt']\n    groups = X_train.groupby('session').size().to_frame('size')['size']\n    X_train = X_train[feature_cols['clicks']]\n    \n    X_valid = train_clicks.loc[train_clicks.fold == valid_fold_num]\n    # X_valid = X_valid.sort_values('session')\n    X_valid = X_valid.sort_values(['session', 'clicks_final_wgt'], ascending=[True, False])\n    \n    y_valid = X_valid['gt']\n    val_groups = X_valid.groupby('session').size().to_frame('size')['size']\n    X_valid = X_valid[feature_cols['clicks']]\n    \n    dtrain = xgb.DMatrix(X_train, label=y_train, group=groups)\n    dvalid = xgb.DMatrix(X_valid, label=y_valid, group=val_groups)\n\n    model = xgb.train(\n        xgb_parms, \n        dtrain=dtrain,\n        evals=[(dtrain,'train'),(dvalid,'valid')],\n        num_boost_round=150,\n        early_stopping_rounds=80,\n        verbose_eval=10,\n    )\n\n    model.save_model(f'XGB_fold{valid_fold_num}_clicks.xgb')\n\ncalc_val_score('clicks')\n\n_ = xgb.plot_importance(model, height=0.2)\n\ndel train_clicks\ndel X_train, y_train, X_valid, y_valid, dtrain, dvalid\ngc.collect()\n\n# 5166908232506884","metadata":{"papermill":{"duration":43.23192,"end_time":"2023-01-31T14:32:50.623803","exception":false,"start_time":"2023-01-31T14:32:07.391883","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:36:26.786674Z","iopub.execute_input":"2023-02-02T16:36:26.786999Z","iopub.status.idle":"2023-02-02T16:37:58.206938Z","shell.execute_reply.started":"2023-02-02T16:36:26.786970Z","shell.execute_reply":"2023-02-02T16:37:58.206001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, ax = plt.subplots(figsize=(30, 30))\n# plot_tree(model, num_trees=100, ax=ax)\n# # fig.savefig('tree.png')\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T16:42:42.242468Z","iopub.execute_input":"2023-02-02T16:42:42.242860Z","iopub.status.idle":"2023-02-02T16:42:44.962213Z","shell.execute_reply.started":"2023-02-02T16:42:42.242823Z","shell.execute_reply":"2023-02-02T16:42:44.961106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trees_df = model.trees_to_dataframe()\n# num_leaves = trees_df.groupby('Tree').size()\n# plt.plot(num_leaves)","metadata":{"papermill":{"duration":0.514426,"end_time":"2023-01-31T14:32:51.277189","exception":false,"start_time":"2023-01-31T14:32:50.762763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:45:08.660488Z","iopub.execute_input":"2023-02-02T16:45:08.660887Z","iopub.status.idle":"2023-02-02T16:45:09.108219Z","shell.execute_reply.started":"2023-02-02T16:45:08.660850Z","shell.execute_reply":"2023-02-02T16:45:09.107276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trees_df.loc[trees_df.Node < 1].Feature.value_counts()\n# trees_df.loc[trees_df.Node < 5].Feature.value_counts()\n# trees_df.loc[trees_df.Node > 10].Feature.value_counts()","metadata":{"papermill":{"duration":0.036797,"end_time":"2023-01-31T14:32:51.339076","exception":false,"start_time":"2023-01-31T14:32:51.302279","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.362502Z","iopub.status.idle":"2023-02-02T16:28:08.362979Z","shell.execute_reply.started":"2023-02-02T16:28:08.362730Z","shell.execute_reply":"2023-02-02T16:28:08.362753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Used params:')\nprint(list(xgb_parms.keys())[4:])\nprint(list(xgb_parms.values())[4:])","metadata":{"papermill":{"duration":0.0338,"end_time":"2023-01-31T14:32:51.528946","exception":false,"start_time":"2023-01-31T14:32:51.495146","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.364611Z","iopub.status.idle":"2023-02-02T16:28:08.365087Z","shell.execute_reply.started":"2023-02-02T16:28:08.364840Z","shell.execute_reply":"2023-02-02T16:28:08.364863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ngc.collect()","metadata":{"papermill":{"duration":0.177089,"end_time":"2023-01-31T14:32:51.732087","exception":false,"start_time":"2023-01-31T14:32:51.554998","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.366385Z","iopub.status.idle":"2023-02-02T16:28:08.367219Z","shell.execute_reply.started":"2023-02-02T16:28:08.366949Z","shell.execute_reply":"2023-02-02T16:28:08.366979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assert False","metadata":{"papermill":{"duration":0.032268,"end_time":"2023-01-31T14:32:51.789415","exception":false,"start_time":"2023-01-31T14:32:51.757147","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.368990Z","iopub.status.idle":"2023-02-02T16:28:08.369486Z","shell.execute_reply.started":"2023-02-02T16:28:08.369220Z","shell.execute_reply":"2023-02-02T16:28:08.369243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_gpu_mem_info()","metadata":{"papermill":{"duration":0.034257,"end_time":"2023-01-31T14:32:51.848811","exception":false,"start_time":"2023-01-31T14:32:51.814554","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.370763Z","iopub.status.idle":"2023-02-02T16:28:08.371600Z","shell.execute_reply.started":"2023-02-02T16:28:08.371315Z","shell.execute_reply":"2023-02-02T16:28:08.371345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_gpu_usage()","metadata":{"papermill":{"duration":0.227499,"end_time":"2023-01-31T14:32:52.100499","exception":false,"start_time":"2023-01-31T14:32:51.873000","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.373331Z","iopub.status.idle":"2023-02-02T16:28:08.373834Z","shell.execute_reply.started":"2023-02-02T16:28:08.373586Z","shell.execute_reply":"2023-02-02T16:28:08.373611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Carts","metadata":{"papermill":{"duration":0.024361,"end_time":"2023-01-31T14:32:52.149594","exception":false,"start_time":"2023-01-31T14:32:52.125233","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train_carts = get_train_df('carts')\n\n# Adding both carts and orders as correct here\ntrain_carts = cudf.merge(\n    train_carts, train_labels.loc[train_labels.type!=0][['session', 'aid', 'gt']], how='left', on=['session', 'aid'])\n\n# Delete all cols that do not improve score\ndel_cols = [\n#     'type', \n    'ts_max', 'ts_min', 'ts_mean', 'ts_med',\n#     'ts_duration', \n    'session_clicks',\n    'session_carts', \n    'session_orders', \n#     'session_length',\n    'repetitions',\n    'aid_num_log', \n    'aid_clicks_num_log', \n    'aid_carts_num_log', \n    'aid_orders_num_log', \n#     'aid_clicks_num_ps_log', \n#     'aid_carts_num_ps_log', \n#     'aid_orders_num_ps_log',\n    \n    'num_clicks_last_2_hours', \n#     'num_carts_last_2_hours',\n    'num_orders_last_2_hours', \n    'cart_order_diff_last_2_hours',\n#     'carts_in_day_1', \n#     'orders_in_day_1', \n#     'carts_in_day_2', \n    'orders_in_day_2', \n    'carts_in_day_3', \n    'orders_in_day_3',\n]\nfor col in del_cols:\n    if col in train_carts.columns:\n        del train_carts[col]\n\ntrain_carts['gt'] = train_carts['gt'].fillna(0)\nprint(train_carts.shape)\n\ntrain_carts = cudf.merge(train_carts, df_fold, on='session', how='left')\n# preview_df(train_carts)\n\nfeature_cols['carts'] = get_feature_cols(train_carts, 'carts')\nprint(len(feature_cols['carts']))\nprint(feature_cols['carts'])","metadata":{"papermill":{"duration":7.444741,"end_time":"2023-01-31T14:32:59.618806","exception":false,"start_time":"2023-01-31T14:32:52.174065","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.375515Z","iopub.status.idle":"2023-02-02T16:28:08.376000Z","shell.execute_reply.started":"2023-02-02T16:28:08.375749Z","shell.execute_reply":"2023-02-02T16:28:08.375772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_parms['learning_rate'] = 0.4\nxgb_parms['max_depth'] = 8\nxgb_parms['gamma'] = 15.0\nxgb_parms['subsample'] = 0.9\nxgb_parms['colsample_bytree'] = 0.75\nxgb_parms['min_child_weight'] = 200\nxgb_parms['max_delta_step'] = 0\nxgb_parms['eta'] = 0.1\nxgb_parms['eval_metric'] =  'ndcg'\n\nfor valid_fold_num in cfg.train_folds:\n    print('Fold', valid_fold_num)\n\n    # X_train = train_carts\n    X_train = train_carts.loc[train_carts.fold != valid_fold_num]\n    X_train = X_train.sort_values('session')\n    \n    y_train = X_train['gt']\n    groups = X_train.groupby('session').size().to_frame('size')['size']\n    X_train = X_train[feature_cols['carts']]\n    \n    X_valid = train_carts.loc[train_carts.fold == valid_fold_num]\n    X_valid = X_valid.sort_values('session')\n    \n    y_valid = X_valid['gt']\n    val_groups = X_valid.groupby('session').size().to_frame('size')['size']\n    X_valid = X_valid[feature_cols['carts']]\n    \n    dtrain = xgb.DMatrix(X_train, label=y_train, group=groups)\n    dvalid = xgb.DMatrix(X_valid, label=y_valid, group=val_groups)\n\n    gc.collect()\n    \n    model = xgb.train(\n        xgb_parms, \n        dtrain=dtrain,\n        evals=[(dtrain,'train'),(dvalid,'valid')],\n        num_boost_round=150,\n        verbose_eval=10,\n        early_stopping_rounds=20,\n    )\n\n    model.save_model(f'XGB_fold{valid_fold_num}_carts.xgb')\n\n_ = xgb.plot_importance(model)\n\ndel train_carts\ndel X_train, y_train, X_valid, y_valid, dtrain, dvalid\ngc.collect()\n\ncalc_val_score('carts')\n# 4057566673020604","metadata":{"papermill":{"duration":41.686797,"end_time":"2023-01-31T14:33:41.331565","exception":false,"start_time":"2023-01-31T14:32:59.644768","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.377677Z","iopub.status.idle":"2023-02-02T16:28:08.378150Z","shell.execute_reply.started":"2023-02-02T16:28:08.377905Z","shell.execute_reply":"2023-02-02T16:28:08.377927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trees_df = model.trees_to_dataframe()\nnum_leaves = trees_df.groupby('Tree').size()\nplt.plot(num_leaves)","metadata":{"papermill":{"duration":0.402821,"end_time":"2023-01-31T14:33:41.762040","exception":false,"start_time":"2023-01-31T14:33:41.359219","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.379607Z","iopub.status.idle":"2023-02-02T16:28:08.380313Z","shell.execute_reply.started":"2023-02-02T16:28:08.380059Z","shell.execute_reply":"2023-02-02T16:28:08.380082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# trees_df.loc[trees_df.Node < 1].Feature.value_counts()\n# trees_df.loc[trees_df.Node < 5].Feature.value_counts()\n# trees_df.loc[trees_df.Node > 10].Feature.value_counts()","metadata":{"papermill":{"duration":0.040682,"end_time":"2023-01-31T14:33:41.830950","exception":false,"start_time":"2023-01-31T14:33:41.790268","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.382034Z","iopub.status.idle":"2023-02-02T16:28:08.382530Z","shell.execute_reply.started":"2023-02-02T16:28:08.382259Z","shell.execute_reply":"2023-02-02T16:28:08.382282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ngc.collect()","metadata":{"papermill":{"duration":0.046277,"end_time":"2023-01-31T14:33:42.124769","exception":false,"start_time":"2023-01-31T14:33:42.078492","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.383820Z","iopub.status.idle":"2023-02-02T16:28:08.384954Z","shell.execute_reply.started":"2023-02-02T16:28:08.384709Z","shell.execute_reply":"2023-02-02T16:28:08.384733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time.sleep(2)\nprint_gpu_mem_info()\nplot_gpu_usage()","metadata":{"papermill":{"duration":2.23001,"end_time":"2023-01-31T14:33:44.382558","exception":false,"start_time":"2023-01-31T14:33:42.152548","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.386286Z","iopub.status.idle":"2023-02-02T16:28:08.387065Z","shell.execute_reply.started":"2023-02-02T16:28:08.386806Z","shell.execute_reply":"2023-02-02T16:28:08.386836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Orders","metadata":{"papermill":{"duration":0.027966,"end_time":"2023-01-31T14:33:44.439062","exception":false,"start_time":"2023-01-31T14:33:44.411096","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = get_train_df('orders')\nprint(len_mils(train))\n\n# Set both carts and orders as correct:\n# train_orders = cudf.merge(train, train_labels.loc[train_labels.type!=0][['session', 'aid', 'gt']], how='left', on=['session', 'aid'])\n# Set only orders as correct:\ntrain_orders = cudf.merge(train, train_labels.loc[train_labels.type==2][['session', 'aid', 'gt']], how='left', on=['session', 'aid'])\n\n# Delete all cols that do not improve score\ndel_cols = [\n#         'type', \n    'ts_max', 'ts_min', 'ts_mean', 'ts_med',\n    'ts_duration', \n    'session_clicks',\n    'session_carts', \n    'session_orders', \n    'session_length',\n    'repetitions',\n    'aid_num_log', \n    'aid_clicks_num_log', \n    'aid_carts_num_log', \n    'aid_orders_num_log', \n#     'aid_clicks_num_ps_log', \n#     'aid_carts_num_ps_log', \n#     'aid_orders_num_ps_log',\n    \n#     'num_clicks_last_2_hours', \n#     'num_carts_last_2_hours',\n#     'num_orders_last_2_hours', \n#     'cart_order_diff_last_2_hours',\n    \n#     'carts_in_day_1', \n#     'orders_in_day_1', \n    'carts_in_day_2', \n    'orders_in_day_2', \n    'carts_in_day_3', \n    'orders_in_day_3',\n    #'type_mean',\n]\nfor col in del_cols:\n    if col in train_orders.columns:\n        del train_orders[col]\n\ntrain_orders['gt'] = train_orders['gt'].fillna(0)\ndel train\n\ntrain_orders = cudf.merge(train_orders, df_fold, on='session', how='left')\n\nfeature_cols['orders'] = get_feature_cols(train_orders, 'orders')\nprint(len(feature_cols['orders']))\nprint(feature_cols['orders'])","metadata":{"papermill":{"duration":5.970805,"end_time":"2023-01-31T14:33:50.438219","exception":false,"start_time":"2023-01-31T14:33:44.467414","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.388638Z","iopub.status.idle":"2023-02-02T16:28:08.389428Z","shell.execute_reply.started":"2023-02-02T16:28:08.389149Z","shell.execute_reply":"2023-02-02T16:28:08.389178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_parms['learning_rate'] = 0.4\nxgb_parms['max_depth'] = 8\nxgb_parms['gamma'] = 15.0\nxgb_parms['subsample'] = 0.8\nxgb_parms['colsample_bytree'] = 1\nxgb_parms['min_child_weight'] = 200\nxgb_parms['max_delta_step'] = 0\nxgb_parms['eta'] = 0.1\n\nfor valid_fold_num in cfg.train_folds:\n    print('Fold', valid_fold_num)\n\n    # X_train = train_orders\n    X_train = train_orders.loc[train_orders.fold != valid_fold_num]\n    X_train = X_train.sort_values('session')\n    \n    y_train = X_train['gt']\n    groups = X_train.groupby('session').size().to_frame('size')['size']\n    X_train = X_train[feature_cols['orders']]\n    \n    X_valid = train_orders.loc[train_orders.fold == valid_fold_num]\n    X_valid = X_valid.sort_values('session')\n    \n    y_valid = X_valid['gt']\n    val_groups = X_valid.groupby('session').size().to_frame('size')['size']\n    X_valid = X_valid[feature_cols['orders']]\n    \n    dtrain = xgb.DMatrix(X_train, label=y_train, group=groups)\n    dvalid = xgb.DMatrix(X_valid, label=y_valid, group=val_groups)\n\n    gc.collect()\n\n    model = xgb.train(\n        xgb_parms,\n        dtrain=dtrain,\n        evals=[(dtrain,'train'),(dvalid,'valid')],\n        num_boost_round=100,\n        verbose_eval=10,\n        early_stopping_rounds=30,\n    )\n\n    model.save_model(f'XGB_fold{valid_fold_num}_orders.xgb')\n\ndel train_orders\ndel X_train, y_train, X_valid, y_valid, dtrain, dvalid\n\n_ = xgb.plot_importance(model)\n\ncalc_val_score('orders')\n\n# 6510660759129224\n# 6514019737999648","metadata":{"papermill":{"duration":33.47338,"end_time":"2023-01-31T14:34:23.940480","exception":false,"start_time":"2023-01-31T14:33:50.467100","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.391042Z","iopub.status.idle":"2023-02-02T16:28:08.391547Z","shell.execute_reply.started":"2023-02-02T16:28:08.391273Z","shell.execute_reply":"2023-02-02T16:28:08.391296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model\ngc.collect()","metadata":{"papermill":{"duration":0.178975,"end_time":"2023-01-31T14:34:24.217197","exception":false,"start_time":"2023-01-31T14:34:24.038222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.392845Z","iopub.status.idle":"2023-02-02T16:28:08.393684Z","shell.execute_reply.started":"2023-02-02T16:28:08.393407Z","shell.execute_reply":"2023-02-02T16:28:08.393436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sleep(2)\nprint_gpu_mem_info()\nplot_gpu_usage()","metadata":{"papermill":{"duration":2.332169,"end_time":"2023-01-31T14:34:42.829587","exception":false,"start_time":"2023-01-31T14:34:40.497418","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.395447Z","iopub.status.idle":"2023-02-02T16:28:08.395937Z","shell.execute_reply.started":"2023-02-02T16:28:08.395694Z","shell.execute_reply":"2023-02-02T16:28:08.395716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assert False","metadata":{"papermill":{"duration":0.037902,"end_time":"2023-01-31T14:34:42.899955","exception":false,"start_time":"2023-01-31T14:34:42.862053","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.397432Z","iopub.status.idle":"2023-02-02T16:28:08.398336Z","shell.execute_reply.started":"2023-02-02T16:28:08.398080Z","shell.execute_reply":"2023-02-02T16:28:08.398110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on validation data","metadata":{"papermill":{"duration":0.035157,"end_time":"2023-01-31T14:34:42.965725","exception":false,"start_time":"2023-01-31T14:34:42.930568","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('Validation:')\nprint('Valid num', cfg.val_fold)","metadata":{"papermill":{"duration":0.041859,"end_time":"2023-01-31T14:34:43.041090","exception":false,"start_time":"2023-01-31T14:34:42.999231","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.399885Z","iopub.status.idle":"2023-02-02T16:28:08.400349Z","shell.execute_reply.started":"2023-02-02T16:28:08.400113Z","shell.execute_reply":"2023-02-02T16:28:08.400136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_names = ['clicks', 'carts', 'orders']\n\npreds = {}\nfor event_name in events_names:\n    print(event_name)\n    \n    model = xgb.Booster()\n    model.load_model(f'XGB_fold{valid_fold_num}_{event_name}.xgb')\n    model.set_param({'predictor': 'gpu_predictor'})\n    \n    preds[event_name] = []\n    for i,f in enumerate(val_candidates_files[event_name]):\n        print(i, end=' ')\n\n        val_candidates = cudf.read_parquet(f)\n        val_candidates = val_candidates.sort_values('session')\n        \n        groups = val_candidates.groupby('session').size().to_frame('size')['size']\n        dtest = xgb.DMatrix(data=val_candidates[feature_cols[event_name]], group=groups)\n\n        preds[event_name].append(model.predict(dtest))\n        del dtest, val_candidates\n    \n    preds[event_name] = np.concatenate(preds[event_name], axis=None)\n    np.save(f'val_preds_{event_name}.npy', preds[event_name])\n    print()","metadata":{"papermill":{"duration":7.267013,"end_time":"2023-01-31T14:34:50.341116","exception":false,"start_time":"2023-01-31T14:34:43.074103","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.401898Z","iopub.status.idle":"2023-02-02T16:28:08.402996Z","shell.execute_reply.started":"2023-02-02T16:28:08.402710Z","shell.execute_reply":"2023-02-02T16:28:08.402736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# gc.collect()\n# time.sleep(2)\n# print_gpu_mem_info()\n# plot_gpu_usage()","metadata":{"papermill":{"duration":2.423636,"end_time":"2023-01-31T14:34:52.798519","exception":false,"start_time":"2023-01-31T14:34:50.374883","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.404567Z","iopub.status.idle":"2023-02-02T16:28:08.405050Z","shell.execute_reply.started":"2023-02-02T16:28:08.404791Z","shell.execute_reply":"2023-02-02T16:28:08.404814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create val submission","metadata":{"papermill":{"duration":0.034372,"end_time":"2023-01-31T14:34:52.947744","exception":false,"start_time":"2023-01-31T14:34:52.913372","status":"completed"},"tags":[]}},{"cell_type":"code","source":"events_names = ['clicks', 'carts', 'orders']\n# events_names = ['clicks']\n\npredictions = {}\nsub = {}\nfor event_name in events_names:\n    print(event_name)\n    \n    # Add predictions to test dataframe\n\n    predictions[event_name] = get_val_data(event_name)[['session','aid']]\n    # print(predictions[event_name].session.nunique())    \n    \n    # predictions[event_name]['pred'] = preds[event_name]\n    predictions[event_name]['pred'] = preds[event_name]\n\n    # Pick top 20 predictions\n\n    predictions[event_name] = predictions[event_name].sort_values(['session','pred'], ascending=[True,False]).reset_index(drop=True)\n    predictions[event_name]['n'] = predictions[event_name].groupby('session').aid.cumcount().astype('int8')\n    predictions[event_name] = predictions[event_name].loc[predictions[event_name].n<20]\n    del predictions[event_name]['pred']\n    del predictions[event_name]['n']\n    # print(predictions[event_name].shape)\n    # display(predictions[event_name].head())\n\n    # Transform (session,aid) pairs to (session,string of all aids) pairs\n    predictions[event_name] = predictions[event_name].to_pandas() # it is faster to do next functions in pandas then in cudf\n    sub[event_name] = predictions[event_name].groupby('session').aid.apply(list)\n    del predictions[event_name]\n    sub[event_name] = sub[event_name].to_frame().reset_index()\n    sub[event_name].aid = sub[event_name].aid.apply(lambda x: \" \".join(map(str,x)))\n    # preview_df(sub[event_name])\n\n    # Add session type for each session num\n    to_add_str = '_' + event_name\n    sub[event_name]['session'] = sub[event_name].session.astype('str') + to_add_str\n\n    gc.collect()\n    time.sleep(2)","metadata":{"papermill":{"duration":39.209457,"end_time":"2023-01-31T14:35:32.190660","exception":false,"start_time":"2023-01-31T14:34:52.981203","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.406526Z","iopub.status.idle":"2023-02-02T16:28:08.407292Z","shell.execute_reply.started":"2023-02-02T16:28:08.407023Z","shell.execute_reply":"2023-02-02T16:28:08.407053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([sub['clicks'], sub['carts'], sub['orders']], axis=0).reset_index(drop=True)\nsubmission.head()\n\nsubmission.columns = ['session_type','labels']\n\nsubmission = submission.sort_values(['session_type']).reset_index(drop=True)\nprint(submission.shape)\nsubmission.head()","metadata":{"papermill":{"duration":1.112736,"end_time":"2023-01-31T14:35:33.634680","exception":false,"start_time":"2023-01-31T14:35:32.521944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.408909Z","iopub.status.idle":"2023-02-02T16:28:08.409400Z","shell.execute_reply.started":"2023-02-02T16:28:08.409135Z","shell.execute_reply":"2023-02-02T16:28:08.409159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Score val submission","metadata":{"papermill":{"duration":0.033829,"end_time":"2023-01-31T14:35:33.703699","exception":false,"start_time":"2023-01-31T14:35:33.669870","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('Valid num', cfg.val_fold)","metadata":{"papermill":{"duration":0.050592,"end_time":"2023-01-31T14:35:33.811286","exception":false,"start_time":"2023-01-31T14:35:33.760694","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.410719Z","iopub.status.idle":"2023-02-02T16:28:08.413742Z","shell.execute_reply.started":"2023-02-02T16:28:08.413469Z","shell.execute_reply":"2023-02-02T16:28:08.413508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntry:\n    pred_df = submission\n\n    # COMPUTE METRIC\n    score = 0\n    weights = {'clicks': 0.10, 'carts': 0.30, 'orders': 0.60}\n    for t in ['clicks','carts','orders']:\n        sub = pred_df.loc[pred_df.session_type.str.contains(t)].copy()\n        sub['session'] = sub.session_type.apply(lambda x: int(x.split('_')[0]))\n        sub.labels = sub.labels.apply(lambda x: [int(i) for i in x.split(' ')[:20]])\n        val_labels = pd.read_parquet(train_labels_path)\n        val_labels = val_labels.loc[val_labels['type']==t]\n\n        val_labels = pd.merge(val_labels, df_fold.to_pandas(), on='session')\n        val_labels = val_labels.loc[val_labels.fold == cfg.val_fold]\n        del val_labels['fold']\n\n        val_labels = val_labels.merge(sub, how='left', on=['session'])\n#         display(val_labels)\n        val_labels['hits'] = val_labels.apply(lambda df: len(set(df.ground_truth).intersection(set(df.labels))), axis=1)\n        val_labels['gt_count'] = val_labels.ground_truth.str.len().clip(0,20)\n        recall = val_labels['hits'].sum() / val_labels['gt_count'].sum()\n        score += weights[t]*recall\n        print(f'{t} recall =',recall)\n\n    print('=============')\n    print('Overall Recall =',score)\n    print('=============')\nexcept Exception as e:\n    print('Exception:')\n    print(e)","metadata":{"papermill":{"duration":27.116723,"end_time":"2023-01-31T14:36:00.961399","exception":false,"start_time":"2023-01-31T14:35:33.844676","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.415230Z","iopub.status.idle":"2023-02-02T16:28:08.415720Z","shell.execute_reply.started":"2023-02-02T16:28:08.415463Z","shell.execute_reply":"2023-02-02T16:28:08.415487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    del model, df, predictions, preds, val_labels, pred_df\nexcept Exception as e:\n    print(e)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T16:28:08.417350Z","iopub.status.idle":"2023-02-02T16:28:08.417854Z","shell.execute_reply.started":"2023-02-02T16:28:08.417607Z","shell.execute_reply":"2023-02-02T16:28:08.417630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict on test data","metadata":{"papermill":{"duration":0.034811,"end_time":"2023-01-31T14:36:02.080465","exception":false,"start_time":"2023-01-31T14:36:02.045654","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('Create test submission:')","metadata":{"papermill":{"duration":0.045238,"end_time":"2023-01-31T14:36:02.161261","exception":false,"start_time":"2023-01-31T14:36:02.116023","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.419454Z","iopub.status.idle":"2023-02-02T16:28:08.419934Z","shell.execute_reply.started":"2023-02-02T16:28:08.419690Z","shell.execute_reply":"2023-02-02T16:28:08.419713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predict","metadata":{"papermill":{"duration":0.034749,"end_time":"2023-01-31T14:36:03.632004","exception":false,"start_time":"2023-01-31T14:36:03.597255","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print('Test folds', cfg.test_folds)","metadata":{"papermill":{"duration":0.096382,"end_time":"2023-01-31T14:36:03.789342","exception":false,"start_time":"2023-01-31T14:36:03.692960","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.421233Z","iopub.status.idle":"2023-02-02T16:28:08.422060Z","shell.execute_reply.started":"2023-02-02T16:28:08.421791Z","shell.execute_reply":"2023-02-02T16:28:08.421819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calc length of test files, it is needed in calculating predictions\n\nlen_test = {}\nfor ev in ['clicks', 'carts', 'orders']:\n    len_test[ev] = 0\n    for f in test_candidates_files[ev]:\n        df = cudf.read_parquet(f)\n        len_test[ev] += len(df)\nprint(len_test)","metadata":{"papermill":{"duration":62.100201,"end_time":"2023-01-31T14:37:05.963166","exception":false,"start_time":"2023-01-31T14:36:03.862965","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.423742Z","iopub.status.idle":"2023-02-02T16:28:08.424552Z","shell.execute_reply.started":"2023-02-02T16:28:08.424250Z","shell.execute_reply":"2023-02-02T16:28:08.424280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_names = ['clicks', 'carts', 'orders']\n\npreds = {}\nfor event_name in events_names:\n    print(event_name)\n    preds[event_name] = np.zeros(len_test[event_name])\n    \n    for valid_fold_num in cfg.test_folds:\n        print('Fold', valid_fold_num, end=' ')\n        \n        model = xgb.Booster()\n        model.load_model(f'XGB_fold{valid_fold_num}_{event_name}.xgb')\n        model.set_param({'predictor': 'gpu_predictor'})\n\n        preds_chunk = []\n        for f in test_candidates_files[event_name]:\n            \n            test = cudf.read_parquet(f)\n            test = test.sort_values('session')\n            groups = test.groupby('session').size().to_frame('size')['size']\n            dtest = xgb.DMatrix(data=test[feature_cols[event_name]], group=groups)\n            \n            preds_chunk.append(model.predict(dtest))\n            del dtest\n\n        preds[event_name] += np.concatenate(preds_chunk, axis=None) / len(cfg.test_folds)\n        print()\n    \n    np.save(f'preds_{event_name}.npy', preds[event_name])\n    print()","metadata":{"papermill":{"duration":23.814229,"end_time":"2023-01-31T14:37:30.045179","exception":false,"start_time":"2023-01-31T14:37:06.230950","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.426131Z","iopub.status.idle":"2023-02-02T16:28:08.426633Z","shell.execute_reply.started":"2023-02-02T16:28:08.426372Z","shell.execute_reply":"2023-02-02T16:28:08.426395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds={}\n# preds['clicks'] = np.load('/kaggle/working/preds_clicks.npy')\n# preds['carts'] = np.load('/kaggle/working/preds_carts.npy')\n# preds['orders'] = np.load('/kaggle/working/preds_orders.npy')","metadata":{"papermill":{"duration":0.046253,"end_time":"2023-01-31T14:37:30.130660","exception":false,"start_time":"2023-01-31T14:37:30.084407","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.428296Z","iopub.status.idle":"2023-02-02T16:28:08.428906Z","shell.execute_reply.started":"2023-02-02T16:28:08.428652Z","shell.execute_reply":"2023-02-02T16:28:08.428676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_gpu_usage()","metadata":{"papermill":{"duration":0.267741,"end_time":"2023-01-31T14:37:30.434332","exception":false,"start_time":"2023-01-31T14:37:30.166591","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.432238Z","iopub.status.idle":"2023-02-02T16:28:08.432751Z","shell.execute_reply.started":"2023-02-02T16:28:08.432499Z","shell.execute_reply":"2023-02-02T16:28:08.432523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create submission","metadata":{"papermill":{"duration":0.042148,"end_time":"2023-01-31T14:37:30.515358","exception":false,"start_time":"2023-01-31T14:37:30.473210","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_test_data(event):\n    test_data = cudf.DataFrame()\n    for f in test_candidates_files[event]:\n        test_data = cudf.concat([test_data, cudf.read_parquet(f).sort_values('session')], ignore_index=True)\n    test_data = test_data.reset_index(drop=True)\n    test_data['session'] = test_data['session'].astype('int32')\n    test_data['aid'] = test_data['aid'].astype('int32')\n    return test_data","metadata":{"papermill":{"duration":0.050316,"end_time":"2023-01-31T14:37:30.608626","exception":false,"start_time":"2023-01-31T14:37:30.558310","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.434043Z","iopub.status.idle":"2023-02-02T16:28:08.434887Z","shell.execute_reply.started":"2023-02-02T16:28:08.434609Z","shell.execute_reply":"2023-02-02T16:28:08.434641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events_names = ['clicks', 'carts', 'orders']\n# events_names = ['clicks']\n\npredictions = {}\nsub = {}\nfor event_name in events_names:\n    print(event_name)\n\n    # Add predictions to test dataframe\n    \n    predictions[event_name] = get_test_data(event_name)[['session','aid']]\n    # print(predictions[event_name].session.nunique())    \n\n    predictions[event_name]['pred'] = preds[event_name]\n\n    # Pick top 20 predictions\n\n    predictions[event_name] = predictions[event_name].sort_values(['session','pred'], ascending=[True,False]).reset_index(drop=True)\n    predictions[event_name]['n'] = predictions[event_name].groupby('session').aid.cumcount().astype('int8')\n    predictions[event_name] = predictions[event_name].loc[predictions[event_name].n<20]\n    del predictions[event_name]['pred']\n    del predictions[event_name]['n']\n    # print(predictions[event_name].shape)\n    # display(predictions[event_name].head())\n\n    # Transform (session,aid) pairs to (session,string of all aids) pairs\n\n    predictions[event_name] = predictions[event_name].to_pandas() # it is faster to do next functions in pandas then in cudf\n    sub[event_name] = predictions[event_name].groupby('session').aid.apply(list)\n    del predictions[event_name]\n    sub[event_name] = sub[event_name].to_frame().reset_index()\n    sub[event_name].aid = sub[event_name].aid.apply(lambda x: \" \".join(map(str,x)))\n    preview_df(sub[event_name])\n\n    # Add session type for each session num\n    to_add_str = '_' + event_name\n    sub[event_name]['session'] = sub[event_name].session.astype('str') + to_add_str\n\n    gc.collect()\n    time.sleep(2)","metadata":{"papermill":{"duration":152.009522,"end_time":"2023-01-31T14:40:02.659721","exception":false,"start_time":"2023-01-31T14:37:30.650199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.436649Z","iopub.status.idle":"2023-02-02T16:28:08.437129Z","shell.execute_reply.started":"2023-02-02T16:28:08.436879Z","shell.execute_reply":"2023-02-02T16:28:08.436902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_gpu_usage(5)","metadata":{"papermill":{"duration":0.2473,"end_time":"2023-01-31T14:40:02.945718","exception":false,"start_time":"2023-01-31T14:40:02.698418","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.438583Z","iopub.status.idle":"2023-02-02T16:28:08.439374Z","shell.execute_reply.started":"2023-02-02T16:28:08.439095Z","shell.execute_reply":"2023-02-02T16:28:08.439126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Concatenate all event types","metadata":{"papermill":{"duration":0.052433,"end_time":"2023-01-31T14:40:03.053520","exception":false,"start_time":"2023-01-31T14:40:03.001087","status":"completed"},"tags":[]}},{"cell_type":"code","source":"submission = pd.concat([sub['clicks'], sub['carts'], sub['orders']], axis=0).reset_index(drop=True)\nsubmission.columns = ['session_type','labels']\nsubmission = submission.sort_values(['session_type']).reset_index(drop=True)\npreview_df(submission, 5)","metadata":{"papermill":{"duration":4.903444,"end_time":"2023-01-31T14:40:09.269005","exception":false,"start_time":"2023-01-31T14:40:04.365561","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.440997Z","iopub.status.idle":"2023-02-02T16:28:08.441504Z","shell.execute_reply.started":"2023-02-02T16:28:08.441227Z","shell.execute_reply":"2023-02-02T16:28:08.441250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Save submission as csv","metadata":{"papermill":{"duration":0.037575,"end_time":"2023-01-31T14:40:09.429977","exception":false,"start_time":"2023-01-31T14:40:09.392402","status":"completed"},"tags":[]}},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"papermill":{"duration":15.036471,"end_time":"2023-01-31T14:40:24.505255","exception":false,"start_time":"2023-01-31T14:40:09.468784","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.442771Z","iopub.status.idle":"2023-02-02T16:28:08.443684Z","shell.execute_reply.started":"2023-02-02T16:28:08.443412Z","shell.execute_reply":"2023-02-02T16:28:08.443442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(submission)","metadata":{"papermill":{"duration":0.054687,"end_time":"2023-01-31T14:40:24.602990","exception":false,"start_time":"2023-01-31T14:40:24.548303","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-02T16:28:08.445229Z","iopub.status.idle":"2023-02-02T16:28:08.446018Z","shell.execute_reply.started":"2023-02-02T16:28:08.445757Z","shell.execute_reply":"2023-02-02T16:28:08.445787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.039571,"end_time":"2023-01-31T14:40:24.687503","exception":false,"start_time":"2023-01-31T14:40:24.647932","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}