{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Как скачать","metadata":{}},{"cell_type":"code","source":"! pip install --upgrade --force-reinstall --no-deps kaggle","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Если делаете через colab, то придется выполнить следующий кусок","metadata":{}},{"cell_type":"code","source":"from google.colab import files ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files.upload()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Загружаете свой kaggle.json(его можно найти в своем профиле kaggle)","metadata":{}},{"cell_type":"code","source":"! mkdir ~/.kaggle ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! cp kaggle.json ~/.kaggle/","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! chmod 600 ~/.kaggle/kaggle.json","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! kaggle competitions download -c om-ml2-21-ranking","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Дальше предполагается, что все скачано","metadata":{}},{"cell_type":"code","source":"! unzip om-ml2-21-ranking.zip -d data/train","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Следующий кусок кода я рекомендую выполнять не в юпитере, поскольку насколько я понял юпитер не умеет в сборку мусора.","metadata":{}},{"cell_type":"code","source":"import json\nimport multiprocessing as mp\nimport os\nfrom collections import OrderedDict\nfrom functools import partial\nfrom operator import itemgetter\n\nimport pandas as pd\n\nDISK_PATH = '<enter you disc path>'\n\n# weights_df = pd.read_csv(DISK_PATH + \"data/train/site_hash_weights.tsv\", sep='\\t', header=None)\n# weights_df = weights_df.rename(columns={1: 'weights'})\n# types_df = pd.read_csv(DISK_PATH + \"data/train/site_hash_types.tsv\", sep='\\t', header=None)\n# types_df = types_df.rename(columns={1: 'cat'})\n# types_df['cat'] = types_df['cat'].astype(\"category\")\n\n\ndef dataframe_from_line(_str):\n    norm_json = json.loads(_str, object_pairs_hook=OrderedDict)\n    first_df = pd.json_normalize(norm_json['user'], record_path='sites_for_user')\n    second_df = pd.json_normalize(norm_json['user'], record_path='theme_events')\n    joined_df = second_df.set_index('site_hash').join(first_df.set_index('site_hash'), on='site_hash')\n\n\n    actions_rank = list(norm_json['actions'].items())\n\n    actions_with_rank = pd.DataFrame.from_dict({'rank': [r[1] for r in actions_rank],\n                                                'site_hash': [r[0] for r in actions_rank]})\n\n    actions_with_rank.loc[:, 'site_hash'] = actions_with_rank['site_hash'].astype('int64')\n\n\n    result = actions_with_rank.set_index('site_hash').join(joined_df)\n\n    result.loc[:, 'clicks'].fillna(-1, inplace=True)\n    result.loc[:, 'clicks'].astype('int32', copy=False)\n    result.loc[:, 'rank'].astype('int32', copy=False)\n    result.loc[:, 'shows'].fillna(-1, inplace=True)\n    result.loc[:, 'shows'].astype('int32', copy=False)\n    result.loc[:, 'last_click_ts'].fillna(-1, inplace=True)\n    result.loc[:, 'last_click_ts'].astype('int32', copy=False)\n    result.loc[:, 'last_show_ts'].fillna(-1, inplace=True)\n    result.loc[:, 'last_show_ts'].astype('int32', copy=False)\n    result.loc[:, 'visits'].fillna(-1, inplace=True)\n    result.loc[:, 'visits'].astype('int32', copy=False)\n    result.loc[:, 'uniq_urls'].fillna(-1, inplace=True)\n    result.loc[:, 'uniq_urls'].astype('int32', copy=False)\n    result.loc[:, 'last_ts'].fillna(-1, inplace=True)\n    result.loc[:, 'last_ts'].astype('int32', copy=False)\n\n    result['session_id'] = norm_json['session_id']\n    result['session_id'].astype(\"string\", copy=False)\n    result['session_ts'] = norm_json['session_ts']\n    result.loc[:, 'session_ts'].astype('int32', copy=False)\n    result['vid'] = norm_json['user']['vid']\n    result['vk_id'] = norm_json['user']['vk_id']\n    result['ok_id'] = norm_json['user']['ok_id']\n    result['email'] = norm_json['user']['email']\n    result['age'] = norm_json['user']['age']\n    result.loc[:, 'age'].astype('int32', copy=False)\n    result['gender'] = norm_json['user']['gender']\n    result.loc[:, 'gender'].astype('int32', copy=False)\n    result['geo'] = norm_json['user']['geo']\n    result.loc[:, 'geo'].astype('int32', copy=False)\n\n    return result\n\n\ndef read_create_and_save(name, prefix, prefix_to_save):\n    with open(prefix + name) as f:\n        content = f.readlines()\n    dfs = []\n    i = 0\n    for _str in content:\n        # if i > 30:\n        #     break\n        i += 1\n        if i % 1000 == 0:\n            print(name + \" i = \" + str(i))\n        try:\n            df = dataframe_from_line(_str)\n        except Exception as e:\n            print(_str)\n            raise e\n        dfs.append(df)\n    res = pd.concat(dfs, axis=0)\n    res.to_pickle(prefix_to_save + name)\n\n\npath_to_json = DISK_PATH + 'data/train/train'\njson_files = [pos_json for pos_json in sorted(os.listdir(path_to_json))]\n\na_pool = mp.Pool(3)\nprod_data = partial(read_create_and_save, prefix=DISK_PATH + 'data/train/train/')\nprod_data = partial(prod_data, prefix_to_save=DISK_PATH + 'data/transformed/')\na_pool.map(prod_data, json_files)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Теперь в папке data/transformed у вас хранятся \"запикленные\" датасеты. Остается их \"склеить\" и сохранить","metadata":{}},{"cell_type":"markdown","source":"Склейку можно сделать в юпитере","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\n\npath_to_files = 'data/transformed/'\ndf_files = [path_to_files + pos_ds for pos_ds in sorted(os.listdir(path_to_files))]\n\nres = []\nfor df in df_files:\n    data = pd.read_pickle(df)\n    res.append(data)\n    \nresult_df = pd.concat(res, axis=0)\ndata = result_df.reset_index()\ndata = data.rename(columns={'index' : 'site_hash'})\ndata.to_pickle('data/transformed/result')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Повторить преобразования для test","metadata":{}},{"cell_type":"markdown","source":"После всех преобразований мне помогла статья:\nhttps://tamaracucumides.medium.com/learning-to-rank-with-lightgbm-code-example-in-python-843bd7b44574\nБез всяких выдумок, осталось только понять как скормить данные модели(хитрыми признаками не пользовался от слова совсем) и сделать предсказание, которое пробьет оба бейзлайна разом.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}