{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Introduction\n\n**I am inspired from [RADEK OSMULSK](https://www.kaggle.com/radek1)'s kernel : [co-visitation matrix](https://www.kaggle.com/radek1/code).**\n\n**In this kernel, I try 'Co-Visitation Items' Architecture.**","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"GLOBAL_SEED = 42\n\nimport os\nos.environ[\"PYTHONIOENCODING\"] = \"utf8\"\nos.environ['PYTHONHASHSEED'] = str(GLOBAL_SEED)\nimport sys\n\nimport pandas as pd\nimport numpy as np\nfrom numpy import random as np_rnd\nimport random as rnd\nimport shutil\nimport gc\nimport datetime\nfrom collections import defaultdict, Counter\nfrom tqdm import tqdm\nfrom multiprocessing import Pool, cpu_count\nimport time\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-01-22T09:23:40.310983Z","iopub.execute_input":"2023-01-22T09:23:40.311457Z","iopub.status.idle":"2023-01-22T09:23:41.291406Z","shell.execute_reply.started":"2023-01-22T09:23:40.311348Z","shell.execute_reply":"2023-01-22T09:23:41.290345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    # python random\n    rnd.seed(seed)\n    # numpy random\n    np_rnd.seed(seed)\n    # tf random\n    try:\n        tf_rnd.set_seed(seed)\n    except:\n        pass\n    # RAPIDS random\n    try:\n        cp.random.seed(seed)\n    except:\n        pass\n    # pytorch random\n    try:\n        torch.manual_seed(seed)\n    except:\n        pass\n\ndef pickleIO(obj, src, op=\"w\"):\n    if op==\"w\":\n        with open(src, op + \"b\") as f:\n            pickle.dump(obj, f)\n    elif op==\"r\":\n        with open(src, op + \"b\") as f:\n            tmp = pickle.load(f)\n        return tmp\n    else:\n        print(\"unknown operation\")\n        return obj\n    \ndef findIdx(data_x, col_names):\n    return [int(i) for i, j in enumerate(data_x) if j in col_names]\n\ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print('Error: Creating directory. ' + directory)\n        \ndef create_submission(df):\n    df = df.reset_index()\n    df[\"type\"] = df[\"type\"].map(CFG.contentType_mapper)\n    df[\"session_type\"] = df[\"session\"].astype(\"str\") + \"_\" + df[\"type\"].astype(\"str\") + \"s\"\n    df = df[[\"session_type\", \"prediction\"]].rename({\"prediction\": \"labels\"}, axis=1)\n    return df\n\ndef create_get_ts(ts):\n    return int((ts.replace(tzinfo=CFG.tz) - CFG.ts_zero).total_seconds())","metadata":{"execution":{"iopub.status.busy":"2023-01-22T09:23:41.293085Z","iopub.execute_input":"2023-01-22T09:23:41.295140Z","iopub.status.idle":"2023-01-22T09:23:41.308132Z","shell.execute_reply.started":"2023-01-22T09:23:41.295108Z","shell.execute_reply":"2023-01-22T09:23:41.307283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    local = False\n    debug = False\n    tz = datetime.timezone.utc\n    ts_zero = datetime.datetime(1970, 1, 1, tzinfo=tz)\n    contentType_mapper = pd.Series([\"clicks\", \"carts\", \"orders\"], index=[0, 1, 2])\n    target_weight = (0.1, 0.3, 0.6)\n\nif CFG.local:\n    CFG.folder_path = \"./dataset/\"\nelse:\n    CFG.folder_path = \"/kaggle/input/\"","metadata":{"execution":{"iopub.status.busy":"2023-01-22T09:23:41.309526Z","iopub.execute_input":"2023-01-22T09:23:41.309978Z","iopub.status.idle":"2023-01-22T09:23:41.323424Z","shell.execute_reply.started":"2023-01-22T09:23:41.309946Z","shell.execute_reply":"2023-01-22T09:23:41.322510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"fraction_of_sessions_to_use = 0.1 if CFG.debug else 1\n\ntrain = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\n\nif fraction_of_sessions_to_use != 1:\n    lucky_sessions_train = train.drop_duplicates(['session']).sample(frac=fraction_of_sessions_to_use, random_state=42)['session']\n    subset_of_train = train[train.session.isin(lucky_sessions_train)]\n    \n    lucky_sessions_test = test.drop_duplicates(['session']).sample(frac=fraction_of_sessions_to_use, random_state=42)['session']\n    subset_of_test = test[test.session.isin(lucky_sessions_test)]\nelse:\n    subset_of_train = train\n    subset_of_test = test\n\nsubset_of_train.index = pd.MultiIndex.from_frame(subset_of_train[['session']])\nsubset_of_test.index = pd.MultiIndex.from_frame(subset_of_test[['session']])\n\n# Concat train & test data\n# This is not information leakage because of considering the predctions session by session\nsubsets = pd.concat([subset_of_train, subset_of_test])\n# subsets = subset_of_train\nsessions = subsets.session.unique()","metadata":{"execution":{"iopub.status.busy":"2023-01-22T09:23:41.325976Z","iopub.execute_input":"2023-01-22T09:23:41.326344Z","iopub.status.idle":"2023-01-22T09:24:23.133211Z","shell.execute_reply.started":"2023-01-22T09:23:41.326312Z","shell.execute_reply":"2023-01-22T09:24:23.131970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test 데이터 세션별 aid 및 event 리스트화\nsession_types = ['clicks', 'carts', 'orders']\ntest_session_AIDs = subset_of_test.reset_index(drop=True).groupby('session')['aid'].apply(list)\ntest_session_types = subset_of_test.reset_index(drop=True).groupby('session')['type'].apply(list)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T09:24:23.134911Z","iopub.execute_input":"2023-01-22T09:24:23.135655Z","iopub.status.idle":"2023-01-22T09:25:25.118470Z","shell.execute_reply.started":"2023-01-22T09:24:23.135622Z","shell.execute_reply":"2023-01-22T09:25:25.117254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, subset_of_train, test, subset_of_test; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-22T09:25:25.119705Z","iopub.execute_input":"2023-01-22T09:25:25.120057Z","iopub.status.idle":"2023-01-22T09:25:25.712355Z","shell.execute_reply.started":"2023-01-22T09:25:25.120026Z","shell.execute_reply":"2023-01-22T09:25:25.711544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate Co-Visitation Items","metadata":{}},{"cell_type":"code","source":"chunk_size = 16384 * 2\nn_aids = 20\ntype_weight_multipliers = {0: 1, 1: 6, 2: 3}\nhour_interval_threshold = 6\nnext_AIDs = defaultdict(Counter)\n\n# session 유니크 값 만큼 반복\nfor i in tqdm(range(0, sessions.shape[0], chunk_size), total=len(range(0, sessions.shape[0], chunk_size))):\n    # chunk size 만큼 session 선택 후 임시 데이터프레임 생성\n    current_chunk = subsets.loc[sessions[i]:sessions[min(sessions.shape[0]-1, i+chunk_size-1)]].reset_index(drop=True)\n    # 세션별로 가장 최근 N개 aid을 선택 (event 타입 상관 안 함)\n    current_chunk = current_chunk.drop_duplicates().groupby('session', as_index=False).nth(list(range(-n_aids, 0))).reset_index(drop=True)\n    # 세션별 M * M 아이템 interaction row를 생성하는 operation\n    consecutive_AIDs = current_chunk.merge(current_chunk, on='session')\n    # 자기 자신과의 아이템 매칭은 제외\n    consecutive_AIDs = consecutive_AIDs[consecutive_AIDs.aid_x != consecutive_AIDs.aid_y]\n    # 두 아이템간의 interaction 시간 텀 계산\n    consecutive_AIDs['days_elapsed'] = np.abs(consecutive_AIDs.ts_y - consecutive_AIDs.ts_x) / 3600\n    # 두 아이템간 interaction 시간 텀이 threshold 값 이하인 아이템만 선택\n#     consecutive_AIDs = consecutive_AIDs[(consecutive_AIDs.days_elapsed > 0) & (consecutive_AIDs.days_elapsed < hour_interval_threshold)]\n    consecutive_AIDs = consecutive_AIDs[(consecutive_AIDs.days_elapsed < hour_interval_threshold)]\n    # 결과를 저장할 next_AIDs 딕셔너리에 키: 카운터 형식으로 저장 (2D 행렬을 대체)\n    for aid_x, aid_y, t in zip(consecutive_AIDs['aid_x'], consecutive_AIDs['aid_y'], consecutive_AIDs['type_y']):\n        next_AIDs[aid_x][aid_y] += type_weight_multipliers[t]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T05:46:03.664307Z","iopub.execute_input":"2023-01-22T05:46:03.664638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del current_chunk, consecutive_AIDs; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling & Inference","metadata":{}},{"cell_type":"code","source":"# CORE HYPER-PARAMERTER 1 - weight by event types\nn_aids = 20\n\noutput = {\n    \"session\": [],\n    \"type\": [],\n    \"rec\": [],\n    \"score\": [],\n}\n\nfor SESS, AIDs, types in tqdm(zip(test_session_AIDs.index, test_session_AIDs.values, test_session_types.values), total=len(test_session_AIDs.index)):\n    # Get more co-visitation items order > cart > click\n    # If you want to analyze & get the items with details, you may need more RAM & CPU resources\n    candidates = []\n    for aid in AIDs[::-1]:\n        candidates += [i for i, j in next_AIDs[aid].most_common(n_aids)]\n    if len(candidates) == 0: candidates = AIDs.copy()\n\n    candidates = Counter(candidates)\n    rec, score = zip(*candidates.most_common(n_aids))\n    \n    output[\"session\"].extend([SESS] * 3)\n    output[\"type\"].extend([0, 1, 2])\n    output[\"rec\"].extend([\" \".join(pd.Series(rec, dtype=\"str\").values)] * 3)\n    output[\"score\"].extend([\" \".join(pd.Series(score, dtype=\"str\").values)] * 3)\n\noutput = pd.DataFrame(output).set_index([\"session\", \"type\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.reset_index().to_parquet(\"./raw_output.parquet\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"output[\"session_type\"] = [str(i[0]) + \"_\" + str(CFG.contentType_mapper[i[1]]) for i in output.index]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/otto-recommender-system/sample_submission.csv\")\nsubmission = submission.set_index(\"session_type\")\nsubmission.loc[output[\"session_type\"].values, \"labels\"] = output[\"rec\"].values\nsubmission = submission.reset_index()\nsubmission.to_csv(\"./submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}