{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Introduction\n\n**Recommendationb By Time Windows**","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"GLOBAL_SEED = 42\n\nimport os\nos.environ[\"PYTHONIOENCODING\"] = \"utf8\"\nos.environ['PYTHONHASHSEED'] = str(GLOBAL_SEED)\nimport sys\n\nimport pandas as pd\nimport numpy as np\nfrom numpy import random as np_rnd\nimport random as rnd\nimport shutil\nimport gc\nimport datetime\nfrom collections import defaultdict, Counter\nfrom tqdm import tqdm\nfrom multiprocessing import Pool, cpu_count\nimport time\nfrom scipy.stats import rankdata\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed=42):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    # python random\n    rnd.seed(seed)\n    # numpy random\n    np_rnd.seed(seed)\n    # tf random\n    try:\n        tf_rnd.set_seed(seed)\n    except:\n        pass\n    # RAPIDS random\n    try:\n        cp.random.seed(seed)\n    except:\n        pass\n    # pytorch random\n    try:\n        torch.manual_seed(seed)\n    except:\n        pass\n\ndef pickleIO(obj, src, op=\"w\"):\n    if op==\"w\":\n        with open(src, op + \"b\") as f:\n            pickle.dump(obj, f)\n    elif op==\"r\":\n        with open(src, op + \"b\") as f:\n            tmp = pickle.load(f)\n        return tmp\n    else:\n        print(\"unknown operation\")\n        return obj\n    \ndef findIdx(data_x, col_names):\n    return [int(i) for i, j in enumerate(data_x) if j in col_names]\n\ndef createFolder(directory):\n    try:\n        if not os.path.exists(directory):\n            os.makedirs(directory)\n    except OSError:\n        print('Error: Creating directory. ' + directory)\n        \ndef create_submission(df):\n    df = df.reset_index()\n    df[\"type\"] = df[\"type\"].map(CFG.contentType_mapper)\n    df[\"session_type\"] = df[\"session\"].astype(\"str\") + \"_\" + df[\"type\"].astype(\"str\") + \"s\"\n    df = df[[\"session_type\", \"prediction\"]].rename({\"prediction\": \"labels\"}, axis=1)\n    return df\n\ndef create_get_ts(ts):\n    return int((ts.replace(tzinfo=CFG.tz) - CFG.ts_zero).total_seconds())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    local = False\n    debug = False\n    tz = datetime.timezone.utc\n    ts_zero = datetime.datetime(1970, 1, 1, tzinfo=tz)\n    contentType_mapper = pd.Series([\"clicks\", \"carts\", \"orders\"], index=[0, 1, 2])\n    target_weight = (0.1, 0.3, 0.6)\n\nif CFG.local:\n    CFG.folder_path = \"./dataset/\"\nelse:\n    CFG.folder_path = \"/kaggle/input/\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"fraction_of_sessions_to_use = 0.1 if CFG.debug else 1\n\ntrain = pd.read_parquet('../input/otto-full-optimized-memory-footprint/train.parquet')\ntest = pd.read_parquet('../input/otto-full-optimized-memory-footprint/test.parquet')\n\nif fraction_of_sessions_to_use != 1:\n    lucky_sessions_train = train.drop_duplicates(['session']).sample(frac=fraction_of_sessions_to_use, random_state=42)['session']\n    subset_of_train = train[train.session.isin(lucky_sessions_train)]\n    \n    lucky_sessions_test = test.drop_duplicates(['session']).sample(frac=fraction_of_sessions_to_use, random_state=42)['session']\n    subset_of_test = test[test.session.isin(lucky_sessions_test)]\nelse:\n    subset_of_train = train\n    subset_of_test = test\n\nsubset_of_train.index = pd.MultiIndex.from_frame(subset_of_train[['session']])\nsubset_of_test.index = pd.MultiIndex.from_frame(subset_of_test[['session']])\n\n# Concat train & test data\n# This is not information leakage because of considering the predctions session by session\nsubsets = pd.concat([subset_of_train, subset_of_test])\n# subsets = subset_of_train\nsessions = subsets.session.unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test 데이터 세션별 aid 및 event 리스트화\nsession_types = ['clicks', 'carts', 'orders']\ntest_session_AIDs = subset_of_test.reset_index(drop=True).groupby('session')['aid'].apply(list)\ntest_session_types = subset_of_test.reset_index(drop=True).groupby('session')['type'].apply(list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train, subset_of_train, test; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"time_info = pd.read_parquet(\"/kaggle/input/otto-extract-time-information/train_test_time_info.parquet\")\n# subsets[[\"month\", \"day\", \"hour\", \"weekday\"]] = time_info.values\nsubsets[[\"hour\", \"weekday\"]] = time_info[[\"hour\", \"weekday\"]].values\ndel time_info; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subsets.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate Time Window Aids","metadata":{}},{"cell_type":"code","source":"# Split interaction point into 1 hour (ex. 1970.1.1 00:00:00 ~ 1970.1.1 00:59:59 -> 0)\nsubsets[\"ts\"] = subsets[\"ts\"] // 3600\n# Get time group splited about test data (for time-efficiency)\ntime_group = subsets[\"ts\"].iloc[-subset_of_test.shape[0]:].unique()\n# Get last interaction time group\ntest_time_group = subsets.iloc[-subset_of_test.shape[0]:].reset_index(drop=True).groupby(\"session\")[\"ts\"].nth(-1)\n# Get last 20 aids by each sessions\nsubsets = subsets.reset_index(drop=True).groupby(\"session\", as_index=False).nth(list(range(-20, 0)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get best aids by each time groups\naids_dic = {}\nfor i in range(time_group.min()-24, time_group.max()+1, 1):\n    Counter(subsets.loc[subsets[\"ts\"] == i, \"aid\"].values).most_common(40)\n    tmp = Counter(subsets.loc[subsets[\"ts\"] == i, \"aid\"].values)\n    tmp_sum = sum(tmp.values())\n    aids_dic[i] = {m: n / tmp_sum for m, n in tmp.most_common(40)}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del subsets, subset_of_test; gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling & Inference","metadata":{}},{"cell_type":"code","source":"n_aids = 20\n# Timeunits for cadidates sampling (ex. if 3, get T-4 ~ T-1 aids for candidates sampling on T+0 )\nsearch_windows = 24\ntype_weight_multipliers = {0: 0.1, 1: 0.6, 2: 0.3}\n\noutput = {\n    \"session\": [],\n    \"type\": [],\n    \"rec\": [],\n    \"score\": [],\n}\n\nfor SESS, AIDs, types, time_group in tqdm(zip(test_session_AIDs.index, test_session_AIDs.values, test_session_types.values, test_time_group.values), total=len(test_session_AIDs.index)):\n\n    # Get common aids for time windows\n    tmp_aids = set.intersection(*map(set, [list(aids_dic[i].keys()) for i in list(range(time_group - search_windows, time_group, 1)) if i in aids_dic.keys()]))\n    if len(tmp_aids) != 0:\n        # Get average of pct change of windows\n        tmp_df = []\n        for i in list(range(time_group - search_windows, time_group, 1)):\n            tmp_df.append([aids_dic[i][j] for j in tmp_aids])\n        if len(tmp_df) != 0:\n            candidates = pd.DataFrame(tmp_df, columns=tmp_aids).pct_change().mean().round(5).to_dict()\n        else:\n            candidates = AIDs.copy()\n    else:\n        candidates = AIDs.copy()\n\n    if len(candidates) == 0: candidates = AIDs.copy()\n        \n    candidates = Counter(candidates)\n    rec, score = zip(*candidates.most_common(n_aids))\n    \n    output[\"session\"].extend([SESS] * 3)\n    output[\"type\"].extend([0, 1, 2])\n    output[\"rec\"].extend([\" \".join(pd.Series(rec, dtype=\"str\").values)] * 3)\n    output[\"score\"].extend([\" \".join(pd.Series(score, dtype=\"str\").values)] * 3)\n\noutput = pd.DataFrame(output).set_index([\"session\", \"type\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.reset_index().to_parquet(\"./raw_output.parquet\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"output[\"session_type\"] = [str(i[0]) + \"_\" + str(CFG.contentType_mapper[i[1]]) for i in output.index]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/otto-recommender-system/sample_submission.csv\")\nsubmission = submission.set_index(\"session_type\")\nsubmission.loc[output[\"session_type\"].values, \"labels\"] = output[\"rec\"].values\nsubmission = submission.reset_index()\nsubmission.to_csv(\"./submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}