{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"scrolled":true},"cell_type":"code","source":"%matplotlib inline\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nDATA_DIR = '../input/'\ntarget_col = 'deal_probability'\nos.listdir(DATA_DIR)","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e79701592a6425431c6f10064d8a7df70e936e01"},"cell_type":"code","source":"import time\nfrom contextlib import contextmanager\nfrom functools import lru_cache\nos.environ['OMP_NUM_THREADS'] = '4'\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.1f} s')\n\ndef reduce_memory(df):\n    for c in df.columns:\n        if df[c].dtype=='int':\n            if df[c].min()<0:\n                if df[c].abs().max()<2**7:\n                    df[c] = df[c].astype('int8')\n                elif df[c].abs().max()<2**15:\n                    df[c] = df[c].astype('int16')\n                elif df[c].abs().max()<2**31:\n                    df[c] = df[c].astype('int32')\n                else:\n                    continue\n            else:\n                if df[c].max()<2**8:\n                    df[c] = df[c].astype('uint8')\n                elif df[c].max()<2**16:\n                    df[c] = df[c].astype('uint16')\n                elif df[c].max()<2**32:\n                    df[c] = df[c].astype('uint32')\n                else:\n                    continue\n        if df[c].dtype=='float64':\n            df[c] = df[c].astype('float32')\n    return df\n\ndef compare_set(a, b):\n    print(len(a), len(b), len(a&b))\n    \ndate_map = {}\nto_datetime = lambda x:pd.to_datetime(x)\ndef get_datetime(df, date_col, to_dayofyear=True):\n    unq = df[date_col].unique().tolist()\n    for u in unq:\n        if u not in date_map:\n            if to_dayofyear:\n                date_map.update({u: to_datetime(u).dayofyear})\n            else:\n                date_map.update({u: to_datetime(u)})\n    df[date_col] = df[date_col].map(date_map.get)\n    return df\ndef fillna_date(df, na_col='activation_date', diff_col='date_from'):\n    mask = df[na_col].isnull()\n    diff = df.loc[~mask, diff_col] - df.loc[~mask, na_col]\n    diff = int(diff.median())\n    print('median diff days', diff)\n    df.loc[mask, na_col] = df.loc[mask, diff_col] - diff\n    df[na_col] = df[na_col].astype('int')\n    return df\ndef get_eval_user_set():\n    with timer('Loading train'):\n        train = pd.read_csv(DATA_DIR+'train.csv', usecols=['user_id'])\n    with timer('Loading test'):\n        test = pd.read_csv(DATA_DIR+'test.csv', usecols=['user_id'])\n    users = set()\n    users |= set(train['user_id'].values)\n    users |= set(test['user_id'].values)\n    print(len(users), 'in total')\n    return users\ndef get_common_active_item_set(train_active_comm, test_active_comm):\n    items = set()\n    items |= set(train_active_comm['item_id'].values)\n    items |= set(test_active_comm['item_id'].values)\n    print(len(items), 'in total')\n    return items","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9723773f0b0d9f6b70cd2ac89e08d2122d25c993"},"cell_type":"code","source":"periods_cols = [\n    'item_id',\n    'activation_date',\n    'date_from',\n    'date_to'\n]\ntrain_periods = pd.read_csv(DATA_DIR+'periods_train.csv', usecols=periods_cols)\ntest_periods = pd.read_csv(DATA_DIR+'periods_test.csv', usecols=periods_cols)\nwith timer('Loading train periods date'):\n    train_periods = get_datetime(train_periods, 'activation_date')\n    train_periods = get_datetime(train_periods, 'date_from')\n    train_periods = get_datetime(train_periods, 'date_to')\nwith timer('Loading test periods date'):\n    test_periods = get_datetime(test_periods, 'activation_date')\n    test_periods = get_datetime(test_periods, 'date_from')\n    test_periods = get_datetime(test_periods, 'date_to')","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"441c2ba8402b81ac3d827eb136881eeaa39244a3"},"cell_type":"code","source":"pd.to_datetime('2017-03-15').dayofyear, pd.to_datetime('2017-04-12').dayofyear\nprint((train_periods['activation_date']<74).sum(), len(train_periods))\nprint((test_periods['activation_date']<102).sum(), len(test_periods))","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5812468d50f04f41c579ef52bda3968b25a20d20"},"cell_type":"code","source":"f = plt.figure(figsize=(12, 4))\nf.add_subplot(1, 2, 1)\n(train_periods['date_from'] - train_periods['activation_date']).hist()\nf.add_subplot(1, 2, 2)\n(test_periods['date_from'] - test_periods['activation_date']).hist()\nplt.show()","execution_count":6,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9583470de9ba78c1807c6a65145d51d4f463af5b"},"cell_type":"code","source":"train_periods = fillna_date(train_periods)\ntest_periods = fillna_date(test_periods)","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"237f0584932abb0eb94c3173d466446e708ad7e1"},"cell_type":"code","source":"train_periods = reduce_memory(train_periods)\ntest_periods = reduce_memory(test_periods)","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"caf08d380d1badd8f751b0b119f0eb484b9c8775"},"cell_type":"code","source":"eval_users = get_eval_user_set()","execution_count":9,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e2e6c60e3f7a842c34193c364df28c6b8ab4ec13"},"cell_type":"code","source":"def get_common_active(df_active, eval_users):\n    print('All shape', df_active.shape)\n    common_indexes = set(df_active['user_id'].values) & eval_users\n    common_indexes = np.array(list(common_indexes))\n    df_active = df_active.reset_index().set_index('user_id')\n    df_active = df_active.loc[common_indexes, :]\n    df_active = df_active.reset_index()\n    print('Common only shape', df_active.shape)\n    return df_active\ndef get_common_period(df_period, active_items):\n    print('All shape', df_period.shape)\n    common_indexes = set(df_period['item_id'].values) & active_items\n    common_indexes = np.array(list(common_indexes))\n    df_period = df_period.reset_index().set_index('item_id')\n    df_period = df_period.loc[common_indexes, :]\n    df_period = df_period.reset_index()\n    print('Common only shape', df_period.shape)\n    return df_period","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eda466c33fa3f4b1575a0f5fb6bfe39a5c50f5ca"},"cell_type":"code","source":"with timer('Loading train_active'):\n    train_active = pd.read_csv(\n        DATA_DIR+'train_active.csv', \n        usecols=['user_id', 'item_id', 'activation_date'])\nwith timer('Loading test_active'):\n    test_active = pd.read_csv(\n        DATA_DIR+'test_active.csv', \n        usecols=['user_id', 'item_id', 'activation_date'])\nwith timer('Loading train'):\n    train = pd.read_csv(\n        DATA_DIR+'train.csv', \n        usecols=['user_id', 'item_id', 'activation_date'])\n    train = get_datetime(train, 'activation_date')\nwith timer('Loading test'):\n    test = pd.read_csv(\n        DATA_DIR+'test.csv', \n        usecols=['user_id', 'item_id', 'activation_date'])\n    test = get_datetime(test, 'activation_date')","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7f1a2b1bb6c5be90e94a5df75661d03f719abdf"},"cell_type":"code","source":"with timer('Getting common users and day of year for train_active'):\n    train_active = get_common_active(train_active, eval_users)\n    train_active = get_datetime(train_active, 'activation_date')\nwith timer('Getting common users and day of year for test_active'):\n    test_active = get_common_active(test_active, eval_users)\n    test_active = get_datetime(test_active, 'activation_date')","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69252d9decc09698a773f300889004ca9c5bab17"},"cell_type":"code","source":"active_items = get_common_active_item_set(train_active, test_active)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24e850f90af95718288d20cf3ec17fef5cf92a38"},"cell_type":"code","source":"with timer('Getting common [items] train period'):\n    train_periods = get_common_period(train_periods, active_items)\nwith timer('Getting common [items] test period'):\n    test_periods = get_common_period(test_periods, active_items)","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"328e6f1a9f51fab1d076ebcf3389e301a6ca8050"},"cell_type":"code","source":"idx_to_save = [train_active,\n               test_active,\n               train_periods,\n               test_periods] \nidx_save_path = ['train_active_raw_index.npy', \n                 'test_active_raw_index.npy', \n                 'train_periods_raw_index.npy', \n                 'test_periods_raw_index.npy']\nfor df, path in zip(idx_to_save, idx_save_path):\n    df = reduce_memory(df)\n    idx = df['index'].values\n    np.save(path, idx)\n    del df['index']; gc.collect();\n    print(path, 'saved')","execution_count":15,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d7a0a92e682d7bdbcf75d06c62e3d128a6e05b7"},"cell_type":"code","source":"print('train activation date start from No.',\n      pd.to_datetime('2017-03-15').dayofyear, \n      'day of year')\nprint('test activation date start from No.', \n      pd.to_datetime('2017-04-12').dayofyear, \n      'day of year')\n\nf = plt.figure(figsize=(18, 12))\nx_range = np.arange(0, 125, 5)\nf.add_subplot(3, 1, 1)\nsns.distplot(train_periods['activation_date']).set_xticks(x_range)\nsns.distplot(test_periods['activation_date']).set_xticks(x_range)\nplt.legend(['train_periods', 'test_periods'], loc=1)\nplt.grid()\n\nf.add_subplot(3, 1, 2)\nsns.distplot(train_periods['date_from']).set_xticks(x_range)\nsns.distplot(test_periods['date_from']).set_xticks(x_range)\nplt.legend(['train_periods', 'test_periods'], loc=1)\nplt.grid()\n\nf.add_subplot(3, 1, 3)\nsns.distplot(train_periods['date_to']).set_xticks(x_range)\nsns.distplot(test_periods['date_to']).set_xticks(x_range)\nplt.legend(['train_periods', 'test_periods'], loc=1)\nplt.grid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ac20ff96d98981e1fb0316399a2c9bfa6689ff5","collapsed":true},"cell_type":"code","source":"f = plt.figure(figsize=(18, 4))\nx_range = np.arange(0, 125, 5)\nsns.distplot(train_active['activation_date']).set_xticks(x_range)\nsns.distplot(test_active['activation_date']).set_xticks(x_range)\nplt.legend(['train_active', 'test_active'], loc=1)\nplt.grid()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"016a656e8869107d59b0377be40ba106d57a7dc7","collapsed":true},"cell_type":"code","source":"f = plt.figure(figsize=(18, 4))\nx_range = np.arange(0, 125, 5)\nsns.distplot(train['activation_date']).set_xticks(x_range)\nsns.distplot(test['activation_date']).set_xticks(x_range)\nplt.legend(['train', 'test'], loc=1)\nplt.grid()","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ca259de7ceead0768bede00943ef582817c93c50"},"cell_type":"code","source":"def get_unique_stats(grp_size):\n    unique_stats = np.unique(grp_size, return_counts=True)\n    unique_stats = np.array(unique_stats).T\n    cols = ['unique', 'unique_counts']\n    unique_stats = pd.DataFrame(unique_stats, columns=cols)\n    return unique_stats","execution_count":17,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"477b9996b08386b94da2a21ba479f0b146f884c4","collapsed":true},"cell_type":"code","source":"train_periods_item_grp = train_periods.groupby('item_id')\nwith timer('Getting groups [items] train period'):\n    train_periods_item_size = train_periods_item_grp.size()\nprint('original shape\\n', train_periods.shape)\ntrain_periods_item_uniqstats = get_unique_stats(train_periods_item_size)\ntrain_periods_item_uniqstats.describe().T","execution_count":18,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c3f7232b292faf185d0f7d710634793ef077003","collapsed":true},"cell_type":"code","source":"test_periods_item_grp = test_periods.groupby('item_id')\nwith timer('Getting groups [items] test period'):\n    test_periods_item_size = test_periods_item_grp.size()\nprint('original shape\\n', test_periods.shape)\ntest_periods_item_uniqstats = get_unique_stats(test_periods_item_size)\ntest_periods_item_uniqstats.describe().T","execution_count":19,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"scrolled":true,"_uuid":"ae50071aea5950d24fb96159160f3c749fa18086","collapsed":true},"cell_type":"code","source":"train_active_user_grp = train_active.groupby('user_id')\nwith timer('Getting groups [users] train active'):\n    train_active_user_size = train_active_user_grp.size()\nprint('original shape\\n', train_active.shape)\ntrain_active_user_uniqstats = get_unique_stats(train_active_user_size)\ntrain_active_user_uniqstats.describe().T","execution_count":20,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"scrolled":true,"_uuid":"025301a43b96da4d06a39aee94242f9e912c0736","collapsed":true},"cell_type":"code","source":"test_active_user_grp = test_active.groupby('user_id')\nwith timer('Getting groups [users] test active'):\n    test_active_user_size = test_active_user_grp.size()\nprint('original shape\\n', test_active.shape)\ntest_active_user_uniqstats = get_unique_stats(test_active_user_size)\ntest_active_user_uniqstats.describe().T","execution_count":21,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"scrolled":true,"_uuid":"85bbcd619a16e05e852f57d75f6825cc89bdb8e2","collapsed":true},"cell_type":"code","source":"train_user_grp = train.groupby('user_id')\nwith timer('Getting groups [users] train'):\n    train_user_size = train_user_grp.size()\nprint('original shape\\n', train.shape)\ntrain_user_uniqstats = get_unique_stats(train_user_size)\ntrain_user_uniqstats.describe().T","execution_count":22,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"scrolled":true,"_uuid":"9e5908a2defa3e375384868bcdbd140ddf671713","collapsed":true},"cell_type":"code","source":"test_user_grp = test.groupby('user_id')\nwith timer('Getting groups [users] test'):\n    test_user_size = test_user_grp.size()\nprint('original shape\\n', test.shape)\ntest_user_uniqstats = get_unique_stats(test_user_size)\ntest_user_uniqstats.describe().T","execution_count":23,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7033c1fb49d6eeba0653f216375963b395edc571"},"cell_type":"code","source":"train_user_size         = train_user_size.rename('user_id_size').reset_index()\ntest_user_size          = test_user_size.rename('user_id_size').reset_index()\ntrain_active_user_size  = train_active_user_size.rename('user_id_size').reset_index()\ntest_active_user_size   = test_active_user_size.rename('user_id_size').reset_index()\ntrain_periods_item_size = train_periods_item_size.rename('item_id_size').reset_index()\ntest_periods_item_size  = test_periods_item_size.rename('item_id_size').reset_index()","execution_count":27,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e2f021ad2dc833a1f5ec9e07c257d334390569c1"},"cell_type":"code","source":"train_active = train_active.merge(train_active_user_size, how='left', on='user_id')\ntest_active = test_active.merge(test_active_user_size, how='left', on='user_id')\ntrain_periods = train_periods.merge(train_periods_item_size, how='left', on='item_id')\ntest_periods = test_periods.merge(test_periods_item_size, how='left', on='item_id')","execution_count":43,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd6d71a1af7c5734f84078c2172acc1a7a58ff34","collapsed":true},"cell_type":"code","source":"dfs_to_save = [train_active,\n               test_active,\n               train_periods,\n               test_periods] \ndfs_save_path = ['train_active_ids_date.csv', \n                 'test_active_ids_date.csv', \n                 'train_periods.csv', \n                 'test_periods.csv']\nfor df, path in zip(dfs_to_save, dfs_save_path):\n    df = reduce_memory(df)\n    df.to_csv(path, index=False)\n    print(path, 'saved')","execution_count":49,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}