{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Working in progress\nData dumped using the CV strategy by Tito and Marisaka Mozz (`cv1_train` file) using multiprocessing. The iterator is `itertuples()` instead of the usual `iterrows()`.\n\n\nReference:\n- https://www.kaggle.com/its7171/lgbm-with-loop-feature-engineering\n- https://www.kaggle.com/ragnar123/riiid-model-lgbm\n- https://www.kaggle.com/ceshine/values-to-numpy-vs-itertuples-vs-iterrows"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom collections import defaultdict\nfrom contextlib import contextmanager\nimport psutil\nimport math\nfrom time import time\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\nimport pickle\nimport random\nimport os\nimport sys\nfrom utils import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parallelization\nWe first compare to see if the `multiprocessing` can recognize the 4 Xeon CPU threads gotten by the system, not just 2 physical cores."},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def get_memory(num_var=10):\n    for name, size in sorted(((name, sys.getsizeof(value)) for name, value in globals().items()), \n                             key= lambda x: -x[1])[:num_var]:\n        print(color(f\"{name:>30}:\", color=Colors.green), \n              color(f\"{get_size(size):>8}\", color=Colors.magenta))\n\nget_system()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import multiprocessing as mp\nnum_cpu = mp.cpu_count()\nprint(f\"Total number of CPU threads: {num_cpu}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEBUG = True\nFOLD = 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEED = 1127 # my boy's bday just for luck\n# Function to seed everything\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading train"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_parquet = '../input/cv-strategy-in-the-kaggle-environment/cv1_train.parquet'\nquestion_file = '../input/riiid-test-answer-prediction/questions.csv'\n\n# Read data\nfeatures = ['timestamp', \n           'user_id', \n           'answered_correctly',\n           'content_id', \n           'content_type_id', \n           'prior_question_elapsed_time', \n           'prior_question_had_explanation']\ntrain_dtypes = {\n    'timestamp': 'int64',\n    'user_id': 'int32', \n    'answered_correctly': 'int8', \n    'content_id': 'int16', \n    'content_type_id':'int8', \n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'bool'\n}\n\n\nwith timer(\"Loading train and valid.\"):\n    df = pd.read_parquet(train_parquet)[features].astype(train_dtypes)\n\n\n# Delete some trianing data to don't have ram problems\nif DEBUG:\n    df = df.iloc[:1_000_000]\nelse:\n    df = df.iloc[-20_000_000:]\n\n# Filter by content_type_id to discard lectures\ndf = df.loc[df.content_type_id == False].reset_index(drop = True)\n\n# Changing dtype to avoid lightgbm error\ndf['prior_question_had_explanation'] = \\\ndf.prior_question_had_explanation.fillna(False).astype('int8')\n\n# Fill prior question elapsed time with the mean\nprior_question_elapsed_time_mean = \\\ndf['prior_question_elapsed_time'].dropna().mean()\ndf['prior_question_elapsed_time']\\\n.fillna(prior_question_elapsed_time_mean, inplace = True)\n\n# Merge with question dataframe\nquestions_df = pd.read_csv(question_file)\nquestions_df['part'] = questions_df['part'].astype(np.int32)\nquestions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32)\n\ndf = pd.merge(df, questions_df[['question_id', 'part']], \n                 left_on = 'content_id', right_on = 'question_id', how = 'left')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature engineering\nThe old feature gen function below, with some modifications. Removing the `if` checking to reduce some overheads. The function is un-enscapsulated to make multiprocessing easier to code with. The new `add_features` function applies to each row of the iterator.\n\nBelow is the new iterator, we first check what it is like for a row"},{"metadata":{"trusted":true},"cell_type":"code","source":"iters = df[['user_id',\n          'answered_correctly', \n          'content_id', \n          'prior_question_elapsed_time', \n          'prior_question_had_explanation',\n          'timestamp']].itertuples()\n\nfor row in iters:\n    print(row, '\\n')\n    print(row[0], '    ', row[1:])\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The `num` in the original function is obtaining the index from `enumerate`, here after resetting the index, the index itself can be used as is directly. "},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Funcion for user stats with loops\ndef add_features(row):\n    \n    '''\n    row[0]: 'user_id',\n    row[1]: 'answered_correctly', \n    row[2]: 'content_id', \n    row[3]: 'prior_question_elapsed_time', \n    row[4]: 'prior_question_had_explanation',\n    row[5]: 'timestamp'\n    '''\n   \n    num = row[0] # index\n    row = row[1:]\n\n    # Client features assignation\n    # ------------------------------------------------------------------\n    if answered_correctly_u_count[row[0]] != 0:\n        answered_correctly_u_avg[num] = \\\n        answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n\n        elapsed_time_u_avg[num] = \\\n        elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n\n        explanation_u_avg[num] = \\\n        explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]\n\n    else:\n        answered_correctly_u_avg[num] = np.nan\n\n        elapsed_time_u_avg[num] = np.nan\n\n        explanation_u_avg[num] = np.nan\n\n    if len(timestamp_u[row[0]]) == 0:\n        timestamp_u_recency_1[num] = np.nan\n        timestamp_u_recency_2[num] = np.nan\n        timestamp_u_recency_3[num] = np.nan\n\n    elif len(timestamp_u[row[0]]) == 1:\n        timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]\n        timestamp_u_recency_2[num] = np.nan\n        timestamp_u_recency_3[num] = np.nan\n\n    elif len(timestamp_u[row[0]]) == 2:\n        timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]\n        timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]\n        timestamp_u_recency_3[num] = np.nan\n\n    elif len(timestamp_u[row[0]]) == 3:\n        timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]\n        timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]\n        timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]\n\n    if len(timestamp_u_incorrect[row[0]]) == 0:\n        timestamp_u_incorrect_recency[num] = np.nan\n    else:\n        timestamp_u_incorrect_recency[num] = \\\n        row[5] - timestamp_u_incorrect[row[0]][0]\n\n    # ------------------------------------------------------------------\n    # Question features assignation\n    if answered_correctly_q_count[row[2]] != 0:\n        answered_correctly_q_avg[num] = \\\n        answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n    else:\n        answered_correctly_q_avg[num] = np.nan\n        elapsed_time_q_avg[num] = np.nan\n        explanation_q_avg[num] = np.nan\n    # ------------------------------------------------------------------\n    # Client Question assignation\n    answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]\n    # ------------------------------------------------------------------\n\n    # ------------------------------------------------------------------\n    # Client features updates\n    answered_correctly_u_count[row[0]] += 1\n    elapsed_time_u_sum[row[0]] += row[3]\n    explanation_u_sum[row[0]] += int(row[4])\n\n    if len(timestamp_u[row[0]]) == 3:\n        timestamp_u[row[0]].pop(0)\n        timestamp_u[row[0]].append(row[5])\n    else:\n        timestamp_u[row[0]].append(row[5])\n\n    # ------------------------------------------------------------------\n    # Question features updates\n    answered_correctly_q_count[row[2]] += 1\n    elapsed_time_q_sum[row[2]] += row[3]\n    explanation_q_sum[row[2]] += int(row[4])\n    # ------------------------------------------------------------------\n    # Client Question updates\n    answered_correctly_uq[row[0]][row[2]] += 1\n\n    # ------------------------------------------------------------------\n    # Flag for training and inference\n    # ------------------------------------------------------------------\n    # Client features updates\n    answered_correctly_u_sum[row[0]] += row[1]\n    if row[1] == 0:\n        if len(timestamp_u_incorrect[row[0]]) == 1:\n            timestamp_u_incorrect[row[0]].pop(0)\n            timestamp_u_incorrect[row[0]].append(row[5])\n        else:\n            timestamp_u_incorrect[row[0]].append(row[5])\n\n    # ------------------------------------------------------------------\n    # Question features updates\n    answered_correctly_q_sum[row[2]] += row[1]\n        # ------------------------------------------------------------------","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_iters = df.groupby(\"user_id\")\nfor group in user_iters:\n    print(group[0], type(group[1])) # group[0] is the user_id\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group[1] # group[1] is the user's DF","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"group[1].index[0] # global index corresponding to the original df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for row in group[1][['user_id',\n                  'answered_correctly', \n                  'content_id', \n                  'prior_question_elapsed_time', \n                  'prior_question_had_explanation',\n                  'timestamp']].itertuples():\n    print(row[0]) # index\n    break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def add_features_user(group,):\n    \n    '''\n    Replacing all row[0] with the user_id\n    '''\n    user_id = group[0]\n    user_df = group[1]\n    print(user_id)\n    for row in user_df[['user_id',\n                  'answered_correctly', \n                  'content_id', \n                  'prior_question_elapsed_time', \n                  'prior_question_had_explanation',\n                  'timestamp']].itertuples():\n        num = row[0]\n        row = row[1:]\n        # Client features assignation\n        # ------------------------------------------------------------------\n        if answered_correctly_u_count[user_id] != 0:\n            answered_correctly_u_avg[num] = \\\n            answered_correctly_u_sum[user_id] / answered_correctly_u_count[user_id]\n\n            elapsed_time_u_avg[num] = \\\n            elapsed_time_u_sum[user_id] / answered_correctly_u_count[user_id]\n\n            explanation_u_avg[num] = \\\n            explanation_u_sum[user_id] / answered_correctly_u_count[user_id]\n\n        else:\n            answered_correctly_u_avg[num] = np.nan\n\n            elapsed_time_u_avg[num] = np.nan\n\n            explanation_u_avg[num] = np.nan\n\n        if len(timestamp_u[user_id]) == 0:\n            timestamp_u_recency_1[num] = np.nan\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n\n        elif len(timestamp_u[user_id]) == 1:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[user_id][0]\n            timestamp_u_recency_2[num] = np.nan\n            timestamp_u_recency_3[num] = np.nan\n\n        elif len(timestamp_u[user_id]) == 2:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[user_id][1]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[user_id][0]\n            timestamp_u_recency_3[num] = np.nan\n\n        elif len(timestamp_u[user_id]) == 3:\n            timestamp_u_recency_1[num] = row[5] - timestamp_u[user_id][2]\n            timestamp_u_recency_2[num] = row[5] - timestamp_u[user_id][1]\n            timestamp_u_recency_3[num] = row[5] - timestamp_u[user_id][0]\n\n        if len(timestamp_u_incorrect[user_id]) == 0:\n            timestamp_u_incorrect_recency[num] = np.nan\n        else:\n            timestamp_u_incorrect_recency[num] = \\\n            row[5] - timestamp_u_incorrect[user_id][0]\n\n        # ------------------------------------------------------------------\n        # Question features assignation\n        if answered_correctly_q_count[row[2]] != 0:\n            answered_correctly_q_avg[num] = \\\n            answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n            explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]\n        else:\n            answered_correctly_q_avg[num] = np.nan\n            elapsed_time_q_avg[num] = np.nan\n            explanation_q_avg[num] = np.nan\n        # ------------------------------------------------------------------\n        # Client Question assignation\n        answered_correctly_uq_count[num] = answered_correctly_uq[user_id][row[2]]\n        # ------------------------------------------------------------------\n\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_count[user_id] += 1\n        elapsed_time_u_sum[user_id] += row[3]\n        explanation_u_sum[user_id] += int(row[4])\n\n        if len(timestamp_u[user_id]) == 3:\n            timestamp_u[user_id].pop(0)\n            timestamp_u[user_id].append(row[5])\n        else:\n            timestamp_u[user_id].append(row[5])\n\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_count[row[2]] += 1\n        elapsed_time_q_sum[row[2]] += row[3]\n        explanation_q_sum[row[2]] += int(row[4])\n        # ------------------------------------------------------------------\n        # Client Question updates\n        answered_correctly_uq[user_id][row[2]] += 1\n\n        # ------------------------------------------------------------------\n        # Flag for training and inference\n        # ------------------------------------------------------------------\n        # Client features updates\n        answered_correctly_u_sum[user_id] += row[1]\n        if row[1] == 0:\n            if len(timestamp_u_incorrect[user_id]) == 1:\n                timestamp_u_incorrect[user_id].pop(0)\n                timestamp_u_incorrect[user_id].append(row[5])\n            else:\n                timestamp_u_incorrect[user_id].append(row[5])\n\n        # ------------------------------------------------------------------\n        # Question features updates\n        answered_correctly_q_sum[row[2]] += row[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# -----------------------------------------------------------------------\n# Client features\nanswered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)\nelapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)\nexplanation_u_avg = np.zeros(len(df), dtype = np.float32)\ntimestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)\ntimestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)\ntimestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)\ntimestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)\n# -----------------------------------------------------------------------\n# Question features\nanswered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)\nelapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)\nexplanation_q_avg = np.zeros(len(df), dtype = np.float32)\n\n# -----------------------------------------------------------------------\n# User Question\nanswered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)\n\n# -----------------------------------------------------------------------","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Client dictionaries, global var to be updated\nanswered_correctly_u_count = defaultdict(int)\nanswered_correctly_u_sum = defaultdict(int)\nelapsed_time_u_sum = defaultdict(int)\nexplanation_u_sum = defaultdict(int)\ntimestamp_u = defaultdict(list)\ntimestamp_u_incorrect = defaultdict(list)\n\n# Question dictionaries, global var to be updated\nanswered_correctly_q_count = defaultdict(int)\nanswered_correctly_q_sum = defaultdict(int)\nelapsed_time_q_sum = defaultdict(int)\nexplanation_q_sum = defaultdict(int)\n\n# Client Question dictionary, if the user has not answer a questions, then the value is a defaultdict(int)\nanswered_correctly_uq = defaultdict(lambda: defaultdict(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pool = mp.Pool(num_cpu) # use 4 CPU threads\nuser_iters = df.groupby(\"user_id\")\nfor group in tqdm(user_iters, total=len(user_iters)):\n    pool.apply_async(add_features_user, group)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iters = df[['user_id',\n          'answered_correctly', \n          'content_id', \n          'prior_question_elapsed_time', \n          'prior_question_had_explanation',\n          'timestamp']].itertuples()\nfor _row in tqdm(iters, total=len(df)):\n    add_features(_row)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dumping features"},{"metadata":{"trusted":true},"cell_type":"code","source":"for item in answered_correctly_u_sum.items():\n    print(item)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('answered_correctly_u_count.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_u_count, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('answered_correctly_u_sum.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_u_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('elapsed_time_u_sum.pickle', 'wb') as f:\n    pickle.dump(elapsed_time_u_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('explanation_u_sum.pickle', 'wb') as f:\n    pickle.dump(explanation_u_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('answered_correctly_q_count.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_q_count, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('answered_correctly_q_sum.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_q_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('elapsed_time_q_sum.pickle', 'wb') as f:\n    pickle.dump(elapsed_time_q_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('explanation_q_sum.pickle', 'wb') as f:\n    pickle.dump(explanation_q_sum, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('timestamp_u.pickle', 'wb') as f:\n    pickle.dump(timestamp_u, f, protocol=pickle.HIGHEST_PROTOCOL)\n    \nwith open('timestamp_u_incorrect.pickle', 'wb') as f:\n    pickle.dump(timestamp_u_incorrect, f, protocol=pickle.HIGHEST_PROTOCOL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answered_correctly_uq_dict = defaultdict(int)\nfor num, row in enumerate(train[['user_id']].values):\n    answered_correctly_uq_dict[row[0]] = answered_correctly_uq[row[0]]\n    \nwith open('answered_correctly_uq_dict.pickle', 'wb') as f:\n    pickle.dump(answered_correctly_uq_dict, f, protocol=pickle.HIGHEST_PROTOCOL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_memory(num_var=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = find_files('pickle', '../working/')\nprint_file_size(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../working/submission.csv')\nsub['answered_correctly'].hist(bins=15);","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}