{"cells":[{"metadata":{},"cell_type":"markdown","source":"
\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nwandb.init(project=\"riiid-challenge-wb\", name=\"exploration\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Import the Rapids suite here - takes abot 1.5 mins\n\nimport sys\n!cp ../input/rapids/rapids.0.15.0 /opt/conda/envs/rapids.tar.gz\n!cd /opt/conda/envs/ && tar -xzvf rapids.tar.gz > /dev/null\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7/site-packages\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib/python3.7\"] + sys.path\nsys.path = [\"/opt/conda/envs/rapids/lib\"] + sys.path \n!cp /opt/conda/envs/rapids/lib/libxgboost.so /opt/conda/lib/\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Regular Libraries\nimport os\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.image as mpimg\nfrom tabulate import tabulate\nimport missingno as msno \nfrom IPython.display import display_html\nfrom PIL import Image\nimport gc\nimport cv2\nfrom scipy.stats import pearsonr\nimport tqdm\n\nimport pydicom # for DICOM images\nfrom skimage.transform import resize\nimport copy\nimport re\n\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\n\n# Color Palette\ncustom_colors = ['#00FFE2', '#00FDFF', '#00BCFF', '#0082FF', '#8000FF', '#B300FF', '#F400FF']\nsns.palplot(sns.color_palette(custom_colors))\n\n# Set Style\nsns.set_style(\"whitegrid\")\nsns.despine(left=True, bottom=True)\n\n# Set tick size\nplt.rc('xtick',labelsize=12)\nplt.rc('ytick',labelsize=12)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*đź“ŚNote: Can't use `Dask-cuDF` because we oly have 1 worker and Memory: 13.96 in the Kaggle GPU Accelerator. If we would have had more than 1 worker, `Dask` would have performed even better :)*"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rapids Imports\nimport cudf\nimport cupy # CuPy is an open-source array library accelerated with NVIDIA CUDA.\n\n\nfrom dask.distributed import Client, wait\nfrom dask_cuda import LocalCUDACluster\n\ncluster = LocalCUDACluster()\nclient = Client(cluster)\nclient","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n\n
\nIn this section we'll use the cudf and cupy libraries provided by RAPIDS, combined with numpy for the plotting part. The notebook runs at the moment in 3 minutes.\n
\n\n# 1. train.csv\n\n* `row_id`: (int64) ID code for the row.\n* `timestamp`: (int64) the time in milliseconds between this user interaction and the first event completion from that user.\n* `user_id`: (int32) ID code for the user.\n* `content_id`: (int16) ID code for the user interaction\n* `content_type_id`: (bool) 0 if the event was a question being posed to the user, 1 if the event was the user watching a lecture.\n* `task_container_id`: (int16) ID code for the *batch of questions or lectures*. (eg. a user might see three questions in a row before seeing the explanations for any of them - those three would all share a task_container_id)\n* `user_answer`: (int8) the user's answer to the question, if any. Read -1 as null, for lectures.\n* `answered_correctly`: (int8) if the user responded correctly. Read -1 as null, for lectures.\n* `prior_question_elapsed_time`: (float32) The average time in milliseconds it took a user to answer each question in the previous question bundle, ignoring any lectures in between (is null for a user's first question bundle or lecture)\n* `prior_question_had_explanation`: (bool) Whether or not the user saw an explanation and the correct response(s) after answering the previous question bundle, ignoring any lectures in between. The value is shared across a single question bundle, and is null for a user's first question bundle or lecture. Typically the first several questions a user sees were part of an onboarding diagnostic test where they did not get any feedback."},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Read in data\ndtypes = {\n \"row_id\": \"int64\",\n \"timestamp\": \"int64\",\n \"user_id\": \"int32\",\n \"content_id\": \"int16\",\n \"content_type_id\": \"boolean\",\n \"task_container_id\": \"int16\",\n \"user_answer\": \"int8\",\n \"answered_correctly\": \"int8\",\n \"prior_question_elapsed_time\": \"float32\", \n \"prior_question_had_explanation\": \"int8\"\n}\n\ntrain = cudf.read_csv('../input/riiid-test-answer-prediction/train.csv', dtype=dtypes)\n\n# # Drop \"row_id\" column as it doesn't give any information\n# train = train.drop(columns = [\"row_id\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> đź“ŚNote: The only 2 columns with missing data (explained in documentation - `NULL` values are present for the first question bundle)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data Information\nprint(\"Rows: {:,}\".format(len(train)), \"\\n\" +\n \"Columns: {}\".format(len(train.columns)))\n\n# Find Missing Data if any\ntotal = len(train)\n\nfor column in train.columns:\n if train[column].isna().sum() != 0:\n print(\"{} has: {:,} ({:.2}%) missing values.\".format(column, train[column].isna().sum(), \n (train[column].isna().sum()/total)*100))\n \n \n# Fill in missing values with \"-1\"\ntrain[\"prior_question_elapsed_time\"] = train[\"prior_question_elapsed_time\"].fillna(-1)\ntrain[\"prior_question_had_explanation\"] = train[\"prior_question_had_explanation\"].fillna(-1)\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.1 Columns individual analysis\n\n* numerical features (distplot): `timestamp`, `prior_question_elapsed_time`\n* categorical features (distplot): `user_id` count, `content_id` count, `task_container_id` count\n* categorical features (barplot): `user_answer` count, `answered_correctly` count, `prior_question_had_explanation` count\n\n### Predefined functionsđź“‚\n\nBecause there is no possibility (yet) to use Rapids for visualization we need to preprocess and convert the data to numpy arrays and plot it afterwards."},{"metadata":{"trusted":true},"cell_type":"code","source":"def distplot_features(df, feature, title, color = custom_colors[4], categorical=True):\n '''Takes a column from the GPU dataframe and plots the distribution (after count).'''\n \n if categorical:\n values = cupy.asnumpy(df[feature].value_counts().values)\n else:\n values = cupy.asnumpy(df[feature].values)\n \n print('Mean: {:,}'.format(np.mean(values)), \"\\n\"\n 'Median: {:,}'.format(np.median(values)), \"\\n\"\n 'Max: {:,}'.format(np.max(values)))\n\n \n fig = plt.figure(figsize = (18, 3))\n \n if categorical:\n sns.distplot(values, hist=False, color = color, kde_kws = {'lw':3})\n else:\n # To speed up the process\n sns.distplot(values[::250000], hist=False, color = color, kde_kws = {'lw':3})\n \n plt.title(title, fontsize=15)\n plt.show();\n \n del values\n gc.collect()\n return fig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def barplot_features(df, feature, title, palette = custom_colors[2:]):\n '''Takes the numerical columns (with less than 10 categories) and plots the barplot.'''\n \n # We need to extract both the name of the category and the no. of appearences\n index = cupy.asnumpy(df[feature].value_counts().reset_index()[\"index\"].values)\n values = cupy.asnumpy(df[feature].value_counts().reset_index()[feature].values) \n\n fig = plt.figure(figsize = (18, 3))\n sns.barplot(x = index, y = values, palette = custom_colors[2:])\n plt.title(title, fontsize=15)\n plt.show();\n \n del index, values\n gc.collect()\n return fig","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspect numerical features"},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_features = ['timestamp', 'prior_question_elapsed_time']\n\nfor feature in numerical_features:\n fig = distplot_features(train, feature=feature, title = feature + \" distribution\", color = custom_colors[1], categorical=False)\n wandb.log({ feature + \" distribution\": fig})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspect Categorical Features: many values"},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_features = ['user_id', 'content_id', 'task_container_id']\n\nfor feature in categorical_features:\n fig = distplot_features(train, feature=feature, title = feature + \" countplot distribution\", color = custom_colors[4], categorical=True)\n wandb.log({feature + \" countplot distribution\": fig})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Inspect Categorical Features: fiew values\n\n> There are only a fiew cases where content_type_id is = 1 (meaning lectures) - which is good, we're not supposed to predict those anyways."},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_for_bar = ['content_type_id', 'user_answer', \n 'answered_correctly', 'prior_question_had_explanation']\n\nfor feature in categorical_for_bar:\n fig = barplot_features(train, feature=feature, title = feature + \" barplot\")\n wandb.log({feature + \" barplot\": fig})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## View the plots saved in W&B dashboard\nYou can see you live dashboard as you log metrics and plots by simply calling `wandb.run`. It displays the dashboard of the currently executing run."},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.run","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.2 Data Processing\n\n> đź“ŚNote: The **outliers** might strongly influence the future models. Hence, we need to carefully handle them. However, by trying to erase the outliers we can erase up to 10% of the data, which is valuable information for training our models.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Total rows we started with\ntotal = len(train)\nfeature = \"timestamp\"\n\n# Compute Outliers\nQ1 = cupy.percentile(train[feature].values, q = 25).item()\nQ3 = cupy.percentile(train[feature].values, q = 75).item()\nIQR = Q3 - Q1\n\n# We'll look only at the upper interval outliers\noutlier_boundry = Q3 + 1.5*IQR\n\nprint('Timestamp: around {:.2}% of the data would be erased.'.format(len(train[train[feature] >= outlier_boundry])/total * 100), \n \"\\n\"+\n 'The outlier boundry is {:,}, which means {:,.5} hrs, which means {:,.5} days.'.format(outlier_boundry, (outlier_boundry / 3.6e+6),\n (outlier_boundry / 3.6e+6)/24))\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> đź“ŚNote: However, I would erase all pupils (`user_id`) that have less than 5 appearences in the data (no prediction can be made on these students) "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Select ids to erase\nids_to_erase = train[\"user_id\"].value_counts().reset_index()[train[\"user_id\"].value_counts().reset_index()[\"user_id\"] < 5]\\\n [\"index\"].values\n\n# Erase the ids\nnew_train = train[~train['user_id'].isin(ids_to_erase)][:1000]\n\nprint(\"We erased {} rows meaning {:.3}% of all data.\".format(len(train)-len(new_train), (1 - len(new_train)/len(train))*100))\ndel ids_to_erase\n# del train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Count how many times the user answered correctly out of all available times\nuser_performance = train.groupby(\"user_id\").agg({ 'row_id': ['count'], 'answered_correctly': ['sum'] }).reset_index()\nuser_performance.columns = [\"user_id\", \"total_count\", \"correct_count\"]\nuser_performance[\"performance\"] = user_performance[\"correct_count\"] / user_performance[\"total_count\"]\n\n# Create intervals for number of appearences\n# between 0 and 1000, 1000 and 2500 and 2500+\ndef condition(x):\n if x <= 1000:\n return 0\n elif (x > 1000) & (x <= 2500):\n return 1\n else:\n return 2\n \nuser_performance[\"total_interval\"] = user_performance[\"total_count\"].applymap(condition)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> đź“ŚNote: So yes, the *average* performance increases along with the number of times one student appears in the data."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert to numpy arrays (so we can plot)\nx = cupy.asnumpy(user_performance[\"total_interval\"].values)\ny = cupy.asnumpy(user_performance[\"performance\"].values)\n\n# Plot\nfig = plt.figure(figsize = (18, 4))\nsns.barplot(x = x, y = y, palette = custom_colors[1:])\nplt.title(\"Performance over number of appearences\", fontsize = 15)\nplt.xticks([0, 1, 2], ['<1000', '1000-2500', '2500+']);\n\nwandb.log({\"Performance over number of appearences\": fig})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.run","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Artifacts Blurb **Here**\n## 1.4 Save and delete\n\n> To keep the notebook as light as possible and to not overload the memory, we save the `train` data in .feather format (lighter, takes about 7 seconds to upload using `cudf`) and delete the dataframes.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checkpoint: save to .parquet\nprint(\"Length of new_train\", len(new_train))\nnew_train.to_parquet('new_train.parquet')\n!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#save it as model artifact on W&B\nartifact = wandb.Artifact(name=\"train_data\", type=\"dataset\")\nartifact.add_file(\"new_train.parquet\")\nwandb.log_artifact(artifact)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Clean the environment\ndel train, new_train\ngc.collect()\n!rm new_train.parquet\n!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nrun = wandb.init()\n\nartifact = run.use_artifact('ivangoncharov/riiid-challenge-wb/train_data:v0', type='dataset')\nartifact_dir = artifact.download()\n\n!ls","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. questions.csv\n\n* `question_id`: foreign key for the train/test `content_id` column, when the content type is question (0).\n* `bundle_id`: code for which questions are served together.\n* `correct_answer`: the answer to the question. Can be compared with the train user_answer column to check if the user was right.\n* `part`: the relevant section of the TOEIC test.\n* `tags`: one or more detailed tag codes for the question. The meaning of the tags will not be provided, but these codes are sufficient for clustering the questions together.\n\n> The Test of English for International Communication (TOEIC) is an international standardized test of English language proficiency for non-native speakers."},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = cudf.read_csv('../input/riiid-test-answer-prediction/questions.csv')\n\n# Data Information\nprint(\"Rows: {:,}\".format(len(questions)), \"\\n\" +\n \"Columns: {}\".format(len(questions.columns)))\n\n# Find Missing Data if any\ntotal = len(questions)\n\nfor column in questions.columns:\n if questions[column].isna().sum() != 0:\n print(\"{} has: {:,} ({:.2}%) missing values.\".format(column, questions[column].isna().sum(), \n (questions[column].isna().sum()/total)*100))\n \n \n# Fill in missing values with \"-1\"\nquestions[\"tags\"] = questions[\"tags\"].fillna(-1)\n\nquestions.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2.1 Inspect the columns\n\n* categorical features (distplot): `question_id` count, `bundle_id` count, `tags` count\n* categorical features (barplot): `correct_answer`, `part`"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- question_id -----\n\n# The table contains an equal number of IDs for each question\nprint('There is a total of {:,} IDs.'.format(len(questions['question_id'].value_counts())), \"\\n\")\n\n# ----- bundle_id -----\nprint('There are {:,} unique bundle IDs.'.format(questions['bundle_id'].nunique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> đź“ŚNote: majority of the questions are from part 5 - if this distribution doesn't match the `test` set, there might be some issues :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"for feature in ['part', 'correct_answer']:\n fig = barplot_features(questions, feature=feature, title=feature + \" - barplot distribution\")\n wandb.log({feature + \" - barplot distribution\": fig})\nfig = distplot_features(questions, 'tags', title = \"Tags - Count Distribution\", color = custom_colors[0], categorical=True)\nwandb.log({\"Tags - Count Distribution\": fig})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Save and delete"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checkpoint: save to parquet\nartifact = wandb.Artifact(name=\"more_train_data\", type=\"dataset\")\nquestions.to_parquet('questions.parquet')\nartifact.add_file('questions.parquet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del questions\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. lectures.csv\n\n* `lecture_id`: foreign key for the train/test `content_id` column, when the content type is lecture (1).\n* `part`: top level category code for the lecture.\n* `tag`: one tag codes for the lecture. The meaning of the tags will not be provided, but these codes are sufficient for clustering the lectures together.\n* `type_of`: brief description of the core purpose of the lecture (`string` - so this data needs to be treated a bit different)\n\n*no missing values*"},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures = cudf.read_csv('../input/riiid-test-answer-prediction/lectures.csv')\n\n# Encode 'type_of' column\nlectures.type_of,codes = lectures['type_of'].factorize()\n\n# Data Information\nprint(\"Rows: {:,}\".format(len(lectures)), \"\\n\" +\n \"Columns: {}\".format(len(lectures.columns)))\nlectures.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.1 Inspect the columns"},{"metadata":{"trusted":true},"cell_type":"code","source":"# ----- lecture_id -----\n# The table contains an equal number of IDs for each question\nprint('There is a total of {:,} IDs.'.format(len(lectures['lecture_id'].value_counts())), \"\\n\")\n\n# There are 151 unique tags\nprint('There are a total of {:,} unique tags IDs.'.format(len(lectures['tag'].value_counts())))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> đź“ŚNote: Again, part 5 is very proeminent."},{"metadata":{"trusted":true},"cell_type":"code","source":"for feature in ['part', 'type_of']:\n fig = barplot_features(lectures, feature=feature, title=feature + \" - barplot distribution\")\n wandb.log({feature + \" - barplot distribution\": fig})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3.3 Save and delete"},{"metadata":{"trusted":true},"cell_type":"code","source":"lectures.to_parquet(\"lectures.parquet\")\nartifact.add_file(\"lectures.parquet\")\ndel lectures\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## View the dashboard in real Time (Blurb)"},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.run","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n\n> Let's look again at the structure of our data:\n\n\n
\n

This section uses the cuML package and XGBoost to compute the predictions.

\n
"},{"metadata":{"trusted":true},"cell_type":"code","source":"cudf.set_allocator(\"managed\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Import the data\ntrain = cudf.read_parquet(\"../input/riiid-answer-correctness-prediction-rapids/new_train.parquet\")\nquestions = cudf.read_parquet(\"../input/riiid-answer-correctness-prediction-rapids/questions.parquet\")\n\n# Lectures we won't load, as we are not supposed to predict for these rows","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# Let's exclude all observations where (content_type_id = 1) & (answered_correctly = -1)\ntrain = train[train['content_type_id'] != 1]\ntrain = train[train['answered_correctly'] != -1].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Feature Engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameters\ntrain_percent = 0.1\ntotal_len = len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split data into train data & feature engineering data (to use for past performance)\n# Timestamp is in descending order - meaning that the last 10% observations have\n# the biggest chance of having had some performance recorded before\n# so looking at the performance in the past we'll try to predict the performance now\n\nfeatures_df = train.iloc[ : int(total_len*(1-train_percent))]\ntrain_df = train.iloc[int(total_len*(1-train_percent)) : ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.1 Feature Engineering - Create Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# --- STUDENT ANSWERS ---\n# Group by student\nuser_answers = features_df[features_df['answered_correctly']!=-1].\\\n groupby('user_id').\\\n agg({'answered_correctly': ['sum', 'mean', 'min', \n 'max', 'count', 'median', \n 'std', 'var']}).\\\n reset_index()\n\nuser_answers.columns = ['user_id', 'user_sum', 'user_mean', 'user_min', 'user_max', \n 'user_count', 'user_median', 'user_std', 'user_var']\n\n\n# --- CONTENT ID ANSWERS ---\n# Group by content\ncontent_answers = features_df[features_df['answered_correctly']!=-1].\\\n groupby('content_id').\\\n agg({'answered_correctly': ['sum', 'mean', 'min', \n 'max', 'count', 'median', \n 'std', 'var']}).\\\n reset_index()\n\ncontent_answers.columns = ['content_id', 'content_sum', 'content_mean', 'content_min', \n 'content_max', 'content_count', 'content_median', 'content_std', \n 'content_var']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Save FE data; we will use it for the `test` set too :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_answers.to_parquet('user_answers.parquet')\ncontent_answers.to_parquet('content_answers.parquet')\nartifact.add_file('user_answers.parquet')\nartifact.add_file('content_answers.parquet')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save the artifacts to cloud\nWe have used artifacts to to track all the files that we've pre-processes. Now let's log these artifacts so that we don't have to repeat these steps"},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.log_artifact(artifact)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train, questions\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Download the Uploaded artifacts [BLURB **here**]"},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nrun = wandb.init()\n\nartifact = run.use_artifact('authors/riiid-challenge-wb/train_data:v0', type='dataset')\nartifact_dir = artifact.download()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.run.finish()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.2 Predefined Functions for Preprocesing¶\n\n> Combine new features with the `train_df`"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Features for ML\nfeatures_to_keep = ['user_sum', 'user_mean', 'user_min', 'user_max', \n 'user_count', 'user_median', 'user_std', 'user_var',\n 'content_sum', 'content_mean', 'content_min', \n 'content_max', 'content_count', 'content_median', 'content_std', \n 'content_var']\ntarget = 'answered_correctly'\nall_features = features_to_keep.copy()\nall_features.append(target)\n\n\n# We need to convert True-False variables to integers\ndef to_bool(x):\n '''For the string variables.'''\n if x == False:\n return 0\n else:\n return 1\n\n \ndef combine_features(data = None):\n '''Combine the features with the Train/Test data.'''\n \n # Add \"past\" information\n features_data = data.merge(user_answers, how = 'left', on = 'user_id')\n features_data = features_data.merge(content_answers, how = 'left', on = 'content_id')\n\n # Apply\n features_data['content_type_id'] = features_data['content_type_id'].applymap(to_bool)\n features_data['prior_question_had_explanation'] = features_data['prior_question_had_explanation'].applymap(to_bool)\n\n # Fill in missing spots\n features_data.fillna(value = -1, inplace = True)\n \n return features_data\n\n\n# Scaling the data did not perform as I expected to - so for now we will exclude it\ndef scale_data(features_data=None, train=True, features_to_keep=None, target=None):\n '''Scales the provided data - if the data is for training, excludes the target column.\n It also chooses the features used in the prediction.'''\n \n data_for_standardization = features_data[features_to_keep]\n matrix = data_for_standardization.as_matrix()\n scaled_matrix = StandardScaler().fit_transform(matrix)\n \n scaled_data = cudf.DataFrame(scaled_matrix)\n scaled_data.columns = data_for_standardization.columns\n \n # We don't want to scale the target also\n if train:\n scaled_data[target] = features_data[target]\n \n return scaled_data","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.3 Apply Functions - getting data ready"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df = combine_features(data=train_df)\n# train_df = scale_data(features_data=train_df, train=True, features_to_keep=features_to_keep, target=target)\n\n# Comment this if you're scaling\ntrain_df = train_df[all_features]\n\nprint(\"Observations in train: {:,}\".format(len(train_df)))\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. XGBoost Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# RAPIDS roc_auc_score is 16x faster than sklearn. - cdeotte\nfrom cuml.metrics import roc_auc_score\nfrom cuml.preprocessing.model_selection import train_test_split\nimport xgboost\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Features, target and train/test split\nX = train_df[features_to_keep]\ny = train_df[target]\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, \n shuffle=False, random_state=13, stratify=y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2.1 Baseline Model ;)\n\n### Helper Function that runs multiple models"},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\n\ndefault_params = {\n 'max_depth' : 4,\n 'max_leaves' : 2**4,\n 'tree_method' : 'gpu_hist',\n 'grow_policy' : 'lossguide',\n 'eta': 0.001\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef train_xgb_model():\n '''Trains an XGB and returns the trained model + ROC value.'''\n wandb.init(project=\"riiid-challenge-wb\", name=\"Baseline-xgboost\", config=default_params)\n config = wandb.config\n params = {\n 'max_depth' : config.max_depth,\n 'max_leaves' : config.max_leaves,\n 'tree_method' : config.tree_method,\n 'grow_policy' : config.grow_policy,\n 'eta' : config.eta,\n 'objective': \"reg:logistic\"\n }\n \n # Create DMatrix - is optimized for both memory efficiency and training speed.\n train_matrix = xgboost.DMatrix(data = X_train, label = y_train)\n \n # Create & Train the model\n model = xgboost.train(params, dtrain=train_matrix, callbacks=[wandb.xgboost.wandb_callback()])\n\n # Make prediction\n predicts = model.predict(xgboost.DMatrix(X_test))\n roc = roc_auc_score(y_test.astype('int32'), predicts)\n wandb.log({\"ROC\": roc})\n print(\" - ROC: {:.5}\".format(roc))\n \n return model, roc\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nmodel1, roc1 = train_xgb_model()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"
\n

We have a ROC score of 0.71628 in less than 10 seconds.

\n

Incredible.

\n
"},{"metadata":{"trusted":true},"cell_type":"code","source":"# save model to file\npickle.dump(model1, open(\"baseline_model.pickle.dat\", \"wb\"))\nartifact = wandb.Artifact(name=\"trained_models\", type=\"model\")\nartifact.add_file(\"baseline_model.pickle.dat\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wandb.run.finish()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. LightGBM Model\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn import metrics\nimport lightgbm as lgbm\nfrom sklearn import metrics\nimport gc\nimport pickle\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We'll do a train | validation | test situation\ntrain, test = train_test_split(train_df, test_size=0.3, shuffle=False, random_state=13)\n\ntrain = train.to_pandas()\ntest = test.to_pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n# -----------\nn_splits = 4\n# -----------\n\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=13)\n\noof = np.zeros(len(train))\npredictions = np.zeros(len(test))\n\n# Covertion to CPU data\nskf_split = skf.split(X=train[features_to_keep], y=cupy.asnumpy(train[target].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param = {\n 'num_leaves': 80,\n 'max_bin': 250,\n 'min_data_in_leaf': 11,\n 'learning_rate': 0.01,\n 'min_sum_hessian_in_leaf': 0.00245,\n 'bagging_fraction': 1.0, \n 'bagging_freq': 5, \n 'feature_fraction': 0.05,\n 'lambda_l1': 4.972,\n 'lambda_l2': 2.276,\n 'min_gain_to_split': 0.65,\n 'max_depth': 14,\n 'save_binary': True,\n 'seed': 1337,\n 'feature_fraction_seed': 1337,\n 'bagging_seed': 1337,\n 'drop_seed': 1337,\n 'data_random_seed': 1337,\n 'objective': 'binary',\n 'boosting_type': 'gbdt',\n 'verbose': 1,\n 'metric': 'auc',\n 'is_unbalance': True,\n 'boost_from_average': False,\n 'device': 'gpu',\n 'gpu_platform_id': 0,\n 'gpu_device_id': 0\n }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport wandb\nfrom wandb.lightgbm import wandb_callback\n\n# Training Loop\ncounter = 1\nfor train_index, valid_index in skf_split:\n wandb.init(project=\"riiid-challenge-wb\", group='lightGBM', \n name='gbm'+str(counter), config=param)\n print(\"==== Fold {} ====\".format(counter))\n \n lgbm_train = lgbm.Dataset(data = train.iloc[train_index, :][features_to_keep].values,\n label = train.iloc[train_index, :][target].values,\n feature_name = features_to_keep,\n free_raw_data = False)\n \n lgbm_valid = lgbm.Dataset(data = train.iloc[valid_index, :][features_to_keep].values,\n label = train.iloc[valid_index, :][target].values,\n feature_name = features_to_keep,\n free_raw_data = False)\n \n lgbm_2 = lgbm.train(params = param, train_set = lgbm_train, valid_sets = [lgbm_valid],\n early_stopping_rounds = 12, num_boost_round=100, verbose_eval=25, \n callbacks=[wandb_callback()])\n \n \n # X_valid to predict\n oof[valid_index] = lgbm_2.predict(train.iloc[valid_index][features_to_keep].values, \n num_iteration = lgbm_2.best_iteration)\n predictions += lgbm_2.predict(test[features_to_keep], \n num_iteration = lgbm_2.best_iteration) / n_splits\n \n counter += 1\n wandb.run.finish()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The Dashboard\nEach W&B project has a dashboard that contains information about all the experiments in that project. Here's an example dashboard of a YOLOv5 project.\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}