{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"dtype = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    'user_answer': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float32',\n    'prior_question_had_explanation': 'boolean'\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature engineering"},{"metadata":{},"cell_type":"markdown","source":"### Target encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\n\n\nper_value_counts = {}\nglobal_counts = Counter()\n\n\ncolumns_target_encode = ['user_id', 'content_id', 'content_type_id', 'task_container_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_or_update(column, value, count, answered_correctly):\n    # column\n    column_data = per_value_counts.get(column, {})\n    per_value_counts[column] = column_data\n    # value\n    value_data = column_data.get(value, Counter())\n    column_data[value] = value_data\n    # counters\n    value_data += Counter({'count': count, 'answered_correctly': answered_correctly})\n\n\ndef update_counts(data, column):\n    agg = data.groupby(column)['answered_correctly'].agg(['count', 'mean'])\n    agg['answered_correctly'] = agg['count'] * agg['mean']\n    for idx,row in agg.iterrows():\n        add_or_update(column, idx, row['count'], row['answered_correctly'])\n\n\ndef update_global_counts(data):\n    global global_counts\n    count = data.shape[0]\n    clicks = data[data['answered_correctly'] == 1].shape[0]\n    global_counts += Counter({'count': count, 'answered_correctly': clicks})\n\n\ndef update_all_counts(data, columns):\n    for column in columns:\n        update_counts(data, column)\n    update_global_counts(data)\n\n\ndef target_encode_value(column, value):\n    counts = per_value_counts.get(column, {}).get(value, Counter())\n    if 'answered_correctly' in counts:\n        return counts['answered_correctly'] / counts['count']\n    else:\n        return global_counts['answered_correctly'] / global_counts['count']\n\n\ndef target_encode(data, columns):\n    out = pd.DataFrame(index=data.index)\n    for column in columns:\n        out[column] = data[column].apply(lambda value: target_encode_value(column, value))\n    return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Standard scaling"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n\ncolumns_std = ['prior_question_elapsed_time']\n\nscaler = StandardScaler()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare x, y"},{"metadata":{"trusted":true},"cell_type":"code","source":"columns_copy = ['prior_question_had_explanation']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_x(data):\n    \n    # copy without changes\n    x = data[columns_copy].fillna(0)\n    \n    # target encode\n    x_target_encode = target_encode(data, columns_target_encode)\n    \n    # std\n    x_std = pd.DataFrame(scaler.transform(data[columns_std]), index=data.index, columns=columns_std).fillna(0)\n    \n    return pd.concat([x, x_target_encode, x_std], axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## SGD"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\nfrom sklearn.metrics import roc_auc_score\n\n\nestimator = SGDClassifier(loss='log')\n\n\nchunksize = 10**6\n\ntrain_count = 0\nvalid_count = 0\n\ntrain_from = 1 * chunksize\nvalidate_from = 90 * chunksize\n\nauc = 0.\n\nfor train in pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', dtype=dtype, chunksize=chunksize, iterator=True):\n    \n    train = train[train['answered_correctly'] != -1]\n    \n    if train_count >= train_from:\n        \n        x_train = make_x(train)\n        y_train = train['answered_correctly']\n        \n        if train_count >= validate_from:\n            y_pred = estimator.predict_proba(x_train)[:, 1]\n            auc += roc_auc_score(y_train, y_pred)\n            valid_count += chunksize\n        \n        estimator.partial_fit(x_train, y_train, classes=[0,1])\n        train_count += chunksize\n        \n        print('Train count:', train_count, 'Validation count:', valid_count)\n        \n    else:\n        \n        train_count += chunksize\n        print('Warmup count:', train_count)\n    \n    update_all_counts(train, columns_target_encode)\n    scaler.partial_fit(train[columns_std])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Validation score:', auc * chunksize / valid_count)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict and submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\n\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (test, sample_prediction) in iter_test:\n    \n    x_test = make_x(test)\n    \n    test['answered_correctly'] = estimator.predict_proba(x_test)[:, 1]\n    \n    env.predict(test.loc[test['content_type_id'] == 0, ['row_id', 'answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}