{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nimport pandas as pd\n\n# You can only call make_env() once, so don't lose it!\nenv = riiideducation.make_env()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can only iterate through a result from `env.iter_test()` once\n# so be careful not to lose it once you start iterating.\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_times_dict_np(df_np):\n    user_times = {}\n    for i, row in enumerate(df_np):\n        if row[1] in user_times:\n            if row[0] > user_times[row[1]]:\n                user_times[row[1]] = row[0]\n        else:\n            user_times[row[1]] = 1\n    return user_times","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_cols_dict(df):\n    cols_lst = list(df.columns)\n    cols_dict = {}\n    for i, col in enumerate(cols_lst):\n        cols_dict[col] = i\n    return cols_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_bool_dict(arr, cols_dict, id_column_name, tag_column_name):\n    bool_dict = {}\n    print(arr.shape[0])\n    for i in range(0,arr.shape[0]):\n        if id_column_name == \"question_id\":\n            ls = [1 if str(t) in str(arr[i,cols_dict[tag_column_name]]).split(\" \") else 0 for t in range(0,188)]\n        elif id_column_name == \"lecture_id\":\n            ls = [1 if str(t)==str(arr[i,cols_dict[tag_column_name]]) else 0 for t in range(0,188)]\n        bool_dict[arr[i,cols_dict[id_column_name]]] = np.array(ls)\n        #print(\"created array for \" + str(i))\n    #print(bool_dict)\n    return bool_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converting categorical data to 0s and 1s, and adding the questions and lectures data to train_df, with\n#NaN for lecture values that don't apply to questions or vice versa\n\ndef categorical_to_binary(new_train_df_np, cols_dict, new_lectures_df_np, new_questions_df_np, \n                          lectures_bool_dict, questions_bool_dict, most_important_tag_coefs, train_data=True): \n    \n    print('here')\n    print(most_important_tag_coefs.shape)\n    num_tags = most_important_tag_coefs.shape[0]\n    print(num_tags)\n    new_train_df_np = np.append(new_train_df_np,(-np.ones((new_train_df_np.shape[0],num_tags))),1)\n    print(new_train_df_np.shape)\n    for i in range(0,new_train_df_np.shape[0]):\n        if new_train_df_np[i,cols_dict['content_type_id']]==0:\n            tags_row = questions_bool_dict[new_train_df_np[i,cols_dict['content_id']]].reshape(188,1)\n        else:\n            tags_row = lectures_bool_dict[new_train_df_np[i,cols_dict['content_id']]].reshape(188,1)\n        #print(tags_row.shape)\n        #print(most_important_tag_coefs.shape)\n        new_train_df_np[i,-num_tags:] = np.matmul(most_important_tag_coefs,tags_row).reshape(1,num_tags)\n    print('encoded tags')\n    \n    return new_train_df_np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalize_elapsed_time_np(train_df_np, cols_dict):\n    for i in range(len(train_df_np)):\n        train_df_np[i, cols_dict[\"prior_question_elapsed_time\"]] = train_df_np[i, cols_dict[\"prior_question_elapsed_time\"]]/300000\n    return  train_df_np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_times_to_float_np(df_np, cols_dict, user_times):\n    for i in range(0, len(df_np)):\n        df_np[i, cols_dict[\"timestamp\"]] = df_np[i, cols_dict[\"timestamp\"]] / user_times[df_np[i, cols_dict[\"user_id\"]]]\n    return df_np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.decomposition import PCA\n\n\ndef find_most_important_tags(new_train_df_np, cols_dict, lectures_bool_dict, questions_bool_dict):\n\n    num_tags = 188\n    tags_arr = -np.ones((new_train_df_np.shape[0],num_tags))\n    print(tags_arr.shape)\n    for i in range(0,tags_arr.shape[0]):\n        if new_train_df_np[i,cols_dict['content_type_id']]==0:\n            tags_arr[i,:] = questions_bool_dict[new_train_df_np[i,cols_dict['content_id']]]\n        else:\n            tags_arr[i,:] = lectures_bool_dict[new_train_df_np[i,cols_dict['content_id']]]\n    \n    pca = PCA(n_components=20)\n    pca.fit(tags_arr)\n    print('pca done')\n    return pca.components_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\ntrain_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', low_memory=False, nrows=50000, \n                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'float64', 'prior_question_elapsed_time': 'float32', \n                             'prior_question_had_explanation': 'boolean',\n                             }\n                      )\n\nlectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv', low_memory=False,\n                             dtype={'lecture_id': 'int64', 'part': 'int8', 'tag': 'int32', 'type_of': 'string'}\n                         )\n\nquestions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv', low_memory=False,\n                              dtype={'question_id': 'int64', 'bundle_id': 'int64', 'correct_answer': 'int8', 'part': 'int8', 'tags': 'string',\n                             }\n                          )\n\ndef get_data_ready(train_df, lectures_df, questions_df, train_data=True, most_important_tag_coefs=None):\n    cols_dict = create_cols_dict(train_df)\n    new_train_df_np = train_df.to_numpy(na_value=-1)\n    user_times = make_times_dict_np(new_train_df_np[:,cols_dict[\"timestamp\"]:cols_dict[\"user_id\"] + 1])\n    print('created user dict')\n    \n    #if(train_data): \n    new_lectures_df_np = lectures_df.to_numpy(na_value=-1)\n    lectures_cols_dict = create_cols_dict(lectures_df)\n    lectures_bool_dict = create_bool_dict(new_lectures_df_np, lectures_cols_dict, \"lecture_id\", \"tag\")\n    \n    new_questions_df_np = questions_df.to_numpy(na_value=-1)      \n    questions_cols_dict = create_cols_dict(questions_df)\n    questions_bool_dict = create_bool_dict(new_questions_df_np, questions_cols_dict, \"question_id\", \"tags\")  \n    \n    print()\n    print(lectures_cols_dict)\n    print(questions_cols_dict)\n    \n    del train_df\n    del lectures_df\n    del questions_df\n    \n    if most_important_tag_coefs is None:\n        most_important_tag_coefs = find_most_important_tags(new_train_df_np, cols_dict, \n        lectures_bool_dict, questions_bool_dict)\n    \n    new_train_df_np = categorical_to_binary(new_train_df_np, cols_dict, new_lectures_df_np, new_questions_df_np,\n                                            lectures_bool_dict, questions_bool_dict, most_important_tag_coefs, train_data)\n    print(\"converted categorical to binary\")\n    new_train_df_np = normalize_elapsed_time_np(new_train_df_np, cols_dict)\n    print('normalized elapsed times')\n    new_train_df_np = convert_times_to_float_np(new_train_df_np, cols_dict, user_times)\n    print('converted times to float')\n#   new_train_df = new_train_df.drop(['timestamp'], axis=1)\n    \n    print(cols_dict)\n        \n    print(\"dropped extraneous columns\")\n    #separate df into X and Y\n    if train_data:\n        y_column = new_train_df_np.T[cols_dict['answered_correctly']]\n        new_train_df_np = np.delete(new_train_df_np, cols_dict['answered_correctly'], axis=1)\n    else:\n        y_column = None\n        \n    if not train_data:\n        new_train_df_np = np.delete(new_train_df_np, cols_dict['prior_group_responses'], axis=1)\n        new_train_df_np = np.delete(new_train_df_np, cols_dict['prior_group_answers_correct'], axis=1)\n    else:\n        new_train_df_np = np.delete(new_train_df_np, cols_dict['user_answer'], axis=1)\n    new_train_df_np = np.delete(new_train_df_np, cols_dict['task_container_id'], axis=1)\n    new_train_df_np = np.delete(new_train_df_np, cols_dict['content_id'], axis=1)\n    new_train_df_np = np.delete(new_train_df_np, cols_dict['user_id'], axis=1)\n    new_train_df_np = np.delete(new_train_df_np, cols_dict['row_id'], axis=1)\n\n    print('cleaned data')\n    \n    return new_train_df_np, y_column, most_important_tag_coefs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(X_train, Y_train, most_important_tag_coefs) = get_data_ready(train_df, lectures_df, questions_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = Y_train.astype('float')\nprint(Y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport sklearn\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neural_network import MLPClassifier\n\nX_train_split, X_CV, Y_train_split, Y_CV = train_test_split(X_train, Y_train, test_size=0.2)\nclf = MLPClassifier(solver='adam', alpha=1e-5, hidden_layer_sizes=(5, 2), max_iter = 10000)\nclf.fit(X_train_split, Y_train_split)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_pred_proba = clf.predict_proba(X_CV)\nY_pred = clf.predict(X_CV)\nprint(Y_pred.shape)\nprint(Y_CV[0:100])\nprint(Y_pred_proba[0:100,:])\nprint(Y_pred[0:100])\naccuracy = clf.score(X_CV,Y_CV)\nprint(accuracy)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#env.predict(sample_prediction_df)\n# first_ind = 0\n# group_num = 0\ndel X_train\ndel Y_train \n\nfor (test_df, sample_prediction_df) in iter_test:\n    # test_df, sample_prediction_df = next(iter_test)\n    (X_test, Y_test, c) = get_data_ready(test_df, lectures_df, questions_df, False, most_important_tag_coefs)\n    X_test = X_test.astype('float')\n    # print(X_test)\n    prediction = clf.predict(X_test)\n    prediction[prediction==-1] = 0.5\n    test_df['answered_correctly'] = prediction\n    test_df['answered_correctly'] = test_df['answered_correctly']\n#   submission['row_id'] = list(range(first_ind, len(submission) + first_ind ))\n#   submission = submission.set_index('row_id')\n#   first_ind += len(submission)\n#   lst = [group_num]*len(submission)\n#   submission['group_num'] = lst\n    submission = test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']]\n    print(submission)\n    env.predict(submission)\n#   group_num += 1","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}