{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport riiideducation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.models.load_model('/kaggle/input/ver6-0/my_model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = pd.read_csv('/kaggle/input/ver6-0/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(data, history=None, get_ans=True):\n    \n    if history is None:\n        history = data\n    \n    # wykluczenie wykładów\n    history_no_lectures = history[history.content_type_id == 0]\n    \n    # wykluczenie pytań\n    history_only_lectures = history[history.content_type_id == 1]\n    \n    # wykluczenie wykładów\n    no_lectures = data[data.content_type_id == 0] \n\n    # wykluczenie pytań\n    only_lectures = data[data.content_type_id == 1] \n    \n    # rozszerzenie tabeli pytań o skuteczność, ilość pytań, ilość poprawnych\n    questions_types = {\n    'question_id': 'int16',\n    'bundle_id': 'int16',\n    'correct_answer': 'int8',\n    'part': 'int8',\n    'tags': 'string'\n    }\n    questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv', dtype = questions_types)\n    questions.tags = [tags.split() if type(tags) is str else tags for tags in questions.tags.values]\n    df = history_no_lectures.groupby(['content_id']).agg({'answered_correctly': ['mean', 'count', 'sum']})\n    df = df['answered_correctly']\n    questions = questions.merge(df, left_on = 'question_id', right_on = 'content_id', how = \"left\")\n    questions = questions.fillna({'mean': 0.0, 'count': 0, 'sum': 0})\n    questions = questions.rename(columns={'mean': 'accuracy', 'count': 'count_all', 'sum': 'count_correct'})\n    \n    \n    # manipulacja tagami\n    questions_with_tags = questions[questions.tags.isna() == False]\n    tags = questions_with_tags.tags.values\n\n    tags_set = set([item for elem in tags for item in elem])\n    tags_list = list(tags_set)\n    tags_df = pd.DataFrame()\n\n    for tag in tags_list:\n        df = questions_with_tags[questions_with_tags.tags.apply(lambda l: tag in l)]\n        tmp_df = df.agg({'count_all': ['sum'], 'count_correct': ['sum']})\n        tmp_df['tag'] = tag\n        tmp_df['amount_questions_with_tag'] = len(df)\n        tmp_df = tmp_df.set_index('tag')\n        tags_df = tags_df.append(tmp_df)\n\n    tags_df['accuracy'] = tags_df['count_correct'] / tags_df['count_all']\n    tags_df = tags_df.sort_values(by='accuracy')\n    \n    \n    struct = [] # dane statystyczne o danych wejściowych (pytaniach)\n    struct_correct = [] # informacja czy na pytanie użytkownik odpowiedział poprawnie\n    # ^ długości tablic te same, odpowiadające sobie indeksy\n\n    for index, row in no_lectures.iterrows():\n        # --- prior_question_had_explanation\n        prior_question_had_explanation = int(row['prior_question_had_explanation']) if type(row['prior_question_had_explanation']) is bool else 0\n\n        # --- all_users_task_container_id_accuracy \n        df = history_no_lectures[history_no_lectures['task_container_id'] == row['task_container_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        all_users_task_container_id_accuracy = df.values[0][0]\n\n        # --- all_users_task_container_id_count\n        all_users_task_container_id_count = df.values[1][0]\n\n        # --- this_user_task_container_id_accuracy\n        df = history_no_lectures[history_no_lectures['task_container_id'] == row['task_container_id']]\n        df = df[df['user_id'] == row['user_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        this_user_task_container_id_accuracy = df.values[0][0]\n\n        # --- this_user_task_container_id_count\n        this_user_task_container_id_count = df.values[1][0]\n\n        # --- all_users_content_id_accuracy\n        df = history_no_lectures[history_no_lectures['content_id'] == row['content_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        all_users_content_id_accuracy = df.values[0][0]\n\n        # --- all_users_content_id_count\n        all_users_content_id_count = df.values[1][0]\n\n        # --- this_user_accuracy\n        df = history_no_lectures[history_no_lectures['user_id'] == row['user_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        this_user_accuracy = df.values[0][0]\n\n        # --- this_user_question_count\n        this_user_question_count = df.values[1][0]\n\n        # --- this_user_lectures_count\n        df = history_no_lectures[history_no_lectures['user_id'] == row['user_id']]\n        this_user_lectures_count = df.shape[0]\n\n        # --- this_user_lectures_count_task_container_id\n        df = df[df['task_container_id'] == row['task_container_id']]\n        this_user_lectures_count_task_container_id = df.shape[0]\n\n        # --- part\n        this_question = questions[questions['question_id'] == row['content_id']]\n        part = this_question['part'].values[0]\n\n        # --- all_users_part_accuracy\n        parts = questions[questions['part'] == part].agg({'count_all': ['sum'], 'count_correct': ['sum']})\n        parts['accuracy'] = parts['count_correct'] / parts['count_all']\n        all_users_part_accuracy = parts['accuracy'].values[0]\n\n        # --- most_difficult_tag_accuracy\n        this_question_tags = this_question.tags.values[0]\n        this_question_tags = tags_df[tags_df.index.isin(this_question_tags)]\n        most_difficult_tag_accuracy = this_question_tags['accuracy'].values[0]\n\n        # --- most_difficult_tag_count\n        most_difficult_tag_count = this_question_tags['count_all'].values[0]\n\n        # --- prior_question_elapsed_time\n        prior_question_elapsed_time = row['prior_question_elapsed_time']\n\n        # --- timestamp\n        timestamp = row['timestamp']\n\n        # --- all_users_answers_sd\n        users_answers = history_no_lectures[history_no_lectures['content_id'] == row['content_id']]\n        value_counts = users_answers['user_answer'].value_counts()\n        A = value_counts.values[value_counts.values == 1][0] if len(value_counts.values[value_counts.values == 1]) > 0 else 0\n        B = value_counts.values[value_counts.values == 2][0] if len(value_counts.values[value_counts.values == 2]) > 0 else 0\n        C = value_counts.values[value_counts.values == 3][0] if len(value_counts.values[value_counts.values == 3]) > 0 else 0\n        D = value_counts.values[value_counts.values == 4][0] if len(value_counts.values[value_counts.values == 4]) > 0 else 0\n        all_users_answers_sd = np.std([A, B, C, D])\n\n        struct.append({\n            'prior_question_had_explanation': prior_question_had_explanation,\n            'all_users_task_container_id_accuracy': all_users_task_container_id_accuracy,\n            'all_users_task_container_id_count': all_users_task_container_id_count,\n            'this_user_task_container_id_accuracy': this_user_task_container_id_accuracy,\n            'this_user_task_container_id_count': this_user_task_container_id_count,\n            'all_users_content_id_accuracy': all_users_content_id_accuracy,\n            'all_users_content_id_count': all_users_content_id_count,\n            'this_user_accuracy': this_user_accuracy,\n            'this_user_question_count': this_user_question_count,\n            'this_user_lectures_count': this_user_lectures_count,\n            'this_user_lectures_count_task_container_id': this_user_lectures_count_task_container_id,\n            'part': part,\n            'all_users_part_accuracy': all_users_part_accuracy,\n            'most_difficult_tag_accuracy': most_difficult_tag_accuracy,\n            'most_difficult_tag_count': most_difficult_tag_count,\n            'prior_question_elapsed_time': prior_question_elapsed_time,\n            'timestamp': timestamp,\n            'all_users_answers_sd': all_users_answers_sd  \n        })\n        \n        if get_ans == True:\n            struct_correct.append(row['answered_correctly'])\n        \n    return struct, struct_correct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def do_predict_df(test_df, sample_prediction_df):\n    if sample_prediction_df.empty:\n        return sample_prediction_df\n\n    data = get_data(test_df, history, False)[0]\n    data = pd.DataFrame(data)\n    data = data.fillna(0)\n    predictions = model.predict(data.values)\n    df_tmp = sample_prediction_df.reset_index()\n    for index in range(len(predictions)):\n        df_tmp.loc[index, 'answered_correctly'] = predictions[index][0]\n        \n    return df_tmp.set_index('group_num')\n\nfor t, (test_df, sample_prediction_df) in enumerate(iter_test):\n    predict_df = do_predict_df(test_df, sample_prediction_df)\n    env.predict(predict_df)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}