{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport tensorflow as tf\n\ntypes = {\n    'row_id': 'int64',\n    'timestamp': 'int64',\n    'user_id': 'int32',\n    'content_id': 'int16',\n    'content_type_id': 'int8',\n    'task_container_id': 'int16',\n    'user_answer': 'int8',\n    'answered_correctly': 'int8',\n    'prior_question_elapsed_time': 'float32', \n    'prior_question_had_explanation': 'boolean'\n}        \n\nnrows = 5000\n\n#train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', dtype=types, nrows=nrows)\n#test = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/example_test.csv', dtype=types, nrows=nrows)\n\n\ndata = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', dtype = types, nrows = 10000)\nx = int(0.7 * data.shape[0])\ny = int(0.3 * data.shape[0])\ntrain = data[0:x]\ntest = data[x:x+y]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_data(data, history=None, get_ans=True):\n    \n    if history is None:\n        history = data\n    \n    # wykluczenie wykładów\n    history_no_lectures = history[history.content_type_id == 0]\n    \n    # wykluczenie pytań\n    history_only_lectures = history[history.content_type_id == 1]\n    \n    # wykluczenie wykładów\n    no_lectures = data[data.content_type_id == 0] \n\n    # wykluczenie pytań\n    only_lectures = data[data.content_type_id == 1] \n    \n    # rozszerzenie tabeli pytań o skuteczność, ilość pytań, ilość poprawnych\n    questions_types = {\n    'question_id': 'int16',\n    'bundle_id': 'int16',\n    'correct_answer': 'int8',\n    'part': 'int8',\n    'tags': 'string'\n    }\n    questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv', dtype = questions_types)\n    questions.tags = [tags.split() if type(tags) is str else tags for tags in questions.tags.values]\n    df = history_no_lectures.groupby(['content_id']).agg({'answered_correctly': ['mean', 'count', 'sum']})\n    df = df['answered_correctly']\n    questions = questions.merge(df, left_on = 'question_id', right_on = 'content_id', how = \"left\")\n    questions = questions.fillna({'mean': 0.0, 'count': 0, 'sum': 0})\n    questions = questions.rename(columns={'mean': 'accuracy', 'count': 'count_all', 'sum': 'count_correct'})\n    \n    \n    # manipulacja tagami\n    questions_with_tags = questions[questions.tags.isna() == False]\n    tags = questions_with_tags.tags.values\n\n    tags_set = set([item for elem in tags for item in elem])\n    tags_list = list(tags_set)\n    tags_df = pd.DataFrame()\n\n    for tag in tags_list:\n        df = questions_with_tags[questions_with_tags.tags.apply(lambda l: tag in l)]\n        tmp_df = df.agg({'count_all': ['sum'], 'count_correct': ['sum']})\n        tmp_df['tag'] = tag\n        tmp_df['amount_questions_with_tag'] = len(df)\n        tmp_df = tmp_df.set_index('tag')\n        tags_df = tags_df.append(tmp_df)\n\n    tags_df['accuracy'] = tags_df['count_correct'] / tags_df['count_all']\n    tags_df = tags_df.sort_values(by='accuracy')\n    \n    \n    struct = [] # dane statystyczne o danych wejściowych (pytaniach)\n    struct_correct = [] # informacja czy na pytanie użytkownik odpowiedział poprawnie\n    # ^ długości tablic te same, odpowiadające sobie indeksy\n\n    for index, row in no_lectures.iterrows():\n        # --- prior_question_had_explanation\n        prior_question_had_explanation = int(row['prior_question_had_explanation']) if type(row['prior_question_had_explanation']) is bool else 0\n\n        # --- all_users_task_container_id_accuracy \n        df = history_no_lectures[history_no_lectures['task_container_id'] == row['task_container_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        all_users_task_container_id_accuracy = df.values[0][0]\n\n        # --- all_users_task_container_id_count\n        all_users_task_container_id_count = df.values[1][0]\n\n        # --- this_user_task_container_id_accuracy\n        df = history_no_lectures[history_no_lectures['task_container_id'] == row['task_container_id']]\n        df = df[df['user_id'] == row['user_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        this_user_task_container_id_accuracy = df.values[0][0]\n\n        # --- this_user_task_container_id_count\n        this_user_task_container_id_count = df.values[1][0]\n\n        # --- all_users_content_id_accuracy\n        df = history_no_lectures[history_no_lectures['content_id'] == row['content_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        all_users_content_id_accuracy = df.values[0][0]\n\n        # --- all_users_content_id_count\n        all_users_content_id_count = df.values[1][0]\n\n        # --- this_user_accuracy\n        df = history_no_lectures[history_no_lectures['user_id'] == row['user_id']]\n        df = df.agg({'answered_correctly': ['mean', 'count']})\n        this_user_accuracy = df.values[0][0]\n\n        # --- this_user_question_count\n        this_user_question_count = df.values[1][0]\n\n        # --- this_user_lectures_count\n        df = history_no_lectures[history_no_lectures['user_id'] == row['user_id']]\n        this_user_lectures_count = df.shape[0]\n\n        # --- this_user_lectures_count_task_container_id\n        df = df[df['task_container_id'] == row['task_container_id']]\n        this_user_lectures_count_task_container_id = df.shape[0]\n\n        # --- part\n        this_question = questions[questions['question_id'] == row['content_id']]\n        part = this_question['part'].values[0]\n\n        # --- all_users_part_accuracy\n        parts = questions[questions['part'] == part].agg({'count_all': ['sum'], 'count_correct': ['sum']})\n        parts['accuracy'] = parts['count_correct'] / parts['count_all']\n        all_users_part_accuracy = parts['accuracy'].values[0]\n\n        # --- most_difficult_tag_accuracy\n        this_question_tags = this_question.tags.values[0]\n        this_question_tags = tags_df[tags_df.index.isin(this_question_tags)]\n        most_difficult_tag_accuracy = this_question_tags['accuracy'].values[0]\n\n        # --- most_difficult_tag_count\n        most_difficult_tag_count = this_question_tags['count_all'].values[0]\n\n        # --- prior_question_elapsed_time\n        prior_question_elapsed_time = row['prior_question_elapsed_time']\n\n        # --- timestamp\n        timestamp = row['timestamp']\n\n        # --- all_users_answers_sd\n        users_answers = history_no_lectures[history_no_lectures['content_id'] == row['content_id']]\n        value_counts = users_answers['user_answer'].value_counts()\n        A = value_counts.values[value_counts.values == 1][0] if len(value_counts.values[value_counts.values == 1]) > 0 else 0\n        B = value_counts.values[value_counts.values == 2][0] if len(value_counts.values[value_counts.values == 2]) > 0 else 0\n        C = value_counts.values[value_counts.values == 3][0] if len(value_counts.values[value_counts.values == 3]) > 0 else 0\n        D = value_counts.values[value_counts.values == 4][0] if len(value_counts.values[value_counts.values == 4]) > 0 else 0\n        all_users_answers_sd = np.std([A, B, C, D])\n\n        struct.append({\n            'prior_question_had_explanation': prior_question_had_explanation,\n            'all_users_task_container_id_accuracy': all_users_task_container_id_accuracy,\n            'all_users_task_container_id_count': all_users_task_container_id_count,\n            'this_user_task_container_id_accuracy': this_user_task_container_id_accuracy,\n            'this_user_task_container_id_count': this_user_task_container_id_count,\n            'all_users_content_id_accuracy': all_users_content_id_accuracy,\n            'all_users_content_id_count': all_users_content_id_count,\n            'this_user_accuracy': this_user_accuracy,\n            'this_user_question_count': this_user_question_count,\n            'this_user_lectures_count': this_user_lectures_count,\n            'this_user_lectures_count_task_container_id': this_user_lectures_count_task_container_id,\n            'part': part,\n            'all_users_part_accuracy': all_users_part_accuracy,\n            'most_difficult_tag_accuracy': most_difficult_tag_accuracy,\n            'most_difficult_tag_count': most_difficult_tag_count,\n            'prior_question_elapsed_time': prior_question_elapsed_time,\n            'timestamp': timestamp,\n            'all_users_answers_sd': all_users_answers_sd  \n        })\n        \n        if get_ans == True:\n            struct_correct.append(row['answered_correctly'])\n        \n    return struct, struct_correct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_struct_raw, train_struct_correct_raw = get_data(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_struct = pd.DataFrame(train_struct_raw)\ntrain_struct = train_struct.fillna(0)\nprint(train_struct.dtypes) # brak object'ów\ntrain_struct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_struct_correct = np.array(train_struct_correct_raw)\nprint(train_struct_correct)\nprint(len(train_struct_correct))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_struct_raw, test_struct_correct_raw = get_data(test, train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_struct = pd.DataFrame(test_struct_raw)\ntest_struct = test_struct.fillna(0)\ntest_struct","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_struct_correct = np.array(test_struct_correct_raw)\nprint(test_struct_correct)\nprint(len(test_struct_correct))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential([tf.keras.layers.Dense(18, activation='sigmoid'), tf.keras.layers.Dense(12, activation='sigmoid'), tf.keras.layers.Dense(6, activation='sigmoid'), tf.keras.layers.Dense(1)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer='adam', loss='mean_squared_error', metrics='AUC')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_struct.values, train_struct_correct, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict(test_struct.values)\npredictions = pd.DataFrame(predictions)\npredictions.to_csv(path_or_buf = 'submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}