{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pandasql as ps\nimport subprocess\nfrom tqdm import tqdm\nimport os\nfrom multiprocessing import Pool, Process, Manager\nfrom sklearn.preprocessing import LabelEncoder\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.metrics import TruePositives, FalsePositives, TrueNegatives, FalseNegatives, AUC, BinaryAccuracy\nfrom tensorflow.keras.layers import  Dense, GRU, Input,Embedding, Flatten, BatchNormalization, Dropout\nfrom tensorflow.keras.optimizers import RMSprop, Adam, SGD \nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.utils import class_weight\nfrom sklearn.preprocessing import MinMaxScaler,OneHotEncoder,StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import make_scorer, roc_auc_score,classification_report\nfrom sklearn.tree import DecisionTreeClassifier\nfrom xgboost import XGBClassifier, plot_importance\nfrom sklearn.neighbors import KNeighborsClassifier, NearestNeighbors\nimport random\nfrom category_encoders import TargetEncoder\nimport random\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nimport warnings\nwarnings.filterwarnings('ignore')\nimport time\nfrom multiprocessing.pool import ThreadPool\nimport multiprocessing as mp\nimport gc\nfrom multiprocessing import Pool","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_train_csv = \"../input/riiid-train-data-multiple-formats/riiid_train.feather\"\npath_question = \"../input/riiid-test-answer-prediction/questions.csv\"\npath_lecture = \"../input/riiid-test-answer-prediction/lectures.csv\"\npath_stat_user = \"../input/riid-zefir/results_u_v2.csv\"\npath_stat_context = \"../input/riid-zefir/results_content_v2.csv\"\nLIST_BD = [\"1/\",\"3/\",\"4/\",\"5/\",\"6/\",\"7/\"]\nLIST_BD2 = [\"1/\",\"2/\",\"3/\",\"4/\",\"5/\",\"6/\",\"7/\",\"8/\"]\nPREOR_TESD_DF = 0\nCONTEXT_STAT3 = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION = pd.read_csv(path_question)\nLECTURES = pd.read_csv(path_lecture)\nDATA = pd.read_feather(path_train_csv)\nUNIQUE_USER = DATA[\"user_id\"].unique().tolist()\n# test_user = [*UNIQUE_USER[:100],\n#              *UNIQUE_USER[60000:61000],\n#              *UNIQUE_USER[110000:110500],\n#              *UNIQUE_USER[360000:360500]\n#             ]\n# train_user = [\n#              *UNIQUE_USER[80000:80110],\n#              *UNIQUE_USER[380000:380110]\n#             ]\n\nUSER_STAT = pd.read_csv(path_stat_user)\n# CONTEXT_STAT = pd.read_csv(path_stat_context)\n# DATA_TRAIN = DATA[DATA.user_id.isin(test_user)]\n# DATA_VAL = DATA[DATA.user_id.isin(train_user)]\nDATA_TRAIN = DATA[5000000:6300000]\nDATA_VAL = DATA[900000:905000]\nTEST = pd.read_csv('../input/riiid-test-answer-prediction/example_test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_VAL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds = DATA_VAL[DATA_VAL.user_id==18789909]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ds1 = ds[ds.content_type_id==0]\nds2 = ds[ds.content_type_id==1]\nds2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LECTURES[\"tag\"].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LECTURES[LECTURES.lecture_id==185]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION[QUESTION.question_id==185]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TODODEL = DATA[:DATA.shape[0]*70//100]\nCONTEXT_STAT = TODODEL.loc[TODODEL.content_type_id==0][[\"content_id\",\"answered_correctly\"]].groupby('content_id').agg([\"mean\",\"sum\",\"count\",\"std\"])\nCONTEXT_STAT.columns = [\"answered_correctly_content_mean\",\"answered_correctly_content_sum\",\"answered_correctly_content_count\",\"answered_correctly_content_std\"]\n# CONTEXT_STAT\nMEAN_ANSVER_CONTENT_CORRECT = CONTEXT_STAT[\"answered_correctly_content_mean\"].mean()\nMEAN_ANSVER_CONTENT_CORRECT_SUM = CONTEXT_STAT[\"answered_correctly_content_sum\"].mean()\nMEAN_ANSVER_CONTENT_CORRECT_COUNT = CONTEXT_STAT[\"answered_correctly_content_count\"].mean()\nMEAN_ANSVER_CONTENT_CORRECT_STD = CONTEXT_STAT[\"answered_correctly_content_std\"].mean()\ndel TODODEL, DATA\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DICT_USER_ID_PATH = {user_id:path_zefir+dir_in_zefir for path_zefir in LIST_BD for dir_in_zefir in os.listdir(\"../input/zefir\"+path_zefir) for user_id in os.listdir(\"../input/zefir\"+path_zefir+dir_in_zefir)}\nDICT_USER_ID_PATH2 = {user_id:path_zefir+dir_in_zefir for path_zefir in LIST_BD2 for dir_in_zefir in os.listdir(\"../input/zefir\"+path_zefir) for user_id in os.listdir(\"../input/zefir\"+path_zefir+dir_in_zefir)}\nKEY = DICT_USER_ID_PATH2.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# start_time= time.time()\n# USER_ID = TEST[\"user_id\"].unique().tolist()\n# # USER_ID = f\n# USER_ID = [191139161]\n# USER_ID2 = {USER:\"../input/zefir\"+DICT_USER_ID_PATH[str(USER)]+\"/\"+str(USER) for USER in USER_ID if str(USER) in KEY}\n# print(USER_ID2)\n# ile_list = list(USER_ID2.values())\n# print(ile_list)\n# if len(USER_ID2)>0:\n#     file_list = list(USER_ID2.values())\n#     print(file_list)\n\n\n#     result_context = 0\n#     ind = 0\n\n\n#     for file in file_list:\n#         if ind==0:\n#             result_context = pd.read_feather(file)\n#             ind=1\n#         else:\n#             result_context = result_context.append(pd.read_feather(file), ignore_index = True)\n\n#     result_context = result_context[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])   \n# print(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# start_time= time.time()\n# USER_ID = TEST[\"user_id\"].unique().tolist()\n# # USER_ID = f\n# position = []\n# USER_ID2 = {USER:\"../input/zefir\"+DICT_USER_ID_PATH[str(USER)]+\"/\"+str(USER) for USER in USER_ID if str(USER) in KEY}\n# file_list = list(USER_ID2.values())\n# # print(file_list)\n# def read_feather(file):\n#     pp = np.full(13523, np.nan)\n#     d = pd.read_feather(file)\n# #     print(d)\n# #     pp[d[\"content_id\"].values] = d[\"answered_correctly\"].values\n#     return d\n# pool = Pool(3)\n# df_collection = pool.map(read_feather, file_list)\n# result_context = 0\n# ind = 0\n# # print(len(df_collection))\n# for d in df_collection:\n#     if ind==0:\n#         result_context = d\n#         ind=1cxv \n#     else:\n#         result_context = result_context.append(d, ignore_index = True)\n\n# result_context = result_context[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\n\n# # df_collection = np.array(df_collection)\n# # CONTEXT_STAT2['answered_correctly_content_mean'] = np.nanmean(df_collection,axis=0)\n# # pool.close()\n# # pool.join()   \n# print(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"USER_STAT.columns = ['user_id',\n                     'answered_correctly_mean',\n                     'answered_correctly_sum',\n                     'answered_correctly_count',\n                     'answered_correctly_std',\n                     'content_type_id_sum',\n                     'content_type_id_count',\n                     'timestamp_max',\n                     'timestamp_mean',\n                     'prior_question_elapsed_time_max',\n                     'prior_question_elapsed_time_mean',\n                     'prior_question_elapsed_time_std',\n                    ]\nUSER_STAT = USER_STAT[1:]\ndata_types_dict = {\n    'user_id': 'uint64',\n    'answered_correctly_mean': 'float32',\n    'answered_correctly_sum': 'uint64',\n    'answered_correctly_count': 'uint64',\n    'answered_correctly_std': 'float32',\n    'content_type_id_sum': 'uint64',\n    'content_type_id_count': 'uint64',\n    'timestamp_max': 'uint64',\n    'timestamp_mean': 'float32',\n    'prior_question_elapsed_time_max': 'uint64',\n    'prior_question_elapsed_time_mean': 'float32',\n    'prior_question_elapsed_time_std': 'float32'\n}\nUSER_STAT = USER_STAT.astype(data_types_dict)\nUSER_STAT = USER_STAT.groupby('user_id').agg({'answered_correctly_mean': 'mean',\n                                              'answered_correctly_sum': 'mean',\n                                              'answered_correctly_count': 'mean',\n                                              'answered_correctly_std': 'mean',\n                                              'content_type_id_sum': 'mean',\n                                              'content_type_id_count': 'mean',\n                                              'timestamp_max': 'mean',\n                                              'timestamp_mean': 'mean',\n                                              'prior_question_elapsed_time_max': 'mean',\n                                              'prior_question_elapsed_time_mean': 'mean',\n                                              'prior_question_elapsed_time_std': 'mean',\n                                             })\n# CONTEXT_STAT.columns = [[\"content_id\",\"answered_correctly_content_mean\",\"answered_correctly_content_count\",\"answered_correctly_content_std\"]]\n# CONTEXT_STAT = CONTEXT_STAT[1:]\n# data_types_dict2 = {\n#     'content_id': 'int64',\n#     'answered_correctly_content_mean': 'float32',\n#     'answered_correctly_content_count': 'unit8',\n#     'answered_correctly_content_std': 'float32'\n# }\n# CONTEXT_STAT = CONTEXT_STAT.astype(data_types_dict2)\n\n\n# CONTEXT_STAT = CONTEXT_STAT.set_index(\"content_id\")\n#FILL NA DATA\n\nMEAN_ANSVER_USER_CORRECT = USER_STAT[\"answered_correctly_mean\"].mean()\nMEAN_SUM = USER_STAT[\"answered_correctly_sum\"].mean()\nMEAN_COUNT = USER_STAT[\"answered_correctly_count\"].mean()\nMEAN_STD = USER_STAT[\"answered_correctly_std\"].mean()\n\nMEAN_CONTENT_TYPE_ID = USER_STAT[\"content_type_id_sum\"].mean()\nMEAN_CONTENT_TYPE_ID_COUNT = USER_STAT[\"content_type_id_count\"].mean()\n\nMEAN_TIMSTEP_MAX = USER_STAT[\"timestamp_max\"].mean()\nMEAN_TIMSTEP_MEAN = USER_STAT[\"timestamp_mean\"].mean()\n\nMEAN_PRIOR_QUESTION_TIME_MAX = USER_STAT[\"prior_question_elapsed_time_max\"].mean()\nMEAN_PRIOR_QUESTION_TIME_MEAN = USER_STAT[\"prior_question_elapsed_time_mean\"].mean()\nMEAN_PRIOR_QUESTION_TIME_STD = USER_STAT[\"prior_question_elapsed_time_std\"].mean()\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"column_question = [str(i)+\"q\" for i in range(189)]\ncolumn_question2 = [str(i)+\"q2\" for i in range(189)]\ncolumn_question3 = [str(i)+\"q3\" for i in range(189)]\ncolumn_question4 = [str(i)+\"q4\" for i in range(189)]\ncolumn_question5 = [str(i)+\"q5\" for i in range(189)]\ncolumn_question6 = [str(i)+\"q6\" for i in range(189)]\n\ncolumn_lectures = [str(i)+\"l\" for i in range(189)]\npart_q=['1_part_q', '2_part_q', '3_part_q', '4_part_q','5_part_q', '6_part_q', '7_part_q']\ncol_fit2 = [\"row_id\",\n            \"timestamp\",\n            \"user_id\",\n            \"content_id\",\n            \"task_container_id\",\n            \"prior_question_elapsed_time\",\n            \"prior_question_had_explanation\",\n#             \"prior_question_had_explanation2\",\n#             \"part_q\",\n#             \"answered_correctly_content_mean\"\n#             \"part\",\n#             \"count_them\",\n#             \"count_them2\",\n#             \"count_them3\",\n#             'count_them4',\n#             'count_them5',\n#             'task_container_id2',\n#             'task_container_id3',\n#             'answer_in_the_past',\n#             'answer_in_the_past2',\n#             'answer_in_the_past3',\n#             'answer_in_the_past4',\n#             'percent1',\n#             'percent2',\n#             'percent3',\n#             'percent4',\n#             'percent5',\n#             'percent6',\n#             'answer_in_the_part',\n#             'answer_in_the_part2',\n#             'answer_in_the_part3',\n#             'answer_in_the_part4',\n#             'count_part1',\n#             'count_part2',\n#             'count_part3',\n#             'count_part4'\n            \n#             'answered_correctly2'\n           ]\ncol_fit2 = [*col_fit2,*[\"answered_correctly_mean\",\"answered_correctly_sum\",\"answered_correctly_count\"]]\n\ncol_fit2 = [*col_fit2,*CONTEXT_STAT.columns.tolist()]\n\n# col_fit2 = [*col_fit2,*column_question]\n# col_fit2 = [*col_fit2,*[\"answered_correctly_content_mean\"]]\n# col_fit2 = [*col_fit2, *[\"concept\", \"intention\", \"question\", \"solving\", \"starter\"]]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def segmentation(dataframe,name_columuns,column,separator,discharge=True):\n    tags_seq = dataframe[column].values.tolist()\n    pp = np.zeros((len(tags_seq), len(name_columuns)), dtype=np.uint8)\n    if discharge:\n        for i,tag_not_split in tqdm(enumerate(tags_seq)):\n            for split_tag in tag_not_split.split(separator):\n                rr= int(split_tag)\n#                 print(rr)\n                pp[i][rr]=1\n    else:\n        for i,tag_not_split in tqdm(enumerate(tags_seq)):\n            mas = tag_not_split.split(separator)\n            pp[i][:len(mas)]=mas\n    return pp\nQUESTION.columns = [\"question_id\",\"bundle_id\",\"correct_answer\",\"part_q\",\"tag_q\"]\nQUESTION[\"part_q2_str\"] = QUESTION['part_q'].astype(\"str\")\nQUESTION['tag_q'] = QUESTION['tag_q'].astype(\"str\")\nLECTURES['tag'] = LECTURES['tag'].astype(\"str\")\nQUESTION['tag_q'] = QUESTION['tag_q'].replace(\"nan\" ,\"188\")\nQUESTION[column_question]=segmentation(QUESTION,column_question,\"tag_q\",\" \",discharge=True)\nLECTURES[column_lectures]=segmentation(LECTURES,column_lectures,\"tag\",\" \",discharge=True)\nvectorizer = CountVectorizer()\nvectorizer2 = CountVectorizer(strip_accents='ascii')\nX = vectorizer.fit_transform(LECTURES['type_of'])\nLECTURES[vectorizer.get_feature_names()] = X.toarray()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmentation2(dataframe,column,split=True,prefix=\"\"):\n    dataframe[column] = dataframe[column].astype(str)\n    dataframe[column]=dataframe[column]+prefix\n    TAGS = dataframe[column].unique()\n    TAGS_UNIQUE = []\n    for tag in tqdm(TAGS):\n        if split:\n            TAGS_UNIQUE.extend(tag.split())\n        else:\n            TAGS_UNIQUE.append(tag)\n    \n    TAGS_UNIQUE = np.unique(TAGS_UNIQUE)\n    dataframe[TAGS_UNIQUE]=0\n    def calculate_all(row):\n        if split:\n            row[row[column].split()]=1\n        else:\n            \n            row[row[column]]=1\n        return row\n    dataframe = dataframe.apply(calculate_all, axis=1)\n    dataframe.drop([column], axis=1, inplace=True)\n    return dataframe\nQUESTION = sigmentation2(QUESTION,\"part_q2_str\",False,\"_part_q\")\n# LECTURES = sigmentation(LECTURES,\"tag\",True,\"_lec\")\n# LECTURES = sigmentation(LECTURES,\"type_of\",False)\n# add_question_coumn = list(QUESTION.columns[4:])\n# add_question_coumn2 = [\"bundle_id\",\"part\"]\n# add_lecures_columns =  ['tag']\n# add_lecures_columns2 = [\"part\",\"concept\",\"intention\",\"solving question\",\"starter\"]\n# add_lecures_columns3 = [\"part2\",\"concept\",\"intention\",\"solving question\",\"starter\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_TRAIN[\"prior_question_had_explanation\"].fillna(False, inplace=True)\nencoders = LabelEncoder()\nencoders.fit(DATA_TRAIN[\"prior_question_had_explanation\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_sklearn():\n#     return SGDClassifier(loss='log')\n#     return NearestNeighbors(n_neighbors=10)\n#     return MLPClassifier(random_state=1, max_iter=300)\n    return XGBClassifier(\n    tree_method=\"hist\",\n    learning_rate=0.08,\n#     gamma=0.1,\n    n_estimators=200,\n    max_depth=3,\n#     min_child_weight=40,\n#     subsample=0.87,\n#     colsample_bytree=0.95,\n#     reg_alpha=0.04,\n#     reg_lambda=0.073,\n    objective='binary:logistic',\n    nthread=4,\n    scale_pos_weight=0.8,\n    seed=27)\n\n#     return DecisionTreeClassifier(min_samples_leaf=28, min_samples_split=28, max_depth=150)\n#     return RandomForestClassifier(n_estimators=200)\n#                                   min_samples_leaf=8, min_samples_split=5, max_depth=60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION[\"content_id\"] = QUESTION['question_id']\nQUESTION.drop([\"question_id\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def constructFitDataFrame(df,question, types=\"TRAIN\"):\n    global CONTEXT_STAT3\n    df[\"prior_question_had_explanation\"].fillna(False, inplace=True)\n    df['prior_question_had_explanation'] = encoders.transform(df['prior_question_had_explanation'])\n    df = df.loc[df['content_type_id'] == 0]\n    df.reset_index(drop=True, inplace=True)\n    df = df.merge(QUESTION, how = 'left', on=['content_id'])\n    #     #все темы на которые отвечал пользователь\n#     all_answer_theme = df.groupby('user_id')[column_question].apply(lambda x: x.cumsum())#\n#     all_answer_theme[\"user_id\"] = df[\"user_id\"]\n#     #предыдущие вопросы\n#     past_answer_theme = df.groupby('user_id')[column_question].apply(lambda x:x.shift(1))\n#     past_answer_theme[\"user_id\"] = df[\"user_id\"]\n#     past_answer_theme[\"prior_question_had_explanation\"] = df[\"prior_question_had_explanation\"]\n#     #предыдущие темы на которые отвечал пользователь\n#     past_all_answer_theme = all_answer_theme.groupby('user_id')[column_question].apply(lambda x:x.shift(1))\n#     past_all_answer_theme[\"user_id\"] = df[\"user_id\"]\n#     #предыдущие вопросы на которые видел ответы\n#     past_answer_theme_see_answer = past_answer_theme[column_question].multiply(past_answer_theme['prior_question_had_explanation'], axis=\"index\")\n#     past_answer_theme_see_answer[\"user_id\"] = df[\"user_id\"]\n#     #все предыдущие темы вопросов на которые видел ответы\n#     cumsum_past_answer_theme_see_answer = past_answer_theme_see_answer.groupby('user_id')[column_question].apply(lambda x: x.cumsum())\n#     # все выпросы на которые в настоящее время видет ответы\n#     question_theme_see_answer = df[column_question].multiply(df['prior_question_had_explanation'], axis=\"index\")\n#     question_theme_see_answer[\"user_id\"] = df[\"user_id\"]\n#     question_theme_see_answer = question_theme_see_answer.groupby('user_id')[column_question].apply(lambda x: x.cumsum())\n    \n#     #все part на которые отвечал пользователь\n#     all_answer_part = df.groupby('user_id')[part_q].apply(lambda x: x.cumsum())#\n#     all_answer_part[\"user_id\"] = df[\"user_id\"]\n#     #предыдущие part\n#     past_answer_part = df.groupby('user_id')[part_q].apply(lambda x:x.shift(1))\n#     past_answer_part[\"user_id\"] = df[\"user_id\"]\n#     past_answer_part[\"prior_question_had_explanation\"] = df[\"prior_question_had_explanation\"]\n#     #предыдущие part на которые отвечал пользователь\n#     past_all_answer_part = all_answer_part.groupby('user_id')[part_q].apply(lambda x:x.shift(1))\n#     past_all_answer_part[\"user_id\"] = df[\"user_id\"]\n#     #предыдущие part на которые видел ответы\n#     past_answer_theme_see_part = past_answer_part[part_q].multiply(past_answer_part['prior_question_had_explanation'], axis=\"index\")\n#     past_answer_theme_see_part[\"user_id\"] = df[\"user_id\"]\n#     #все предыдущие темы вопросов на которые видел ответы\n#     cumsum_past_answer_part_see_answer = past_answer_theme_see_part.groupby('user_id')[part_q].apply(lambda x: x.cumsum())\n#     # все выпросы на которые в настоящее время видет ответы\n#     question_theme_see_part = df[part_q].multiply(df['prior_question_had_explanation'], axis=\"index\")\n#     question_theme_see_part[\"user_id\"] = df[\"user_id\"]\n#     question_theme_see_part = question_theme_see_part.groupby('user_id')[part_q].apply(lambda x: x.cumsum())\n    \n#     d=0\n#     d2=0\n#     d3=0\n#     d4=0\n#     part1=0\n#     part2=0\n#     part3=0\n#     part4=0\n#     for i in range(1,8):\n#         part = df[str(i)+'_part_q']\n#         see_answer = df[\"prior_question_had_explanation\"]\n#         column = str(i)+'_part_q'\n#         if i==0:\n#             part1=part*past_answer_part[column].gt(0)*see_answer\n#             part2 =part*past_all_answer_part[column].gt(0)\n#             part3 =part*cumsum_past_answer_part_see_answer[column].gt(0)\n#             part4 = part*question_theme_see_part[column].gt(0)\n#         else:\n#             part1 +=part*past_answer_part[column].gt(0)*see_answer\n#             part2 +=part*past_all_answer_part[column].gt(0)\n#             part3 +=part*cumsum_past_answer_part_see_answer[column].gt(0)\n#             part4 += part*question_theme_see_part[column].gt(0)\n        \n        \n#     for i in range(189):\n#         theme = df[str(i)+\"q\"]\n#         see_answer = df[\"prior_question_had_explanation\"]\n#         column = str(i)+\"q\"\n#         if i==0:\n#             d=theme*past_answer_theme[column].gt(0)*see_answer\n#             d2 =theme*past_all_answer_theme[column].gt(0)\n#             d3 =theme*cumsum_past_answer_theme_see_answer[column].gt(0)\n#             d4 = theme*question_theme_see_answer[column].gt(0)\n#         else:\n#             d+=theme*past_answer_theme[column].gt(0)*see_answer\n#             d2 +=theme*past_all_answer_theme[column].gt(0)\n#             d3 +=theme*cumsum_past_answer_theme_see_answer[column].gt(0)\n#             d4 += theme*question_theme_see_answer[column].gt(0)\n#     df[\"count_them\"] = df[column_question].sum(axis=1) #count theme in the question\n#     df[\"count_them2\"] = all_answer_theme[column_question].sum(axis=1) # all the theme in the question\n#     df['count_them3'] = all_answer_theme[column_question].gt(0).sum(axis=1) # unique count theme in question\n#     df[\"count_them4\"] =  df[\"count_them2\"].shift(1)\n#     df[\"count_them5\"] =  df[\"count_them3\"].shift(1)\n    \n#     df['answer_in_the_past'] = d #видел ли ответы строго на предыдущие вопросы\n#     df['answer_in_the_past2'] = d2 # отвечал ли на темы вопросов в прошлом\n#     df['answer_in_the_past3'] = d3 # отвечал ли на темы вопросов в прошлом и видел когда-то на них ответы\n#     df['answer_in_the_past4'] = d4 # отвечал на эти вопорсы и видел ответы в момент ответа\n#     df['percent1'] = d*100/df[\"count_them\"] # процент знания сторого предыдущих тем вопросов\n#     df['percent2'] = d2*100/df[\"count_them\"] # процент опыта ответов на прошлые вопросы вне зависимоти от того видел ли ответы\n#     df['percent3'] = d3*100/df[\"count_them\"] # процент знания тем вопосов за весь период обучения\n#     df['percent4'] = d4*100/df[\"count_them\"] # порцент знания\n#     df['percent5'] = df[\"count_them3\"]*100/df[\"count_them\"]\n#     df['percent6'] = df[\"count_them4\"]*100/df[\"count_them\"]\n#     df['percent7'] = df[\"count_them5\"]*100/df[\"count_them\"]\n    \n    \n#     df['answer_in_the_part'] = part1 #видел ли ответы строго на предыдущие part\n#     df['answer_in_the_part2'] = part2 #видел ли ответы строго на предыдущие part\n#     df['answer_in_the_part3'] = part3 #видел ли ответы строго на предыдущие part\n#     df['answer_in_the_part4'] = part4 #видел ли ответы строго на предыдущие part\n    \n# #     df[\"count_part\"] = df[column_question].sum(axis=1) #count theme in the question\n#     df[\"count_part1\"] = all_answer_part[part_q].sum(axis=1) # all the theme in the question\n#     df['count_part2'] = all_answer_part[part_q].gt(0).sum(axis=1) # unique count theme in question\n#     df[\"count_part3\"] =  df[\"count_part1\"].shift(1)\n#     df[\"count_part4\"] =  df[\"count_part2\"].shift(1)\n    \n    \n    \n#     df[\"prior_question_had_explanation2\"] = df.groupby('user_id')[\"prior_question_had_explanation\"].apply(lambda x: x.cumsum())\n#     df = df.loc[df['content_type_id'] == 0]\n#     # #     df.reset_index(drop=True, inplace=True)\n#     df['task_container_id2'] = df.groupby('user_id')['task_container_id'].transform(lambda x:x.shift(1))\n#     df.loc[df.task_container_id==df.task_container_id2,[\"task_container_id3\"]] = 1\n#     df = df.merge(USER_STAT, on=['user_id'], how=\"left\")\n#     RAND_CORR = random.uniform(MEAN_ANSVER_USER_CORRECT, 0.65)\n#     df[\"answered_correctly_mean\"].fillna(RAND_CORR,inplace=True)\n#     df[\"answered_correctly_sum\"].fillna(MEAN_SUM,inplace=True)\n#     df[\"answered_correctly_count\"].fillna(MEAN_COUNT,inplace=True)\n#     df[\"answered_correctly_std\"].fillna(MEAN_STD,inplace=True)\n\n\n#     df[\"content_type_id_sum\"].fillna(MEAN_CONTENT_TYPE_ID,inplace=True)\n#     df[\"content_type_id_count\"].fillna(MEAN_CONTENT_TYPE_ID_COUNT,inplace=True)\n\n#     df[\"timestamp_max\"].fillna(MEAN_TIMSTEP_MAX,inplace=True)\n#     df[\"timestamp_mean\"].fillna(MEAN_TIMSTEP_MEAN,inplace=True)\n#     df[\"prior_question_elapsed_time_max\"].fillna(MEAN_PRIOR_QUESTION_TIME_MAX,inplace=True)\n#     df[\"prior_question_elapsed_time_mean\"].fillna(MEAN_PRIOR_QUESTION_TIME_MEAN,inplace=True)\n#     df[\"prior_question_elapsed_time_std\"].fillna(MEAN_PRIOR_QUESTION_TIME_STD,inplace=True)\n\n    \n    USER_ID = df[\"user_id\"].unique().tolist()\n    USER_ID2 = {USER:\"../input/zefir\"+DICT_USER_ID_PATH2[str(USER)]+\"/\"+str(USER) for USER in USER_ID if str(USER) in KEY}\n    CONTEXT_STAT2 = 0\n    #если есть хотябы один пользователь в базе zefir 1-8\n    #то загружаю их в CONTEXT_STAT2\n    if len(USER_ID2)>0:\n        file_list = list(USER_ID2.values())\n        ind = 0\n        for file in file_list:\n            user_id = file.split(\"/\")\n            user_id = int(user_id[len(user_id)-1])\n            if ind==0:\n                CONTEXT_STAT2 = pd.read_feather(file)\n                CONTEXT_STAT2[\"user_id\"] = user_id\n                ind=1\n            else:\n                CS = pd.read_feather(file)\n                CS[\"user_id\"] = user_id\n                CONTEXT_STAT2 = CONTEXT_STAT2.append(CS, ignore_index = True)\n        #если есть данные из набора test то добавляю их в CONTEXT_STAT2\n        if type(CONTEXT_STAT3)!=int:\n                ADD_INFORMATION = CONTEXT_STAT3[CONTEXT_STAT3.user_id.isin(USER_ID)]\n                ADD_INFORMATION = ADD_INFORMATION.loc[ADD_INFORMATION.content_type_id==0]\n                CONTEXT_STAT2 = CONTEXT_STAT2.append(ADD_INFORMATION, ignore_index=True)\n        # статистика  по ответам пользователя        \n        USER_STAT2 = CONTEXT_STAT2[['user_id','answered_correctly']].groupby(\"user_id\").agg([\"mean\",\"sum\",\"count\"])\n        USER_STAT2.columns = [\"answered_correctly_mean\",\"answered_correctly_sum\",\"answered_correctly_count\"]\n        #статистика по ответам пользователей на темы\n#         CONTEXT_STAT2 = CONTEXT_STAT2[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\n#         CONTEXT_STAT2.columns = [\"answered_correctly_content_mean\"]\n        #добовляем статистику по ответам пользователя\n        df = df.merge(USER_STAT2, on=['user_id'], how=\"left\")\n        #если новых пользователй больше 2\n#         if len(USER_ID)-len(USER_ID2)>2:\n        #добовляем в статистику CONTEXT_STAT2 общую статистику за train\n#             CONTEXT_STAT2.reset_index(inplace=True)\n#             CONTEXT_STAT2 = CONTEXT_STAT2.append(CONTEXT_STAT.reset_index(), ignore_index = True)\n#             CONTEXT_STAT2 = CONTEXT_STAT2[['content_id','answered_correctly_content_mean']].groupby(['content_id']).agg(['mean'])\n#             CONTEXT_STAT2.columns = [\"answered_correctly_content_mean\"]\n        #добавляем статистику по темам\n#         df = df.merge(CONTEXT_STAT2, on=['content_id'],  how=\"left\")\n    #если нет в базе zefir пользователей\n    if len(USER_ID2)==0:\n        #если есть данные из набора test \n        if type(CONTEXT_STAT3)!=int:\n                #то получаю их\n                ADD_INFORMATION = CONTEXT_STAT3[CONTEXT_STAT3.user_id.isin(USER_ID)]\n                ADD_INFORMATION = ADD_INFORMATION.loc[ADD_INFORMATION.content_type_id==0]\n                #получаю статистику пользоватлей по ответам\n                USER_STAT2 = ADD_INFORMATION[['user_id','answered_correctly']].groupby(\"user_id\").agg([\"mean\",\"sum\",\"count\"])\n                #получаю статистику по ответам на темы\n#                 CONTEXT_STAT2 = ADD_INFORMATION[['content_id','answered_correctly']].groupby(['content_id']).agg(['mean'])\n#                 CONTEXT_STAT2.columns = [\"answered_correctly_content_mean\"]\n                USER_STAT2.columns = [\"answered_correctly_mean\",\"answered_correctly_sum\",\"answered_correctly_count\"]\n                #если новых пользователей больше 2\n#                 if len(USER_ID)-len(ADD_INFORMATION[\"user_id\"].unique().tolist())>2:\n#                     CONTEXT_STAT2.reset_index(inplace=True)\n#                     CONTEXT_STAT2 = CONTEXT_STAT2.append(CONTEXT_STAT.reset_index(), ignore_index = True)\n#                     CONTEXT_STAT2 = CONTEXT_STAT2[['content_id','answered_correctly_content_mean']].groupby(['content_id']).agg(['mean'])\n#                     CONTEXT_STAT2.columns = [\"answered_correctly_content_mean\"]\n                df = df.merge(USER_STAT2, on=['user_id'], how=\"left\")\n#                 df = df.merge(CONTEXT_STAT2, on=['content_id'],  how=\"left\")\n#         else:\n            \n    df = df.merge(CONTEXT_STAT, on=['content_id'],  how=\"left\")\n    RAND_CORR = random.uniform(MEAN_ANSVER_USER_CORRECT, 0.65)\n    df[\"answered_correctly_mean\"].fillna(RAND_CORR,inplace=True)\n    df[\"answered_correctly_sum\"].fillna(MEAN_SUM,inplace=True)\n    df[\"answered_correctly_count\"].fillna(MEAN_COUNT,inplace=True)\n    df[\"answered_correctly_content_mean\"].fillna(MEAN_ANSVER_CONTENT_CORRECT,inplace=True)\n    df[\"answered_correctly_content_sum\"].fillna(MEAN_ANSVER_CONTENT_CORRECT_SUM,inplace=True)\n    df[\"answered_correctly_content_count\"].fillna(MEAN_ANSVER_CONTENT_CORRECT_COUNT,inplace=True)\n    df[\"answered_correctly_content_std\"].fillna(MEAN_ANSVER_CONTENT_CORRECT_STD,inplace=True)\n    df.fillna(0, inplace=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\ntrain = constructFitDataFrame(DATA_TRAIN,QUESTION)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\nCONTEXT_STAT3 = 0\nprint(DATA_VAL[DATA_VAL.content_type_id==0].shape)\nval = constructFitDataFrame(DATA_VAL,QUESTION)\ntrain_x = train[col_fit2]\nval_x = val[col_fit2]\nprint(train_x.info())\nprint(val_x.info())\ntrain_y = train[\"answered_correctly\"]\nval_y = val[\"answered_correctly\"]\nprint(val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # # d = train[column_question]*train[column_question2]\n# train[train.user_id==115][[\"tag_q\",\"answer_in_the_past\",'count_them2',\"count_them4\",\"answer_in_the_past2\",'answer_in_the_past3',\"prior_question_had_explanation\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_model_sklearn()\nmodel.fit(train_x,train_y)\nprint(\"ROC AUC TRAIN: \",make_scorer(roc_auc_score, needs_proba=True)(model, train_x, train_y))\npred = model.predict_proba(val_x)\nprint(\"ROC AUC: \",roc_auc_score(val_y, pred[:,1]))\nprint(pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CONTEXT_STAT3 = 0\n# sh = DATA_VAL.shape[0]\n# preds = []\n# for step in tqdm(range(0,sh,19)):\n#     pred_data = DATA_VAL[step:step+19]\n#     pred_data = constructFitDataFrame(pred_data,QUESTION)\n#     val_xx = pred_data[col_fit2]\n#     val_yy = pred_data[\"answered_correctly\"]\n#     pred = model.predict_proba(val_xx)\n#     preds = [*preds,*pred[:,1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as pl\npl.figure(figsize=(15,8))\nsns.barplot(y=train_x.columns, x=model.feature_importances_, )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train,train_x,train_y,val_x,val_y,DATA_TRAIN,DATA_VAL\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# start_time= time.time()\n# example_test = pd.read_csv(\"../input/riiid-test-answer-prediction/example_test.csv\")\n# example_test = constructFitDataFrame(example_test,QUESTION,\"TEST\")\n# example_test['answered_correctly'] =  model.predict_proba(example_test[col_fit2])[:,1]\n# print(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# start_time= time.time()\nCONTEXT_STAT3 = 0\nCOLUMNS_SAVE = [\"content_id\",\"answered_correctly\",\"user_id\",\"content_type_id\"]\nfor (test_df, sample_prediction_df) in iter_test:\n    PREIOR = eval(test_df.iloc[0][\"prior_group_answers_correct\"])\n    if type(PREOR_TESD_DF)==int:\n        PREOR_TESD_DF = test_df\n    else:\n        PREOR_TESD_DF[\"answered_correctly\"] = PREIOR\n        if type(CONTEXT_STAT3)==int:\n            CONTEXT_STAT3 = PREOR_TESD_DF[COLUMNS_SAVE]\n        else:\n            CONTEXT_STAT3 = CONTEXT_STAT3.append(PREOR_TESD_DF[COLUMNS_SAVE], ignore_index = True)\n        PREOR_TESD_DF=test_df\n    test_df = constructFitDataFrame(test_df,QUESTION,\"TEST\")\n    test_df['answered_correctly'] =  model.predict_proba(test_df[col_fit2])[:,1]\n#     display(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n# print(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}