{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pandasql as ps\nimport subprocess\nfrom tqdm import tqdm\nimport os\nfrom multiprocessing import Pool, Process, Manager\nfrom sklearn.preprocessing import LabelEncoder\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.metrics import TruePositives, FalsePositives, TrueNegatives, FalseNegatives, AUC, BinaryAccuracy\nfrom tensorflow.keras.layers import  Dense, GRU, Input,Embedding, Flatten, BatchNormalization, Dropout\nfrom tensorflow.keras.optimizers import RMSprop, Adam, SGD \nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.utils import class_weight\nfrom sklearn.preprocessing import MinMaxScaler,OneHotEncoder,StandardScaler\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import make_scorer, roc_auc_score,classification_report\nfrom sklearn.tree import DecisionTreeClassifier\nfrom xgboost import XGBClassifier, plot_importance\nfrom sklearn.neighbors import KNeighborsClassifier, NearestNeighbors\nimport random\nfrom category_encoders import TargetEncoder\nimport random\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nimport warnings\nwarnings.filterwarnings('ignore')\nimport time\nfrom multiprocessing.pool import ThreadPool\nimport multiprocessing as mp\nimport gc\nfrom multiprocessing import Pool","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_train_csv = \"../input/riiid-train-data-multiple-formats/riiid_train.feather\"\npath_question = \"../input/riiid-test-answer-prediction/questions.csv\"\npath_lecture = \"../input/riiid-test-answer-prediction/lectures.csv\"\npath_stat_user = \"../input/riid-zefir/results_u_v2.csv\"\npath_stat_context = \"../input/riid-zefir/results_content_v2.csv\"\nLIST_BD = [\"1/\",\"3/\",\"4/\",\"5/\",\"6/\",\"7/\"]\nLIST_BD2 = [\"11/\",\"12/\",\"13/\",\"14/\",\"15/\",\"16/\",\"17/\",\"18/\"]\nPREOR_TESD_DF = 0\nCONTEXT_STAT3 = 0\nPREOR_TESD_USER=0\nDISCT_USER_STAT2= {}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION = pd.read_csv(path_question)\nLECTURES = pd.read_csv(path_lecture)\nDATA = pd.read_feather(path_train_csv)\nUNIQUE_USER = DATA[\"user_id\"].unique().tolist()\n# test_user = [*UNIQUE_USER[:100],\n#              *UNIQUE_USER[60000:61000],\n#              *UNIQUE_USER[110000:110500],\n#              *UNIQUE_USER[360000:360500]\n#             ]\n# train_user = [\n#              *UNIQUE_USER[80000:80110],\n#              *UNIQUE_USER[380000:380110]\n#             ]\n\nUSER_STAT = pd.read_csv(path_stat_user)\n# CONTEXT_STAT = pd.read_csv(path_stat_context)\n# DATA_TRAIN = DATA[DATA.user_id.isin(test_user)]\n# DATA_VAL = DATA[DATA.user_id.isin(train_user)]\nDATA_TRAIN = DATA[3000000:4300000]\nDATA_VAL = DATA[900000:905000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# s = DATA_TRAIN[[\"user_id\",\"content_id\"]]\n# d = s.set_index(\"user_id\").to_dict()[\"content_id\"]\n# US = pd.DataFrame.from_dict(d, orient='index', columns=['foo'])\n# US.index.name=\"user_id\"\n# d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s = pd.read_feather(\"../input/zefir11/0/115\")\nmas = eval(s[\"correct\"].values[0])\nd  = {}\nd[115] = (np.mean(mas), np.sum(mas), len(mas))\nUS = pd.DataFrame.from_dict(d, orient='index', columns=['mean','sum','count'])\nUS.index.name=\"user_id\"\nUS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# s = pd.read_feather(\"../input/zefir11/0/115\")\n# # s[\"user_id\"] = 12\n# s2 = pd.read_feather(\"../input/zefir11/100/26769053\")\n# ss = s.append(s2, ignore_index=True)\n# ss[\"user_id\"] = 12\n# # s3 = pd.read_feather(\"../input/zefir11/100/26791627\")\n# # s4 = pd.read_feather(\"../input/zefir11/100/26813870\")\n\n# # s3 = s3.append(s4, ignore_index=True)\n# # s3[\"user_id\"]=13\n# # ss = ss.append(s3, ignore_index=True)\n# # ss.reset_index(drop=True, inplace=True)\n# ss\n# # f = eval(\"[\"+ss.groupby(\"user_id\")['correct'].apply(lambda x: \"*\" + (x + ' ,*').cumsum().str.strip()).loc[1][:-3]+\"]\")\n# # ss[\"cum\"] = ss[[\"user_id\",\"correct\"]].groupby(\"user_id\")['correct'].apply(lambda x: \"*\" + (x + ' ,*').cumsum().str.strip())\n# # ss[\"cum\"] = ss.groupby(\"user_id\")[\"cum\"]\n# # ss = ss.drop_duplicates(subset=['user_id'], keep='last')\n# # eval(\"[\"+ss[\"cum\"].values[0][:-3]+\"]\")\n# # for user in ss[\"user_id\"].unique():\n# #     arr = ss[ss.user_id==user][\"correct\"].values\n# #     ret = []\n# #     for a in arr:\n# #         ret = [*ret, *eval(a)]\n    \n# # np.mean(ret)\n# # ss[[\"user_id\",\"correct\"]].groupby(\"user_id\")['correct'].apply(lambda x: \"*\" + (x + ' ,*').cumsum().str.strip())\n# ss[\"cum\"] = ss[[\"user_id\",\"correct\"]].groupby(\"user_id\")['correct'].apply(lambda x:np.mean(eval(x.str.strip())) ) \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# DICT_USER_ID_PATH = {user_id:path_zefir+dir_in_zefir for path_zefir in LIST_BD for dir_in_zefir in os.listdir(\"../input/zefir\"+path_zefir) for user_id in os.listdir(\"../input/zefir\"+path_zefir+dir_in_zefir)}\nDICT_USER_ID_PATH2 = {user_id:path_zefir+dir_in_zefir for path_zefir in LIST_BD2 for dir_in_zefir in os.listdir(\"../input/zefir\"+path_zefir) for user_id in os.listdir(\"../input/zefir\"+path_zefir+dir_in_zefir)}\nKEY = DICT_USER_ID_PATH2.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TODODEL = DATA[:DATA.shape[0]*79//100]\nCONTEXT_STAT = TODODEL.loc[TODODEL.content_type_id==0][[\"content_id\",\"answered_correctly\"]].groupby('content_id').agg([\"mean\",\"sum\",\"count\",\"std\"])\nCONTEXT_STAT.columns = [\"answered_correctly_content_mean\",\"answered_correctly_content_sum\",\"answered_correctly_content_count\",\"answered_correctly_content_std\"]\n# CONTEXT_STAT\nMEAN_ANSVER_CONTENT_CORRECT = CONTEXT_STAT[\"answered_correctly_content_mean\"].mean()\nMEAN_ANSVER_CONTENT_CORRECT_SUM = CONTEXT_STAT[\"answered_correctly_content_sum\"].mean()\nMEAN_ANSVER_CONTENT_CORRECT_COUNT = CONTEXT_STAT[\"answered_correctly_content_count\"].mean()\nMEAN_ANSVER_CONTENT_CORRECT_STD = CONTEXT_STAT[\"answered_correctly_content_std\"].mean()\ndel TODODEL, DATA\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# USER_STAT.reset_index(inplace=True)\nUSER_STAT.columns = ['user_id',\n                     'answered_correctly_mean',\n                     'answered_correctly_sum',\n                     'answered_correctly_count',\n                     'answered_correctly_std',\n                     'content_type_id_sum',\n                     'content_type_id_count',\n                     'timestamp_max',\n                     'timestamp_mean',\n                     'prior_question_elapsed_time_max',\n                     'prior_question_elapsed_time_mean',\n                     'prior_question_elapsed_time_std',\n                    ]\n\nUSER_STAT = USER_STAT[1:]\ndata_types_dict = {\n    'user_id': 'uint64',\n    'answered_correctly_mean': 'float32',\n    'answered_correctly_sum': 'uint64',\n    'answered_correctly_count': 'uint64',\n    'answered_correctly_std': 'float32',\n    'content_type_id_sum': 'uint64',\n    'content_type_id_count': 'uint64',\n    'timestamp_max': 'uint64',\n    'timestamp_mean': 'float32',\n    'prior_question_elapsed_time_max': 'uint64',\n    'prior_question_elapsed_time_mean': 'float32',\n    'prior_question_elapsed_time_std': 'float32'\n}\nUSER_STAT = USER_STAT.astype(data_types_dict)\nUSER_STAT = USER_STAT.groupby('user_id').agg({'answered_correctly_mean': 'mean',\n                                              'answered_correctly_sum': 'mean',\n                                              'answered_correctly_count': 'mean',\n                                              'answered_correctly_std': 'mean',\n                                              'content_type_id_sum': 'mean',\n                                              'content_type_id_count': 'mean',\n                                              'timestamp_max': 'mean',\n                                              'timestamp_mean': 'mean',\n                                              'prior_question_elapsed_time_max': 'mean',\n                                              'prior_question_elapsed_time_mean': 'mean',\n                                              'prior_question_elapsed_time_std': 'mean',\n                                             })\n\nMEAN_ANSVER_USER_CORRECT = USER_STAT[\"answered_correctly_mean\"].mean()\nMEAN_SUM = USER_STAT[\"answered_correctly_sum\"].mean()\nMEAN_COUNT = USER_STAT[\"answered_correctly_count\"].mean()\nMEAN_STD = USER_STAT[\"answered_correctly_std\"].mean()\n\nMEAN_CONTENT_TYPE_ID = USER_STAT[\"content_type_id_sum\"].mean()\nMEAN_CONTENT_TYPE_ID_COUNT = USER_STAT[\"content_type_id_count\"].mean()\n\nMEAN_TIMSTEP_MAX = USER_STAT[\"timestamp_max\"].mean()\nMEAN_TIMSTEP_MEAN = USER_STAT[\"timestamp_mean\"].mean()\n\nMEAN_PRIOR_QUESTION_TIME_MAX = USER_STAT[\"prior_question_elapsed_time_max\"].mean()\nMEAN_PRIOR_QUESTION_TIME_MEAN = USER_STAT[\"prior_question_elapsed_time_mean\"].mean()\nMEAN_PRIOR_QUESTION_TIME_STD = USER_STAT[\"prior_question_elapsed_time_std\"].mean()\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"column_question = [str(i)+\"q\" for i in range(189)]\ncolumn_question2 = [str(i)+\"q2\" for i in range(189)]\ncolumn_question3 = [str(i)+\"q3\" for i in range(189)]\ncolumn_question4 = [str(i)+\"q4\" for i in range(189)]\ncolumn_question5 = [str(i)+\"q5\" for i in range(189)]\ncolumn_question6 = [str(i)+\"q6\" for i in range(189)]\n\ncolumn_lectures = [str(i)+\"l\" for i in range(189)]\npart_q=['1_part_q', '2_part_q', '3_part_q', '4_part_q','5_part_q', '6_part_q', '7_part_q']\ncol_fit2 = [\"row_id\",\n            \"timestamp\",\n            \"user_id\",\n            \"content_id\",\n            \"task_container_id\",\n            \"prior_question_elapsed_time\",\n            \"prior_question_had_explanation\",\n#             \"prior_question_had_explanation2\",\n#             \"part_q\",\n#             \"answered_correctly_content_mean\"\n#             \"part\",\n            \n#             'task_container_id2',\n#             'task_container_id3',\n            \n#             'answer_in_the_part',\n#             'answer_in_the_part2',\n#             'answer_in_the_part3',\n#             'answer_in_the_part4',\n#             'count_part1',\n#             'count_part2',\n#             'count_part3',\n#             'count_part4'\n            \n#             'answered_correctly2'\n           ]\nnew_columns = [\"count_them\",\n#             \"count_them2\",\n#             \"count_them3\",\n#             'count_them4',\n#             'count_them5',\n#             'answer_in_the_past',\n#             'answer_in_the_past2',\n#             'answer_in_the_past3',\n#             'answer_in_the_past4',\n#             'percent1',\n#             'percent2',\n#             'percent3',\n#             'percent4',\n#             'percent5',\n#             'percent6',\n#             'percent7'\n              ]\n# col_fit2 = [*col_fit2,*new_columns]\ncol_fit2 = [*col_fit2,*[\"answered_correctly_mean\",\"answered_correctly_sum\",\"answered_correctly_count\"]]\n\ncol_fit2 = [*col_fit2,*CONTEXT_STAT.columns.tolist()]\n\n# col_fit2 = [*col_fit2,*column_question]\n# col_fit2 = [*col_fit2,*[\"answered_correctly_content_mean\"]]\n# col_fit2 = [*col_fit2, *[\"concept\", \"intention\", \"question\", \"solving\", \"starter\"]]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def segmentation(dataframe,name_columuns,column,separator,discharge=True):\n    tags_seq = dataframe[column].values.tolist()\n    pp = np.zeros((len(tags_seq), len(name_columuns)), dtype=np.uint8)\n    if discharge:\n        for i,tag_not_split in tqdm(enumerate(tags_seq)):\n            for split_tag in tag_not_split.split(separator):\n                rr= int(split_tag)\n#                 print(rr)\n                pp[i][rr]=1\n    else:\n        for i,tag_not_split in tqdm(enumerate(tags_seq)):\n            mas = tag_not_split.split(separator)\n            pp[i][:len(mas)]=mas\n    return pp\nQUESTION.columns = [\"question_id\",\"bundle_id\",\"correct_answer\",\"part_q\",\"tag_q\"]\nQUESTION[\"part_q2_str\"] = QUESTION['part_q'].astype(\"str\")\nQUESTION['tag_q'] = QUESTION['tag_q'].astype(\"str\")\nLECTURES['tag'] = LECTURES['tag'].astype(\"str\")\nQUESTION['tag_q'] = QUESTION['tag_q'].replace(\"nan\" ,\"188\")\nQUESTION[column_question]=segmentation(QUESTION,column_question,\"tag_q\",\" \",discharge=True)\nLECTURES[column_lectures]=segmentation(LECTURES,column_lectures,\"tag\",\" \",discharge=True)\nvectorizer = CountVectorizer()\nvectorizer2 = CountVectorizer(strip_accents='ascii')\nX = vectorizer.fit_transform(LECTURES['type_of'])\nLECTURES[vectorizer.get_feature_names()] = X.toarray()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmentation2(dataframe,column,split=True,prefix=\"\"):\n    dataframe[column] = dataframe[column].astype(str)\n    dataframe[column]=dataframe[column]+prefix\n    TAGS = dataframe[column].unique()\n    TAGS_UNIQUE = []\n    for tag in tqdm(TAGS):\n        if split:\n            TAGS_UNIQUE.extend(tag.split())\n        else:\n            TAGS_UNIQUE.append(tag)\n    \n    TAGS_UNIQUE = np.unique(TAGS_UNIQUE)\n    dataframe[TAGS_UNIQUE]=0\n    def calculate_all(row):\n        if split:\n            row[row[column].split()]=1\n        else:\n            \n            row[row[column]]=1\n        return row\n    dataframe = dataframe.apply(calculate_all, axis=1)\n    dataframe.drop([column], axis=1, inplace=True)\n    return dataframe\nQUESTION = sigmentation2(QUESTION,\"part_q2_str\",False,\"_part_q\")\n# LECTURES = sigmentation(LECTURES,\"tag\",True,\"_lec\")\n# LECTURES = sigmentation(LECTURES,\"type_of\",False)\n# add_question_coumn = list(QUESTION.columns[4:])\n# add_question_coumn2 = [\"bundle_id\",\"part\"]\n# add_lecures_columns =  ['tag']\n# add_lecures_columns2 = [\"part\",\"concept\",\"intention\",\"solving question\",\"starter\"]\n# add_lecures_columns3 = [\"part2\",\"concept\",\"intention\",\"solving question\",\"starter\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_TRAIN[\"prior_question_had_explanation\"].fillna(False, inplace=True)\nencoders = LabelEncoder()\nencoders.fit(DATA_TRAIN[\"prior_question_had_explanation\"].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model_sklearn():\n#     return SGDClassifier(loss='log')\n#     return NearestNeighbors(n_neighbors=10)\n#     return MLPClassifier(random_state=1, max_iter=300)\n    return XGBClassifier(\n    tree_method=\"hist\",\n    learning_rate=0.08,\n#     gamma=0.1,\n    n_estimators=200,\n    max_depth=2,\n#     min_child_weight=40,\n#     subsample=0.87,\n#     colsample_bytree=0.95,\n#     reg_alpha=0.04,\n#     reg_lambda=0.073,\n    objective='binary:logistic',\n    nthread=4,\n    scale_pos_weight=0.8,\n    seed=27)\n\n#     return DecisionTreeClassifier(min_samples_leaf=28, min_samples_split=28, max_depth=150)\n#     return RandomForestClassifier(n_estimators=200)\n#                                   min_samples_leaf=8, min_samples_split=5, max_depth=60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION[\"content_id\"] = QUESTION['question_id']\nQUESTION.drop([\"question_id\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"QUESTION","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ind_save_data =0\nPREOR_TESD_DF = 0\ndef constructFitDataFrame(df,question, types=\"TRAIN\"):\n    global ind_save_data, PREOR_TESD_DF\n    df[\"prior_question_had_explanation\"].fillna(False, inplace=True)\n    df['prior_question_had_explanation'] = encoders.transform(df['prior_question_had_explanation'])\n    \n    keys_middle = DISCT_USER_STAT2.keys() \n    if types==\"TEST\":\n        PREIOR = eval(df.iloc[0][\"prior_group_answers_correct\"])\n        if ind_save_data==0:\n            PREOR_TESD_DF = df\n            ind_save_data=1\n        else:\n            PREOR_TESD_DF[\"answered_correctly\"] = PREIOR\n            for user_id,answered_correctly in PREOR_TESD_DF.loc[PREOR_TESD_DF.content_type_id==0][[\"user_id\",\"answered_correctly\"]].to_numpy():\n                if user_id in keys_middle:\n                    DISCT_USER_STAT2[user_id].append(answered_correctly)\n                else:\n                    DISCT_USER_STAT2[user_id] = [answered_correctly]\n            PREOR_TESD_DF=df     \n    keys_middle = DISCT_USER_STAT2.keys()                \n    df = df.loc[df['content_type_id'] == 0]\n#     df[\"is_test\"]=1\n    \n    df.reset_index(drop=True, inplace=True)\n    USER_ID = df[\"user_id\"].unique().tolist()\n    \n  \n   \n    USER_ID2 = {USER:\"../input/zefir\"+DICT_USER_ID_PATH2[str(USER)]+\"/\"+str(USER) for USER in USER_ID if str(USER) in KEY}\n    CONTEXT_STAT2 = 0\n    DISCT_USER_STAT = {}\n    DISCT_USER_STAT2_middle = DISCT_USER_STAT2.copy()\n    \n#     if type(CONTEXT_STAT3)!=int:\n#                 ADD_INFORMATION = CONTEXT_STAT3[CONTEXT_STAT3.user_id.isin(USER_ID)]\n#                 DISCT_USER_STAT2 = ADD_INFORMATION.loc[ADD_INFORMATION.content_type_id==0][[\"user_id\",\"correct\"]].set_index(\"user_id\").to_dict()[\"correct\"]\n    \n    if len(USER_ID2)>0:\n        file_list = list(USER_ID2.values())\n        ind = 0\n        for file in file_list:\n            user_id = file.split(\"/\")\n            user_id = int(user_id[len(user_id)-1])\n            CONTEXT_STAT2 = pd.read_feather(file)\n            if user_id in keys_middle:\n                mas = [*[int(i) for i in CONTEXT_STAT2[\"correct\"].values[0][1:-1].split(\",\")],\n                       *DISCT_USER_STAT2[user_id]]\n                DISCT_USER_STAT[user_id] = (np.mean(mas), np.sum(mas), len(mas))\n                USER_ID.remove(user_id)\n            else:\n                mas = [int(i) for i in CONTEXT_STAT2[\"correct\"].values[0][1:-1].split(\",\")]\n                DISCT_USER_STAT[user_id] = (np.mean(mas), np.sum(mas), len(mas))          \n        for u in USER_ID:\n            if u in keys_middle:\n                mas = DISCT_USER_STAT2[u]\n                DISCT_USER_STAT[u] = (np.mean(mas), np.sum(mas), len(mas))\n        \n        USER_STAT2 = pd.DataFrame.from_dict(DISCT_USER_STAT, orient='index', columns=[\"answered_correctly_mean\",\n                                                                \"answered_correctly_sum\",\n                                                                \"answered_correctly_count\"])\n        USER_STAT2.index.name=\"user_id\"\n        df = df.merge(USER_STAT2, on=['user_id'], how=\"left\")\n    else: #TODO else\n        for u in USER_ID:\n            if u in keys_middle:\n                mas = DISCT_USER_STAT2[u]\n                DISCT_USER_STAT[u] = (np.mean(mas), np.sum(mas), len(mas))\n        if len(DISCT_USER_STAT):\n            USER_STAT2 = pd.DataFrame.from_dict(DISCT_USER_STAT, orient='index', columns=[\"answered_correctly_mean\",\n                                                                \"answered_correctly_sum\",\n                                                                \"answered_correctly_count\"])\n            USER_STAT2.index.name=\"user_id\"\n            df = df.merge(USER_STAT2, on=['user_id'], how=\"left\")\n            \n#         #если есть данные из набора test \n#         if type(CONTEXT_STAT3)!=int:\n#                 #то получаю их\n#                 ADD_INFORMATION = CONTEXT_STAT3[CONTEXT_STAT3.user_id.isin(USER_ID)]\n# #                 CONTEXT_STAT2 = ADD_INFORMATION.loc[ADD_INFORMATION.content_type_id==0]\n#                 DISCT_USER_STAT2 = ADD_INFORMATION.loc[ADD_INFORMATION.content_type_id==0][[\"user_id\",\"correct\"]].set_index(\"user_id\").to_dict()[\"correct\"]\n    \n   \n                \n                #\n#                 CONTEXT_STAT2 = CONTEXT_STAT2.append(ADD_INFORMATION, ignore_index=True)\n#         CONTEXT_STAT2[\"cum\"] = CONTEXT_STAT2[[\"user_id\",\"correct\"]].groupby(\"user_id\")['correct'].apply(lambda x: \"*\" + (x + ' ,*').cumsum().str.strip())\n#         CONTEXT_STAT2 = CONTEXT_STAT2.drop_duplicates(subset=['user_id'], keep='last')\n    \n#         for user in CONTEXT_STAT2[\"user_id\"].unique():\n#             arr = CONTEXT_STAT2[CONTEXT_STAT2.user_id==user][\"correct\"].values\n#             ret = []\n#             for a in arr:\n#                 ret = [*ret, *eval(a)]\n#             df.loc[df.user_id==user,[\"answered_correctly_mean\",\n#                                      \"answered_correctly_sum\",\n#                                      \"answered_correctly_count\"]] = [np.mean(ret),\n#                                                                      np.sum(ret),\n#                                                                      len(ret)]\n            \n        \n        # статистика  по ответам пользователя        \n#         USER_STAT2 = CONTEXT_STAT2[['user_id','answered_correctly']].groupby(\"user_id\").agg([\"mean\",\"sum\",\"count\"])\n#         USER_STAT2.columns = [\"answered_correctly_mean\",\"answered_correctly_sum\",\"answered_correctly_count\"] \n#         df = df.merge(USER_STAT2, on=['user_id'], how=\"left\")\n        \n        \n#     if len(USER_ID2)==0:\n#         #если есть данные из набора test \n#         if type(CONTEXT_STAT3)!=int:\n#                 #то получаю их\n#                 ADD_INFORMATION = CONTEXT_STAT3[CONTEXT_STAT3.user_id.isin(USER_ID)]\n#                 CONTEXT_STAT2 = ADD_INFORMATION.loc[ADD_INFORMATION.content_type_id==0]\n#                 DISCT_USER_STAT2 = ADD_INFORMATION[[\"user_id\",\"correct\"]].set_index(\"user_id\").to_dict()[\"correct\"]\n#                 USE\n#                 #получаю статистику пользоватлей по ответам\n#                 for user in CONTEXT_STAT2[\"user_id\"].unique():\n#                     arr = CONTEXT_STAT2[CONTEXT_STAT2.user_id==user][\"correct\"].values\n#                     ret = []\n#                     for a in arr:\n#                         ret = [*ret, *eval(a)]\n                    \n#                     df.loc[df.user_id==user,[\"answered_correctly_mean\",\n#                                              \"answered_correctly_sum\",\n#                                             \"answered_correctly_count\"]] = [np.mean(ret),\n#                                                                             np.sum(ret),\n#                                                                             len(ret)]\n#                 USER_STAT2 = CONTEXT_STAT2[['user_id','answered_correctly']].groupby(\"user_id\").agg([\"mean\",\"sum\",\"count\"])\n#                 USER_STAT2.columns = [\"answered_correctly_mean\",\"answered_correctly_sum\",\"answered_correctly_count\"]\n#                 df = df.merge(USER_STAT2, on=['user_id'], how=\"left\")\n   \n#     if type(CONTEXT_STAT2)!=int:\n#         if CONTEXT_STAT2.shape[0]>0:\n#             if types==\"TEST\":\n#                 df = CONTEXT_STAT2.append(df, ignore_index=True)\n            \n#             df = df.merge(QUESTION, how = 'left', on=['content_id'])\n#             display(df)\n#             все темы на которые отвечал пользователь\n#             all_answer_theme = df.groupby('user_id')[column_question].apply(lambda x: x.cumsum())#\n#             all_answer_theme[\"user_id\"] = df[\"user_id\"]\n            \n            #     #предыдущие вопросы\n#             past_answer_theme = df.groupby('user_id')[column_question].apply(lambda x:x.shift(1))\n#             past_answer_theme[\"user_id\"] = df[\"user_id\"]\n#             past_answer_theme[\"prior_question_had_explanation\"] = df[\"prior_question_had_explanation\"]\n                #предыдущие темы на которые отвечал пользователь\n#             past_all_answer_theme = all_answer_theme.groupby('user_id')[column_question].apply(lambda x:x.shift(1))\n#             past_all_answer_theme[\"user_id\"] = df[\"user_id\"]\n            #     #предыдущие вопросы на которые видел ответы\n#             past_answer_theme_see_answer = past_answer_theme[column_question].multiply(past_answer_theme['prior_question_had_explanation'], axis=\"index\")\n#             past_answer_theme_see_answer[\"user_id\"] = df[\"user_id\"]\n#             display(past_answer_theme_see_answer)\n                #все предыдущие темы вопросов на которые видел ответы\n#             cumsum_past_answer_theme_see_answer = past_answer_theme_see_answer.groupby('user_id')[column_question].apply(lambda x: x.cumsum())\n    # все выпросы на которые в настоящее время видет ответы\n#             question_theme_see_answer = df[column_question].multiply(df['prior_question_had_explanation'], axis=\"index\")\n#             question_theme_see_answer[\"user_id\"] = df[\"user_id\"]\n#             question_theme_see_answer = question_theme_see_answer.groupby('user_id')[column_question].apply(lambda x: x.cumsum())\n#             d=0\n# #             d2=0\n#             d3=0\n# #             d4=0\n#             for i in range(189):\n#                 theme = df[str(i)+\"q\"]\n#                 see_answer = df[\"prior_question_had_explanation\"]\n#                 column = str(i)+\"q\"\n#                 if i==0:\n#                     d=theme*past_answer_theme[column].gt(0)*see_answer\n# #                     d2 =theme*past_all_answer_theme[column].gt(0)\n#                     d3 =theme*cumsum_past_answer_theme_see_answer[column].gt(0)\n# #                     d4 = theme*question_theme_see_answer[column].gt(0)\n#                 else:\n#                     d+=theme*past_answer_theme[column].gt(0)*see_answer\n# #                     d2 +=theme*past_all_answer_theme[column].gt(0)\n#                     d3 +=theme*cumsum_past_answer_theme_see_answer[column].gt(0)\n#                     d4 += theme*question_theme_see_answer[column].gt(0)\n#             df[\"count_them\"] = df[column_question].sum(axis=1) #count theme in the question\n#             df[\"count_them2\"] = all_answer_theme[column_question].sum(axis=1) # all the theme in the question\n#             df['count_them3'] = all_answer_theme[column_question].gt(0).sum(axis=1) # unique count theme in question\n#             df[\"count_them4\"] =  df[\"count_them2\"].shift(1)\n#             df[\"count_them5\"] =  df[\"count_them3\"].shift(1)\n\n#             df['answer_in_the_past'] = d #видел ли ответы строго на предыдущие вопросы\n#             df['answer_in_the_past2'] = d2 # отвечал ли на темы вопросов в прошлом\n#             df['answer_in_the_past3'] = d3 # отвечал ли на темы вопросов в прошлом и видел когда-то на них ответы\n#             df['answer_in_the_past4'] = d4 # отвечал на эти вопорсы и видел ответы в момент ответа\n#             df['percent1'] = d*100/df[\"count_them\"] # процент знания сторого предыдущих тем вопросов\n#             df['percent2'] = d2*100/df[\"count_them\"] # процент опыта ответов на прошлые вопросы вне зависимоти от того видел ли ответы\n#             df['percent3'] = d3*100/df[\"count_them\"] # процент знания тем вопосов за весь период обучения\n#             df['percent4'] = d4*100/df[\"count_them\"] # порцент знания\n#             df['percent5'] = df[\"count_them3\"]*100/df[\"count_them\"]\n#             df['percent6'] = df[\"count_them4\"]*100/df[\"count_them\"]\n#             df['percent7'] = df[\"count_them5\"]*100/df[\"count_them\"]\n#         else:\n#             df[new_columns]=0\n#     else:\n#             df[new_columns]=0\n            \n#     df[\"prior_question_had_explanation2\"] = df.groupby('user_id')[\"prior_question_had_explanation\"].apply(lambda x: x.cumsum())\n#     df['task_container_id2'] = df.groupby('user_id')['task_container_id'].transform(lambda x:x.shift(1))\n#     df.loc[df.task_container_id==df.task_container_id2,[\"task_container_id3\"]] = 1      \n        \n\n\n\n#     #все предыдущие темы вопросов на которые видел ответы\n#     cumsum_past_answer_theme_see_answer = past_answer_theme_see_answer.groupby('user_id')[column_question].apply(lambda x: x.cumsum())\n#     # все выпросы на которые в настоящее время видет ответы\n#     question_theme_see_answer = df[column_question].multiply(df['prior_question_had_explanation'], axis=\"index\")\n#     question_theme_see_answer[\"user_id\"] = df[\"user_id\"]\n#     question_theme_see_answer = question_theme_see_answer.groupby('user_id')[column_question].apply(lambda x: x.cumsum())\n    \n#     #все part на которые отвечал пользователь\n#     all_answer_part = df.groupby('user_id')[part_q].apply(lambda x: x.cumsum())#\n#     all_answer_part[\"user_id\"] = df[\"user_id\"]\n#     #предыдущие part\n#     past_answer_part = df.groupby('user_id')[part_q].apply(lambda x:x.shift(1))\n#     past_answer_part[\"user_id\"] = df[\"user_id\"]\n#     past_answer_part[\"prior_question_had_explanation\"] = df[\"prior_question_had_explanation\"]\n#     #предыдущие part на которые отвечал пользователь\n#     past_all_answer_part = all_answer_part.groupby('user_id')[part_q].apply(lambda x:x.shift(1))\n#     past_all_answer_part[\"user_id\"] = df[\"user_id\"]\n#     #предыдущие part на которые видел ответы\n#     past_answer_theme_see_part = past_answer_part[part_q].multiply(past_answer_part['prior_question_had_explanation'], axis=\"index\")\n#     past_answer_theme_see_part[\"user_id\"] = df[\"user_id\"]\n#     #все предыдущие темы вопросов на которые видел ответы\n#     cumsum_past_answer_part_see_answer = past_answer_theme_see_part.groupby('user_id')[part_q].apply(lambda x: x.cumsum())\n#     # все выпросы на которые в настоящее время видет ответы\n#     question_theme_see_part = df[part_q].multiply(df['prior_question_had_explanation'], axis=\"index\")\n#     question_theme_see_part[\"user_id\"] = df[\"user_id\"]\n#     question_theme_see_part = question_theme_see_part.groupby('user_id')[part_q].apply(lambda x: x.cumsum())\n    \n#     d=0\n#     d2=0\n#     d3=0\n#     d4=0\n#     part1=0\n#     part2=0\n#     part3=0\n#     part4=0\n#     for i in range(1,8):\n#         part = df[str(i)+'_part_q']\n#         see_answer = df[\"prior_question_had_explanation\"]\n#         column = str(i)+'_part_q'\n#         if i==0:\n#             part1=part*past_answer_part[column].gt(0)*see_answer\n#             part2 =part*past_all_answer_part[column].gt(0)\n#             part3 =part*cumsum_past_answer_part_see_answer[column].gt(0)\n#             part4 = part*question_theme_see_part[column].gt(0)\n#         else:\n#             part1 +=part*past_answer_part[column].gt(0)*see_answer\n#             part2 +=part*past_all_answer_part[column].gt(0)\n#             part3 +=part*cumsum_past_answer_part_see_answer[column].gt(0)\n#             part4 += part*question_theme_see_part[column].gt(0)\n        \n        \n#     for i in range(189):\n#         theme = df[str(i)+\"q\"]\n#         see_answer = df[\"prior_question_had_explanation\"]\n#         column = str(i)+\"q\"\n#         if i==0:\n#             d=theme*past_answer_theme[column].gt(0)*see_answer\n#             d2 =theme*past_all_answer_theme[column].gt(0)\n#             d3 =theme*cumsum_past_answer_theme_see_answer[column].gt(0)\n#             d4 = theme*question_theme_see_answer[column].gt(0)\n#         else:\n#             d+=theme*past_answer_theme[column].gt(0)*see_answer\n#             d2 +=theme*past_all_answer_theme[column].gt(0)\n#             d3 +=theme*cumsum_past_answer_theme_see_answer[column].gt(0)\n#             d4 += theme*question_theme_see_answer[column].gt(0)\n#     df[\"count_them\"] = df[column_question].sum(axis=1) #count theme in the question\n#     df[\"count_them2\"] = all_answer_theme[column_question].sum(axis=1) # all the theme in the question\n#     df['count_them3'] = all_answer_theme[column_question].gt(0).sum(axis=1) # unique count theme in question\n#     df[\"count_them4\"] =  df[\"count_them2\"].shift(1)\n#     df[\"count_them5\"] =  df[\"count_them3\"].shift(1)\n    \n#     df['answer_in_the_past'] = d #видел ли ответы строго на предыдущие вопросы\n#     df['answer_in_the_past2'] = d2 # отвечал ли на темы вопросов в прошлом\n#     df['answer_in_the_past3'] = d3 # отвечал ли на темы вопросов в прошлом и видел когда-то на них ответы\n#     df['answer_in_the_past4'] = d4 # отвечал на эти вопорсы и видел ответы в момент ответа\n#     df['percent1'] = d*100/df[\"count_them\"] # процент знания сторого предыдущих тем вопросов\n#     df['percent2'] = d2*100/df[\"count_them\"] # процент опыта ответов на прошлые вопросы вне зависимоти от того видел ли ответы\n#     df['percent3'] = d3*100/df[\"count_them\"] # процент знания тем вопосов за весь период обучения\n#     df['percent4'] = d4*100/df[\"count_them\"] # порцент знания\n#     df['percent5'] = df[\"count_them3\"]*100/df[\"count_them\"]\n#     df['percent6'] = df[\"count_them4\"]*100/df[\"count_them\"]\n#     df['percent7'] = df[\"count_them5\"]*100/df[\"count_them\"]\n    \n    \n#     df['answer_in_the_part'] = part1 #видел ли ответы строго на предыдущие part\n#     df['answer_in_the_part2'] = part2 #видел ли ответы строго на предыдущие part\n#     df['answer_in_the_part3'] = part3 #видел ли ответы строго на предыдущие part\n#     df['answer_in_the_part4'] = part4 #видел ли ответы строго на предыдущие part\n    \n# #     df[\"count_part\"] = df[column_question].sum(axis=1) #count theme in the question\n#     df[\"count_part1\"] = all_answer_part[part_q].sum(axis=1) # all the theme in the question\n#     df['count_part2'] = all_answer_part[part_q].gt(0).sum(axis=1) # unique count theme in question\n#     df[\"count_part3\"] =  df[\"count_part1\"].shift(1)\n#     df[\"count_part4\"] =  df[\"count_part2\"].shift(1)\n    \n    \n    \n#     \n\n    \n   \n#     df = df[df.is_test==1]        \n    df = df.merge(CONTEXT_STAT, on=['content_id'],  how=\"left\")\n    RAND_CORR = random.uniform(MEAN_ANSVER_USER_CORRECT, 0.65)\n    df[\"answered_correctly_mean\"].fillna(RAND_CORR,inplace=True)\n    df[\"answered_correctly_sum\"].fillna(MEAN_SUM,inplace=True)\n    df[\"answered_correctly_count\"].fillna(MEAN_COUNT,inplace=True)\n    df[\"answered_correctly_content_mean\"].fillna(MEAN_ANSVER_CONTENT_CORRECT,inplace=True)\n    df[\"answered_correctly_content_sum\"].fillna(MEAN_ANSVER_CONTENT_CORRECT_SUM,inplace=True)\n    df[\"answered_correctly_content_count\"].fillna(MEAN_ANSVER_CONTENT_CORRECT_COUNT,inplace=True)\n    df[\"answered_correctly_content_std\"].fillna(MEAN_ANSVER_CONTENT_CORRECT_STD,inplace=True)\n    df.fillna(0, inplace=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\ntrain = constructFitDataFrame(DATA_TRAIN,QUESTION)\ntrain[col_fit2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%time\nCONTEXT_STAT3 = 0\nprint(DATA_VAL[DATA_VAL.content_type_id==0].shape)\nval = constructFitDataFrame(DATA_VAL,QUESTION)\ntrain_x = train[col_fit2]\nval_x = val[col_fit2]\nprint(train_x.info())\nprint(val_x.info())\ntrain_y = train[\"answered_correctly\"]\nval_y = val[\"answered_correctly\"]\nprint(val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # # d = train[column_question]*train[column_question2]\n# train[train.user_id==115][[\"tag_q\",\"answer_in_the_past\",'count_them2',\"count_them4\",\"answer_in_the_past2\",'answer_in_the_past3',\"prior_question_had_explanation\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_model_sklearn()\nmodel.fit(train_x,train_y)\nprint(\"ROC AUC TRAIN: \",make_scorer(roc_auc_score, needs_proba=True)(model, train_x, train_y))\npred = model.predict_proba(val_x)\nprint(\"ROC AUC: \",roc_auc_score(val_y, pred[:,1]))\nprint(pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CONTEXT_STAT3 = 0\n# sh = DATA_VAL.shape[0]\n# preds = []\n# for step in tqdm(range(0,sh,19)):\n#     pred_data = DATA_VAL[step:step+19]\n#     pred_data = constructFitDataFrame(pred_data,QUESTION)\n#     val_xx = pred_data[col_fit2]\n#     val_yy = pred_data[\"answered_correctly\"]\n#     pred = model.predict_proba(val_xx)\n#     preds = [*preds,*pred[:,1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as pl\npl.figure(figsize=(15,8))\nsns.barplot(y=train_x.columns, x=model.feature_importances_, )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train,train_x,train_y,val_x,val_y,DATA_TRAIN,DATA_VAL\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# start_time= time.time()\n# example_test = pd.read_csv(\"../input/riiid-test-answer-prediction/example_test.csv\")\n# example_test = constructFitDataFrame(example_test,QUESTION,\"TEST\")\n# example_test['answered_correctly'] =  model.predict_proba(example_test[col_fit2])[:,1]\n# print(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time= time.time()\nCONTEXT_STAT3 = 0\nCOLUMNS_SAVE = [\"content_id\",\"correct\",\"user_id\",\"content_type_id\",\"prior_question_had_explanation\"]\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df = constructFitDataFrame(test_df,QUESTION,\"TEST\")\n    test_df['answered_correctly'] =  model.predict_proba(test_df[col_fit2])[:,1]\n#     display(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}