{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import classification_report\n\nfrom lightgbm import LGBMClassifier\nfrom lightgbm import plot_importance, plot_metric, early_stopping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions = pd.read_csv('../input/feature-engineering-datasets/questions_fe.csv')\nquestion_stats = pd.read_csv('../input/feature-engineering-datasets/questions_stats_fe.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TO DO:\n#\n#MODIFICARE AGGIORNAMENTO USER PERFORMANCE USANDO DATATABLE (FARE COUNT E SUM SUI CAMPI DI INTERESSE)\n\n#given the dataset, I retrieve the cumulative history of students\ndef get_last_performance( data,last_record=pd.DataFrame()):\n    last_record.index= -1 - last_record.index\n    df = data[data['content_type_id']==0].merge(questions[['question_id','part','kmean_cluster']], left_on='content_id', right_on = 'question_id', how = 'left')\n    df = df[['timestamp','user_id','answered_correctly','part','row_id','kmean_cluster']]\n    df['part_answer'] = 'part_'+ df.part.astype(str)  + '_' + df.answered_correctly.astype(str)\n    df['cluster_answer'] = 'cluster_'+ df.kmean_cluster.astype(str)  + '_' + df.answered_correctly.astype(str)\n    \n    user_performance = df[['user_id','row_id']].merge(\n        pd.get_dummies(df['part_answer']),\n        left_index = True, \n        right_index = True).merge(\n        pd.get_dummies(df['cluster_answer']),\n        left_index = True, \n        right_index = True).set_index('row_id')\n    try:\n        user_performance = user_performance.append(last_record.drop(columns=['timestamp'],inplace=False))\n    except:\n            None\n    user_performance = user_performance.groupby('user_id').cumsum().merge(\n        df[['user_id','row_id','timestamp']].set_index('row_id'), left_index = True, right_index = True)\n    user_performance_part_last = last_record.append(user_performance).groupby('user_id',as_index=False).max().fillna(0)\n    \n    return user_performance, user_performance_part_last","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_last_performance(pd.read_csv('../input/riiid-test-answer-prediction/train.csv',nrows=100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_col = ['answered_correctly']\nfeature_col = [\n 'timestamp',\n 'prior_question_elapsed_time',\n    'prior_question_had_explanation',\n    'part',\n    'kmean_cluster',\n    'content_type_id',\n 'part_1_avg',\n #'part_1_count',\n 'part_2_avg',\n #'part_2_count',\n 'part_3_avg',\n #'part_3_count',\n 'part_4_avg',\n #'part_4_count',\n 'part_5_avg',\n #'part_5_count',\n 'part_6_avg',\n #'part_6_count',\n 'part_7_avg',\n #'part_7_count',\n 'cluster_0_avg',\n #'cluster_0_count',\n 'cluster_1_avg',\n #'cluster_1_count',\n 'cluster_2_avg',\n #'cluster_2_count',\n 'cluster_3_avg',\n #'cluster_3_count',\n 'cluster_4_avg',\n #'cluster_4_count',\n    'cluster_5_avg',\n #'cluster_5_count',\n 'cluster_6_avg',\n #'cluster_6_count',\n 'bundle_mean_answered_correctly',\n 'bundle_std_answered_correctly',\n 'bundle_perc_students',\n #'bundle_times',\n 'part_mean_answered_correctly',\n 'part_std_answered_correctly',\n 'part_perc_students',\n #'part_times',\n 'cluster_mean_answered_correctly',\n 'cluster_std_answered_correctly',\n 'cluster_perc_students'\n# 'cluster_times'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm = LGBMClassifier(\n    num_leaves = 21,\n    max_depth = 5,\n    n_estimators = 100,\n    min_child_samples = 1000, \n    subsample=0.7, \n    n_jobs= -1,\n    min_data_in_leaf = 100,\n    is_higher_better = True,\n    first_metric_only = True,\n    feature_fraction = 0.5,\n    lambda_l1 = 0.1,\n    learning_rate = 0.1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\niter_df =  pd.read_csv('../input/riiid-test-answer-prediction/train.csv',chunksize = 1000000)\n\nuser_last_performance = pd.DataFrame()\ndebug = []\nscore_old = 0\n\nfor i,df in enumerate(iter_df):\n    #print('Running iteration: {}'.format(i))\n    gc.collect()\n    user_stats = df[['row_id','user_id','content_id','content_type_id',\n                                                  'timestamp','answered_correctly',\n                                                  'prior_question_elapsed_time',\n                                                  'prior_question_had_explanation']]\n    user_performance, user_last_performance = get_last_performance(df, user_last_performance)\n    print(user_performance)\n    data = user_stats.merge(\n        user_performance.reset_index().rename(columns={'index':'row_id'}),\n        on =['row_id','user_id','timestamp'], how = 'inner'\n    ).merge(\n        question_stats, left_on = 'content_id', right_on = 'question_id'\n    ).merge(\n        questions, left_on = 'question_id', right_on = 'question_id'\n     ).set_index('row_id')\n    \n    #Rescaling some features\n    data = data.astype({'prior_question_had_explanation':'bool'})\n    data['prior_question_had_explanation'] = data['prior_question_had_explanation'].fillna(False)\n    \n    data.timestamp/=(365*24*60*60*100)\n    data.prior_question_elapsed_time/=(60*60*24)\n    data = data.fillna(-1)\n    \n    # Calculate stats performance on parts\n    for part in questions.part.unique():\n        data['part_'+str(part)+'_avg'] =  data['part_'+str(part)+'_1'].divide(data['part_'+str(part)+'_0'] + data['part_'+str(part)+'_1'])\n        data['part_'+str(part)+'_count'] = data['part_'+str(part)+'_0'] + data['part_'+str(part)+'_1']\n        data.drop(columns = ['part_'+str(part)+'_0', 'part_'+str(part)+'_1'],inplace = True)\n    data = data.fillna(0.5)\n    \n    # Calculate stats performance on clusters\n    \n    for part in questions.kmean_cluster.unique():\n        data['cluster_'+str(part)+'_avg'] =  data['cluster_'+str(part)+'_1'].divide(data['cluster_'+str(part)+'_0'] + data['cluster_'+str(part)+'_1'])\n        data['cluster_'+str(part)+'_count'] = data['cluster_'+str(part)+'_0'] + data['cluster_'+str(part)+'_1']\n        data.drop(columns = ['cluster_'+str(part)+'_0', 'cluster_'+str(part)+'_1'],inplace = True)\n    data = data.fillna(0.5)\n    \n    # Model Training & Validation in Loop\n    \n    X_train,X_test,Y_train, Y_test = train_test_split(data[feature_col], data[target_col], test_size = 0.2, shuffle = False)\n    \n    \n    \n    if i == 0:\n        lgbm.fit(X_train, Y_train,callbacks = [early_stopping])\n    score_train = roc_auc_score(Y_train.values, lgbm.predict_proba(X_train)[:,1])\n    if ((score_old - score_train > 0.15) | (score_train < 0.745)) & (i%25==0):\n        lgbm.fit(X_train, Y_train,callbacks = [early_stopping])\n        score_train = roc_auc_score(Y_train.values, lgbm.predict_proba(X_train)[:,1])\n    score_test = roc_auc_score(Y_test.values, lgbm.predict_proba(X_test)[:,1])\n    #if i%10==0:\n    print('\\nRun: {}'.format(i))\n    print('Score Train: {}'.format(score_train))\n    print('Score Test: {}'.format(score_test))\n\n    debug.append([i, len(df),len(user_stats),len(user_performance),len(user_last_performance),score_train, score_test])\n    score_old = score_train\n    \n    del user_performance, score_train, score_test, data, X_train, X_test, Y_train, Y_test\n    #print('\\nClosed iteration: {}'.format(i))\n    #if i == 10:\n    #    break\n\ndebug_df = pd.DataFrame(debug, columns = [\n    'run', 'len_chunk', 'len_user_stats', 'len_performance','num_users','score_train','score_test'\n])\n\ndebug_df[['score_train','score_test']].plot.line()    \n\nplot_importance(lgbm, figsize  = (18,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# salvataggio del modello lgbm\nlgbm.booster_.save_model('./model.txt')\n# salvataggio user_last_performance\nuser_last_performance.to_csv('./user_stats.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null\n\nimport lightgbm as lgb\nimport datatable as dt\nfrom datatable import f, by\nimport numpy as np ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_last_performance = dt.fread('./user_stats.csv')\nquestion_stats = dt.fread('../input/feature-engineering-datasets/questions_stats_fe.csv')\nquestions = dt.fread('../input/feature-engineering-datasets/questions_fe.csv')\nquestions.names={'question_id':'content_id'}\nquestions.key='content_id'\nquestion_stats.names={'question_id':'content_id'}\nquestion_stats.key='content_id'\nuser_last_performance.key='user_id'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aggregation_dict = {}\naggregation_perf_dict = {}\nfor part in dt.unique(questions[:,'part']).to_list()[0]:\n    for i in (0,1):\n        aggregation_dict['part_'+str(part)+'_'+str(i)] = dt.sum((f.answered_correctly==0)&(f.part==part))\n        aggregation_perf_dict['part_'+str(part)+'_'+str(i)] = dt.sum(f['part_'+str(part)+'_'+str(i)])\nfor part in dt.unique(questions[:,'kmean_cluster']).to_list()[0]:\n    for i in (0,1):\n        aggregation_dict['cluster_'+str(part)+'_'+str(i)] = dt.sum((f.answered_correctly==0)&(f.part==part))\n        aggregation_perf_dict['cluster_'+str(part)+'_'+str(i)] = dt.sum(f['cluster_'+str(part)+'_'+str(i)])\naggregation_dict['timestamp'] = dt.max(f.timestamp)\naggregation_perf_dict['timestamp'] = dt.max(f.timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#MODIFICARE DATASET CON DATATABLE MIGLIORANDO L'EFFICIENZA\n\ndef enrich_dataset(dataset, last_record):\n    \n    # RICEVO IL DATASET SU CUI APPLICARE I RISULTATI\n    \n    #AGGIUNGO GLI ATTRIBUTI PRECALCOLATI PER TUTENTE\n    \n    #AGGIUNGO GLI ATTRIBUTI DAI VARI DATASETS\n    \n    #RESTITUISCO DATASET ARRICCHITO\n    \n    data = dt.Frame(\n        dataset[['content_type_id','row_id','user_id','content_id','timestamp'\n                 ,'prior_question_elapsed_time','prior_question_had_explanation']].fillna(-60*60*24)\n    )\n    content_type_id,row_id, user_id, content_id, timestamp, prior_question_elapsed_time,prior_question_had_explanation = data.export_names()\n    data = data[:,{'row_id':row_id, \n                   'user_id':user_id, \n                   'content_id':content_id, \n                   'timestamp':timestamp/(365*24*60*60*100), \n                   'prior_question_elapsed_time':prior_question_elapsed_time/(60*60*24),\n                  'content_type_id':content_type_id,\n                  'prior_question_had_explanation':prior_question_had_explanation}]\n    data = data[:,:,dt.join(question_stats)]\n    X = user_last_performance[:, dt.f[:].remove(dt.f['timestamp'])]\n    X.key = 'user_id'\n    data = data[:,:,dt.join(X)]\n    \n    del X\n\n    for part in dt.unique(questions[:,'part']).to_list()[0]:\n        col1 = 'part_'+str(part)+'_avg'\n        col2 = 'part_'+str(part)+'_count'\n        part_1 = 'part_'+str(part)+'_1'\n        part_0 = 'part_'+str(part)+'_0'\n        data[:,col1] = data[:,dt.f[part_1]/(dt.f[part_0]+dt.f[part_1])]\n        data[:,col2] = data[:,dt.f[part_0] + dt.f[part_1]]\n        del data[:, part_1]\n        del data[:, part_0]\n    for part in dt.unique(questions[:,'kmean_cluster']).to_list()[0]:\n        col1 = 'cluster_'+str(part)+'_avg'\n        col2 = 'cluster_'+str(part)+'_count'\n        part_1 = 'cluster_'+str(part)+'_1'\n        part_0 = 'cluster_'+str(part)+'_0'\n        data[:,col1] = data[:,dt.f[part_1]/(dt.f[part_0]+dt.f[part_1])]\n        data[:,col2] = data[:,dt.f[part_0] + dt.f[part_1]]\n        del data[:, part_1]\n        del data[:, part_0]\n    return data.to_pandas().fillna(0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_last_performance(last_df, user_last):\n    from datatable import f, by \n    data = dt.Frame(last_df.loc[last_df['content_type_id']==0,['timestamp','user_id','row_id','answered_correctly','content_id']])\n    data.key = 'row_id'\n    X = data[:,:,dt.join(questions)]\n    del X[:,['content_id','bundle_id','tags']]\n    X.key = 'row_id'\n    data = X[:,aggregation_dict,by('user_id')]\n    data.rbind(user_last)\n    X = data[:,aggregation_perf_dict,by('user_id')]    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = {'row_id':'int64','answered_correctly':'float64'}\n\nimport gc\n\nlast_df = pd.DataFrame()\n\nfor (test_df,current_prediction_df) in iter_test:\n    \n    gc.collect()\n    \n    if last_df.shape[0]>0:\n        last_df['answered_correctly'] = np.array(eval(test_df['prior_group_answers_correct'].iloc[0]))\n        #%time\n        user_last_performance = get_last_performance(last_df, user_last_performance)\n    data = enrich_dataset(test_df, user_last_performance)\n    data = data.loc[data.content_type_id == 0, : ]\n    \n    assert len(data)==len(current_prediction_df)\n    \n    current_prediction_df.answered_correctly = lgbm.predict(data[feature_col].fillna(-1)).ravel()\n    del data\n    \n    env.predict(current_prediction_df.fillna(0.5).astype(d))\n    last_df = test_df.copy(deep=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}