{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install ../input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null\n\nimport lightgbm as lgb\nimport datatable as dt\nfrom datatable import f, by\nimport numpy as np ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"lgbm = lgb.Booster(model_file = '../input/model-building-with-cumulative-features/model.txt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"user_last_performance = dt.fread('../input/model-building-with-cumulative-features/user_stats.csv')\nquestion_stats = dt.fread('../input/feature-engineering-datasets/questions_stats_fe.csv')\nquestions = dt.fread('../input/feature-engineering-datasets/questions_fe.csv')\nquestions.names={'question_id':'content_id'}\nquestions.key='content_id'\nquestion_stats.names={'question_id':'content_id'}\nquestion_stats.key='content_id'\nuser_last_performance.key='user_id'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aggregation_dict = {}\naggregation_perf_dict = {}\nfor part in dt.unique(questions[:,'part']).to_list()[0]:\n    for i in (0,1):\n        aggregation_dict['part_'+str(part)+'_'+str(i)] = dt.sum((f.answered_correctly==0)&(f.part==part))\n        aggregation_perf_dict['part_'+str(part)+'_'+str(i)] = dt.sum(f['part_'+str(part)+'_'+str(i)])\nfor part in dt.unique(questions[:,'kmean_cluster']).to_list()[0]:\n    for i in (0,1):\n        aggregation_dict['cluster_'+str(part)+'_'+str(i)] = dt.sum((f.answered_correctly==0)&(f.part==part))\n        aggregation_perf_dict['cluster_'+str(part)+'_'+str(i)] = dt.sum(f['cluster_'+str(part)+'_'+str(i)])\naggregation_dict['timestamp'] = dt.max(f.timestamp)\naggregation_perf_dict['timestamp'] = dt.max(f.timestamp)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"#MODIFICARE DATASET CON DATATABLE MIGLIORANDO L'EFFICIENZA\n\ndef enrich_dataset(dataset, last_record):\n    \n    # RICEVO IL DATASET SU CUI APPLICARE I RISULTATI\n    \n    #AGGIUNGO GLI ATTRIBUTI PRECALCOLATI PER TUTENTE\n    \n    #AGGIUNGO GLI ATTRIBUTI DAI VARI DATASETS\n    \n    #RESTITUISCO DATASET ARRICCHITO\n    \n    data = dt.Frame(\n        dataset[['content_type_id','row_id','user_id','content_id','timestamp','prior_question_elapsed_time']].fillna(-60*60*24)\n    )\n    content_type_id,row_id, user_id, content_id, timestamp, prior_question_elapsed_time = data.export_names()\n    data = data[:,{'row_id':row_id, \n                   'user_id':user_id, \n                   'content_id':content_id, \n                   'timestamp':timestamp/(365*24*60*60*100), \n                   'prior_question_elapsed_time':prior_question_elapsed_time/(60*60*24),\n                  'content_type_id':content_type_id}]\n    data = data[:,:,dt.join(question_stats)]\n    X = user_last_performance[:, dt.f[:].remove(dt.f['timestamp'])]\n    X.key = 'user_id'\n    data = data[:,:,dt.join(X)]\n    \n    del X\n\n    for part in dt.unique(questions[:,'part']).to_list()[0]:\n        col1 = 'part_'+str(part)+'_avg'\n        col2 = 'part_'+str(part)+'_count'\n        part_1 = 'part_'+str(part)+'_1'\n        part_0 = 'part_'+str(part)+'_0'\n        data[:,col1] = data[:,dt.f[part_1]/(dt.f[part_0]+dt.f[part_1])]\n        data[:,col2] = data[:,dt.f[part_0] + dt.f[part_1]]\n        del data[:, part_1]\n        del data[:, part_0]\n    for part in dt.unique(questions[:,'kmean_cluster']).to_list()[0]:\n        col1 = 'cluster_'+str(part)+'_avg'\n        col2 = 'cluster_'+str(part)+'_count'\n        part_1 = 'cluster_'+str(part)+'_1'\n        part_0 = 'cluster_'+str(part)+'_0'\n        data[:,col1] = data[:,dt.f[part_1]/(dt.f[part_0]+dt.f[part_1])]\n        data[:,col2] = data[:,dt.f[part_0] + dt.f[part_1]]\n        del data[:, part_1]\n        del data[:, part_0]\n    return data.to_pandas().fillna(0.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_col = ['answered_correctly']\nfeature_col = [\n 'timestamp',\n 'prior_question_elapsed_time',\n 'part_1_avg',\n #'part_1_count',\n 'part_2_avg',\n #'part_2_count',\n 'part_3_avg',\n #'part_3_count',\n 'part_4_avg',\n #'part_4_count',\n 'part_5_avg',\n #'part_5_count',\n 'part_6_avg',\n #'part_6_count',\n 'part_7_avg',\n #'part_7_count',\n 'cluster_0_avg',\n #'cluster_0_count',\n 'cluster_1_avg',\n #'cluster_1_count',\n 'cluster_2_avg',\n #'cluster_2_count',\n 'cluster_3_avg',\n #'cluster_3_count',\n 'cluster_4_avg',\n #'cluster_4_count',\n    'cluster_5_avg',\n #'cluster_5_count',\n 'cluster_6_avg',\n #'cluster_6_count',\n 'bundle_mean_answered_correctly',\n 'bundle_std_answered_correctly',\n 'bundle_perc_students',\n #'bundle_times',\n 'part_mean_answered_correctly',\n 'part_std_answered_correctly',\n 'part_perc_students',\n #'part_times',\n 'cluster_mean_answered_correctly',\n 'cluster_std_answered_correctly',\n 'cluster_perc_students'\n# 'cluster_times'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_last_performance(last_df, user_last):\n    from datatable import f, by \n    data = dt.Frame(last_df.loc[last_df['content_type_id']==0,['timestamp','user_id','row_id','answered_correctly','content_id']])\n    data.key = 'row_id'\n    X = data[:,:,dt.join(questions)]\n    del X[:,['content_id','bundle_id','tags']]\n    X.key = 'row_id'\n    data = X[:,aggregation_dict,by('user_id')]\n    data.rbind(user_last_performance)\n    X = data[:,aggregation_perf_dict,by('user_id')]    \n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d = {'row_id':'int64','answered_correctly':'float64'}\n\nimport gc\n\nlast_df = pd.DataFrame()\n\nfor (test_df,current_prediction_df) in iter_test:\n    \n    gc.collect()\n    \n    if last_df.shape[0]>0:\n        last_df['answered_correctly'] = np.array(eval(test_df['prior_group_answers_correct'].iloc[0]))\n        #%time\n        X = get_last_performance(last_df, user_last_performance)\n        user_last_performance = X\n        del X\n    data = enrich_dataset(test_df, user_last_performance)\n    data = data[data.content_type_id == 0]\n    \n    assert len(data)==len(current_prediction_df)\n    \n    current_prediction_df.answered_correctly = lgbm.predict(data[feature_col].fillna(-1)).ravel()\n    del data\n    \n    env.predict(current_prediction_df.fillna(0.5).astype(d))\n    last_df = test_df.copy(deep=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}