{"cells":[{"metadata":{},"cell_type":"markdown","source":"I was able to get improvement in inference speed by using numpy instead of pandas"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\nimport riiideducation\nenv = riiideducation.make_env()\niter_tester = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_c = pd.read_parquet('../input/riiid-data-prparation/results_question.parq')\nresults_u = pd.read_parquet('../input/riiid-data-prparation/results_user.parq')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgbm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = lgbm.Booster(model_file='../input/riiid-training/booster_075291.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"col1 = ['answered_correctly_content', 'content_id']\ncol2 = ['answered_correctly_user', 'questions_attempted', 'user_id']\nrem =  ['prior_question_elapsed_time'] \npk = ['user_id','content_id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"used_cols = rem+col1+col2\ninit_cols = pk+rem","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# required only for numpy \nresults_u = results_u.values\nresults_c = results_c.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"inference using numpy"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('time on each step:')\nt = time.time()\nfor test_df, sample_prediction_df in iter_tester:\n    test_df = test_df.reset_index()\n    test_df['answered_correctly'] = 0.6571448171182185\n    nump = test_df[init_cols].values\n    for i in range(len(test_df)):\n        uinf = results_u[np.where(nump[i,0]==results_u[:,2])[0]]\n        qinf = results_c[np.where(nump[i,1]==results_c[:,1])[0]]\n        if len(uinf)==0 or len(qinf)==0:\n            continue\n        test_df.loc[i,'answered_correctly'] = model.predict(np.hstack((nump[i,2].reshape(1,-1),qinf,uinf)))\n    env.predict(test_df.loc[test_df['content_type_id']==0,['row_id','answered_correctly']])\n    print(time.time()-t)\n    t = time.time()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inference using pandas"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('time on each step:')\nt = time.time()\nfor test_df, sample_prediction_df in iter_tester:\n    test_df = test_df.reset_index()\n    test_df = test_df.merge(results_c,on='content_id',right_index=True)\n    temp = test_df.merge(results_u,on='user_id',right_index=True)\n    test_df['answered_correctly'] = 0.6571448171182185\n    test_df.loc[temp.index.to_list(),'answered_correctly'] = model.predict(temp[used_cols])\n    env.predict(test_df.loc[test_df['content_type_id']==0,['row_id','answered_correctly']])\n    print(time.time()-t)\n    t = time.time()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}