{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# import cudf\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# import matplotlib.pyplot as plt\nimport pickle\nimport os\n# from sklearn.cluster import KMeans\n# from pandas.api.types import CategoricalDtype\nfrom sklearn.preprocessing import StandardScaler\nimport time\nimport tensorflow as tf\nfrom tensorflow import keras\nassert tf.__version__ >= \"2.0\"\nprint(np.__version__)\nprint(pd.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\n# import resource","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from multiprocessing import Pool, TimeoutError","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.random.seed(42) # generating random see\n# tf.random.set_seed(42) # setting random seed\ninputDataFolder ='/kaggle/input/prediction-with-score/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\ndef sizeof_fmt(num, suffix='B'):\n    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n        if abs(num) < 1024.0:\n            return \"%3.1f %s%s\" % (num, unit, suffix)\n        num /= 1024.0\n    return \"%.1f %s%s\" % (num, 'Yi', suffix)\n\ndef printVariableSize(li):\n    for name, size in sorted(((name, sys.getsizeof(value)) for name, value in li.items()),\n                             key= lambda x: -x[1])[:10]:\n        print(\"{:>30}: {:>8}\".format(name, sizeof_fmt(size)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\noutfile = open(inputDataFolder+'user_history_group_dict.pkl','rb')\ntrain = pickle.load(outfile)\noutfile.close()\n# train = cudf.from_pandas(train)\ntrain_keys = train.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[115].info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\noutfile = open(inputDataFolder+'user_score.pkl','rb')\nuser_score = pickle.load(outfile)\noutfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"user_score= user_score.astype('float32')\nuser_score.dtype","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\noutfile = open(inputDataFolder+'question_weights.pkl','rb')\nquestion_weights = pickle.load(outfile)\noutfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\noutfile = open(inputDataFolder+'content_cat.pkl','rb')\ncontent_df = pickle.load(outfile).drop(columns=['new_content_id'])\noutfile.close()\n# content_df = cudf.from_pandas(content_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"content_df=content_df.astype('float32')\ncontent_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\noutfile = open(inputDataFolder+'sc_timestamp.pkl','rb')\nsc_timestamp = pickle.load(outfile)\noutfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\noutfile = open(inputDataFolder+'new_content_id_index_map.pkl','rb')\nnew_content_id_index_map = pickle.load(outfile)\noutfile.close()\n# new_content_id_index_map = pd.from_pandas(new_content_id_index_map)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_content_id_index_map_dict = { new_content_id_index_map.iloc[i,1]:new_content_id_index_map.iloc[i,0] for i in range(len(new_content_id_index_map))}\nnew_content_id_index_map_dict_keys = new_content_id_index_map_dict.keys()\n# new_content_id_index_map_dict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\noutfile = open(inputDataFolder+'valid_df.pkl','rb')\nvalid_df = pickle.load(outfile)\noutfile.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"printVariableSize(globals())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def formatData(testData):\n\n    testData['prior_question_had_explanation']=testData['prior_question_had_explanation'].fillna(False)\n    testData['prior_question_had_explanation']=testData['prior_question_had_explanation'].astype(int)\n\n    testData['new_content_id'] = testData['content_type_id'].astype(str)+'_'+testData['content_id'].astype(str)\n\n    testData.loc[:,['timestamp','prior_question_elapsed_time']] = sc_timestamp.transform(testData.loc[:,['timestamp','prior_question_elapsed_time']])\n\n    testData = testData[['user_id','timestamp', 'new_content_id', 'content_type_id', \n                  'prior_question_had_explanation', 'answered_correctly']]\n    \n    testData=testData.astype({'timestamp':'float32', 'prior_question_had_explanation': 'float32','content_type_id':'float32'})\n\n    for i in range(len(testData)):\n        if testData.iloc[i,2] in new_content_id_index_map_dict_keys:\n            testData.iloc[i,2] = new_content_id_index_map_dict[testData.iloc[i,2]]\n        else:\n            testData.iloc[i,2] = 0\n\n\n    return testData","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_user_score = .5\ndepthOfHistory = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def testData_preProcessing(testData):\n\n    testData_np = testData.to_numpy()\n    X_submit = np.empty((len(testData), 207))\n    X_submit[:,:6] = testData_np\n    for i in np.arange(len(X_submit)):\n        if X_submit[i,0] in user_score.index:\n            X_submit[i,6]= user_score[X_submit[i,0]]  \n        else:\n            X_submit[i,6]= new_user_score\n        X_submit[i,7:]= content_df.loc[X_submit[i,2],:]\n\n    X_submit=np.delete(X_submit, [2,5],1)\n\n    testData_filtered=[]\n\n    for i in X_submit:\n        \n        if i[0] in train_keys:\n\n            lenOfHist = len(train[i[0]])\n            if lenOfHist >= depthOfHistory:\n                heightOfArray = depthOfHistory\n            else:\n                heightOfArray = lenOfHist\n\n            tempHist = np.zeros((heightOfArray, 207))\n\n            tempHist[:,:7] = train[i[0]].tail(depthOfHistory).astype({'timestamp':'float32', 'prior_question_had_explanation': 'float32','content_type_id':'float32'}).to_numpy()\n               \n            tempHist[:,7:]= content_df.loc[tempHist[:,6],:]\n\n            tempHist=np.delete(tempHist, [4,6],1)\n\n            temp_all = np.append(tempHist,[i], axis=0)\n\n\n        else:\n            temp_all=[i]\n        \n        temp_all=np.delete(temp_all, 0,1)\n        testData_filtered.append([temp_all])\n\n  \n    return testData_filtered\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef genTestData1(data):\n\n    data.reset_index(drop=True,inplace=True)\n\n    data = testData_preProcessing(data)\n\n    for g in data:\n\n        yield tf.convert_to_tensor(g, dtype=np.float32)\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def genTestData(data):\n    while True:\n        \n        yield tf.convert_to_tensor(testData_filtered_dummy[0], dtype=np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preProcessData(data):\n\n    data.reset_index(drop=True,inplace=True)\n    return testData_preProcessing(data)\n\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = keras.models.load_model(inputDataFolder+'latest_model.h5')\n# model = keras.models.load_model(inputDataFolder+'best_model_20.h5')\nmodel.call = tf.function(model.call, experimental_relax_shapes=True)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"env = riiideducation.make_env()\niter_test = env.iter_test()\n# iter_test = valid_gen()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor (test_df, sample_prediction_df) in iter_test:\n\n    test_df.reset_index(drop=True,inplace=True)\n\n    test_df['answered_correctly']=-1\n    if len(test_df)>0:\n\n        test_df_mod =formatData(test_df)\n\n\n        X_pred= test_df_mod[test_df_mod['content_type_id'] == 0]\n        len_X_pred = len(X_pred)\n\n        if len_X_pred >0:\n\n\n            data = preProcessData(X_pred)\n\n            y_pred =[]\n\n            for d in data:\n\n                train_dataset=tf.convert_to_tensor(d , dtype=np.float32)\n                y_pred_temp = model(train_dataset)\n                \n                y_pred.append(y_pred_temp[0][0])\n\n            y_pred = np.asarray(y_pred)\n\n\n            test_df.loc[test_df['content_type_id'] == 0,'answered_correctly'] = np.nan_to_num(y_pred,nan=0.9,posinf=.9,neginf=0.1)\n\n            env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n\n        else :\n            env.predict(np.array([], dtype = np.float32))\n\n    else:\n        env.predict(np.array([], dtype = np.float32))\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}