{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 引入库"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datatable as dt\nimport psutil\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.model_selection import train_test_split\nimport riiideducation\nfrom pandas_profiling import ProfileReport\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import MinMaxScaler\nimport lightgbm as lgb\nfrom sklearn.ensemble import AdaBoostRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom scipy.stats import ttest_ind\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV\nfrom scipy.stats import randint as sp_randint\nfrom scipy.stats import uniform as sp_uniform\nfrom sklearn.metrics import mean_absolute_error\nfrom matplotlib import pyplot as plt \n%matplotlib inline\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 读入数据"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.DataFrame()\ncounter = 1\nfor chunk in pd.read_csv('../input/riiid-test-answer-prediction/train.csv', chunksize=1000000, low_memory=False, nrows=10000000):\n    print('Reading chunck {}'.format(counter))\n    train_df = pd.concat([train_df, chunk], ignore_index=True)\n    counter += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/riiid-test-answer-prediction/example_test.csv')\nquestions_df = pd.read_csv('../input/riiid-test-answer-prediction/questions.csv')\nlectures_df = pd.read_csv('../input/riiid-test-answer-prediction/lectures.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 分析数据-train_df"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df['answered_correctly'].isna().sum())\nprint(train_df.shape[0])\nprint('There are {:0.4f}% of missing data in answered_correctly. '.format(train_df['answered_correctly'].isna().sum()/train_df.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df['prior_question_elapsed_time'].isna().sum())\nprint(train_df.shape[0])\nprint('There are {:0.4f}% of missing data in prior_question_elapsed_time. '.format(train_df['prior_question_elapsed_time'].isna().sum()/train_df.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_df['prior_question_had_explanation'].isna().sum())\nprint(train_df.shape[0])\nprint('There are {:0.4f}% of missing data in prior_question_had_explanation. '.format(train_df['prior_question_had_explanation'].isna().sum()/train_df.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 填补Nan"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Code from https://www.kaggle.com/dmikar/baseline-for-riiid-lightgbm\nmean_prior = train_df['prior_question_elapsed_time'].astype(\"float64\").mean()\nprint(f'{mean_prior} is filled for the missing data in prior_question_elapsed_time. ')\n\ntrain_df['prior_question_elapsed_time'].fillna(mean_prior, inplace = True)\ntrain_df['prior_question_had_explanation'].fillna(False, inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# 分析数据-test_df"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['prior_question_had_explanation'].dtype","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 分析数据-questions_df"},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df['tag'] = questions_df['tags'].str.split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"There are {train_df['user_id'].unique().shape[0]} users in training data.\") \nprint(f\"There are {test_df['user_id'].unique().shape[0]} users in testing data.\") \ntotal_users = pd.concat([train_df['user_id'], test_df['user_id']])\nprint(f\"There are {total_users.unique().shape[0]} users in either data frames.\" )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train_df.loc[0:1000000,['answered_correctly']]\nX = train_df.loc[0:1000000,['task_container_id', 'content_type_id','prior_question_elapsed_time', 'prior_question_had_explanation']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mini_max_scaler = MinMaxScaler()\nX = mini_max_scaler.fit_transform(X)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(X, y, test_size =0.3, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_train_df['prior_question_had_explanation'].isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_df = X_train_df.astype('float64')\nX_val_df = X_val_df.astype('float64')\ny_train_df = y_train_df.astype('float64')\ny_val_df = y_val_df.astype('float64')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import Sequential\nfrom keras.layers import Dense\nimport lightgbm as lgbm\n# 定义模型\nmodel=lgbm.LGBMClassifier(num_leaves=60,learning_rate=0.05,n_estimators=40)\nmodel.fit(X_train_df,y_train_df)\ny_pre=model.predict(X_val_df)\ncounty = 0\ncount = 0\ny_val_df = y_val_df.values\nprint(y_val_df)\nfor i in range(len(y_pre)):\n    count+=1\n    if(y_pre[i] == y_val_df[i]):\n        county+=1\nprint(county/count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# nb_epoch = 5\n# model.fit(X_train_df, y_train_df, epochs=nb_epoch, validation_data=(X_val_df, y_val_df), batch_size=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ex = pd.read_csv('../input/riiid-test-answer-prediction/example_test.csv')\nex = ex[ex['content_type_id'] == 0]\nex1 = ex[['user_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']]\n# 这里做你的处理\nmean_prior_test = ex1['prior_question_elapsed_time'].astype(\"float64\").mean()\nex1['prior_question_elapsed_time'].fillna(mean_prior_test, inplace=True)\nex1['prior_question_had_explanation'].fillna(False, inplace=True)\nex1 = ex1.astype('float64')\nex1 = mini_max_scaler.fit_transform(ex1)\nprint(ex1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = model.predict(ex1)\nex['answered_correctly']=res\n# res = sum(res, [])\nress = ex[['row_id', 'answered_correctly']]\nprint(ress)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import riiideducation\nenv = riiideducation.make_env()\niter_test = env.iter_test()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for (sample_test,sample_prediction_df) in iter_test:\n    sample_test = sample_test[sample_test['content_type_id'] == 0]\n    sample_test1 = sample_test[['user_id', 'content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation']]\n    # 这里做你的处理\n    mean_prior_test = sample_test1['prior_question_elapsed_time'].astype(\"float64\").mean()\n    sample_test1['prior_question_elapsed_time'].fillna(mean_prior_test, inplace=True)\n    sample_test1['prior_question_had_explanation'].fillna(False, inplace=True)\n    sample_test1 = sample_test1.astype('float64')\n    sample_test1 = mini_max_scaler.fit_transform(sample_test1)\n    \n    res = model.predict(sample_test1)\n    sample_test['answered_correctly']=res\n    \n    env.predict(sample_test[['row_id','answered_correctly']])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}