{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport riiideducation\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nfrom sklearn import model_selection\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"dtype={\n        'row_id': 'int64', \n        'timestamp': 'int64', \n        'user_id': 'int32', \n        'content_id': 'int16', \n        'content_type_id': 'int8',\n        'task_container_id': 'int16', \n        'user_answer': 'int8', \n        'answered_correctly': 'int8', \n        'prior_question_elapsed_time': 'float32', \n        'prior_question_had_explanation': 'boolean'\n    } \ntrain = pd.read_csv('../input/riiid-test-answer-prediction/train.csv', dtype = dtype, low_memory = False, nrows= 10**6)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dataframe_information(dataframe):\n    \n    # Shape\n    print(dataframe.shape)\n    \n    # Information Values\n    Information_df = pd.DataFrame()\n    \n    feature_list = []\n    count_list = []\n    percent_list = []\n    unique_value_list = []\n    mean_list = []\n    median_list = []\n    mode_list = []\n    \n    for col in dataframe.columns:\n        \n        feature_list.append(col)\n        count_list.append(dataframe[col].isnull().sum())\n        percent_list.append(dataframe[col].isnull().sum() * 100/len(dataframe))\n        unique_value_list.append(dataframe[col].nunique())\n        mean_list.append(int(dataframe[col].mean()))\n        median_list.append(int(dataframe[col].median()))\n        mode_list.append(int(dataframe[col].mode()[0]))\n        \n\n    Information_df['feature'] = feature_list\n    Information_df['Number of missing values'] = count_list\n    Information_df['Percentage_of_missing'] = percent_list\n    Information_df['Unique_values_for_each_column'] = unique_value_list\n    Information_df['Mean for the variable'] = mean_list\n    Information_df['Median for the variable'] = median_list\n    Information_df['Mode for the variable'] = mode_list\n    \n    return Information_df\n\ndataframe_information(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['row_id','timestamp', 'user_id', 'content_id','content_type_id', 'task_container_id', 'prior_question_elapsed_time',\n       'prior_question_had_explanation']\n\ntarget = ['answered_correctly']\ndef preprocessing(dataframe, features, target):\n    \n    # Basic data information\n    print(dataframe.shape)\n    \n    # Filling Null values\n    dataframe = dataframe.fillna(0)\n    \n    # Change the categorical to numerical\n    mapping = {False: 0, True: 1}\n    dataframe.loc[:, 'prior_question_had_explanation'] = dataframe.loc[:, 'prior_question_had_explanation'].map(mapping)\n    \n    # Dropping Unnecessary columns\n    decided_col_list = ['row_id','timestamp', 'user_id', 'content_id','content_type_id', 'task_container_id','user_answer', 'answered_correctly', 'prior_question_elapsed_time',\n       'prior_question_had_explanation']\n\n\n    for col in dataframe.columns:\n        if str(col) not in decided_col_list:\n            dataframe = dataframe.drop(col, axis = 1)\n    \n    # Missing Values and infinite values replacement    \n    dataframe.loc[:, 'prior_question_elapsed_time'] = dataframe.loc[:, 'prior_question_elapsed_time'].replace(np.nan, dataframe['prior_question_elapsed_time'].mode()[0])\n    dataframe.loc[:, 'prior_question_had_explanation'] = dataframe.loc[:, 'prior_question_had_explanation'].replace(np.nan, dataframe['prior_question_had_explanation'].median())\n    \n    # Standardize the values:\n    features_tobe_standardized = ['timestamp', 'prior_question_elapsed_time']\n    dataframe.timestamp = (dataframe.timestamp - dataframe.timestamp.mean()) / dataframe.timestamp.std()\n    dataframe.prior_question_elapsed_time = (dataframe.prior_question_elapsed_time - dataframe.prior_question_elapsed_time.mean()) / dataframe.prior_question_elapsed_time.std()            \n    \n    \n    # Removing answered_correctly which is equal to -1\n    #dataframe = dataframe[dataframe['answered_correctly'] != -1]\n    \n    # Groupby user id\n    #dataframe = dataframe.groupby('user_id')['answered_correctly'].sum()\n    #dataframe['intelligence'] = dataframe\n    return dataframe, dataframe[features]\n\ntrain = preprocessing(train, features, target)[0]\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"****EXPLORATORY DATA ANALYSIS****"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train['answered_correctly'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import cross_val_score\n\ndef cross_validation(model, X, y, num_folds):\n    kfolds = model_selection.KFold(n_splits = num_folds)\n    results = cross_val_score(model, X, y, cv = kfolds, scoring = 'neg_mean_squared_error')\n    print(f'Mean: {results.mean()}, Standard Deviation: {results.std()}')\n    return [results.mean(), results.std()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet, SGDClassifier\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.metrics import roc_auc_score\n\nmodels = [LinearRegression(),Ridge(), Lasso(), ElasticNet(), DecisionTreeRegressor(), SGDClassifier(loss = 'log')]\nregressor = models[4]\nregressor.fit(train[features], train[target])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Spot check the algorithms\n# for model in models:\n#     cross_validation(model, train[features], train[target], 7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    env = riiideducation.make_env()\nexcept:\n    pass\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction) in iter_test:\n    X_test = preprocessing(test_df, features, target)[1]\n    test_df['answered_correctly'] = regressor.predict(X_test)\n    test_df = test_df.fillna(0)\n    env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])\n    print(test_df.head(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}