{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport gc\n\nimport numpy as np \nimport pandas as pd \nfrom sklearn.preprocessing import MinMaxScaler, OrdinalEncoder\nimport category_encoders as ce","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"class DataLoader():\n    def __init__(self):\n        self.train = None\n        self.dev = None\n        self.test = None\n        self.X_train = None\n        self.X_dev = None\n        self.X_test = None\n        self.y_train = None\n        self.y_dev = None\n        self.prediction_dev = None\n        self.prediction_test = None\n        \n        self.sub = None\n        self.feature_columns = [\n                   'timestamp',\n                   'user_id',\n                   'content_id',\n                   'content_type_id',\n                   'task_container_id',\n                   'prior_question_elapsed_time',\n                   'prior_question_had_explanation'\n                  ]\n        self.label_column = 'answered_correctly'\n        self.cat_features = [\n                'user_id',\n                'content_id',\n                'content_type_id',\n                'task_container_id', \n                'prior_question_had_explanation']\n        self.numeric_columns = ['timestamp', 'prior_question_elapsed_time']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PreProcess():\n    def __init__(self):\n        self.min_max_scaler = None\n        self.test_size=0.2\n        \n    # Create a column with the number of missing values in each row:    \n    def set_missing_values_column(df):\n        df['nb_missing_values'] = df.shape[1] - df.count(axis=1)\n        return df\n    \n    def preprocess_MinMaxScaler(self,dd,which):\n        df1 = dd.train[dd.numeric_features]\n        if which == 'test':\n            df1 = dd.test[dd.numeric_features]\n\n        if self.min_max_scaler is None:\n            self.min_max_scaler = MinMaxScaler()\n            self.min_max_scaler.fit(df1)\n        df_cols = self.min_max_scaler.transform(df1)\n        if which == 'train':\n            dd.train[dd.numeric_features] = df_cols\n        elif which == 'test':\n            dd.test[dd.numeric_features] = df_cols\n        return dd\n\n    def preprocess_MinMaxScaler_all(self, dd):\n        scaler = MinMaxScaler()\n        scaler.fit(dd.train[dd.numeric_features])\n        dd.train[dd.numeric_features] = scaler.transform(dd.train[dd.numeric_features])\n        dd.test[dd.numeric_features] = scaler.transform(dd.test[dd.numeric_features])\n        self.min_max_scaler = scaler\n        return dd\n\n    def preprocess_OrdinalEncoder_all(self, dd):\n        scaler = OrdinalEncoder()\n        scaler.fit(dd.train[dd.cat_features])\n        dd.train[dd.cat_features] = scaler.transform(dd.train[dd.cat_features])\n        dd.test[dd.cat_features] = scaler.transform(dd.test[dd.cat_features])\n        return dd\n    \n    def preprocess_target_encoder(self, dd):\n        encoder = ce.TargetEncoder(cols=dd.cat_features)\n        gc.collect()\n        encoder.fit(dd.train[dd.feature_columns], dd.train[dd.label_column])\n        gc.collect()\n        dd.train[dd.feature_columns] = encoder.transform(dd.train[dd.feature_columns], dd.train[dd.label_column])\n        gc.collect() \n        dd.test[dd.feature_columns] = encoder.transform(dd.test[dd.feature_columns])\n        gc.collect()\n        return dd\n\n    def custom_train_test_split(self, dd):\n        dd.X_train, dd.X_dev, dd.y_train, dd.y_dev = train_test_split(dd.train[dd.feature_columns], dd.train[dd.label_column], test_size=self.test_size, random_state=42)\n        dd.train=None\n        gc.collect()\n        return dd","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}