{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"# -*- coding: utf-8 -*-\nfrom collections import Counter\nfrom curses import raw\nfrom sklearn.preprocessing import PolynomialFeatures\nfrom matplotlib.pyplot import axis\nfrom numba import jit\nimport numba\nimport itertools\nimport random\nimport time\nfrom time import time\nimport warnings\nfrom itertools import combinations\nimport re\nimport matplotlib.pylab as plt\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom regex import X\nimport seaborn as sns\nfrom rich import print\nfrom tqdm import tqdm, trange, tqdm_notebook\nimport copy\nimport math\nfrom datetime import datetime  # 注意这里datetime的位置\nfrom datetime import timedelta  # 注意这里timedelta的位置\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.metrics import r2_score, median_absolute_error, mean_absolute_error, accuracy_score\nwarnings.filterwarnings('ignore')\nimport sys\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class my_feature_engineering():\n    #! 这里的init方法用来封装进一个数据，之后所有这个类中的方法都是直接访问这个数据。\n    def __init__(self, data):\n        self.data = copy.deepcopy(data)\n        self.eps = np.finfo('float').eps\n\n    # 缺失值填充, 替换inf和NaN。\n\n    def fill_na_inf(self, num_cols):\n        # df.dropna(inplace=True)\n        self.data.replace(np.NaN, 0, inplace=True)\n        for col in num_cols:\n            if np.inf in self.data[col].values:  # 找到所有的inf所在的数据列。\n                self.data[col].replace(np.inf, self.data[col][np.isfinite(self.data[col])].max(\n                ), inplace=True)  # ! 这里不用max，否则就是inf本身。而是用除了inf以外最大的实数值来代替。\n            self.data[col].fillna(self.data[col].mean())  # 填充缺失值。\n\n        return self.data\n\n    # 建立移动平均数列\n\n    def moving_average(self, num_cols):\n        for col in num_cols:\n            self.data[f'MA_{col}'] = self.data[col].rolling(3).mean()\n            self.data[f'MA_{col}'].fillna(\n                self.data[col].mean(), inplace=True)\n        return self.data\n\n    # 建立高阶特征，包括：X^0.2, X^2, X^3, log(X)。\n    # 如果要方法类中封装的数据，需要用self.data来引用。\n\n    def high_order_feature(self, num_cols):  # NOTE: 单变量high order。\n        for col in num_cols:\n            try:\n                self.data[f'{col}^0.5'] = abs(\n                    self.data[col].values)**0.5  # 不能是负数。\n                self.data[f'{col}^2'] = self.data[col]**2\n                # self.data[f'{col}^3'] = self.data[col]**3\n                # 数据必须>=0, +eps防止ln0的出现。\n                self.data[f'ln({col})'] = np.log(\n                    abs(self.data[col].values) + self.eps)\n            except:\n                pass\n        return self.data\n\n    # 对连续型数据进行分组/分箱/离散化。\n    def feature_group(self, num_cols):\n        from sklearn.cluster import MiniBatchKMeans\n        from yellowbrick.cluster.elbow import kelbow_visualizer\n        for col in (num_cols):\n            try:\n                X = self.data[col].values.reshape(-1, 1)\n                k = kelbow_visualizer(MiniBatchKMeans(\n                    random_state=111), X, k=(2, 10), show=False).elbow_value_\n                clf = MiniBatchKMeans(n_clusters=k)\n                clf.fit(X)\n                self.data[f'{col}_label'] = clf.predict(X)\n            except:\n                pass\n        return self.data\n\n    # 连续型数据的标准化/归一化。\n    def normalization(self, num_cols):\n        from sklearn.preprocessing import StandardScaler\n        for col in num_cols:\n            try:\n                X = self.data[col].values.reshape(-1, 1)\n                scaler = StandardScaler()\n                scaler.fit(X)\n                self.data[f'{col}_Zscore'] = scaler.transform(X)\n            except:\n                pass\n        return self.data\n\n    # NOTE： 离散数据的编码。\n    def category_encoding(self, category_col, original_col_drop=False):\n        self.data.replace(np.NaN, 0, inplace=True)\n        from sklearn.preprocessing import LabelEncoder\n        LE = LabelEncoder()\n        self.list_encoding = []\n        try:\n            for col in (category_col):\n                self.data[f'{col}_encode'] = LE.fit_transform(self.data[col])\n                self.list_encoding.append(f'{col}_encode')\n                if original_col_drop == True:\n                    self.data.drop(col, inplace=True, axis=1)\n        except Exception as e:\n            print(e)\n            pass\n\n        return self.data, self.list_encoding\n\n    '''2个变量的特征工程，2 variables feature engineering'''\n    # 2个变量的四则运算建立新的特征。\n\n    def two_var_arithmetic(self, num_cols):\n        for cols in itertools.combinations(num_cols, 2):\n            var_1, var_2 = cols[0], cols[1]\n            self.data[f'{var_1}_{var_2}_add'] = self.data[var_1] + \\\n                self.data[var_2]\n            self.data[f'{var_1}_{var_2}_minus'] = self.data[var_1] - \\\n                self.data[var_2]\n            self.data[f'{var_1}_{var_2}_multiply'] = self.data[var_1] * \\\n                self.data[var_2]\n\n        return self.data\n\n    # 2个变量的高阶特征，通过sklearn中的PolynomialFeatures函数。\n    def two_var_highorder(self, num_cols):\n        self.data.replace(np.NaN, 0, inplace=True)\n        from sklearn.preprocessing import PolynomialFeatures\n        PF = PolynomialFeatures(\n            degree=[2, 3], include_bias=False, interaction_only=True)  # ! 如果在之前单变量的highorder函数中已经设置了平方和立方项，那这里的intteraction_only=True. degree从2开始，否则会将产生原始数列一样的数列，其他特征工程函数可能会报错。\n        try:\n            pf_res = PF.fit_transform(self.data[num_cols])\n            col_names = PF.get_feature_names(self.data[num_cols].columns)\n            # NOTE: 注意这里的数据其实是新增的数据，不能直接赋值给self.data, 这样会覆盖原来的数据。因此需要用pd.concate做列的拼接。\n            self.data = pd.concat(\n                [self.data, pd.DataFrame(pf_res, columns=col_names)], axis=1)\n        except:\n            pass\n\n        return self.data\n\n    def stats_feature(self, num_cols, category_col):\n        for num_col in (num_cols):\n            for cat_col in category_col:\n                try:\n                    feat_dict_sum = dict(self.data.groupby([cat_col])[\n                        num_col].sum().iteritems())\n                    self.data[f'{cat_col}_{num_col}_sum'] = [\n                        feat_dict_sum.get(key) for key in self.data[cat_col]]\n\n                    feat_dict_mean = dict(self.data.groupby([cat_col])[\n                        num_col].mean().iteritems())\n                    self.data[f'{cat_col}_{num_col}_mean'] = [\n                        feat_dict_mean.get(key) for key in self.data[cat_col]]\n\n                    feat_dict_std = dict(self.data.groupby([cat_col])[\n                        num_col].std().iteritems())\n                    self.data[f'{cat_col}_{num_col}_std'] = [\n                        feat_dict_std.get(key) for key in self.data[cat_col]]\n\n                    feat_dict_count = dict(self.data.groupby([cat_col])[\n                        num_col].count().iteritems())\n                    self.data[f'{cat_col}_{num_col}_count'] = [\n                        feat_dict_count.get(key) for key in self.data[cat_col]]\n\n                    # feat_dict_min = dict(self.data.groupby([cat_col])[\n                    #     num_col].min().iteritems())\n                    # self.data[f'{cat_col}_{num_col}_min'] = [\n                    #     feat_dict_min.get(key) for key in self.data[cat_col]]\n\n                    # feat_dict_max = dict(self.data.groupby([cat_col])[\n                    #     num_col].max().iteritems())\n                    # self.data[f'{cat_col}_{num_col}_max'] = [\n                    #     feat_dict_max.get(key) for key in self.data[cat_col]]\n\n                    # feat_dict_nunique = dict(self.data.groupby([cat_col])[\n                    #     num_col].nunique().iteritems())\n                    # self.data[f'{cat_col}_{num_col}_nunique'] = [\n                    #     feat_dict_nunique.get(key) for key in self.data[cat_col]]\n                except:\n                    pass\n\n        return self.data\n\n    # 第一分位数/第三分位数特征\n    def quantile_feature(self, num_cols, category_col):\n        for n_col in num_cols:\n            for c_col in category_col:\n                try:\n                    q1 = self.data.groupby([c_col])[n_col].quantile(\n                        [0.25]).reset_index()\n                    self.data[f'{c_col}_{n_col}_q1'] = [\n                        q1[n_col][q1[c_col] == c].values for c in self.data[c_col]]\n                    self.data[f'{c_col}_{n_col}_q1'] = [\n                        float(x) for x in self.data[f'{c_col}_{n_col}_q1']]\n\n                    q3 = self.data.groupby([c_col])[n_col].quantile(\n                        [0.75]).reset_index()\n                    self.data[f'{c_col}_{n_col}_q3'] = [\n                        q3[n_col][q3[c_col] == c].values for c in self.data[c_col]]\n                    self.data[f'{c_col}_{n_col}_q3'] = [\n                        float(x) for x in self.data[f'{c_col}_{n_col}_q3']]\n                except Exception as e:\n                    print(e)\n                    pass\n        return self.data\n\n    '''Manul Feature Engineering Session'''\n    # 2 category variables multiply to construct a new feature column.\n\n    def two_category_multiply(self, to_multiply_data, to_multiply_col, encode_rename=False):\n        for col in itertools.combinations(to_multiply_col, 2):\n            var_1, var_2 = col[0], col[1]\n            to_multiply_data[f'{var_1}_{var_2}_multiply'] = to_multiply_data[var_1] * \\\n                to_multiply_data[var_2]\n\n        if encode_rename == True:\n            self.data.columns = [re.sub('_encode', '', x)\n                                 for x in self.data.columns.values]\n\n        return self.data\n\n    def find_outliner(self, num_col, remove_option=False):\n        std_num = 3\n        print('remove option', remove_option)\n        abnormal = []\n        row_line = []\n        try:\n            for col in (num_col):\n                col_outliners = []\n                std = np.std(self.data[col].values)\n                mean = np.average(self.data[col].values)\n                upper_limit = (mean + std_num*std)\n                lower_limit = (mean - std_num*std)\n                row_line += [self.data[self.data[col].isin([x])].index for x in self.data[col].values if (\n                    (x > upper_limit) or (x < lower_limit))]\n                abnormal += col_outliners\n\n            if remove_option == True:\n                row_line = np.array(row_line)\n                # !把list of listx展开成一个list。\n                row_index = [item for sublist in row_line for item in sublist]\n                self.data = self.data.drop(row_index, axis=0)\n        except:\n            pass\n\n        self.data = self.fill_na_inf(num_cols=num_col)\n\n        return self.data\n\n    # 建立时间特征\n    def time_feature(self, time_cols):\n        for t_col in time_cols:\n            try:\n                self.data[t_col] = pd.to_datetime(self.data[t_col])\n                # self.data[f'{t_col}_year'] = self.data[t_col].dt.year\n                self.data[f'{t_col}_quarter'] = self.data[t_col].dt.quarter\n                self.data[f'{t_col}_month'] = self.data[t_col].dt.month\n                self.data[f'{t_col}_weekofyear'] = self.data[t_col].dt.weekofyear\n                self.data[f'{t_col}_day'] = self.data[t_col].dt.day\n                self.data[f'{t_col}_weekday'] = self.data[t_col].dt.weekday\n                # NOTE: weekday的星期一从0开始，5&6是周末。\n                self.data[f'{t_col}_weekend'] = [\n                    1 if x > 4 else 0 for x in self.data[t_col].dt.weekday]\n                # self.data[f'{t_col}_hour'] = self.data[t_col].dt.hour\n            except Exception as e:\n                print(e)\n                pass\n\n        return self.data\n\n    #! 以下建立一个统一的函数来选择性运行每个特征工程函数。\n\n    def my_feature_auto(\n            self,\n            num_col,\n            category_col,\n            time_col,\n            fill_na_inf=False,\n            moving_average=False,\n            high_order_feature=False,\n            feature_group=False,\n            normalization=False,\n            category_encoding=False,\n            original_col_drop=False,\n            two_var_arithmetic=False,\n            two_var_highorder=False,\n            two_category_multiply=False,\n            encode_rename=False,\n            stats_feature=False,\n            quantile_feature=False,\n            find_outliner=False,\n            remove_option=False,\n            time_feature=False,\n            col_order=False):\n        # 计算整个Pipeline运行的总时长。\n        import time\n        start_time = pd.to_datetime(time.strftime(\n            \"%Y-%m-%d %H:%M:%S\", time.localtime()))\n\n        n = 1  # * 记录运行的函数个数\n\n        # 首先，处理各种异常数据，包括：缺失值，NaN，inf等。\n        if fill_na_inf == True:\n            self.data = self.fill_na_inf(num_col)\n            print(n, ':fill_na_inf complete!')\n            n += 1\n\n        if moving_average == True:\n            self.data = self.moving_average(num_col)\n            print(n, ':moving_average complete!')\n            n += 1\n\n        # 建立高阶特征，包括：X^0.2, X^2, X^3, log(X)\n        if high_order_feature == True:\n            self.data = self.high_order_feature(num_col)\n            print(n, ':high_order_feature complete!')\n            n += 1\n\n        # 对连续型数据进行分组/分箱/离散化。\n        if feature_group == True:\n            self.data = self.feature_group(num_col)\n            print(n, ':feature_group complete!')\n            n += 1\n\n        # 连续型数据的标准化/归一化。\n        if normalization == True:\n            self.data = self.normalization(num_col)\n            print(n, ':normalization complete!')\n            n += 1\n\n        # 离散数据的编码。\n        if category_encoding == True:\n            self.data, list_encoding = self.category_encoding(\n                category_col, original_col_drop)\n            print(n, ':category_encoding complete!')\n            n += 1\n\n        if two_var_arithmetic == True:\n            self.data = self.two_var_arithmetic(num_col)\n            print(n, ':two_var_arithmetic complete!')\n            n += 1\n\n        if two_var_highorder == True:\n            self.data = self.two_var_highorder(num_col)\n            print(n, ':two_var_highorder complete!')\n            n += 1\n\n        if two_category_multiply == True:\n            self.data = self.two_category_multiply(\n                to_multiply_data=self.data, to_multiply_col=list_encoding, encode_rename=True)\n            print(n, ':two_category_multiply complete!')\n            n += 1\n\n        if stats_feature == True:\n            self.data = self.stats_feature(num_col, category_col)\n            print(n, ':stats_feature complete!')\n            n += 1\n\n        if quantile_feature == True:\n            self.data = self.quantile_feature(num_col, category_col)\n            print(n, ':quantile_feature complete!')\n            n += 1\n\n        # 新的数据self.dataframe的列进行排序。\n        if col_order == True:\n            self.data = pd.DataFrame(\n                self.data, columns=list(sorted(self.data.columns)))\n            print(n, ':col_order complete!')\n            n += 1\n\n        if find_outliner == True:\n            self.data = self.find_outliner(num_col, remove_option)\n            print(n, ':find_outliner complete!')\n            n += 1\n\n        if time_feature == True:\n            self.data = self.time_feature(time_col)\n            print(n, ':time_feature complete!')\n            n += 1\n\n        #! 最终再执行一次再处理各种异常数据，包括：缺失值，NaN，inf等。\n        if fill_na_inf == True:\n            self.data = self.fill_na_inf(num_col)\n            print(n, ':fill_na_inf complete, AGAIN!')\n            n += 1\n\n        print('Finally, Auto Feature Engineering Complete!')\n        end_time = pd.to_datetime(time.strftime(\n            \"%Y-%m-%d %H:%M:%S\", time.localtime()))\n        print('Time Used:', (end_time - start_time))\n        print('total feature columns:', len(self.data.columns.values))\n\n        return self.data\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## A complete function to reduce memory usage of data file.\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                #NOTE: check the data upper/lower limits of all kinds of system int/float settings. \n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(\n        100 * (start_mem - end_mem) / start_mem))\n\n    return df\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''construct the final submission data'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### load the test data and the pre-trained model file. \ndf_test = pd.read_csv('../input/amex-default-prediction/test_data.csv', chunksize=10000)\nmodel_list = pd.read_pickle('../input/pickle/saved_model_p4.pickle')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### construct the final submission data file. \nz = 0\ny_pred_list = []\ntarget_pred = pd.Series()\ndf_submit = pd.DataFrame()\n\nfor df_test_partial in tqdm_notebook(df_test):\n    z +=1\n    if z < 3:\n        df_test_partial = reduce_mem_usage(df_test_partial)\n        # print(df_test_partial.info())\n        s = (df_test_partial.dtypes == 'float16')\n        num_cols = list(s[s].index) + ['B_31']\n        cat_cols = ['D_63', 'D_64']\n        time_cols = ['S_2']\n        customer_ID = ['customer_ID']\n\n        '''feature engineering'''\n        mfe = my_feature_engineering(df_test_partial)  # NOTE：初始化, 传入原始数据df。\n        data = mfe.my_feature_auto(num_col=num_cols, category_col=cat_cols, time_col=time_cols, fill_na_inf=True, moving_average=False, feature_group=False, normalization=False, high_order_feature=True, category_encoding=True,\n                                original_col_drop=True, encode_rename=False, two_var_arithmetic=True, two_var_highorder=False, two_category_multiply=False, stats_feature=True, quantile_feature=False, find_outliner=False, remove_option=False, time_feature=True, col_order=False)\n        data.dropna(inplace=True)\n    #         print(data.info())\n\n        train_col = data.columns.to_list()\n        try:\n            t64 = (data.dtypes == 'datetime64[ns]')\n            obj = (data.dtypes == 'object')\n            obj_col = (obj[obj].index)\n            time_col = (t64[t64].index)\n            target_col = 'target'\n            train_col.remove('customer_ID')\n            train_col.remove('S_2')\n            train_col.remove(target_col)\n            train_col.remove(time_col)\n        except Exception as e:\n            print(e)\n            pass\n\n        ### final train data setup.\n        print('start predicting!')\n        for model in (model_list[:2]): #load the pre-trained model in advance. \n            pred_res = model.predict(data[train_col], predict_disable_shape_check=True, num_threads=20)\n            y_pred_list.append(pred_res)\n\n        y_pred_all = pd.DataFrame(y_pred_list).T\n        # print(y_pred_list)\n        y_pred = np.mean(y_pred_all, axis=1)\n        y_pred = pd.Series(np.round(y_pred, 0))\n    #     print(y_pred)\n        print(pd.Series(y_pred).value_counts())\n\n        target_pred = pd.concat([target_pred, y_pred], axis=0)\n        \n        df_temp = pd.DataFrame(\n            {\n            'customer_ID': df_test_partial['customer_ID'].values,\n            'prediction': y_pred\n            }\n        )\n\n        df_submit = pd.concat([df_submit, df_temp], axis=0)\n   \n\n        try:\n            del data, df_test_partial, mfe\n            gc.collect()\n        except Exception as e:\n            print(e)\n            pass","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(target_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# /kaggle/working/\n\npd.to_pickle(target_pred, \"/kaggle/working/target_pred_z1.pickle\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.to_pickle(target_pred, \"/Users/yunshi/Downloads/Kaggle_American Express_Default Prediction/target_pred.pickle\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}