{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 从原始数据导入 压缩成pkl文件","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:09:25.555438Z","iopub.execute_input":"2022-10-08T10:09:25.556234Z","iopub.status.idle":"2022-10-08T10:09:25.581968Z","shell.execute_reply.started":"2022-10-08T10:09:25.556112Z","shell.execute_reply":"2022-10-08T10:09:25.581098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 导入数据 压缩float64和int64","metadata":{}},{"cell_type":"code","source":"%%time\ntrain_data = pd.DataFrame()\nwith pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', chunksize=10**5) as reader:\n    for counter, chunk in enumerate(reader):\n        for column in chunk.columns:\n            if str(chunk[column].dtype) == \"float64\":\n                chunk[column] = chunk[column].astype(np.float32)\n            if str(chunk[column].dtype) == \"int64\":\n                chunk[column] = chunk[column].astype(np.int32)\n        train_data = pd.concat([train_data, chunk])\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:09:25.583838Z","iopub.execute_input":"2022-10-08T10:09:25.584673Z","iopub.status.idle":"2022-10-08T10:19:53.233070Z","shell.execute_reply.started":"2022-10-08T10:09:25.584637Z","shell.execute_reply":"2022-10-08T10:19:53.231970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# train_data = pd.DataFrame()\n# with pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', chunksize=10**5) as reader:\n#     for counter, chunk in enumerate(reader):\n# #         for column in chunk.columns:\n# #             if str(chunk[column].dtype) == \"float64\":\n# #                 chunk[column] = chunk[column].astype(np.float32)\n# #             if str(chunk[column].dtype) == \"int64\":\n# #                 chunk[column] = chunk[column].astype(np.int32)\n#         train_data = pd.concat([train_data, chunk])\n#         gc.collect()\n# train_data.memory_usage().sum() / 1024**2","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:19:53.234573Z","iopub.execute_input":"2022-10-08T10:19:53.235292Z","iopub.status.idle":"2022-10-08T10:19:53.240955Z","shell.execute_reply.started":"2022-10-08T10:19:53.235246Z","shell.execute_reply":"2022-10-08T10:19:53.239868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.to_pickle(\"train_data.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:19:53.243412Z","iopub.execute_input":"2022-10-08T10:19:53.243862Z","iopub.status.idle":"2022-10-08T10:20:06.823973Z","shell.execute_reply.started":"2022-10-08T10:19:53.243819Z","shell.execute_reply":"2022-10-08T10:20:06.823059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(\"/kaggle/input/amex-default-prediction/train_labels.csv\")\ntrain_labels.to_pickle(\"train_labels.pkl\")","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:20:06.825065Z","iopub.execute_input":"2022-10-08T10:20:06.825390Z","iopub.status.idle":"2022-10-08T10:20:07.995759Z","shell.execute_reply.started":"2022-10-08T10:20:06.825360Z","shell.execute_reply":"2022-10-08T10:20:07.994751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 对'int16', 'int32', 'int64', 'float16', 'float32', 'float64'压缩\n最后得到的pkl文件大约为2G","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:20:07.997057Z","iopub.execute_input":"2022-10-08T10:20:07.997402Z","iopub.status.idle":"2022-10-08T10:20:08.011199Z","shell.execute_reply.started":"2022-10-08T10:20:07.997371Z","shell.execute_reply":"2022-10-08T10:20:08.010362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_mem_red = reduce_mem_usage(train_data)","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:20:08.012295Z","iopub.execute_input":"2022-10-08T10:20:08.012640Z","iopub.status.idle":"2022-10-08T10:20:21.761907Z","shell.execute_reply.started":"2022-10-08T10:20:08.012609Z","shell.execute_reply":"2022-10-08T10:20:21.760602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train_mem_red","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:23:37.693032Z","iopub.execute_input":"2022-10-08T10:23:37.693939Z","iopub.status.idle":"2022-10-08T10:23:37.698473Z","shell.execute_reply.started":"2022-10-08T10:23:37.693893Z","shell.execute_reply":"2022-10-08T10:23:37.697625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 根据IDgroupby然后aggregate\n数值型变量用'mean', 'std', 'min', 'max', 'last'\n\n类别变量用'count', 'last', 'nunique'","metadata":{}},{"cell_type":"code","source":"train = train_data.merge(train_labels, left_on='customer_ID', right_on='customer_ID')","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:26:13.068245Z","iopub.execute_input":"2022-10-08T10:26:13.068965Z","iopub.status.idle":"2022-10-08T10:26:23.523034Z","shell.execute_reply.started":"2022-10-08T10:26:13.068920Z","shell.execute_reply":"2022-10-08T10:26:23.522023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = train.drop(['customer_ID', 'S_2', 'target'], axis=1).columns.to_list()\ncat_features = [\n    \"B_30\",\n    \"B_38\",\n    \"D_114\",\n    \"D_116\",\n    \"D_117\",\n    \"D_120\",\n    \"D_126\",\n    \"D_63\",\n    \"D_64\",\n    \"D_66\",\n    \"D_68\",\n]\nnum_features = [col for col in features if col not in cat_features]","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:26:25.515435Z","iopub.execute_input":"2022-10-08T10:26:25.515863Z","iopub.status.idle":"2022-10-08T10:26:37.086375Z","shell.execute_reply.started":"2022-10-08T10:26:25.515830Z","shell.execute_reply":"2022-10-08T10:26:37.085445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_num_agg = train.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\ntrain_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\ntrain_cat_agg = train.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\ntrain_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\ntrain_target = (train.groupby(\"customer_ID\").tail(1).set_index('customer_ID', drop=True).sort_index()[\"target\"])\ntrain = pd.concat([train_num_agg, train_cat_agg, train_target], axis=1)\n\n#train.to_pickle(\"../data/train_agg.pkl\", compression=\"gzip\")","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:36:14.750511Z","iopub.execute_input":"2022-10-08T10:36:14.750949Z","iopub.status.idle":"2022-10-08T10:36:14.777110Z","shell.execute_reply.started":"2022-10-08T10:36:14.750915Z","shell.execute_reply":"2022-10-08T10:36:14.775402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-10-08T10:36:43.516265Z","iopub.execute_input":"2022-10-08T10:36:43.517287Z","iopub.status.idle":"2022-10-08T10:36:43.523952Z","shell.execute_reply.started":"2022-10-08T10:36:43.517248Z","shell.execute_reply":"2022-10-08T10:36:43.522825Z"},"trusted":true},"execution_count":null,"outputs":[]}]}