{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport joblib\nimport gc\nimport itertools\nfrom itertools import combinations\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\n\nimport lightgbm as lgb\nfrom sklearn.preprocessing import LabelEncoder\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\n\n\npd.set_option('display.width', 1000)\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\nimport warnings; warnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:06.787173Z","iopub.execute_input":"2022-08-03T11:12:06.787685Z","iopub.status.idle":"2022-08-03T11:12:06.797616Z","shell.execute_reply.started":"2022-08-03T11:12:06.787640Z","shell.execute_reply":"2022-08-03T11:12:06.796548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -qq investpy","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:06.839464Z","iopub.execute_input":"2022-08-03T11:12:06.840086Z","iopub.status.idle":"2022-08-03T11:12:19.139150Z","shell.execute_reply.started":"2022-08-03T11:12:06.840051Z","shell.execute_reply":"2022-08-03T11:12:19.137515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import investpy","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:19.142721Z","iopub.execute_input":"2022-08-03T11:12:19.143250Z","iopub.status.idle":"2022-08-03T11:12:19.149541Z","shell.execute_reply.started":"2022-08-03T11:12:19.143199Z","shell.execute_reply":"2022-08-03T11:12:19.148229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def percent_to_float(row):\n    act = row.actual\n    return float(act[0:3])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_difference(data, num_features):\n    df1 = []\n    customer_ids = []\n    \n    for customer_id, df in tqdm(data.groupby(['customer_ID'])):\n        diff_df1 = df[num_features].diff(1).iloc[[-1]].values.astype(np.float16)#float32をfloat16に変えた\n        df1.append(diff_df1)\n        customer_ids.append(customer_id)\n    \n    df1 = np.concatenate(df1, axis = 0)\n    df1 = pd.DataFrame(df1, columns = [col + '_diff1' for col in df[num_features].columns])\n    df1['customer_ID'] = customer_ids\n    return df1","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:19.151406Z","iopub.execute_input":"2022-08-03T11:12:19.152233Z","iopub.status.idle":"2022-08-03T11:12:19.165370Z","shell.execute_reply.started":"2022-08-03T11:12:19.152184Z","shell.execute_reply":"2022-08-03T11:12:19.164409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# trainデータの前処理","metadata":{}},{"cell_type":"code","source":"def read_preprocess_traindata(train_, train_labels_):    \n    # 特徴量の指定(customerとS_2以外のすべて)\n    features = train_.drop(['customer_ID', 'target', 'S_2'], axis = 1).columns.to_list()\n    # カテゴリ変数の指定\n    cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]\n    \n    # 数値変数の個数(特徴量の中からカテゴリ変数で指定したものを除く)\n    num_features = [col for col in features if col not in cat_features]\n    \n    # floatデータのメモリ削減(もともとfloat32だったものをfloat16まで減らす)\n    print('Starting reducing memory. (num feature: cast float32 to float16)')\n    float_cols = list(train_.dtypes[train_.dtypes == 'float32'].index)\n    for col in tqdm(float_cols):\n        train_[col] = train_[col].astype(np.float16)\n    \n    # 特徴量の生成\n    print('Starting training feature engineer...')    \n    train_num_agg = train_.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    train_num_agg.columns = ['_'.join(x) for x in train_num_agg.columns]\n    train_num_agg.reset_index(inplace = True)\n    train_cat_agg = train_.groupby(\"customer_ID\")[cat_features].agg(['count', 'last', 'nunique'])\n    train_cat_agg.columns = ['_'.join(x) for x in train_cat_agg.columns]\n    train_cat_agg.reset_index(inplace = True)\n    \n    print('Starting reducing memory. (new features:cast 64->8)')\n    # floatデータのメモリ削減(もともとfloat32だったものをfloat16まで減らす)\n    cols = list(train_num_agg.dtypes[train_num_agg.dtypes == 'float64'].index)    \n    for col in tqdm(cols):\n        train_num_agg[col] = train_num_agg[col].astype(np.float16)\n        \n    # intデータのメモリ削減(もともとint32だったものをint8まで減らす)\n    cols = list(train_cat_agg.dtypes[train_cat_agg.dtypes == 'int64'].index)    \n    for col in tqdm(cols):\n        train_cat_agg[col] = train_cat_agg[col].astype(np.int8)\n        \n    print('Starting calculate difference...')\n    train_diff = get_difference(train_, num_features)\n    train_ = train_num_agg.merge(train_cat_agg, how = 'inner', on = 'customer_ID').merge(train_diff, how = 'inner', on = 'customer_ID').merge(train_labels_, how = 'inner', on = 'customer_ID')\n    \n    del train_num_agg, train_cat_agg, train_diff\n    gc.collect()\n    \n    # floatデータのメモリ削減(もともとfloat32だったものをfloat16まで減らす)\n    print('Starting reducing memory. (num feature: cast float16 to float32)')\n    float_cols = list(train_.dtypes[train_.dtypes == 'float16'].index)\n    for col in tqdm(float_cols):\n        train_[col] = train_[col].astype(np.float32)\n    \n    print('Saving file ...')\n    train_.to_parquet('train_fe.parquet')\n    print('Finished Saving file.')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:19.167465Z","iopub.execute_input":"2022-08-03T11:12:19.167792Z","iopub.status.idle":"2022-08-03T11:12:19.183815Z","shell.execute_reply.started":"2022-08-03T11:12:19.167763Z","shell.execute_reply":"2022-08-03T11:12:19.182915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 相関の高いデータはdropする\nDrop_columns_list = ['D_119', 'D_118', 'D_61', 'D_107', 'D_55', 'D_131', 'D_132',\n                     'D_74', 'D_75', 'D_143', 'D_141', 'D_104', 'D_77', 'S_7', \n                     'S_24', 'B_13', 'B_18', 'B_33', 'B_20', 'B_15', 'B_37', 'B_23', 'B_11', #ここまでは相関が高いため削除、以上はNULLが多くて削除\n                     'D_42', 'D_49', 'D_50', 'D_53', 'D_56', 'S_9', 'B_17', 'D_66', 'D_73',\n                     'D_76', 'D_77', 'R_9', 'D_82', 'B_29', 'D_87', 'D_88', 'D_105', 'D_106', \n                     'R_26', 'D_108', 'D_110', 'D_111', 'B_39', 'B_42', 'D_132', 'D_134', \n                     'D_135', 'D_136', 'D_137', 'D_138', 'D_142']","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:19.186657Z","iopub.execute_input":"2022-08-03T11:12:19.187010Z","iopub.status.idle":"2022-08-03T11:12:19.200715Z","shell.execute_reply.started":"2022-08-03T11:12:19.186978Z","shell.execute_reply":"2022-08-03T11:12:19.199795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = [\n        \"B_30\",\n        \"B_38\",\n        \"D_114\",\n        \"D_116\",\n        \"D_117\",\n        \"D_120\",\n        \"D_126\",\n        \"D_63\",\n        \"D_64\",\n        \"D_66\",\n        \"D_68\",\n    ]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:19.202108Z","iopub.execute_input":"2022-08-03T11:12:19.203193Z","iopub.status.idle":"2022-08-03T11:12:19.215692Z","shell.execute_reply.started":"2022-08-03T11:12:19.203145Z","shell.execute_reply":"2022-08-03T11:12:19.214642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Drop_columns_list = set(Drop_columns_list) - set(cat_features)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:19.217645Z","iopub.execute_input":"2022-08-03T11:12:19.218021Z","iopub.status.idle":"2022-08-03T11:12:19.231774Z","shell.execute_reply.started":"2022-08-03T11:12:19.217988Z","shell.execute_reply":"2022-08-03T11:12:19.230621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## trainデータの前処理を実行","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet('../input/amex-parquet/train_data.parquet')\ntrain_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:19.233832Z","iopub.execute_input":"2022-08-03T11:12:19.234272Z","iopub.status.idle":"2022-08-03T11:12:44.003608Z","shell.execute_reply.started":"2022-08-03T11:12:19.234235Z","shell.execute_reply":"2022-08-03T11:12:44.001115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"orig_len = len(train)\ntrain = train.drop(Drop_columns_list, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:44.008532Z","iopub.execute_input":"2022-08-03T11:12:44.008966Z","iopub.status.idle":"2022-08-03T11:12:45.536489Z","shell.execute_reply.started":"2022-08-03T11:12:44.008916Z","shell.execute_reply":"2022-08-03T11:12:45.535321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ECONOMIC = ['ISM Non-Manufacturing PMI', 'ISM Manufacturing PMI', \"Core PCE Price Index\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:45.538076Z","iopub.execute_input":"2022-08-03T11:12:45.538556Z","iopub.status.idle":"2022-08-03T11:12:45.543909Z","shell.execute_reply.started":"2022-08-03T11:12:45.538522Z","shell.execute_reply":"2022-08-03T11:12:45.542349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train => 2017 to 2018\neconomic_data = investpy.economic_calendar(countries=['united states'], from_date='01/01/2017', to_date='28/03/2018')\neconomic_data = economic_data[economic_data[\"importance\"]==\"high\"]\neconomic_data = economic_data[[\"date\", \"event\", \"actual\"]]\nCore_PCE_df = economic_data[economic_data['event'].str.contains(ECONOMIC[2])]\nCore_PCE_df[\"actual\"] = Core_PCE_df.apply(percent_to_float, axis=1)\n\neconomic_data = economic_data[economic_data['event'].str.contains(ECONOMIC[0]) | \n                              economic_data['event'].str.contains(ECONOMIC[1])]\n\n\neconomic_data = pd.concat([economic_data, Core_PCE_df])\neconomic_data['actual'] = economic_data['actual'].astype(np.float32)\neconomic_data = economic_data.reset_index(drop=\"True\")\ndisplay(economic_data.head(5))\ndisplay(economic_data.tail(5))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:12:57.912459Z","iopub.execute_input":"2022-08-03T11:12:57.913064Z","iopub.status.idle":"2022-08-03T11:13:15.370626Z","shell.execute_reply.started":"2022-08-03T11:12:57.913012Z","shell.execute_reply":"2022-08-03T11:13:15.369471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"S_2_unique = train[\"S_2\"].unique()\nS2_to_ISM_list = []\n\nfor date_str in S_2_unique:\n    yyyy = date_str[0:4]\n    mm = date_str[5:7]\n    dd = date_str[8:10]\n    economic_date_df = economic_data[economic_data[\"date\"].str.contains(f\"{mm}/{yyyy}\")]\n    ISM_Manu_value = economic_date_df[economic_date_df[\"event\"].str.contains(ECONOMIC[0])][\"actual\"].values[0]\n    ISM_NonManu_value = economic_date_df[economic_date_df[\"event\"].str.contains(ECONOMIC[1])][\"actual\"].values[0]\n    if len(economic_date_df[economic_date_df[\"event\"].str.contains(ECONOMIC[2])]) != 0:\n        Core_PCE_value = economic_date_df[economic_date_df[\"event\"].str.contains(ECONOMIC[2])][\"actual\"].values[0]\n    else:\n        next_m = int(mm) - 1\n        next_df = economic_data[economic_data[\"date\"].str.contains(f\"{next_m}/{yyyy}\")]\n        Core_PCE_value = next_df[next_df[\"event\"].str.contains(ECONOMIC[2])][\"actual\"].values[0]\n    \n    S2_to_ISM_list.append([date_str, ISM_Manu_value, ISM_NonManu_value, Core_PCE_value])\nS2_to_ISM_df = pd.DataFrame(S2_to_ISM_list, columns=[\"S_2\", \"ISM_Manu\", \"ISM_NonManu\", \"Core_PCE_value\"])\ndisplay(S2_to_ISM_df.head(5))\nS2_to_ISM_df.to_csv(\"S2_to_ISM.csv\", index=False)\n\ndel economic_data, economic_date_df","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:13:29.036799Z","iopub.execute_input":"2022-08-03T11:13:29.037315Z","iopub.status.idle":"2022-08-03T11:13:30.542746Z","shell.execute_reply.started":"2022-08-03T11:13:29.037277Z","shell.execute_reply":"2022-08-03T11:13:30.541457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.merge(train, S2_to_ISM_df, on=\"S_2\", how=\"left\")\nprint(orig_len, len(train))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:13:34.181748Z","iopub.execute_input":"2022-08-03T11:13:34.182167Z","iopub.status.idle":"2022-08-03T11:13:49.224588Z","shell.execute_reply.started":"2022-08-03T11:13:34.182135Z","shell.execute_reply":"2022-08-03T11:13:49.223159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[[\"S_2\",\"ISM_Manu\",\"ISM_NonManu\", \"Core_PCE_value\"]].head(5)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:13:49.227730Z","iopub.execute_input":"2022-08-03T11:13:49.228241Z","iopub.status.idle":"2022-08-03T11:13:52.710665Z","shell.execute_reply.started":"2022-08-03T11:13:49.228193Z","shell.execute_reply":"2022-08-03T11:13:52.709342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"read_preprocess_traindata(train, train_labels)\ndel train, train_labels","metadata":{"execution":{"iopub.status.busy":"2022-08-03T11:13:52.712164Z","iopub.execute_input":"2022-08-03T11:13:52.712610Z","iopub.status.idle":"2022-08-03T11:14:01.510795Z","shell.execute_reply.started":"2022-08-03T11:13:52.712569Z","shell.execute_reply":"2022-08-03T11:14:01.509007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}