{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-06T21:09:25.729018Z","iopub.execute_input":"2022-06-06T21:09:25.729604Z","iopub.status.idle":"2022-06-06T21:09:25.758032Z","shell.execute_reply.started":"2022-06-06T21:09:25.729489Z","shell.execute_reply":"2022-06-06T21:09:25.757079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- takes as base this dataset: https://www.kaggle.com/datasets/raddar/amex-data-integer-dtypes-parquet-format\n- feat engineering from here: https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created","metadata":{}},{"cell_type":"code","source":"# took from: https://www.kaggle.com/code/cdeotte/xgboost-starter-0-793\n\ndef build_features(df):\n    # FEATURE ENGINEERING FROM \n    # https://www.kaggle.com/code/huseyincot/amex-agg-data-how-it-created\n    \n    all_cols = [c for c in list(df.columns) if c not in ['customer_ID','S_2']]\n    cat_features = [\"B_30\",\"B_38\",\"D_114\",\"D_116\",\"D_117\",\"D_120\",\"D_126\",\"D_63\",\"D_64\",\"D_66\",\"D_68\"]\n    num_features = [col for col in all_cols if col not in cat_features]\n\n    df_num_agg = df.groupby(\"customer_ID\")[num_features].agg(['mean', 'std', 'min', 'max', 'last'])\n    df_num_agg.columns = ['_'.join(x) for x in df_num_agg.columns]\n\n    df_cat_agg = df.groupby(\"customer_ID\")[cat_features].agg(['last', 'nunique'])\n    df_cat_agg.columns = ['_'.join(x) for x in df_cat_agg.columns]\n\n    df = pd.concat([df_num_agg, df_cat_agg], axis=1)\n    del df_num_agg, df_cat_agg\n    gc.collect()\n\n    print('shape after engineering', df.shape )\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-06-06T21:09:26.485312Z","iopub.execute_input":"2022-06-06T21:09:26.486166Z","iopub.status.idle":"2022-06-06T21:09:26.496453Z","shell.execute_reply.started":"2022-06-06T21:09:26.486110Z","shell.execute_reply":"2022-06-06T21:09:26.495280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***\n## preproc on train","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\n\ncid = train.pop(\"customer_ID\")\ntrain[\"customer_ID\"] = cid.str[-16:].apply(lambda x: int(x,16))\n\ndel cid\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T21:09:43.403996Z","iopub.execute_input":"2022-06-06T21:09:43.404485Z","iopub.status.idle":"2022-06-06T21:10:07.334400Z","shell.execute_reply.started":"2022-06-06T21:09:43.404443Z","shell.execute_reply":"2022-06-06T21:10:07.333304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_agg = build_features(train)\ntrain_agg.to_parquet(\"train_agg.parquet\")\n\ndel train,train_agg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T21:10:18.032059Z","iopub.execute_input":"2022-06-06T21:10:18.032777Z","iopub.status.idle":"2022-06-06T21:12:34.479696Z","shell.execute_reply.started":"2022-06-06T21:10:18.032736Z","shell.execute_reply":"2022-06-06T21:12:34.478525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***\n### preproc on test","metadata":{}},{"cell_type":"code","source":"test = pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/test.parquet\")\n\ncid = test.pop(\"customer_ID\")\ntest[\"customer_ID\"] = cid.str[-16:].apply(lambda x: int(x,16))\n\ndel cid\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T21:12:56.421242Z","iopub.execute_input":"2022-06-06T21:12:56.421687Z","iopub.status.idle":"2022-06-06T21:13:44.493244Z","shell.execute_reply.started":"2022-06-06T21:12:56.421654Z","shell.execute_reply":"2022-06-06T21:13:44.491891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_agg = build_features(test)\n\ndel test,test_agg\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T21:13:44.495117Z","iopub.execute_input":"2022-06-06T21:13:44.495979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***","metadata":{}}]}