{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a id=\"top\"></a> <br>\n## contents\n1. [Data ETL](#1)\n1. [Create Vertical Federation](#2)\n1. [Create Horizontal Federation](#3)","metadata":{}},{"cell_type":"markdown","source":"<a id=\"1\"></a> <br>\n## 1- Data ETL\n\n###### [Go to top](#top)","metadata":{}},{"cell_type":"markdown","source":"This notebook was mainly copied from [this notebook](https://www.kaggle.com/code/chauhuynh/my-first-kernel-3-699). All credits belongs to the original author.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom tqdm import tqdm_notebook as tqdm\n\nimport random\nseed = 1414\nrandom.seed(seed)\nnp.random.seed(seed)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-06T12:34:58.773783Z","iopub.execute_input":"2022-06-06T12:34:58.774111Z","iopub.status.idle":"2022-06-06T12:34:58.810552Z","shell.execute_reply.started":"2022-06-06T12:34:58.774067Z","shell.execute_reply":"2022-06-06T12:34:58.809844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls /kaggle/input/elofederatedlearningdataetltraintest","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:34:59.132623Z","iopub.execute_input":"2022-06-06T12:34:59.133290Z","iopub.status.idle":"2022-06-06T12:34:59.897021Z","shell.execute_reply.started":"2022-06-06T12:34:59.133147Z","shell.execute_reply":"2022-06-06T12:34:59.896018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/elo-merchant-category-recommendation/train.csv')\n#df_test = pd.read_csv('../input/test.csv')\ndf_hist_trans = pd.read_csv('../input/elo-merchant-category-recommendation/historical_transactions.csv')\ndf_new_merchant_trans = pd.read_csv('../input/elo-merchant-category-recommendation/new_merchant_transactions.csv')","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2022-06-06T12:34:59.898744Z","iopub.execute_input":"2022-06-06T12:34:59.899107Z","iopub.status.idle":"2022-06-06T12:36:50.913184Z","shell.execute_reply.started":"2022-06-06T12:34:59.899045Z","shell.execute_reply":"2022-06-06T12:36:50.911945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/elofederatedlearningdataetltraintest/horizontalsplit-0-0-lower_table.table.csv')\ntest_df['test'] = 1\ntest_df = test_df[['card_id', 'test']]","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:36:50.914867Z","iopub.execute_input":"2022-06-06T12:36:50.915259Z","iopub.status.idle":"2022-06-06T12:36:51.170567Z","shell.execute_reply.started":"2022-06-06T12:36:50.915185Z","shell.execute_reply":"2022-06-06T12:36:51.169495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in [df_hist_trans,df_new_merchant_trans]:\n    df['category_2'].fillna(1.0,inplace=True)\n    df['category_3'].fillna('A',inplace=True)\n    df['merchant_id'].fillna('M_ID_00a6ca8a8a',inplace=True)","metadata":{"_uuid":"71f89a3b8a93b2f2feb2cd0a45f860cde33687be","execution":{"iopub.status.busy":"2022-06-06T12:36:51.172869Z","iopub.execute_input":"2022-06-06T12:36:51.173304Z","iopub.status.idle":"2022-06-06T12:36:57.121392Z","shell.execute_reply.started":"2022-06-06T12:36:51.173258Z","shell.execute_reply":"2022-06-06T12:36:57.120016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['outliers'] = 0\ndf_train.loc[df_train['target'] < -30, 'outliers'] = 1\ndf_train['outliers'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:36:57.123791Z","iopub.execute_input":"2022-06-06T12:36:57.124262Z","iopub.status.idle":"2022-06-06T12:36:57.161478Z","shell.execute_reply.started":"2022-06-06T12:36:57.124215Z","shell.execute_reply":"2022-06-06T12:36:57.160346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_train = df_train[df_train['outliers'] == 1]","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:36:57.163233Z","iopub.execute_input":"2022-06-06T12:36:57.163541Z","iopub.status.idle":"2022-06-06T12:36:57.172453Z","shell.execute_reply.started":"2022-06-06T12:36:57.163489Z","shell.execute_reply":"2022-06-06T12:36:57.171781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[df_train['outliers'] != 1]","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:36:57.173642Z","iopub.execute_input":"2022-06-06T12:36:57.174059Z","iopub.status.idle":"2022-06-06T12:36:57.193230Z","shell.execute_reply.started":"2022-06-06T12:36:57.174001Z","shell.execute_reply":"2022-06-06T12:36:57.192133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:36:57.194549Z","iopub.execute_input":"2022-06-06T12:36:57.194845Z","iopub.status.idle":"2022-06-06T12:36:57.200150Z","shell.execute_reply.started":"2022-06-06T12:36:57.194777Z","shell.execute_reply":"2022-06-06T12:36:57.199307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:36:57.201527Z","iopub.execute_input":"2022-06-06T12:36:57.201773Z","iopub.status.idle":"2022-06-06T12:36:57.213061Z","shell.execute_reply.started":"2022-06-06T12:36:57.201723Z","shell.execute_reply":"2022-06-06T12:36:57.211741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]","metadata":{"_uuid":"dda90662d05e22310dd713df106ea07f4b8bccfc","execution":{"iopub.status.busy":"2022-06-06T12:36:57.214482Z","iopub.execute_input":"2022-06-06T12:36:57.215035Z","iopub.status.idle":"2022-06-06T12:36:57.224248Z","shell.execute_reply.started":"2022-06-06T12:36:57.214975Z","shell.execute_reply":"2022-06-06T12:36:57.223615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in [df_hist_trans,df_new_merchant_trans]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    df['category_1'] = df['category_1'].map({'Y':1, 'N':0}) \n    #https://www.kaggle.com/c/elo-merchant-category-recommendation/discussion/73244\n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","metadata":{"_uuid":"690ba01a38f524e9345b419200f588f937bc067a","execution":{"iopub.status.busy":"2022-06-06T12:36:57.225504Z","iopub.execute_input":"2022-06-06T12:36:57.226010Z","iopub.status.idle":"2022-06-06T12:37:42.763666Z","shell.execute_reply.started":"2022-06-06T12:36:57.225952Z","shell.execute_reply":"2022-06-06T12:37:42.762320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\n\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['authorized_flag'] = ['sum', 'mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_hist_trans[col+'_mean'] = df_hist_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']    \n\nnew_columns = get_new_columns('hist',aggs)\ndf_hist_trans_group = df_hist_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['hist_purchase_date_diff'] = (df_hist_trans_group['hist_purchase_date_max'] - df_hist_trans_group['hist_purchase_date_min']).dt.days\ndf_hist_trans_group['hist_purchase_date_average'] = df_hist_trans_group['hist_purchase_date_diff']/df_hist_trans_group['hist_card_id_size']\ndf_hist_trans_group['hist_purchase_date_uptonow'] = (datetime.datetime.today() - df_hist_trans_group['hist_purchase_date_max']).dt.days\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\n#df_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\ndel df_hist_trans_group;gc.collect()","metadata":{"_uuid":"ddf1d5bb0ade2b22b0f072c208c1506ea64503ea","execution":{"iopub.status.busy":"2022-06-06T12:37:42.765298Z","iopub.execute_input":"2022-06-06T12:37:42.765570Z","iopub.status.idle":"2022-06-06T12:44:16.892227Z","shell.execute_reply.started":"2022-06-06T12:37:42.765524Z","shell.execute_reply":"2022-06-06T12:44:16.890984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggs = {}\nfor col in ['month','hour','weekofyear','dayofweek','year','subsector_id','merchant_id','merchant_category_id']:\n    aggs[col] = ['nunique']\naggs['purchase_amount'] = ['sum','max','min','mean','var']\naggs['installments'] = ['sum','max','min','mean','var']\naggs['purchase_date'] = ['max','min']\naggs['month_lag'] = ['max','min','mean','var']\naggs['month_diff'] = ['mean']\naggs['weekend'] = ['sum', 'mean']\naggs['category_1'] = ['sum', 'mean']\naggs['card_id'] = ['size']\n\nfor col in ['category_2','category_3']:\n    df_new_merchant_trans[col+'_mean'] = df_new_merchant_trans.groupby([col])['purchase_amount'].transform('mean')\n    aggs[col+'_mean'] = ['mean']\n    \nnew_columns = get_new_columns('new_hist',aggs)\ndf_hist_trans_group = df_new_merchant_trans.groupby('card_id').agg(aggs)\ndf_hist_trans_group.columns = new_columns\ndf_hist_trans_group.reset_index(drop=False,inplace=True)\ndf_hist_trans_group['new_hist_purchase_date_diff'] = (df_hist_trans_group['new_hist_purchase_date_max'] - df_hist_trans_group['new_hist_purchase_date_min']).dt.days\ndf_hist_trans_group['new_hist_purchase_date_average'] = df_hist_trans_group['new_hist_purchase_date_diff']/df_hist_trans_group['new_hist_card_id_size']\ndf_hist_trans_group['new_hist_purchase_date_uptonow'] = (datetime.datetime.today() - df_hist_trans_group['new_hist_purchase_date_max']).dt.days\ndf_train = df_train.merge(df_hist_trans_group,on='card_id',how='left')\n#df_test = df_test.merge(df_hist_trans_group,on='card_id',how='left')\ndel df_hist_trans_group;gc.collect()","metadata":{"_uuid":"f7f5625db40db4395374991124fb796c9decd60b","execution":{"iopub.status.busy":"2022-06-06T12:44:16.893662Z","iopub.execute_input":"2022-06-06T12:44:16.893942Z","iopub.status.idle":"2022-06-06T12:44:39.238872Z","shell.execute_reply.started":"2022-06-06T12:44:16.893894Z","shell.execute_reply":"2022-06-06T12:44:39.238223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_hist_trans;gc.collect()\ndel df_new_merchant_trans;gc.collect()\ndf_train.head(5)","metadata":{"_uuid":"a075cc90ab1322829e4fad3ff39fce307c5db93c","execution":{"iopub.status.busy":"2022-06-06T12:44:39.239798Z","iopub.execute_input":"2022-06-06T12:44:39.240177Z","iopub.status.idle":"2022-06-06T12:44:40.319287Z","shell.execute_reply.started":"2022-06-06T12:44:39.240139Z","shell.execute_reply":"2022-06-06T12:44:40.318420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:44:40.321950Z","iopub.execute_input":"2022-06-06T12:44:40.322478Z","iopub.status.idle":"2022-06-06T12:44:40.328422Z","shell.execute_reply.started":"2022-06-06T12:44:40.322415Z","shell.execute_reply":"2022-06-06T12:44:40.327566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in [df_train]:\n    df['first_active_month'] = pd.to_datetime(df['first_active_month'])\n    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n    df['month'] = df['first_active_month'].dt.month\n    df['elapsed_time'] = (datetime.datetime.today() - df['first_active_month']).dt.days\n    df['hist_first_buy'] = (df['hist_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_hist_first_buy'] = (df['new_hist_purchase_date_min'] - df['first_active_month']).dt.days\n    for f in ['hist_purchase_date_max','hist_purchase_date_min','new_hist_purchase_date_max',\\\n                     'new_hist_purchase_date_min']:\n        df[f] = df[f].astype(np.int64) * 1e-9\n    df['card_id_total'] = df['new_hist_card_id_size']+df['hist_card_id_size']\n    df['purchase_amount_total'] = df['new_hist_purchase_amount_sum']+df['hist_purchase_amount_sum']","metadata":{"_uuid":"ce2082fc1fb0e3f8f7d27fc166aa7a8351b65504","execution":{"iopub.status.busy":"2022-06-06T12:44:40.330221Z","iopub.execute_input":"2022-06-06T12:44:40.330837Z","iopub.status.idle":"2022-06-06T12:44:40.614974Z","shell.execute_reply.started":"2022-06-06T12:44:40.330758Z","shell.execute_reply":"2022-06-06T12:44:40.614093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_columns = [c for c in df_train.columns if c not in ['first_active_month', 'outliers']]\ndf_train = df_train[df_train_columns]","metadata":{"_uuid":"c4f20f27679889542acfd60d1f1ac381b201ac43","execution":{"iopub.status.busy":"2022-06-06T12:44:40.616746Z","iopub.execute_input":"2022-06-06T12:44:40.617122Z","iopub.status.idle":"2022-06-06T12:44:40.747487Z","shell.execute_reply.started":"2022-06-06T12:44:40.617046Z","shell.execute_reply":"2022-06-06T12:44:40.746582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:44:40.748725Z","iopub.execute_input":"2022-06-06T12:44:40.749029Z","iopub.status.idle":"2022-06-06T12:44:40.754471Z","shell.execute_reply.started":"2022-06-06T12:44:40.748973Z","shell.execute_reply":"2022-06-06T12:44:40.753636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.merge(test_df,on='card_id',how='left')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:44:40.755638Z","iopub.execute_input":"2022-06-06T12:44:40.755919Z","iopub.status.idle":"2022-06-06T12:44:41.258391Z","shell.execute_reply.started":"2022-06-06T12:44:40.755868Z","shell.execute_reply":"2022-06-06T12:44:41.257494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['test'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:44:41.259627Z","iopub.execute_input":"2022-06-06T12:44:41.259917Z","iopub.status.idle":"2022-06-06T12:44:41.268809Z","shell.execute_reply.started":"2022-06-06T12:44:41.259861Z","shell.execute_reply":"2022-06-06T12:44:41.268106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:44:41.270122Z","iopub.execute_input":"2022-06-06T12:44:41.270374Z","iopub.status.idle":"2022-06-06T12:44:41.279555Z","shell.execute_reply.started":"2022-06-06T12:44:41.270325Z","shell.execute_reply":"2022-06-06T12:44:41.278720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:44:41.280734Z","iopub.execute_input":"2022-06-06T12:44:41.281028Z","iopub.status.idle":"2022-06-06T12:44:41.459105Z","shell.execute_reply.started":"2022-06-06T12:44:41.280978Z","shell.execute_reply":"2022-06-06T12:44:41.457900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.to_csv('elo-ETL-data.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:44:41.460612Z","iopub.execute_input":"2022-06-06T12:44:41.460891Z","iopub.status.idle":"2022-06-06T12:45:13.218537Z","shell.execute_reply.started":"2022-06-06T12:44:41.460836Z","shell.execute_reply":"2022-06-06T12:45:13.217181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train['test']==1].drop(['test'],axis=1).to_csv('elo-ETL-data-test.csv')\ndf_train[df_train['test']!=1].drop(['test'],axis=1).to_csv('elo-ETL-data-train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:13.220577Z","iopub.execute_input":"2022-06-06T12:45:13.220990Z","iopub.status.idle":"2022-06-06T12:45:44.322464Z","shell.execute_reply.started":"2022-06-06T12:45:13.220909Z","shell.execute_reply":"2022-06-06T12:45:44.321290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a> <br>\n## 2- Create Vertical Federation\n\n###### [Go to top](#top)","metadata":{}},{"cell_type":"code","source":"df_train_columns_without_id = [c for c in df_train.columns if c not in ['card_id','target','test']]\nround(len(df_train_columns_without_id)*0.6)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.324077Z","iopub.execute_input":"2022-06-06T12:45:44.324483Z","iopub.status.idle":"2022-06-06T12:45:44.331565Z","shell.execute_reply.started":"2022-06-06T12:45:44.324315Z","shell.execute_reply":"2022-06-06T12:45:44.330440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_columns_60 = random.sample(df_train_columns_without_id, round(len(df_train_columns_without_id)*0.6))\ndf_train_columns_40 = [c for c in df_train.columns if c not in df_train_columns_60]\ndf_train_columns_60 = df_train_columns_60 + ['card_id', 'test']\nprint(df_train_columns_60)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.333268Z","iopub.execute_input":"2022-06-06T12:45:44.333641Z","iopub.status.idle":"2022-06-06T12:45:44.345617Z","shell.execute_reply.started":"2022-06-06T12:45:44.333561Z","shell.execute_reply":"2022-06-06T12:45:44.344641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train_columns_40)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.346840Z","iopub.execute_input":"2022-06-06T12:45:44.347197Z","iopub.status.idle":"2022-06-06T12:45:44.357269Z","shell.execute_reply.started":"2022-06-06T12:45:44.347159Z","shell.execute_reply":"2022-06-06T12:45:44.356593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train_columns_60].shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.358211Z","iopub.execute_input":"2022-06-06T12:45:44.358587Z","iopub.status.idle":"2022-06-06T12:45:44.399450Z","shell.execute_reply.started":"2022-06-06T12:45:44.358549Z","shell.execute_reply":"2022-06-06T12:45:44.398700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_train_columns = [c for c in extra_train.columns if c not in ['first_active_month', 'outliers', 'target','test']]","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.400520Z","iopub.execute_input":"2022-06-06T12:45:44.400997Z","iopub.status.idle":"2022-06-06T12:45:44.405532Z","shell.execute_reply.started":"2022-06-06T12:45:44.400940Z","shell.execute_reply":"2022-06-06T12:45:44.404730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_train[extra_train_columns].shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.406472Z","iopub.execute_input":"2022-06-06T12:45:44.406935Z","iopub.status.idle":"2022-06-06T12:45:44.423363Z","shell.execute_reply.started":"2022-06-06T12:45:44.406896Z","shell.execute_reply":"2022-06-06T12:45:44.422677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_train[extra_train_columns].head()","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.424929Z","iopub.execute_input":"2022-06-06T12:45:44.425175Z","iopub.status.idle":"2022-06-06T12:45:44.445408Z","shell.execute_reply.started":"2022-06-06T12:45:44.425130Z","shell.execute_reply":"2022-06-06T12:45:44.444394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_v = pd.concat([df_train[df_train_columns_60],extra_train[extra_train_columns]])\nfed_60_v.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.446698Z","iopub.execute_input":"2022-06-06T12:45:44.446972Z","iopub.status.idle":"2022-06-06T12:45:44.559906Z","shell.execute_reply.started":"2022-06-06T12:45:44.446924Z","shell.execute_reply":"2022-06-06T12:45:44.558702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_v.to_csv('elo-ETL-data-60-vertical.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_v_test = fed_60_v[fed_60_v['test']==1]\nfed_60_v_test = fed_60_v_test.drop(['test'],axis=1)\nfed_60_v_test.to_csv('elo-ETL-data-60-vertical-test.csv')\nfor i,each in enumerate([c for c in df_train_columns_60 if c not in ['card_id','target']]):\n    #print(i)\n    fed_60_v_test.rename(columns={each:f'x{i}'},inplace=True)\n    \n#fed_60_v_test.to_csv('elo-ETL-data-60-vertical-test-x.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:44.561556Z","iopub.execute_input":"2022-06-06T12:45:44.561946Z","iopub.status.idle":"2022-06-06T12:45:48.044686Z","shell.execute_reply.started":"2022-06-06T12:45:44.561872Z","shell.execute_reply":"2022-06-06T12:45:48.043499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_v_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:48.046310Z","iopub.execute_input":"2022-06-06T12:45:48.046683Z","iopub.status.idle":"2022-06-06T12:45:48.054072Z","shell.execute_reply.started":"2022-06-06T12:45:48.046605Z","shell.execute_reply":"2022-06-06T12:45:48.053046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_v_train = fed_60_v[fed_60_v['test']!=1]\nfed_60_v_train = fed_60_v_train.drop(['test'],axis=1)\nfed_60_v_train.to_csv('elo-ETL-data-60-vertical-train.csv')\nfor i,each in enumerate([c for c in df_train_columns_60 if c not in ['card_id','target']]):\n    #print(i)\n    fed_60_v_train.rename(columns={each:f'x{i}'},inplace=True)\n    \n#fed_60_v_train.to_csv('elo-ETL-data-60-vertical-train-x.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:45:48.055935Z","iopub.execute_input":"2022-06-06T12:45:48.056609Z","iopub.status.idle":"2022-06-06T12:46:19.930067Z","shell.execute_reply.started":"2022-06-06T12:45:48.056545Z","shell.execute_reply":"2022-06-06T12:46:19.929362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_v_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:19.931341Z","iopub.execute_input":"2022-06-06T12:46:19.931787Z","iopub.status.idle":"2022-06-06T12:46:19.937179Z","shell.execute_reply.started":"2022-06-06T12:46:19.931744Z","shell.execute_reply":"2022-06-06T12:46:19.936362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_40_v = df_train[df_train_columns_40]\nfed_40_v.to_csv('elo-ETL-data-40-vertical.csv')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_40_v_test = fed_40_v[fed_40_v['test']==1]\nfed_40_v_test = fed_40_v_test.drop(['test'],axis=1)\nfed_40_v_test.to_csv('elo-ETL-data-40-vertical-test.csv')\nfor i,each in enumerate([c for c in df_train_columns_40 if c not in ['card_id','target']]):\n    #print(i)\n    fed_40_v_test.rename(columns={each:f'x{i}'},inplace=True)\n    \n#fed_40_v_test.to_csv('elo-ETL-data-40-vertical-test-x.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:19.938360Z","iopub.execute_input":"2022-06-06T12:46:19.938619Z","iopub.status.idle":"2022-06-06T12:46:22.609520Z","shell.execute_reply.started":"2022-06-06T12:46:19.938567Z","shell.execute_reply":"2022-06-06T12:46:22.608750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_40_v_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:22.610754Z","iopub.execute_input":"2022-06-06T12:46:22.611251Z","iopub.status.idle":"2022-06-06T12:46:22.617404Z","shell.execute_reply.started":"2022-06-06T12:46:22.611185Z","shell.execute_reply":"2022-06-06T12:46:22.616595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_40_v_train = fed_40_v[fed_40_v['test']!=1]\nfed_40_v_train = fed_40_v_train.drop(['test'],axis=1)\nfed_40_v_train.to_csv('elo-ETL-data-40-vertical-train.csv')\nfor i,each in enumerate([c for c in df_train_columns_40 if c not in ['card_id','target']]):\n    #print(i)\n    fed_40_v_train.rename(columns={each:f'x{i}'},inplace=True)\n    \n#fed_40_v_train.to_csv('elo-ETL-data-40-vertical-train-x.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:22.618938Z","iopub.execute_input":"2022-06-06T12:46:22.619446Z","iopub.status.idle":"2022-06-06T12:46:46.730609Z","shell.execute_reply.started":"2022-06-06T12:46:22.619219Z","shell.execute_reply":"2022-06-06T12:46:46.729765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_40_v_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:46.732335Z","iopub.execute_input":"2022-06-06T12:46:46.732595Z","iopub.status.idle":"2022-06-06T12:46:46.737811Z","shell.execute_reply.started":"2022-06-06T12:46:46.732543Z","shell.execute_reply":"2022-06-06T12:46:46.737137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3\"></a> <br>\n## 3- Create Horizontal Federation\n\n###### [Go to top](#top)","metadata":{}},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:46.739296Z","iopub.execute_input":"2022-06-06T12:46:46.739538Z","iopub.status.idle":"2022-06-06T12:46:46.758302Z","shell.execute_reply.started":"2022-06-06T12:46:46.739490Z","shell.execute_reply":"2022-06-06T12:46:46.757045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_t = df_train[df_train['test']==1]\ndf_t.drop(['test'],axis=1).to_csv('elo-ETL-data-horizontal-test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:46.759786Z","iopub.execute_input":"2022-06-06T12:46:46.760324Z","iopub.status.idle":"2022-06-06T12:46:49.890883Z","shell.execute_reply.started":"2022-06-06T12:46:46.760266Z","shell.execute_reply":"2022-06-06T12:46:49.889790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_t.drop(['test'],axis=1).shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:49.892613Z","iopub.execute_input":"2022-06-06T12:46:49.893009Z","iopub.status.idle":"2022-06-06T12:46:49.907726Z","shell.execute_reply.started":"2022-06-06T12:46:49.892935Z","shell.execute_reply":"2022-06-06T12:46:49.906998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train[df_train['test']!=1].drop(['test'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:49.909128Z","iopub.execute_input":"2022-06-06T12:46:49.909595Z","iopub.status.idle":"2022-06-06T12:46:49.997628Z","shell.execute_reply.started":"2022-06-06T12:46:49.909541Z","shell.execute_reply":"2022-06-06T12:46:49.996879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_h = df_train.sample(n=round(df_train.shape[0]*0.6), random_state=seed, axis=0)\nfed_60_h.to_csv('elo-ETL-data-60-horizontal-train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:46:49.998787Z","iopub.execute_input":"2022-06-06T12:46:49.999080Z","iopub.status.idle":"2022-06-06T12:47:06.858381Z","shell.execute_reply.started":"2022-06-06T12:46:49.999028Z","shell.execute_reply":"2022-06-06T12:47:06.857604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_60_h.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:47:06.860213Z","iopub.execute_input":"2022-06-06T12:47:06.860904Z","iopub.status.idle":"2022-06-06T12:47:06.868461Z","shell.execute_reply.started":"2022-06-06T12:47:06.860815Z","shell.execute_reply":"2022-06-06T12:47:06.867329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_40_h = pd.concat([df_train, fed_60_h, fed_60_h]).drop_duplicates(keep=False)  #df1-df2\nfed_40_h.to_csv('elo-ETL-data-40-horizontal-train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:47:06.869926Z","iopub.execute_input":"2022-06-06T12:47:06.870186Z","iopub.status.idle":"2022-06-06T12:47:19.875692Z","shell.execute_reply.started":"2022-06-06T12:47:06.870133Z","shell.execute_reply":"2022-06-06T12:47:19.875026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fed_40_h.shape","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:47:19.876792Z","iopub.execute_input":"2022-06-06T12:47:19.877305Z","iopub.status.idle":"2022-06-06T12:47:19.884157Z","shell.execute_reply.started":"2022-06-06T12:47:19.877242Z","shell.execute_reply":"2022-06-06T12:47:19.882847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i,each in enumerate([c for c in fed_60_h.columns if c not in ['card_id','target']]):\n    #print(i)\n    fed_60_h.rename(columns={each:f'x{i}'},inplace=True)\n    fed_40_h.rename(columns={each:f'x{i}'},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:47:19.885999Z","iopub.execute_input":"2022-06-06T12:47:19.886362Z","iopub.status.idle":"2022-06-06T12:47:22.689533Z","shell.execute_reply.started":"2022-06-06T12:47:19.886285Z","shell.execute_reply":"2022-06-06T12:47:22.688047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fed_60_h.to_csv('elo-ETL-data-60-horizontal-train-x.csv')\n#fed_40_h.to_csv('elo-ETL-data-40-horizontal-train-x.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-06T12:47:22.690924Z","iopub.execute_input":"2022-06-06T12:47:22.691175Z","iopub.status.idle":"2022-06-06T12:47:50.739893Z","shell.execute_reply.started":"2022-06-06T12:47:22.691130Z","shell.execute_reply":"2022-06-06T12:47:50.739092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}