{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Create more of 1k features with datatable\n\n## The goal this is code is to user that are starting it is competitions ou those one that are with difficult in you developed  the pré-processing it is data.\n\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom datatable import (dt, f, by, ifelse, update, sort,join,\n                       count, min, max, mean, sum, rowsum,sd,last)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:13:50.560322Z","iopub.execute_input":"2022-07-20T16:13:50.560744Z","iopub.status.idle":"2022-07-20T16:13:50.568255Z","shell.execute_reply.started":"2022-07-20T16:13:50.560709Z","shell.execute_reply":"2022-07-20T16:13:50.566792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## This stage is load the data of train.","metadata":{}},{"cell_type":"code","source":"train_data_path = '../input/amex-default-prediction/train_data.csv'\ndf_train = dt.fread(train_data_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:13:52.752449Z","iopub.execute_input":"2022-07-20T16:13:52.752864Z","iopub.status.idle":"2022-07-20T16:15:29.959439Z","shell.execute_reply.started":"2022-07-20T16:13:52.752816Z","shell.execute_reply":"2022-07-20T16:15:29.957735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:15:29.962940Z","iopub.execute_input":"2022-07-20T16:15:29.963826Z","iopub.status.idle":"2022-07-20T16:15:29.976941Z","shell.execute_reply.started":"2022-07-20T16:15:29.963783Z","shell.execute_reply":"2022-07-20T16:15:29.975134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## This stage is calculated the date diff in days for each customer_id","metadata":{}},{"cell_type":"code","source":"tt=df_train[:, {'MIN_DATE': min(f.S_2)}, by(['customer_ID'])]\n\ntt.key = 'customer_ID'\n\ndf_train=df_train[:, :, join(tt)]\n\ndf_train['days']=(df_train[:,'S_2'].to_pandas().values-df_train[:,'MIN_DATE'].to_pandas().values)/ np.timedelta64(1, 'D')\n\ndel df_train[:, ['MIN_DATE']]","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:15:33.916347Z","iopub.execute_input":"2022-07-20T16:15:33.916725Z","iopub.status.idle":"2022-07-20T16:15:42.993397Z","shell.execute_reply.started":"2022-07-20T16:15:33.916695Z","shell.execute_reply":"2022-07-20T16:15:42.992145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:15:44.222131Z","iopub.execute_input":"2022-07-20T16:15:44.222543Z","iopub.status.idle":"2022-07-20T16:15:44.231331Z","shell.execute_reply.started":"2022-07-20T16:15:44.222509Z","shell.execute_reply":"2022-07-20T16:15:44.230432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Then the data is sorted by customer_id and days in descending order.","metadata":{}},{"cell_type":"code","source":"df_train=df_train[:, :, dt.sort(f.customer_ID,f.days)]","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:15:50.981978Z","iopub.execute_input":"2022-07-20T16:15:50.982989Z","iopub.status.idle":"2022-07-20T16:15:57.696426Z","shell.execute_reply.started":"2022-07-20T16:15:50.982936Z","shell.execute_reply":"2022-07-20T16:15:57.695237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The select the numeric featrures for application the transformation on the groupby. ","metadata":{}},{"cell_type":"code","source":"var=[i for i in list(df_train.names) if sum(i==np.array(['D_63','D_64','customer_ID','S_2']))==0]\n","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:15:59.838864Z","iopub.execute_input":"2022-07-20T16:15:59.839424Z","iopub.status.idle":"2022-07-20T16:15:59.854061Z","shell.execute_reply.started":"2022-07-20T16:15:59.839383Z","shell.execute_reply":"2022-07-20T16:15:59.851469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### In this is stage is generated a script in string to be applied on groupby the DataTable, below have the transformation in each numeric column, follow the instruction:\n\n### 1. Average \n### 2. Max\n### 3. Amplitude\n### 4. Stanrd Desviation \n### 5. The last value the of laste date for each \"customer_id\"\n### 6. Average of Gaussiana to each numeric column.\n\n### You have all freendom in create you the features the from of script below.","metadata":{}},{"cell_type":"code","source":"init=\"df_train[:,{'DATE_MAX':max(f.S_2),'QTD_REGISTER':dt.count(),'D_63_LAST':last(f.D_63),'D_64_LAST':last(f.D_64),\"\n\nend=\"}, by(['customer_ID'])]\"\n\n\n\nfor idx,j in enumerate(['_AVG','_MAX','_AMPL','_STD','_SUM','_LAST','_GAUSS']):\n    for idx1,i in enumerate(var):\n        if idx1<(len(var)-1):\n            if j=='_AVG':\n                init=init+\"'\"+i+j+\"': mean(f.\"+i+\"),\"\n            elif j=='_MAX':\n                init=init+\"'\"+i+j+\"': max(f.\"+i+\"),\"\n            elif j=='_AMPL':\n                init=init+\"'\"+i+j+\"': max(f.\"+i+\")-min(f.\"+i+\"),\"\n            elif j=='_STD':\n                init=init+\"'\"+i+j+\"': sd(f.\"+i+\"),\"\n            elif j=='_LAST':\n                init=init+\"'\"+i+j+\"': last(f.\"+i+\"),\"\n            elif j=='_GAUSS':\n                init=init+\"'\"+i+j+\"': mean((f.\"+i+\"-mean(f.\"+i+\"))/sd(f.\"+i+\")),\"\n        else:\n            if j=='_AVG':\n                init=init+\"'\"+i+j+\"': mean(f.\"+i+\"),\"\n            elif j=='_MAX':\n                init=init+\"'\"+i+j+\"': max(f.\"+i+\"),\"\n            elif j=='_AMPL':\n                init=init+\"'\"+i+j+\"': max(f.\"+i+\")-min(f.\"+i+\"),\"\n            elif j=='_STD':\n                init=init+\"'\"+i+j+\"': sd(f.\"+i+\"),\"\n            elif j=='_LAST':\n                init=init+\"'\"+i+j+\"': last(f.\"+i+\"),\"\n            elif j=='_GAUSS':\n                init=init+\"'\"+i+j+\"': mean((f.\"+i+\"-mean(f.\"+i+\"))/sd(f.\"+i+\"))\"\n                \n\ninit=init+end","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:16:07.567970Z","iopub.execute_input":"2022-07-20T16:16:07.568387Z","iopub.status.idle":"2022-07-20T16:16:07.587930Z","shell.execute_reply.started":"2022-07-20T16:16:07.568355Z","shell.execute_reply":"2022-07-20T16:16:07.586496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The function \"eval\" have with finality read a script in string","metadata":{}},{"cell_type":"code","source":"df_table=eval(init)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:16:13.771506Z","iopub.execute_input":"2022-07-20T16:16:13.772096Z","iopub.status.idle":"2022-07-20T16:18:40.880275Z","shell.execute_reply.started":"2022-07-20T16:16:13.772047Z","shell.execute_reply":"2022-07-20T16:18:40.879119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_table.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:24:39.853342Z","iopub.execute_input":"2022-07-20T16:24:39.854103Z","iopub.status.idle":"2022-07-20T16:24:39.866461Z","shell.execute_reply.started":"2022-07-20T16:24:39.854064Z","shell.execute_reply":"2022-07-20T16:24:39.865330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## In the end we have 11128 features.","metadata":{}},{"cell_type":"code","source":"df_table.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:24:45.543134Z","iopub.execute_input":"2022-07-20T16:24:45.543547Z","iopub.status.idle":"2022-07-20T16:24:45.551592Z","shell.execute_reply.started":"2022-07-20T16:24:45.543516Z","shell.execute_reply":"2022-07-20T16:24:45.550422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Below is apply the target on final data.","metadata":{}},{"cell_type":"code","source":"train_labels_path = '../input/amex-default-prediction/train_labels.csv'\ndf_train_labels = dt.fread(train_labels_path)\n\ndf_train_labels.key = 'customer_ID'\ndf_table=df_table[:, :, join(df_train_labels)]\n","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:24:51.766974Z","iopub.execute_input":"2022-07-20T16:24:51.767435Z","iopub.status.idle":"2022-07-20T16:24:52.923700Z","shell.execute_reply.started":"2022-07-20T16:24:51.767386Z","shell.execute_reply":"2022-07-20T16:24:52.922477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_table.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:24:56.084680Z","iopub.execute_input":"2022-07-20T16:24:56.085138Z","iopub.status.idle":"2022-07-20T16:24:56.094896Z","shell.execute_reply.started":"2022-07-20T16:24:56.085100Z","shell.execute_reply":"2022-07-20T16:24:56.093522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_table.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:24:58.496309Z","iopub.execute_input":"2022-07-20T16:24:58.496689Z","iopub.status.idle":"2022-07-20T16:24:58.504840Z","shell.execute_reply.started":"2022-07-20T16:24:58.496659Z","shell.execute_reply":"2022-07-20T16:24:58.503518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## On end we save the data.","metadata":{}},{"cell_type":"code","source":"df_table.to_csv('train_new_feats.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-20T16:25:45.719218Z","iopub.execute_input":"2022-07-20T16:25:45.719649Z","iopub.status.idle":"2022-07-20T16:26:52.854623Z","shell.execute_reply.started":"2022-07-20T16:25:45.719614Z","shell.execute_reply":"2022-07-20T16:26:52.852841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## We have applied also on the dataset of test without the necessity of increase memory ram of computer.","metadata":{}}]}