{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective\n\nIn this notebook, I have optimised some of the preprocessing methods that you can find [here](https://www.kaggle.com/code/raimondomelis/preprocessing-of-data-in-chunks-right-way). In the previous notebook I explained the theory, but now we will see how to perform preprocessing with the GPU accelerator. In particular, we are going to compare GPU P100 vs GPU T4 x2. \n\n**TIP:** never perform all preprocessing on the GPU (in Kaggle it is limited), but it is convenient to also use the CPU simultaneously.","metadata":{}},{"cell_type":"markdown","source":"# Calculations performed\n\nI performed some of the standard feature engineering calculations: **count, last, nunique**","metadata":{}},{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:07:04.144588Z","iopub.execute_input":"2022-11-11T11:07:04.145480Z","iopub.status.idle":"2022-11-11T11:07:04.173584Z","shell.execute_reply.started":"2022-11-11T11:07:04.145393Z","shell.execute_reply":"2022-11-11T11:07:04.172788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport operator as op\nimport numpy as np\nimport cupy as cp\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport cudf\nimport time\n\n\nimport warnings \nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:09:08.765423Z","iopub.execute_input":"2022-11-11T11:09:08.765987Z","iopub.status.idle":"2022-11-11T11:09:11.311392Z","shell.execute_reply.started":"2022-11-11T11:09:08.765951Z","shell.execute_reply":"2022-11-11T11:09:11.310175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GPU P100 \n\nIn the right-hand panel, set P100 as GPU accelerator","metadata":{}},{"cell_type":"code","source":"#Dividing our dataset into N° parts\nnum_parts = 4\ndef read_preprocess_divide(num_parts):\n    #wanted columns\n    columns = ['ip', 'channel', 'click_time']\n    dtypes = {\n             'ip'      : 'int32',\n             'channel' : 'int16',\n             'click_time' : 'datetime64[us]',\n             }\n    df = cudf.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', usecols=columns, dtype=dtypes)\n    all_rows = len(df)\n    chunk = all_rows//num_parts\n    #sort the dataset by ip and reset the index\n    df = df.sort_values(by=['ip', 'click_time']).reset_index(drop = True)\n    return df, all_rows, chunk \n    \n\ndef window(df):\n    #calculate the most common value with the \"mode\", and the \"window\"\n    most_common = df['ip'].mode().values.tolist()[0]\n    window = len(df[df['ip'] == most_common])+1\n    return window\n\ndef feature_engineering(df,start,new_end):\n    if new_end is not None:\n        end = new_end+1\n    else:\n        end = None\n    features = [c for c in list(df.columns) if c not in ['ip','click_time']]\n    cat_function = ['count', 'last', 'nunique']    \n    new_chunk = df[start:end].groupby('ip')[features].agg(cat_function)\n    new_chunk.columns = ['_'.join(x) for x in new_chunk.columns]\n    new_chunk.reset_index(inplace = True)\n    diff_num_features = [f'diff_{col}' for col in features]\n    df = df.to_pandas()\n    ips = df[start:end]['ip'].values\n    new_chunk_diff = df[start:end].groupby('ip')[features].diff().add_prefix('diff_')\n    new_chunk_diff.insert(0,'ip',ips)\n    new_chunk_diff = cudf.DataFrame(new_chunk_diff)\n    new_chunk_diff = new_chunk_diff.groupby('ip')[diff_num_features].agg(cat_function)\n    new_chunk_diff.columns = ['_'.join(x) for x in new_chunk_diff.columns]\n    new_chunk_diff.reset_index(inplace = True)\n    new_chunk = new_chunk.merge(new_chunk_diff, how = 'inner', on = 'ip')\n    new_chunk = new_chunk.sort_values(by=['ip']).reset_index(drop = True)\n    return new_chunk","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:09:15.509984Z","iopub.execute_input":"2022-11-11T11:09:15.510336Z","iopub.status.idle":"2022-11-11T11:09:15.523003Z","shell.execute_reply.started":"2022-11-11T11:09:15.510307Z","shell.execute_reply":"2022-11-11T11:09:15.521890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf, all_rows, chunk = read_preprocess_divide(num_parts)","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:09:19.202253Z","iopub.execute_input":"2022-11-11T11:09:19.202614Z","iopub.status.idle":"2022-11-11T11:10:27.638258Z","shell.execute_reply.started":"2022-11-11T11:09:19.202584Z","shell.execute_reply":"2022-11-11T11:10:27.637186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#function to select a safe window of rows\nwindow = window(df)\n#new dataframe to append the results of the for loop\nnew_df=cudf.DataFrame()\n#set start = 0\nstart = 0\nfor p in range(0,num_parts):\n    end = p*chunk + chunk\n    if end < all_rows:\n        chunk_window = df[start:end].tail(window)\n        second_last_unique = chunk_window['ip'].unique().values.tolist()[-2]\n        new_end = chunk_window[chunk_window['ip'] == second_last_unique].tail(1).index[0]\n        print(f\"Processing {(new_end+1)-start} rows of chunk N° {p+1}\")\n        new_chunk = feature_engineering(df,start,new_end)\n    else:\n        print(f\"Processing {all_rows-(new_end+1)} rows of chunk N° {p+1}\")\n        new_chunk = feature_engineering(df,start,None)\n    start = new_end+1\n    new_df = new_df.append(new_chunk, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:11:43.728618Z","iopub.execute_input":"2022-11-11T11:11:43.729310Z","iopub.status.idle":"2022-11-11T11:14:01.152091Z","shell.execute_reply.started":"2022-11-11T11:11:43.729272Z","shell.execute_reply":"2022-11-11T11:14:01.151015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:14:15.232740Z","iopub.execute_input":"2022-11-11T11:14:15.233514Z","iopub.status.idle":"2022-11-11T11:14:15.318000Z","shell.execute_reply.started":"2022-11-11T11:14:15.233480Z","shell.execute_reply":"2022-11-11T11:14:15.316492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GPU T4 x2\n\nIn the right-hand panel, set T4 x2 as GPU accelerator","metadata":{}},{"cell_type":"markdown","source":"# Importing Libraries","metadata":{}},{"cell_type":"code","source":"import gc\nimport operator as op\nimport numpy as np\nimport cupy as cp\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport cudf\nimport time\n\n\nimport warnings \nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:19:13.393816Z","iopub.execute_input":"2022-11-11T11:19:13.394506Z","iopub.status.idle":"2022-11-11T11:19:18.801584Z","shell.execute_reply.started":"2022-11-11T11:19:13.394365Z","shell.execute_reply":"2022-11-11T11:19:18.800608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dividing our dataset into N° parts\nnum_parts = 4\ndef read_preprocess_divide(num_parts):\n    #wanted columns\n    columns = ['ip', 'channel', 'click_time']\n    dtypes = {\n             'ip'      : 'int32',\n             'channel' : 'int16',\n             'click_time' : 'datetime64[us]',\n             }\n    df = cudf.read_csv('../input/talkingdata-adtracking-fraud-detection/train.csv', usecols=columns, dtype=dtypes)\n    all_rows = len(df)\n    chunk = all_rows//num_parts\n    #sort the dataset by ip and reset the index\n    df = df.sort_values(by=['ip', 'click_time']).reset_index(drop = True)\n    return df, all_rows, chunk \n    \n\ndef window(df):\n    #calculate the most common value with the \"mode\", and the \"window\"\n    most_common = df['ip'].mode().values.tolist()[0]\n    window = len(df[df['ip'] == most_common])+1\n    return window\n\ndef feature_engineering(df,start,new_end):\n    if new_end is not None:\n        end = new_end+1\n    else:\n        end = None\n    features = [c for c in list(df.columns) if c not in ['ip','click_time']]\n    cat_function = ['count', 'last', 'nunique']    \n    new_chunk = df[start:end].groupby('ip')[features].agg(cat_function)\n    new_chunk.columns = ['_'.join(x) for x in new_chunk.columns]\n    new_chunk.reset_index(inplace = True)\n    diff_num_features = [f'diff_{col}' for col in features]\n    df = df.to_pandas()\n    ips = df[start:end]['ip'].values\n    new_chunk_diff = df[start:end].groupby('ip')[features].diff().add_prefix('diff_')\n    new_chunk_diff.insert(0,'ip',ips)\n    new_chunk_diff = cudf.DataFrame(new_chunk_diff)\n    new_chunk_diff = new_chunk_diff.groupby('ip')[diff_num_features].agg(cat_function)\n    new_chunk_diff.columns = ['_'.join(x) for x in new_chunk_diff.columns]\n    new_chunk_diff.reset_index(inplace = True)\n    new_chunk = new_chunk.merge(new_chunk_diff, how = 'inner', on = 'ip')\n    new_chunk = new_chunk.sort_values(by=['ip']).reset_index(drop = True)\n    return new_chunk","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:19:43.745079Z","iopub.execute_input":"2022-11-11T11:19:43.745823Z","iopub.status.idle":"2022-11-11T11:19:43.761196Z","shell.execute_reply.started":"2022-11-11T11:19:43.745786Z","shell.execute_reply":"2022-11-11T11:19:43.760019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndf, all_rows, chunk = read_preprocess_divide(num_parts)","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:19:50.755371Z","iopub.execute_input":"2022-11-11T11:19:50.755745Z","iopub.status.idle":"2022-11-11T11:21:24.966688Z","shell.execute_reply.started":"2022-11-11T11:19:50.755714Z","shell.execute_reply":"2022-11-11T11:21:24.965585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#function to select a safe window of rows\nwindow = window(df)\n#new dataframe to append the results of the for loop\nnew_df=cudf.DataFrame()\n#set start = 0\nstart = 0\nfor p in range(0,num_parts):\n    end = p*chunk + chunk\n    if end < all_rows:\n        chunk_window = df[start:end].tail(window)\n        second_last_unique = chunk_window['ip'].unique().values.tolist()[-2]\n        new_end = chunk_window[chunk_window['ip'] == second_last_unique].tail(1).index[0]\n        print(f\"Processing {(new_end+1)-start} rows of chunk N° {p+1}\")\n        new_chunk = feature_engineering(df,start,new_end)\n    else:\n        print(f\"Processing {all_rows-(new_end+1)} rows of chunk N° {p+1}\")\n        new_chunk = feature_engineering(df,start,None)\n    start = new_end+1\n    new_df = new_df.append(new_chunk, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:22:16.117744Z","iopub.execute_input":"2022-11-11T11:22:16.118112Z","iopub.status.idle":"2022-11-11T11:24:44.271931Z","shell.execute_reply.started":"2022-11-11T11:22:16.118081Z","shell.execute_reply":"2022-11-11T11:24:44.270822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_df","metadata":{"execution":{"iopub.status.busy":"2022-11-11T11:26:13.494441Z","iopub.execute_input":"2022-11-11T11:26:13.494825Z","iopub.status.idle":"2022-11-11T11:26:13.601619Z","shell.execute_reply.started":"2022-11-11T11:26:13.494794Z","shell.execute_reply":"2022-11-11T11:26:13.600400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# RESULTS\n\n**GPU P100**\n\n- Read, preprocess and divide took **1min 8s**\n- The processing with feature engineering took **2min 17s**\n\n**GPU T4 x2**\n\n- Read, preprocess and divide took **1min 34s**\n- The processing with feature engineering took **2min 28s**\n\n\nGPU P100 apparently seems more powerful, but let us remember that GPU T4 x2 was designed for image processing and neural networks. My advice for novices is to use only CPU and GPU P100 for tabular datasets, and GPU T4 x2 for images","metadata":{}}]}