{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport cudf\nimport cupy as cp\nimport dask_cudf\nimport os\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-26T17:17:27.567361Z","iopub.execute_input":"2022-07-26T17:17:27.567642Z","iopub.status.idle":"2022-07-26T17:17:27.574715Z","shell.execute_reply.started":"2022-07-26T17:17:27.567601Z","shell.execute_reply":"2022-07-26T17:17:27.573675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dask.distributed import Client\nclient = Client(n_workers=4, threads_per_worker=4, memory_limit='15GB')\nclient","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:17:27.575988Z","iopub.execute_input":"2022-07-26T17:17:27.576315Z","iopub.status.idle":"2022-07-26T17:17:31.571282Z","shell.execute_reply.started":"2022-07-26T17:17:27.576282Z","shell.execute_reply":"2022-07-26T17:17:31.570011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cdf=dask_cudf.read_csv('../input/amex-default-prediction/train_data.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-26T16:33:51.675216Z","iopub.execute_input":"2022-07-26T16:33:51.675857Z","iopub.status.idle":"2022-07-26T16:33:51.682731Z","shell.execute_reply.started":"2022-07-26T16:33:51.675808Z","shell.execute_reply":"2022-07-26T16:33:51.681486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdf=dask_cudf.read_csv('../input/amex-default-prediction/train_data.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:17:37.127312Z","iopub.execute_input":"2022-07-26T17:17:37.127676Z","iopub.status.idle":"2022-07-26T17:17:39.341403Z","shell.execute_reply.started":"2022-07-26T17:17:37.127617Z","shell.execute_reply":"2022-07-26T17:17:39.339586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdf1=cdf.drop_duplicates('customer_ID')#.compute()\ncdf2=cdf1.drop('S_2',axis=1)\ncdf3=cdf2.drop('customer_ID', axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:17:42.886501Z","iopub.execute_input":"2022-07-26T17:17:42.887355Z","iopub.status.idle":"2022-07-26T17:17:44.508331Z","shell.execute_reply.started":"2022-07-26T17:17:42.887319Z","shell.execute_reply":"2022-07-26T17:17:44.507321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols=cdf3.columns.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:17:52.154991Z","iopub.execute_input":"2022-07-26T17:17:52.155799Z","iopub.status.idle":"2022-07-26T17:17:52.162698Z","shell.execute_reply.started":"2022-07-26T17:17:52.155762Z","shell.execute_reply":"2022-07-26T17:17:52.161278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols.remove('D_64')","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:17:53.836744Z","iopub.execute_input":"2022-07-26T17:17:53.837183Z","iopub.status.idle":"2022-07-26T17:17:53.844334Z","shell.execute_reply.started":"2022-07-26T17:17:53.837149Z","shell.execute_reply":"2022-07-26T17:17:53.843302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:17:56.440335Z","iopub.execute_input":"2022-07-26T17:17:56.440759Z","iopub.status.idle":"2022-07-26T17:17:56.452381Z","shell.execute_reply.started":"2022-07-26T17:17:56.440725Z","shell.execute_reply":"2022-07-26T17:17:56.451294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols.remove('D_63')","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:18:21.444599Z","iopub.execute_input":"2022-07-26T17:18:21.444966Z","iopub.status.idle":"2022-07-26T17:18:21.45239Z","shell.execute_reply.started":"2022-07-26T17:18:21.444938Z","shell.execute_reply":"2022-07-26T17:18:21.451304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:18:32.937255Z","iopub.execute_input":"2022-07-26T17:18:32.937647Z","iopub.status.idle":"2022-07-26T17:18:32.947853Z","shell.execute_reply.started":"2022-07-26T17:18:32.937598Z","shell.execute_reply":"2022-07-26T17:18:32.946511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdf4=cdf3.drop(cols,axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:18:35.741292Z","iopub.execute_input":"2022-07-26T17:18:35.741607Z","iopub.status.idle":"2022-07-26T17:18:35.761849Z","shell.execute_reply.started":"2022-07-26T17:18:35.741579Z","shell.execute_reply":"2022-07-26T17:18:35.760673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncdf5=cdf4.compute()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:18:38.69418Z","iopub.execute_input":"2022-07-26T17:18:38.694501Z","iopub.status.idle":"2022-07-26T17:21:00.083896Z","shell.execute_reply.started":"2022-07-26T17:18:38.694474Z","shell.execute_reply":"2022-07-26T17:21:00.082677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdf5","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:23:16.432904Z","iopub.execute_input":"2022-07-26T17:23:16.433404Z","iopub.status.idle":"2022-07-26T17:23:16.529615Z","shell.execute_reply.started":"2022-07-26T17:23:16.433358Z","shell.execute_reply":"2022-07-26T17:23:16.528635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(cols)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T14:39:38.008017Z","iopub.execute_input":"2022-07-26T14:39:38.008857Z","iopub.status.idle":"2022-07-26T14:39:38.017514Z","shell.execute_reply.started":"2022-07-26T14:39:38.008817Z","shell.execute_reply":"2022-07-26T14:39:38.015485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nD_63=cdf3['D_63'].compute()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:48:47.069904Z","iopub.execute_input":"2022-07-26T12:48:47.070273Z","iopub.status.idle":"2022-07-26T12:51:14.962907Z","shell.execute_reply.started":"2022-07-26T12:48:47.070239Z","shell.execute_reply":"2022-07-26T12:51:14.961666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nD_64=cdf3['D_64'].compute()","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:51:14.966603Z","iopub.execute_input":"2022-07-26T12:51:14.966933Z","iopub.status.idle":"2022-07-26T12:53:31.279204Z","shell.execute_reply.started":"2022-07-26T12:51:14.966893Z","shell.execute_reply":"2022-07-26T12:53:31.277048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:46:03.768587Z","iopub.execute_input":"2022-07-26T12:46:03.769164Z","iopub.status.idle":"2022-07-26T12:46:03.837194Z","shell.execute_reply.started":"2022-07-26T12:46:03.769128Z","shell.execute_reply":"2022-07-26T12:46:03.836116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-07-26T12:46:00.881057Z","iopub.execute_input":"2022-07-26T12:46:00.88164Z","iopub.status.idle":"2022-07-26T12:46:00.957132Z","shell.execute_reply.started":"2022-07-26T12:46:00.881574Z","shell.execute_reply":"2022-07-26T12:46:00.955849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D_63=cdf5['D_63'].astype('category')\nD_64=cdf5['D_64'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:26:12.441068Z","iopub.execute_input":"2022-07-26T17:26:12.441395Z","iopub.status.idle":"2022-07-26T17:26:12.508073Z","shell.execute_reply.started":"2022-07-26T17:26:12.441366Z","shell.execute_reply":"2022-07-26T17:26:12.507099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D_63= cudf.get_dummies(D_63)\nD_64=cudf.get_dummies(D_64)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:26:15.500104Z","iopub.execute_input":"2022-07-26T17:26:15.50042Z","iopub.status.idle":"2022-07-26T17:26:17.126879Z","shell.execute_reply.started":"2022-07-26T17:26:15.500391Z","shell.execute_reply":"2022-07-26T17:26:17.124167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"D_63","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:26:24.868065Z","iopub.execute_input":"2022-07-26T17:26:24.8684Z","iopub.status.idle":"2022-07-26T17:26:24.939477Z","shell.execute_reply.started":"2022-07-26T17:26:24.868372Z","shell.execute_reply":"2022-07-26T17:26:24.938645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new=cudf.concat([D_63,D_64],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:26:30.339206Z","iopub.execute_input":"2022-07-26T17:26:30.339524Z","iopub.status.idle":"2022-07-26T17:26:30.345551Z","shell.execute_reply.started":"2022-07-26T17:26:30.339496Z","shell.execute_reply":"2022-07-26T17:26:30.344493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new.to_csv('d_64_D_63_hot_test.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-26T17:26:46.29437Z","iopub.execute_input":"2022-07-26T17:26:46.294818Z","iopub.status.idle":"2022-07-26T17:26:46.353383Z","shell.execute_reply.started":"2022-07-26T17:26:46.294779Z","shell.execute_reply":"2022-07-26T17:26:46.352321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-07-26T07:26:52.583085Z","iopub.execute_input":"2022-07-26T07:26:52.583418Z","iopub.status.idle":"2022-07-26T07:26:52.588349Z","shell.execute_reply.started":"2022-07-26T07:26:52.583388Z","shell.execute_reply":"2022-07-26T07:26:52.587101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}