{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom dask import dataframe as dd\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-15T17:06:33.028365Z","iopub.execute_input":"2022-08-15T17:06:33.029915Z","iopub.status.idle":"2022-08-15T17:06:33.960972Z","shell.execute_reply.started":"2022-08-15T17:06:33.029728Z","shell.execute_reply":"2022-08-15T17:06:33.960001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.set_option('display.height', 1000)\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T17:06:35.469523Z","iopub.execute_input":"2022-08-15T17:06:35.469946Z","iopub.status.idle":"2022-08-15T17:06:35.475708Z","shell.execute_reply.started":"2022-08-15T17:06:35.469911Z","shell.execute_reply":"2022-08-15T17:06:35.474482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_parquet('../input/amex-parquet/train_data.parquet')","metadata":{"execution":{"iopub.status.busy":"2022-08-15T17:06:36.568878Z","iopub.execute_input":"2022-08-15T17:06:36.569303Z","iopub.status.idle":"2022-08-15T17:07:14.020909Z","shell.execute_reply.started":"2022-08-15T17:06:36.569266Z","shell.execute_reply":"2022-08-15T17:07:14.019645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df[train_df['customer_ID']=='0000099d6bd597052cdcda90ffabf56573fe9d7c79be5fbac11a8ed792feb62a']['S_2']","metadata":{"execution":{"iopub.status.busy":"2022-08-15T12:18:54.284691Z","iopub.execute_input":"2022-08-15T12:18:54.285095Z","iopub.status.idle":"2022-08-15T12:18:54.289681Z","shell.execute_reply.started":"2022-08-15T12:18:54.285062Z","shell.execute_reply":"2022-08-15T12:18:54.288372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#temp = train_df.groupby(['customer_ID']).agg({'S_2':['count']}).reset_index()\n#temp['S_2'].max()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-15T12:18:54.764404Z","iopub.execute_input":"2022-08-15T12:18:54.764826Z","iopub.status.idle":"2022-08-15T12:18:54.769984Z","shell.execute_reply.started":"2022-08-15T12:18:54.764787Z","shell.execute_reply":"2022-08-15T12:18:54.768680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df[train_df['customer_ID']=='0000099d6bd597052cdcda90ffabf56573fe9d7c79be5fbac11a8ed792feb62a']","metadata":{"execution":{"iopub.status.busy":"2022-08-15T12:18:58.589471Z","iopub.execute_input":"2022-08-15T12:18:58.589913Z","iopub.status.idle":"2022-08-15T12:18:58.595650Z","shell.execute_reply.started":"2022-08-15T12:18:58.589866Z","shell.execute_reply":"2022-08-15T12:18:58.594736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\n\nnumerical_cols = []\nfor col in list(train_df.columns):\n    if col not in categorical_cols:\n        numerical_cols.append(col)\n#numerical_cols","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_cols.remove('target')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']]","metadata":{"execution":{"iopub.status.busy":"2022-08-15T17:54:00.826522Z","iopub.execute_input":"2022-08-15T17:54:00.827214Z","iopub.status.idle":"2022-08-15T17:54:01.280531Z","shell.execute_reply.started":"2022-08-15T17:54:00.827172Z","shell.execute_reply":"2022-08-15T17:54:01.279246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def impute_categorical_cols(col):\n    train_df[col].fillna(train_df[col].mode()[0], inplace=True)\n    print('NA imputed...')\n\ncols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nfor col in cols:\n    impute_categorical_cols(col)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T17:54:12.381548Z","iopub.execute_input":"2022-08-15T17:54:12.381979Z","iopub.status.idle":"2022-08-15T17:54:13.961525Z","shell.execute_reply.started":"2022-08-15T17:54:12.381943Z","shell.execute_reply":"2022-08-15T17:54:13.960172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convertToObject(col, df):\n    df[col] = df[col].astype(object)\n    print('datatype converted...')\n\ncols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_66', 'D_68']\nfor col in cols:\n    convertToObject(col, train_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T17:54:16.321305Z","iopub.execute_input":"2022-08-15T17:54:16.321735Z","iopub.status.idle":"2022-08-15T17:54:47.933446Z","shell.execute_reply.started":"2022-08-15T17:54:16.321702Z","shell.execute_reply":"2022-08-15T17:54:47.932172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T17:56:04.786696Z","iopub.execute_input":"2022-08-15T17:56:04.787169Z","iopub.status.idle":"2022-08-15T17:56:12.368542Z","shell.execute_reply.started":"2022-08-15T17:56:04.787135Z","shell.execute_reply":"2022-08-15T17:56:12.367524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']]","metadata":{"execution":{"iopub.status.busy":"2022-08-15T11:40:01.899572Z","iopub.execute_input":"2022-08-15T11:40:01.900426Z","iopub.status.idle":"2022-08-15T11:40:02.619713Z","shell.execute_reply.started":"2022-08-15T11:40:01.900377Z","shell.execute_reply":"2022-08-15T11:40:02.618381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def frequency_counts(col, df):\n    print(df[col].value_counts())\n    \ncols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nfor col in cols:\n    print(col)\n    frequency_counts(col, train_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-15T10:45:36.360678Z","iopub.execute_input":"2022-08-15T10:45:36.361154Z","iopub.status.idle":"2022-08-15T10:45:37.270822Z","shell.execute_reply.started":"2022-08-15T10:45:36.361112Z","shell.execute_reply":"2022-08-15T10:45:37.269368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[cols].dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-15T17:56:23.341996Z","iopub.execute_input":"2022-08-15T17:56:23.342416Z","iopub.status.idle":"2022-08-15T17:56:24.222709Z","shell.execute_reply.started":"2022-08-15T17:56:23.342380Z","shell.execute_reply":"2022-08-15T17:56:24.221556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nencoded_dummies = pd.get_dummies(train_df[cols])\ntrain_df.drop(cols, axis=1, inplace=True)\ntrain_df = pd.concat([train_df, encoded_dummies], axis=1)\ndel encoded_dummies","metadata":{"execution":{"iopub.status.busy":"2022-08-15T12:20:49.431755Z","iopub.execute_input":"2022-08-15T12:20:49.432756Z","iopub.status.idle":"2022-08-15T12:21:03.422767Z","shell.execute_reply.started":"2022-08-15T12:20:49.432697Z","shell.execute_reply":"2022-08-15T12:21:03.420937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imputeNumericalCols(df, col):\n    df[col].fillna()","metadata":{"execution":{"iopub.status.busy":"2022-08-15T12:38:54.445459Z","iopub.execute_input":"2022-08-15T12:38:54.445922Z","iopub.status.idle":"2022-08-15T12:38:58.668566Z","shell.execute_reply.started":"2022-08-15T12:38:54.445885Z","shell.execute_reply":"2022-08-15T12:38:58.667266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}