{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport pickle\n\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-16T05:53:20.137356Z","iopub.execute_input":"2022-08-16T05:53:20.137823Z","iopub.status.idle":"2022-08-16T05:53:20.145199Z","shell.execute_reply.started":"2022-08-16T05:53:20.137766Z","shell.execute_reply":"2022-08-16T05:53:20.143571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.read_parquet(\"../input/amex-data-integer-dtypes-parquet-format/train.parquet\")\ntrain_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:53:20.518998Z","iopub.execute_input":"2022-08-16T05:53:20.519392Z","iopub.status.idle":"2022-08-16T05:53:41.889016Z","shell.execute_reply.started":"2022-08-16T05:53:20.519362Z","shell.execute_reply":"2022-08-16T05:53:41.888080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_features = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68']\nbin_features = ['B_31', 'D_87']\nnumeric_features = [colname for colname in train_df.columns if colname not in cat_features+bin_features+['customer_ID', 'S_2', 'target']]\n\nprint(len(numeric_features))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:53:41.890416Z","iopub.execute_input":"2022-08-16T05:53:41.891306Z","iopub.status.idle":"2022-08-16T05:53:41.897691Z","shell.execute_reply.started":"2022-08-16T05:53:41.891272Z","shell.execute_reply":"2022-08-16T05:53:41.896798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def divide_int(s, v):\n    s = s.apply(lambda x: -1 if x==-1 else x/v)\n    s = s.astype(np.float32)\n    return s","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:53:41.899109Z","iopub.execute_input":"2022-08-16T05:53:41.899955Z","iopub.status.idle":"2022-08-16T05:53:41.907928Z","shell.execute_reply.started":"2022-08-16T05:53:41.899921Z","shell.execute_reply":"2022-08-16T05:53:41.906949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"div_map={\n    'B_4': 78,\n    'B_16':12,\n    'B_20': 17,\n    'B_22': 2,\n    \n    'D_39': 34,\n    'D_44': 8,\n    'D_49': 71,\n    'D_51': 3,\n    \n    'D_59': 48,\n    'D_65': 38,\n    'D_70': 4,\n    'D_72': 3,\n    \n    \n    'D_74':4,\n    'D_75':15,\n    'D_78':2,\n    'D_79':2,\n    'D_80':5,\n    \n    'D_82': 2,\n    'D_84': 2,\n    'D_89': 9,\n    'D_91': 2,\n    \n    'D_106': 23,\n    'D_107': 3,\n    'D_111': 2,\n    'D_113': 5,\n    \n    'D_122': 7,\n    'D_124': 22,\n    'D_136': 4,\n    'D_138': 2,\n    \n    'D_145': 11,\n    'R_3': 10,\n    'R_5': 2,\n    'R_9': 6,\n    \n    'R_11': 2,\n    'R_13': 31,\n    'R_16': 2,\n    'R_17': 35,\n    'R_18': 31,\n    \n    'R_26': 28,\n    'S_11': 25,\n    'S_15': 10,\n    \n    'B_19': 100,\n    'S_13': 1034,\n    'S_8': 3166\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:53:41.910417Z","iopub.execute_input":"2022-08-16T05:53:41.911065Z","iopub.status.idle":"2022-08-16T05:53:41.919311Z","shell.execute_reply.started":"2022-08-16T05:53:41.911021Z","shell.execute_reply":"2022-08-16T05:53:41.918278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k,v in div_map.items():\n    print(k, \" --> before --> {:.5f}\".format(train_df[k].max()) )\n    train_df[k] = divide_int(train_df[k], v)\n    print(k, \" --> after --> {:.5f}\".format(train_df[k].max()) )\n    print()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:53:41.920364Z","iopub.execute_input":"2022-08-16T05:53:41.921071Z","iopub.status.idle":"2022-08-16T05:55:07.282531Z","shell.execute_reply.started":"2022-08-16T05:53:41.921037Z","shell.execute_reply":"2022-08-16T05:55:07.281374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"na_df = []\nfor colname in numeric_features:\n    s = train_df[colname]\n    na_percent = len(s[s.isna()])/len(s)\n    na_df.append({\n        'colname': colname,\n        'na_percent': na_percent\n    })\n\nna_df = pd.DataFrame.from_dict(na_df)\nna_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:55:07.283928Z","iopub.execute_input":"2022-08-16T05:55:07.284288Z","iopub.status.idle":"2022-08-16T05:55:08.790243Z","shell.execute_reply.started":"2022-08-16T05:55:07.284249Z","shell.execute_reply":"2022-08-16T05:55:08.789041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nstat_df=[]\nfor colname in numeric_features:\n    s = train_df[colname]\n    s = s[s.isna()==False]\n    stat_df.append({\n        'colname': colname,\n        \n        'q01': np.quantile(s, .01),\n        'q05': np.quantile(s, .05),\n        'q10': np.quantile(s, .1),\n        \n        'q90': np.quantile(s, .90),\n        'q95': np.quantile(s, .95),\n        'q99': np.quantile(s, .99),\n        \n        'min_value': s.min(),\n        'median_value': s.median(),\n        'max_value': s.max()\n    })\n\nstat_df = pd.DataFrame.from_dict(stat_df)\nstat_df = stat_df.merge(na_df)\n\nstat_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:55:08.791542Z","iopub.execute_input":"2022-08-16T05:55:08.792330Z","iopub.status.idle":"2022-08-16T05:56:02.643962Z","shell.execute_reply.started":"2022-08-16T05:55:08.792295Z","shell.execute_reply":"2022-08-16T05:56:02.643146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_df['rmax_1'] = np.abs(stat_df['q95'].div(stat_df['q90']))\nstat_df['rmax_2'] = np.abs(stat_df['q99'].div(stat_df['q95']))\nstat_df['rmax_3'] = np.abs(stat_df['max_value'].div(stat_df['q99']))\n\nstat_df['rmin_1'] = np.abs(stat_df['q05'].div(stat_df['q10']))\nstat_df['rmin_2'] = np.abs(stat_df['q01'].div(stat_df['q05']))\nstat_df['rmin_3'] = np.abs(stat_df['min_value'].div(stat_df['q01']))\n\nstat_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:56:02.646315Z","iopub.execute_input":"2022-08-16T05:56:02.647230Z","iopub.status.idle":"2022-08-16T05:56:02.676189Z","shell.execute_reply.started":"2022-08-16T05:56:02.647193Z","shell.execute_reply":"2022-08-16T05:56:02.674963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_df1 = stat_df[(stat_df.rmin_2>10) & \n                  (stat_df.rmin_2!=np.inf) & \n                  (stat_df.q01!=-1) & \n                  (stat_df.q05!=-1)].copy()\n\n\nmin_df2 = stat_df[(stat_df.rmin_3>10) & \n                  (stat_df.rmin_3!=np.inf) & \n                  (stat_df.min_value != np.inf) &\n                  (stat_df.q01!=-1)\n                 ].copy()\n\n\nprint(min_df1.shape, min_df2.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:56:02.677889Z","iopub.execute_input":"2022-08-16T05:56:02.678260Z","iopub.status.idle":"2022-08-16T05:56:02.691469Z","shell.execute_reply.started":"2022-08-16T05:56:02.678227Z","shell.execute_reply":"2022-08-16T05:56:02.690371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_df2[['colname', 'q01', 'min_value', 'rmin_2', 'rmin_3']].head()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:56:02.692572Z","iopub.execute_input":"2022-08-16T05:56:02.693570Z","iopub.status.idle":"2022-08-16T05:56:02.711238Z","shell.execute_reply.started":"2022-08-16T05:56:02.693534Z","shell.execute_reply":"2022-08-16T05:56:02.710412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_df1 = stat_df[(stat_df.rmax_2 > 10) & \n                  (stat_df.rmax_2 != np.inf)].copy()\n\nmax_df2 = stat_df[(stat_df.rmax_3 > 10) & \n                  (stat_df.rmax_3 != np.inf) &\n                  (stat_df.colname.isin(max_df1.colname)==False)\n                 ].copy()\n\nprint(len(max_df1), len(max_df2))","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:56:02.712429Z","iopub.execute_input":"2022-08-16T05:56:02.712936Z","iopub.status.idle":"2022-08-16T05:56:02.722548Z","shell.execute_reply.started":"2022-08-16T05:56:02.712896Z","shell.execute_reply":"2022-08-16T05:56:02.721368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stat_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:58:44.058932Z","iopub.execute_input":"2022-08-16T05:58:44.059969Z","iopub.status.idle":"2022-08-16T05:58:44.081757Z","shell.execute_reply.started":"2022-08-16T05:58:44.059927Z","shell.execute_reply":"2022-08-16T05:58:44.080946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clip_map = {}\n\nfor _,row in min_df2.iterrows():\n    colname = row.colname\n    min_value = row.min_value\n    q01 = row.q01\n    median_value = row.median_value\n    \n    k1 = q01 - 2*np.abs(median_value-q01)\n    k2 = q01 - 2*np.abs(q01)\n    \n    clip_map[colname]={}\n    clip_map[colname]['min_value'] = min(k1, k2)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:58:46.489251Z","iopub.execute_input":"2022-08-16T05:58:46.489895Z","iopub.status.idle":"2022-08-16T05:58:46.499925Z","shell.execute_reply.started":"2022-08-16T05:58:46.489857Z","shell.execute_reply":"2022-08-16T05:58:46.498525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _,row in max_df1.iterrows():\n    colname = row.colname\n    max_value = row.max_value\n    \n    q95 = row.q95\n    median_value = row.median_value\n    \n    k1 = q95 + 2*np.abs(median_value-q95)\n    k2 = q95 + 2*np.abs(q95)\n    \n    if clip_map.get(colname,None) is None:\n        clip_map[colname]={}\n    clip_map[colname]['max_value'] = max(k1, k2)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:58:48.638950Z","iopub.execute_input":"2022-08-16T05:58:48.640045Z","iopub.status.idle":"2022-08-16T05:58:48.648460Z","shell.execute_reply.started":"2022-08-16T05:58:48.639994Z","shell.execute_reply":"2022-08-16T05:58:48.646965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _,row in max_df2.iterrows():\n    colname = row.colname\n    max_value = row.max_value\n    \n    q99 = row.q99\n    median_value = row.median_value\n    \n    k1 = q99 + 2*np.abs(median_value-q99)\n    k2 = q99 + 2*np.abs(q99)\n    \n    if clip_map.get(colname,None) is None:\n        clip_map[colname]={}\n    clip_map[colname]['max_value'] = max(k1, k2)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:59:28.620028Z","iopub.execute_input":"2022-08-16T05:59:28.620807Z","iopub.status.idle":"2022-08-16T05:59:28.633604Z","shell.execute_reply.started":"2022-08-16T05:59:28.620765Z","shell.execute_reply":"2022-08-16T05:59:28.632611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"sequence_clip_map.pkl\", 'wb') as file:\n    pickle.dump(clip_map, file)","metadata":{"execution":{"iopub.status.busy":"2022-08-16T05:59:43.294817Z","iopub.execute_input":"2022-08-16T05:59:43.297584Z","iopub.status.idle":"2022-08-16T05:59:43.309442Z","shell.execute_reply.started":"2022-08-16T05:59:43.297513Z","shell.execute_reply":"2022-08-16T05:59:43.308020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}