{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":35332,"databundleVersionId":3723648,"sourceType":"competition"}],"dockerImageVersionId":30197,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tqdm \nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T11:35:52.112815Z","iopub.execute_input":"2024-11-21T11:35:52.114018Z","iopub.status.idle":"2024-11-21T11:35:52.120935Z","shell.execute_reply.started":"2024-11-21T11:35:52.113943Z","shell.execute_reply":"2024-11-21T11:35:52.119366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Input file and chunk size\ninput_file = '/kaggle/input/amex-default-prediction/train_data.csv'\noutput_file = '/kaggle/working/train_data_rounded.parquet'\nchunk_size = 100000  # Number of rows per chunk\n\n# Initialize a list to store processed chunks\nprocessed_chunks = []\n\n# Process the file in chunks\ni = 1\nfor chunk in pd.read_csv(input_file, chunksize=chunk_size):\n    print(f\"Chunk {i}\")\n    i += 1\n    # Select columns that are floats\n    float_cols = chunk.select_dtypes(include=['float']).columns\n    \n    # Round down float columns to 3 decimal places\n    for col in float_cols:\n        chunk[col] = np.floor(chunk[col] * 100)  # Round down to 3 decimals\n\n    # convert all Nan to -inf16\n    chunk[float_cols] = chunk[float_cols].fillna(-32768)\n    \n    # Convert float columns to float16\n    chunk[float_cols] = chunk[float_cols].astype('int16')\n    \n    # Append processed chunk to list\n    processed_chunks.append(chunk)\n\n# Concatenate all processed chunks and save to Parquet format\nprocessed_data = pd.concat(processed_chunks, ignore_index=True)\nprocessed_data.to_parquet(output_file, index=False)\n\nprint(f\"Processed data saved to {output_file}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T11:35:52.122574Z","iopub.execute_input":"2024-11-21T11:35:52.122974Z","iopub.status.idle":"2024-11-21T11:45:34.863163Z","shell.execute_reply.started":"2024-11-21T11:35:52.122931Z","shell.execute_reply":"2024-11-21T11:45:34.861701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# Input file and chunk size\ninput_file = '/kaggle/input/amex-default-prediction/test_data.csv'\noutput_file = '/kaggle/working/test_data_rounded.parquet'\nchunk_size = 100000  # Number of rows per chunk\n\n# Initialize a list to store processed chunks\nprocessed_chunks = []\n\n# Process the file in chunks\ni = 1\nfor chunk in pd.read_csv(input_file, chunksize=chunk_size):\n    print(f\"Chunk {i}\")\n    i += 1\n    # Select columns that are floats\n    float_cols = chunk.select_dtypes(include=['float']).columns\n    \n    # Round down float columns to 3 decimal places\n    for col in float_cols:\n        chunk[col] = np.floor(chunk[col] * 1000)  # Round down to 3 decimals\n\n    # convert all Nan to -inf16\n    chunk[float_cols] = chunk[float_cols].fillna(-32768)\n    \n    # Convert float columns to float16\n    chunk[float_cols] = chunk[float_cols].astype('int16')\n    \n    # Append processed chunk to list\n    processed_chunks.append(chunk)\n\n# Concatenate all processed chunks and save to Parquet format\nprocessed_data = pd.concat(processed_chunks, ignore_index=True)\nprocessed_data.to_parquet(output_file, index=False)\n\nprint(f\"Processed data saved to {output_file}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T11:45:34.864732Z","iopub.execute_input":"2024-11-21T11:45:34.865208Z","iopub.status.idle":"2024-11-21T12:09:13.516457Z","shell.execute_reply.started":"2024-11-21T11:45:34.865162Z","shell.execute_reply":"2024-11-21T12:09:13.513017Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"awjdawkjdajkwdjkawkdj","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.522613Z","iopub.execute_input":"2024-11-21T12:09:13.523266Z","iopub.status.idle":"2024-11-21T12:09:13.555839Z","shell.execute_reply.started":"2024-11-21T12:09:13.523213Z","shell.execute_reply":"2024-11-21T12:09:13.552581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# chunk_size = 100000  # Number of rows per chunk\n# for chunk in pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv', chunksize=chunk_size):\n#     # Process each chunk\n#     print(chunk.head())\n#     print(len(chunk))\n#     print(type(chunk[\"B_2\"]))\n#     print(chunk[\"B_2\"].nunique())\n#     print(chunk[\"B_2\"].round(3).nunique())\n#     plt.hist(chunk[(chunk[\"B_2\"] < 0.01) & (chunk[\"B_1\"] > 0.0)][\"B_1\"], bins=100)\n#     break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.558328Z","iopub.status.idle":"2024-11-21T12:09:13.559225Z","shell.execute_reply.started":"2024-11-21T12:09:13.558844Z","shell.execute_reply":"2024-11-21T12:09:13.558882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# x = pd.read_csv('/kaggle/input/amex-default-prediction/train_data.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.561644Z","iopub.status.idle":"2024-11-21T12:09:13.562344Z","shell.execute_reply.started":"2024-11-21T12:09:13.562106Z","shell.execute_reply":"2024-11-21T12:09:13.562131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def floorify(x, lo):\n    \"\"\"example: x in [0, 0.01] -> x := 0\"\"\"\n    return lo if x <= lo+0.01 and x >= lo else x\n\ndef floorify_zeros(x):\n    \"\"\"look around values [0,0.01] and determine if in proximity it's categorical. If yes - floorify\"\"\"\n    has_zeros = len([t for t in x if t>=0 and t<=0.01])>0 \n    no_proximity = len([t for t in x if t<0 and t>=-0.01])==0 and len([t for t in x if t>0.01 and t<=0.02])==0\n    if not no_proximity:\n        return x\n    if not has_zeros:\n        return x\n    x = [floorify(t, 0.0) for t in x]\n    return x\n\ndef floorify_ones(x):\n    \"\"\"look around values [1,1.01] and determine if in proximity it's categorical. If yes - floorify\"\"\"    \n    has_ones = len([t for t in x if t>=1 and t<=1.01])>0 \n    no_proximity = len([t for t in x if t<1 and t>=0.99])==0 and len([t for t in x if t>1.01 and t<=1.02])==0\n    if not no_proximity:\n        return x\n    if not has_ones:\n        return x\n    x = [floorify(t, 1.0) for t in x]\n    return x\n\ndef convert_na(x):\n    \"\"\"nan -> -1 if positive values\"\"\"\n    if np.nanmin(x)>=0:\n        return [-1 if np.isnan(t) else t for t in x]\n\ndef convert_to_int(x):\n    \"\"\"float -> int8 if possible\"\"\"\n    q = convert_na(x)\n    if set(np.unique(q)).union({-1,0,1}) == {-1,0,1}:\n        return [np.int8(t) for t in q]\n    return x\n\ndef floorify_ones_and_zeros(t):\n    \"\"\"do everything\"\"\"\n    t = floorify_zeros(t)\n    t = floorify_ones(t)\n    t = convert_to_int(t)\n    return t\n\ndef floorify_frac(x, interval=1):\n    \"\"\"convert to int if float appears ordinal\"\"\"\n    xt = (np.floor(x/interval+1e-6)).fillna(-1)\n    if np.max(xt)<=127:\n        return xt.astype(np.int8)\n    return xt.astype(np.int16)  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.564292Z","iopub.status.idle":"2024-11-21T12:09:13.565198Z","shell.execute_reply.started":"2024-11-21T12:09:13.564893Z","shell.execute_reply":"2024-11-21T12:09:13.564922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x['B_4'] = floorify_frac(x['B_4'],1/78)\nx['B_16'] = floorify_frac(x['B_16'],1/12)\nx['B_20'] = floorify_frac(x['B_20'],1/17)\nx['B_22'] = floorify_frac(x['B_22'],1/2)\nx['B_30'] = floorify_frac(x['B_30'])\nx['B_31'] = floorify_frac(x['B_31'])\nx['B_32'] = floorify_frac(x['B_32'])\nx['B_33'] = floorify_frac(x['B_33'])\nx['B_38'] = floorify_frac(x['B_38'])\nx['B_41'] = floorify_frac(x['B_41'])\nx['D_39'] = floorify_frac(x['D_39'],1/34)\nx['D_44'] = floorify_frac(x['D_44'],1/8)\nx['D_49'] = floorify_frac(x['D_49'],1/71)\nx['D_51'] = floorify_frac(x['D_51'],1/3)\nx['D_59'] = floorify_frac(x['D_59']+5/48,1/48)\nx['D_65'] = floorify_frac(x['D_65'],1/38)\nx['D_66'] = floorify_frac(x['D_66'])\nx['D_68'] = floorify_frac(x['D_68'])\nx['D_70'] = floorify_frac(x['D_70'],1/4)\nx['D_72'] = floorify_frac(x['D_72'],1/3)\nx['D_74'] = floorify_frac(x['D_74'],1/14)\nx['D_75'] = floorify_frac(x['D_75'],1/15)\nx['D_78'] = floorify_frac(x['D_78'],1/2)\nx['D_79'] = floorify_frac(x['D_79'],1/2)\nx['D_80'] = floorify_frac(x['D_80'],1/5)\nx['D_81'] = floorify_frac(x['D_81'])\nx['D_82'] = floorify_frac(x['D_82'],1/2)\nx['D_83'] = floorify_frac(x['D_83'])\nx['D_84'] = floorify_frac(x['D_84'],1/2)\nx['D_86'] = floorify_frac(x['D_86'])\nx['D_87'] = floorify_frac(x['D_87'])\nx['D_89'] = floorify_frac(x['D_89'],1/9)\nx['D_91'] = floorify_frac(x['D_91'],1/2)\nx['D_92'] = floorify_frac(x['D_92'])\nx['D_93'] = floorify_frac(x['D_93'])\nx['D_94'] = floorify_frac(x['D_94'])\nx['D_96'] = floorify_frac(x['D_96'])\nx['D_103'] = floorify_frac(x['D_103'])\nx['D_106'] = floorify_frac(x['D_106'],1/23)\nx['D_107'] = floorify_frac(x['D_107'],1/3)\nx['D_108'] = floorify_frac(x['D_108'])\nx['D_109'] = floorify_frac(x['D_109'])\nx['D_111'] = floorify_frac(x['D_111'],1/2)\nx['D_113'] = floorify_frac(x['D_113'],1/5)\nx['D_114'] = floorify_frac(x['D_114'])\nx['D_116'] = floorify_frac(x['D_116'])\nx['D_117'] = floorify_frac(x['D_117']+1)\nx['D_120'] = floorify_frac(x['D_120'])\nx['D_122'] = floorify_frac(x['D_122'],1/7)\nx['D_123'] = floorify_frac(x['D_123'])\nx['D_124'] = floorify_frac(x['D_124']+1/22,1/22)\nx['D_125'] = floorify_frac(x['D_125'])\nx['D_126'] = floorify_frac(x['D_126']+1)\nx['D_127'] = floorify_frac(x['D_127'])\nx['D_129'] = floorify_frac(x['D_129'])\nx['D_135'] = floorify_frac(x['D_135'])\nx['D_136'] = floorify_frac(x['D_136'],1/4)\nx['D_137'] = floorify_frac(x['D_137'])\nx['D_138'] = floorify_frac(x['D_138'],1/2)\nx['D_139'] = floorify_frac(x['D_139'])\nx['D_140'] = floorify_frac(x['D_140'])\nx['D_143'] = floorify_frac(x['D_143'])\nx['D_145'] = floorify_frac(x['D_145'],1/11)\nx['R_2'] = floorify_frac(x['R_2'])\nx['R_3'] = floorify_frac(x['R_3'],1/10)\nx['R_4'] = floorify_frac(x['R_4'])\nx['R_5'] = floorify_frac(x['R_5'],1/2)\nx['R_8'] = floorify_frac(x['R_8'])\nx['R_9'] = floorify_frac(x['R_9'],1/6)\nx['R_10'] = floorify_frac(x['R_10'])\nx['R_11'] = floorify_frac(x['R_11'],1/2)\nx['R_13'] = floorify_frac(x['R_13'],1/31)\nx['R_15'] = floorify_frac(x['R_15'])\nx['R_16'] = floorify_frac(x['R_16'],1/2)\nx['R_17'] = floorify_frac(x['R_17'],1/35)\nx['R_18'] = floorify_frac(x['R_18'],1/31)\nx['R_19'] = floorify_frac(x['R_19'])\nx['R_20'] = floorify_frac(x['R_20'])\nx['R_21'] = floorify_frac(x['R_21'])\nx['R_22'] = floorify_frac(x['R_22'])\nx['R_23'] = floorify_frac(x['R_23'])\nx['R_24'] = floorify_frac(x['R_24'])\nx['R_25'] = floorify_frac(x['R_25'])\nx['R_26'] = floorify_frac(x['R_26'],1/28)\nx['R_28'] = floorify_frac(x['R_28'])\nx['S_6'] = floorify_frac(x['S_6'])\nx['S_11'] = floorify_frac(x['S_11']+5/25,1/25)\nx['S_15'] = floorify_frac(x['S_15']+3/10,1/10)\nx['S_18'] = floorify_frac(x['S_18'])\nx['S_20'] = floorify_frac(x['S_20'])\n\nx['D_63'] = x['D_63'].apply(lambda t: {'CR':0, 'XZ':1, 'XM':2, 'CO':3, 'CL':4, 'XL':5}[t]).astype(np.int8)\nx['D_64'] = x['D_64'].apply(lambda t: {np.nan:-1, 'O':0, '-1':1, 'R':2, 'U':3}[t]).astype(np.int8)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.567Z","iopub.status.idle":"2024-11-21T12:09:13.567649Z","shell.execute_reply.started":"2024-11-21T12:09:13.567333Z","shell.execute_reply":"2024-11-21T12:09:13.567375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# can be rounded up as each bin has AUC of 0.5\nx['B_19'] = np.floor(x['B_19']*100).fillna(-1).astype(np.int8)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.569736Z","iopub.status.idle":"2024-11-21T12:09:13.570397Z","shell.execute_reply.started":"2024-11-21T12:09:13.570102Z","shell.execute_reply":"2024-11-21T12:09:13.570131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# one value overlaps, but the split can identified by S_11\nx.loc[x.S_13.between(0.67, 0.7) & (x.S_11.isin([15,16,17])),'S_13'] = 0.6789168283158535\nfloor_vals = (0, 0.0377176456223467, 0.2804642206328049, 0.4013539714415651, 0.4206963381303189, 0.5067698438641042, \n              0.5261121975338173, 0.5551258157960416, 0.6218568673028206, 0.6876208933830246, 0.8433269036807703, 1)\nfor c in floor_vals:\n    x['S_13'] = x['S_13'].apply(lambda t: floorify(t,c))\nx['S_13'] = np.round(x['S_13']*1034).fillna(-1).astype(np.int16)    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.572677Z","iopub.status.idle":"2024-11-21T12:09:13.57334Z","shell.execute_reply.started":"2024-11-21T12:09:13.573044Z","shell.execute_reply":"2024-11-21T12:09:13.573074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# this one has many more value overlaps, but the splits can be identified by S_15\nx.loc[(x.S_8>=0.30) & (x.S_8<=0.35) & (x.S_15<=6),'S_8'] = 0.3224889650033656\nx.loc[(x.S_8>=0.30) & (x.S_8<=0.35) & (x.S_15==7),'S_8'] = 0.3145925513763017\nx.loc[(x.S_8>=0.45) & (x.S_8<=0.477) & (x.S_15==3),'S_8'] = 0.4570436553944634\nx.loc[(x.S_8>=0.45) & (x.S_8<=0.477) & (x.S_15==5),'S_8'] = 0.4636765662005172\nx.loc[(x.S_8>=0.45) & (x.S_8<=0.477) & (x.S_15==6),'S_8'] = 0.4592546209653157\nx.loc[(x.S_8>=0.55) & (x.S_8<=0.65) & (x.S_15==5),'S_8'] = 0.5938092592144236\nx.loc[(x.S_8>=0.55) & (x.S_8<=0.65) & (x.S_15==4),'S_8'] = 0.5994946974629933\nx.loc[(x.S_8>=0.55) & (x.S_8<=0.65) & (x.S_15<=2),'S_8'] = 0.6017056828901041\nx.loc[(x.S_8>=0.73) & (x.S_8<=0.78) & (x.S_15==3),'S_8'] = 0.7441567340107059\nx.loc[(x.S_8>=0.73) & (x.S_8<=0.78) & (x.S_15==5),'S_8'] = 0.7517372106519937\nx.loc[(x.S_8>=0.73) & (x.S_8<=0.78) & (x.S_15==4),'S_8'] = 0.7586861099807893\nx.loc[(x.S_8>=0.91) & (x.S_8<=0.98) & (x.S_15==4),'S_8'] = 0.9147189165383852\nx.loc[(x.S_8>=0.91) & (x.S_8<=0.98) & (x.S_15<=2),'S_8'] = 0.9327230426634736\nx.loc[(x.S_8>=0.91) & (x.S_8<=0.98) & (x.S_15==3),'S_8'] = 0.935565546481781\nx.loc[(x.S_8>=1.12) & (x.S_8<=1.17) & (x.S_15<=2),'S_8'] = 1.1440303975988897\nx.loc[(x.S_8>=1.12) & (x.S_8<=1.17) & (x.S_15==3),'S_8'] = 1.151926881019957\nfloor_vals = (0, 0.1017056275625063, 0.119709415455368, 0.1667719530078215, 0.2438408100936861, \n              0.3578648754166172, 0.4055590769093041, 0.4772583808904347, 0.4876816287061991, \n              0.6620341135675392, 0.7005685574395781, 0.8509160456526623, 1, 1.0145299163657109, \n              1.1051803467580654, 1.2214158871037435)\nfor c in floor_vals:    \n    x['S_8'] = x['S_8'].apply(lambda t: floorify(t,c))\nx['S_8'] = np.round(x['S_8']*3166).fillna(-1).astype(np.int16)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.574669Z","iopub.status.idle":"2024-11-21T12:09:13.575266Z","shell.execute_reply.started":"2024-11-21T12:09:13.574964Z","shell.execute_reply":"2024-11-21T12:09:13.575011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = x.select_dtypes(include=[float]).columns\nfor col in tqdm.tqdm(cols):\n    x[col] = floorify_ones_and_zeros(x[col])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.577692Z","iopub.status.idle":"2024-11-21T12:09:13.578142Z","shell.execute_reply.started":"2024-11-21T12:09:13.577938Z","shell.execute_reply":"2024-11-21T12:09:13.577956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in x.select_dtypes(include=[float]).columns.tolist():\n    x[col] = x[col].astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.580555Z","iopub.status.idle":"2024-11-21T12:09:13.58195Z","shell.execute_reply.started":"2024-11-21T12:09:13.581645Z","shell.execute_reply":"2024-11-21T12:09:13.581688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x.to_parquet('parquet/train.parquet')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-21T12:09:13.583403Z","iopub.status.idle":"2024-11-21T12:09:13.583884Z","shell.execute_reply.started":"2024-11-21T12:09:13.583653Z","shell.execute_reply":"2024-11-21T12:09:13.583673Z"}},"outputs":[],"execution_count":null}]}