{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\n\nimport random\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nimport joblib\nimport itertools\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nfrom tqdm.notebook import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-07T18:54:13.717357Z","iopub.execute_input":"2022-07-07T18:54:13.717983Z","iopub.status.idle":"2022-07-07T18:54:16.215127Z","shell.execute_reply.started":"2022-07-07T18:54:13.717897Z","shell.execute_reply":"2022-07-07T18:54:16.213419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_feats=['B_30', 'B_38', 'D_114', 'D_116', \n           'D_117', 'D_120', 'D_126', 'D_63', \n           'D_64', 'D_66', 'D_68']\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:54:16.216881Z","iopub.execute_input":"2022-07-07T18:54:16.217271Z","iopub.status.idle":"2022-07-07T18:54:16.223112Z","shell.execute_reply.started":"2022-07-07T18:54:16.217237Z","shell.execute_reply":"2022-07-07T18:54:16.221474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_memory_usage(df):\n    start_mem = df.memory_usage().sum() / 1024 ** 2\n    print(\"Initial Memory chunk: {:.3f}\".format(start_mem))\n   \n   # df['customer_ID'] =df['customer_ID'].apply(lambda x: int(x[-16:],16)).astype('int32')\n    df.S_2=pd.to_datetime(df.S_2)\n    for cat_col in cat_feats:\n        encoder = LabelEncoder()\n        df[cat_col] = encoder.fit_transform(df[cat_col])\n    df[cat_feats]=df[cat_feats].astype(np.int8)\n    #df=df.fillna(-1)\n    SKIP=['customer_ID'+'S_2']+cat_feats\n    for col in df.columns:\n        if col in SKIP: \n            continue\n        type_ = df[col].dtype\n\n        if str(type_)[:3] == \"int\":\n            df[col] = df[col].astype(np.int8)\n        elif str(type_)[:5] == \"float\":\n            df[col] = df[col].astype(np.float16)\n\n    end_mem = df.memory_usage().sum() / 1024 ** 2\n    print(\"Final Memory chunk: {:.3f}\".format(end_mem))\n    print(\"Reduced by: {:.2f}\".format((start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:55:10.523972Z","iopub.execute_input":"2022-07-07T18:55:10.524498Z","iopub.status.idle":"2022-07-07T18:55:10.535778Z","shell.execute_reply.started":"2022-07-07T18:55:10.524454Z","shell.execute_reply":"2022-07-07T18:55:10.534802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\nchunksize=1.5*(10**5)\ntrain_df = pd.DataFrame()\nwith pd.read_csv('../input/amex-default-prediction/train_data.csv', chunksize=chunksize) as reader:\n    counter=0\n    for chunk in reader:\n        print(counter, flush=True)\n        chunk=reduce_memory_usage(chunk)\n        train_df = pd.concat([train_df, chunk])\n        counter+=1","metadata":{"execution":{"iopub.status.busy":"2022-07-07T18:55:14.135979Z","iopub.execute_input":"2022-07-07T18:55:14.136743Z","iopub.status.idle":"2022-07-07T19:05:56.232633Z","shell.execute_reply.started":"2022-07-07T18:55:14.136697Z","shell.execute_reply":"2022-07-07T19:05:56.231559Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntrain_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:05:56.234761Z","iopub.execute_input":"2022-07-07T19:05:56.235526Z","iopub.status.idle":"2022-07-07T19:05:56.413042Z","shell.execute_reply.started":"2022-07-07T19:05:56.235476Z","shell.execute_reply":"2022-07-07T19:05:56.411768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:13:31.016190Z","iopub.execute_input":"2022-07-07T19:13:31.017127Z","iopub.status.idle":"2022-07-07T19:13:31.162506Z","shell.execute_reply.started":"2022-07-07T19:13:31.017077Z","shell.execute_reply":"2022-07-07T19:13:31.160917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=pd.read_csv('../input/amex-default-prediction/train_labels.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:13:33.911617Z","iopub.execute_input":"2022-07-07T19:13:33.912281Z","iopub.status.idle":"2022-07-07T19:13:35.079697Z","shell.execute_reply.started":"2022-07-07T19:13:33.912244Z","shell.execute_reply":"2022-07-07T19:13:35.078763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.target=labels.target.astype('int8')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:13:35.509028Z","iopub.execute_input":"2022-07-07T19:13:35.509941Z","iopub.status.idle":"2022-07-07T19:13:35.515102Z","shell.execute_reply.started":"2022-07-07T19:13:35.509899Z","shell.execute_reply":"2022-07-07T19:13:35.514287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.merge(train_df, labels, on='customer_ID')\ntrain_df['customer_ID']=train_df['customer_ID'].apply(lambda x: int(x[-16:],16))\ntrain_df['customer_ID']=train_df['customer_ID'].astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:13:37.977204Z","iopub.execute_input":"2022-07-07T19:13:37.977741Z","iopub.status.idle":"2022-07-07T19:13:54.807266Z","shell.execute_reply.started":"2022-07-07T19:13:37.977701Z","shell.execute_reply":"2022-07-07T19:13:54.805836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:05.680788Z","iopub.execute_input":"2022-07-07T19:14:05.681201Z","iopub.status.idle":"2022-07-07T19:14:05.703954Z","shell.execute_reply.started":"2022-07-07T19:14:05.681167Z","shell.execute_reply":"2022-07-07T19:14:05.703084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('./Amex date pickled')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:42.879909Z","iopub.execute_input":"2022-07-07T19:14:42.880378Z","iopub.status.idle":"2022-07-07T19:14:42.886006Z","shell.execute_reply.started":"2022-07-07T19:14:42.880337Z","shell.execute_reply":"2022-07-07T19:14:42.884720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_pickle('./Amex date pickled/train.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:14:47.153483Z","iopub.execute_input":"2022-07-07T19:14:47.153935Z","iopub.status.idle":"2022-07-07T19:14:51.420690Z","shell.execute_reply.started":"2022-07-07T19:14:47.153901Z","shell.execute_reply":"2022-07-07T19:14:51.419819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df\ndel labels\ndel chunk","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:21:02.132635Z","iopub.execute_input":"2022-07-07T19:21:02.133049Z","iopub.status.idle":"2022-07-07T19:21:02.232292Z","shell.execute_reply.started":"2022-07-07T19:21:02.133016Z","shell.execute_reply":"2022-07-07T19:21:02.231472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:21:08.216852Z","iopub.execute_input":"2022-07-07T19:21:08.217253Z","iopub.status.idle":"2022-07-07T19:21:08.355816Z","shell.execute_reply.started":"2022-07-07T19:21:08.217218Z","shell.execute_reply":"2022-07-07T19:21:08.354386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunksize=1.5*(10**5)\ntest_df = pd.DataFrame()\nwith pd.read_csv('../input/amex-default-prediction/test_data.csv', chunksize=chunksize) as reader:\n    counter=0\n    for chunk in tqdm(reader):\n        print(counter, flush=True)\n        chunk=reduce_memory_usage(chunk)\n        test_df = pd.concat([test_df, chunk])\n        counter+=1","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:22:30.464756Z","iopub.execute_input":"2022-07-07T19:22:30.465721Z","iopub.status.idle":"2022-07-07T19:44:25.982771Z","shell.execute_reply.started":"2022-07-07T19:22:30.465678Z","shell.execute_reply":"2022-07-07T19:44:25.981482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['customer_ID']=test_df['customer_ID'].apply(lambda x: int(x[-16:],16))\ntest_df['customer_ID']=test_df['customer_ID'].astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:45:36.782744Z","iopub.execute_input":"2022-07-07T19:45:36.783221Z","iopub.status.idle":"2022-07-07T19:45:48.578640Z","shell.execute_reply.started":"2022-07-07T19:45:36.783183Z","shell.execute_reply":"2022-07-07T19:45:48.577450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:46:10.158768Z","iopub.execute_input":"2022-07-07T19:46:10.159264Z","iopub.status.idle":"2022-07-07T19:46:10.377150Z","shell.execute_reply.started":"2022-07-07T19:46:10.159225Z","shell.execute_reply":"2022-07-07T19:46:10.376316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:46:21.790846Z","iopub.execute_input":"2022-07-07T19:46:21.791301Z","iopub.status.idle":"2022-07-07T19:46:21.818209Z","shell.execute_reply.started":"2022-07-07T19:46:21.791264Z","shell.execute_reply":"2022-07-07T19:46:21.816825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_pickle('./Amex date pickled/test.pkl')","metadata":{"execution":{"iopub.status.busy":"2022-07-07T19:47:36.583774Z","iopub.execute_input":"2022-07-07T19:47:36.585025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}