{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-04T19:51:01.944432Z","iopub.execute_input":"2023-01-04T19:51:01.944865Z","iopub.status.idle":"2023-01-04T19:51:01.960373Z","shell.execute_reply.started":"2023-01-04T19:51:01.94483Z","shell.execute_reply":"2023-01-04T19:51:01.95851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np  # linear algebra\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport sys\nimport pickle\nimport glob\nfrom sklearn.preprocessing import LabelEncoder\n\npd.set_option('display.max_columns', None)\nimport random\n\nrandom.seed(75)\nfrom tqdm.notebook import tqdm_notebook\nfrom functools import partial, reduce\n\n### warnings setting\nimport sys\nimport warnings\n\nif not sys.warnoptions:\n    warnings.simplefilter(\"ignore\")\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n\n#### model\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import roc_auc_score, roc_curve, auc\nimport catboost\nfrom catboost import Pool, CatBoostClassifier\nimport lightgbm as lgb\nimport joblib\nimport pickle\nfrom tqdm.notebook import tqdm_notebook\nimport uuid\n\n##### LOGGING Stettings #####\nimport logging\n\n# Create logger\nlogger = logging.getLogger()\nlogger.setLevel(logging.INFO)\n# Create STDERR handler\nhandler = logging.StreamHandler(sys.stderr)\n# Create formatter and add it to the handler\nformatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S', )\nhandler.setFormatter(formatter)\n# Set STDERR handler as the only handler\nlogger.handlers = [handler]\n\n#### plots\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.colors\n\nsns.set(rc={'axes.facecolor': '#f9ecec', 'figure.facecolor': '#f9ecec'})\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nfrom plotly.offline import init_notebook_mode\n\n### Plotly settings\ntheme_palette = {\n    'base': '#a3d3eb',\n    'complementary': '#ebbba3',\n    'triadic': '#eba3d3',\n    'backgound': '#f6fbfd'\n}\n\ntemp = dict(layout=go.Layout(font=dict(family=\"Ubuntu\", size=14),\n                             height=600,\n                             legend=dict(  #traceorder='reversed',\n                                 orientation=\"v\",\n                                 y=1.15,\n                                 x=0.9),\n                             plot_bgcolor=theme_palette['backgound'],\n                             paper_bgcolor=theme_palette['backgound']))","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:33:28.456286Z","iopub.execute_input":"2023-01-04T19:33:28.456734Z","iopub.status.idle":"2023-01-04T19:33:32.074143Z","shell.execute_reply.started":"2023-01-04T19:33:28.456695Z","shell.execute_reply":"2023-01-04T19:33:32.072526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_parquet('/kaggle/input/jiegouhua714/train.parquet')\n","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:56:56.50847Z","iopub.execute_input":"2023-01-04T09:56:56.508908Z","iopub.status.idle":"2023-01-04T09:57:10.822899Z","shell.execute_reply.started":"2023-01-04T09:56:56.508868Z","shell.execute_reply":"2023-01-04T09:57:10.822123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Define some features by category\nfeatures = train.drop(['customer_ID', 'S_2'], axis=1).columns.to_list()\ncat_vars = ['B_30', 'B_38', 'D_114', 'D_116', 'D_117', 'D_120', 'D_126', 'D_63', 'D_64', 'D_66', 'D_68', 'month',\n            'day_of_week']\nnum_vars = list(filter(lambda x: x not in cat_vars, features))\n\n# devide nums vars by AmEx Categories\ndelequincy_vars = filter(lambda x: x.startswith('D') and x not in cat_vars, features)\nspend_vars = filter(lambda x: (x.startswith('S')) and (x not in cat_vars), features)\npayment_vars = filter(lambda x: x.startswith('P') and x not in cat_vars, features)\nbalance_vars = filter(lambda x: x.startswith('B') and x not in cat_vars, features)\nrisk_vars = filter(lambda x: x.startswith('R') and x not in cat_vars, features)\n\nwith open('features.pkl', 'wb') as f:\n    pickle.dump(features, f)\n\nwith open('cat_vars.pkl', 'wb') as f:\n    pickle.dump(cat_vars, f)\n\nwith open('num_vars.pkl', 'wb') as f:\n    pickle.dump(num_vars, f)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T17:52:57.570028Z","iopub.execute_input":"2023-01-04T17:52:57.570535Z","iopub.status.idle":"2023-01-04T17:52:57.600416Z","shell.execute_reply.started":"2023-01-04T17:52:57.570496Z","shell.execute_reply":"2023-01-04T17:52:57.598236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BatchGenerator:\n    def __init__(self, df, batch_feature='customer_ID', keep_features=features, n_batchs=750):\n        self.df = df\n        self.batch_feature = batch_feature\n        self.keep_feature = list(set([batch_feature] + keep_features))\n        self.n_batchs = n_batchs\n\n    def __iter__(self):\n        unique_vals = self.df[self.batch_feature].unique()\n        batch_size = int(np.ceil(len(unique_vals) / self.n_batchs))\n        groups = self.df.groupby(self.batch_feature).groups\n        n_batchs = min(self.n_batchs, int(np.ceil(len(unique_vals) / batch_size)))\n        for i in range(n_batchs):\n            keys = unique_vals[i * batch_size:(i + 1) * batch_size]\n            idx = [i for s in keys for i in groups[s]]\n            if i == n_batchs - 1:\n                keys = unique_vals[(i + 1) * batch_size:]\n                idx = idx + [i for s in keys for i in groups[s]]\n            yield self.df.loc[idx, self.keep_feature]\n\n\nweek_days = {1: 'Mon', 2: 'Tue', 3: 'Wen', 4: 'Thu', 5: 'Fri', 6: 'Sat', 7: 'Sun'}","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:58:11.732737Z","iopub.execute_input":"2023-01-04T09:58:11.733096Z","iopub.status.idle":"2023-01-04T09:58:11.744411Z","shell.execute_reply.started":"2023-01-04T09:58:11.733067Z","shell.execute_reply":"2023-01-04T09:58:11.742959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_date_vars(df, date_var='S_2', sort_by=['customer_ID', 'S_2'], week_days=week_days):\n    # change to datetime\n    df[date_var] = pd.to_datetime(df[date_var])\n    # sort by custoner ther by date\n    df = df.sort_values(by=sort_by)\n    # extract some date characteristics\n    # year has not a very\n    # month\n    df['month'] = df[date_var].dt.month\n    # day of week\n    df['day_of_week'] = df[date_var].apply(lambda x: x.isocalendar()[-1])\n    return df\n\n\ngroup_names = [\"delequincy_vars\", \"spend_vars\", \"payment_vars\", \"balance_vars\", \"risk_vars\"]\n","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:58:24.141068Z","iopub.execute_input":"2023-01-04T09:58:24.141426Z","iopub.status.idle":"2023-01-04T09:58:24.149091Z","shell.execute_reply.started":"2023-01-04T09:58:24.141396Z","shell.execute_reply":"2023-01-04T09:58:24.147967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def row_rise_aggregation(df,\n                         group_vars=[delequincy_vars, spend_vars, payment_vars, balance_vars, risk_vars],\n                         group_names=group_names,\n                         save=True):\n    print('shape before row_rise_aggregation', df.shape)\n    for group_name, group_var in zip(group_names, group_vars):\n        df[group_name + '_sum'] = df[group_var].sum(axis=1)\n        df[group_name + '_mean'] = df[group_var].mean(axis=1)\n        df[group_name + '_missing'] = df.isnull().sum(axis=1)\n    print('shape after row_rise_aggregation', df.shape)\n    if save:\n        df.reset_index(drop=False).to_feather(f\"row_agg_{str(uuid.uuid4())}.ftr\")\n        return df['customer_ID'].nunique()\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:58:31.468529Z","iopub.execute_input":"2023-01-04T09:58:31.468941Z","iopub.status.idle":"2023-01-04T09:58:31.476085Z","shell.execute_reply.started":"2023-01-04T09:58:31.468868Z","shell.execute_reply":"2023-01-04T09:58:31.475253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def column_rise_aggregation(df, num_vars=num_vars, cat_vars=cat_vars, save=True):\n    print('shape before column_rise_aggregation', df.shape)\n    group_names = filter(lambda x: '_vars' in x, df.columns)\n    num_agg = df.groupby(\"customer_ID\")[list(set(list(num_vars) + list(group_names)))].agg(\n        ['mean', 'std', 'min', 'max', 'last'])\n    num_agg.columns = ['_'.join(x) for x in num_agg.columns]\n\n    cat_agg = df.groupby(\"customer_ID\")[list(set(list(cat_vars) + ['month', 'day_of_week']))].agg(\n        ['count', 'last', 'nunique', pd.Series.mode])\n    cat_agg.columns = ['_'.join(x) for x in cat_agg.columns]\n\n    mode_cols = filter(lambda x: x.endswith('_mode'), cat_agg.columns)\n    for col in mode_cols:\n        cat_agg[col] = cat_agg[col].apply(lambda x: random.choice(str(x).strip('[]').split()))\n    #concat the two dataframes\n    df = pd.concat([num_agg, cat_agg], axis=1)\n    del num_agg, cat_agg\n\n    gc.collect()\n    print('shape after column_rise_aggregation', df.shape)\n    if save:\n        df.reset_index(drop=False).to_feather(f\"col_agg_{str(uuid.uuid4())}.ftr\")\n        return len(df)  #df['customer_ID'].nunique()\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:58:42.407976Z","iopub.execute_input":"2023-01-04T09:58:42.408352Z","iopub.status.idle":"2023-01-04T09:58:42.419477Z","shell.execute_reply.started":"2023-01-04T09:58:42.408319Z","shell.execute_reply":"2023-01-04T09:58:42.4176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from https://www.kaggle.com/code/ragnar123/amex-lgbm-dart-cv-0-7977\ndef get_difference(df, num_features):\n    res = []\n    customer_ids = []\n    for customer_id, df in tqdm_notebook(df.groupby(['customer_ID'])):\n        # Get the differences\n        diff_df = df[num_features].diff(1).iloc[[-1]].values.astype(np.float32)\n        # Append to lists\n        res.append(diff_df)\n        customer_ids.append(customer_id)\n    # Concatenate\n    res = np.concatenate(res, axis=0)\n    # Transform to dataframe\n    res = pd.DataFrame(res, columns=[col + '_diff1' for col in df[num_features].columns])\n    # Add customer id\n    res['customer_ID'] = customer_ids\n    print('final shape', res.shape)\n    #       df = df.merge(res, on='customer_ID', how='inner')\n    #       df.reset_index(drop=False).to_feather(f\"diff_{str(uuid.uuid4())}.ftr\")\n    return res  #df['customer_ID'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:59:18.055022Z","iopub.execute_input":"2023-01-04T09:59:18.055806Z","iopub.status.idle":"2023-01-04T09:59:18.064289Z","shell.execute_reply.started":"2023-01-04T09:59:18.055773Z","shell.execute_reply":"2023-01-04T09:59:18.062586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_partition(df, prefix='train'):\n    global c\n    df.reset_index(drop=True).to_feather(f'{prefix}_{c}.ftr')\n    c = c + 1\n    return df['customer_ID'].nunique()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:59:25.123626Z","iopub.execute_input":"2023-01-04T09:59:25.124015Z","iopub.status.idle":"2023-01-04T09:59:25.129829Z","shell.execute_reply.started":"2023-01-04T09:59:25.123983Z","shell.execute_reply":"2023-01-04T09:59:25.128578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_BATCHS = 100\nc = 0\n\nsamples_df = BatchGenerator(train, batch_feature='customer_ID', keep_features=features + ['S_2'], n_batchs=N_BATCHS)\nprocessed_elements = sum(map(partial(save_partition, prefix='train'), tqdm_notebook(samples_df, total=N_BATCHS)))\n\ndel train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T09:59:46.936053Z","iopub.execute_input":"2023-01-04T09:59:46.936397Z","iopub.status.idle":"2023-01-04T10:01:42.849103Z","shell.execute_reply.started":"2023-01-04T09:59:46.936369Z","shell.execute_reply":"2023-01-04T10:01:42.847942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed = []\nn_paths = 0\nfor path in tqdm_notebook(glob.glob('train_*.ftr')):\n    # apply on train\n    sample_df = pd.read_feather(path)\n    diff_df = get_difference(sample_df, num_features=num_vars)\n    sample_df = sample_df.merge(diff_df, on='customer_ID', how='inner')\n\n    #   if sample_df.shape[1]<300:\n    sample_df = extract_date_vars(sample_df)\n    all_num_vars = num_vars + list(map(lambda x: x + '_diff1', num_vars))\n    sample_df = row_rise_aggregation(sample_df, save=False)\n    sample_df = column_rise_aggregation(sample_df, num_vars=all_num_vars, save=False)\n\n    \n    for col in tqdm_notebook(num_vars):\n        try:\n            sample_df[f'{col}_last_mean_diff'] = sample_df[f'{col}_last'] - sample_df[f'{col}_mean']\n        except:\n            pass\n\n    sample_df = sample_df.reset_index()\n\n    sample_df.reset_index(drop=True).to_feather(path)\n    \n\ntrain = pd.concat(map(lambda sample_df: pd.read_feather(sample_df), tqdm_notebook(glob.glob('train_*.ftr'))))","metadata":{"execution":{"iopub.status.busy":"2023-01-04T10:06:53.462778Z","iopub.execute_input":"2023-01-04T10:06:53.463286Z","iopub.status.idle":"2023-01-04T10:34:38.413849Z","shell.execute_reply.started":"2023-01-04T10:06:53.46325Z","shell.execute_reply":"2023-01-04T10:34:38.412676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Left join with labels:\nlabels = pd.read_csv('/kaggle/input/amex-default-prediction/train_labels.csv')\nprint(labels.shape, labels['customer_ID'].nunique())\nlabels = labels.set_index('customer_ID')\ntrain = train.set_index('customer_ID')\ntrain['target'] = labels['target']\n# del labels\ngc.collect()\n# save result\ntrain.reset_index().to_feather('train_fin_df.ftr')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T11:57:33.82446Z","iopub.execute_input":"2023-01-04T11:57:33.824885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_BATCHS = 300\nc=0\n\ntest = pd.read_parquet('/kaggle/input/jiegouhua714/test.parquet')\nn_cid = test.customer_ID.nunique()\nprint(test.shape, n_cid )\nsamples_df = BatchGenerator(test, batch_feature='customer_ID', keep_features=features+['S_2'], n_batchs=N_BATCHS)\nprocessed_elements = sum(map(partial(save_partition, prefix='test'), tqdm_notebook(samples_df, total=N_BATCHS)))\n    \ndel test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T10:38:26.29473Z","iopub.execute_input":"2023-01-04T10:38:26.295163Z","iopub.status.idle":"2023-01-04T10:46:57.648387Z","shell.execute_reply.started":"2023-01-04T10:38:26.295134Z","shell.execute_reply":"2023-01-04T10:46:57.64713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed = 0\n\n      \nfor path in tqdm_notebook(glob.glob('test_*.ftr')):\n    # apply on train\n    sample_df = pd.read_feather(path)\n    diff_df = get_difference(sample_df, num_features=num_vars)\n    sample_df = sample_df.merge(diff_df, on='customer_ID', how='inner')\n        \n #   if sample_df.shape[1]<300:\n    sample_df = extract_date_vars(sample_df)\n    all_num_vars = num_vars + list(map(lambda x: x+'_diff1', num_vars))\n    sample_df = row_rise_aggregation(sample_df, save=False)\n    sample_df = column_rise_aggregation(sample_df, num_vars=all_num_vars, save=False)\n        \n    print(\"diff between last and mean transaction\")\n    for col in tqdm_notebook(num_vars):\n        try:\n            sample_df[f'{col}_last_mean_diff'] = sample_df[f'{col}_last'] - sample_df[f'{col}_mean']\n        except:\n            pass\n\n    sample_df = sample_df.reset_index()\n        \n    sample_df.reset_index(drop=True).to_feather(path)\n    print(\"save processed\", path)\n        \n    processed += sample_df.customer_ID.nunique()\n        \ntest = pd.concat(map(lambda sample_df: pd.read_feather(sample_df), tqdm_notebook(glob.glob('test_*.ftr'))))\ntest.reset_index(drop=True).to_feather('test_fin_df.ftr')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T10:47:02.137189Z","iopub.execute_input":"2023-01-04T10:47:02.137638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.concat(map(lambda sample_df: pd.read_feather(sample_df), tqdm_notebook(glob.glob('test_*.ftr'))))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:34:02.149474Z","iopub.execute_input":"2023-01-04T19:34:02.149944Z","iopub.status.idle":"2023-01-04T19:35:17.701205Z","shell.execute_reply.started":"2023-01-04T19:34:02.149903Z","shell.execute_reply":"2023-01-04T19:35:17.700303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:35:42.818696Z","iopub.execute_input":"2023-01-04T19:35:42.819116Z","iopub.status.idle":"2023-01-04T19:35:43.112109Z","shell.execute_reply.started":"2023-01-04T19:35:42.81908Z","shell.execute_reply":"2023-01-04T19:35:43.110698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.set_index('customer_ID')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T20:00:35.171166Z","iopub.execute_input":"2023-01-04T20:00:35.171599Z","iopub.status.idle":"2023-01-04T20:00:43.240524Z","shell.execute_reply.started":"2023-01-04T20:00:35.171561Z","shell.execute_reply":"2023-01-04T20:00:43.238915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-01-04T12:14:50.108809Z","iopub.execute_input":"2023-01-04T12:14:50.10977Z","iopub.status.idle":"2023-01-04T12:14:50.190474Z","shell.execute_reply.started":"2023-01-04T12:14:50.109719Z","shell.execute_reply":"2023-01-04T12:14:50.186226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.reset_index(drop=True).to_feather('test_fin_df.ftr')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.concat(map(lambda sample_df: pd.read_feather(sample_df), tqdm_notebook(glob.glob('train_*.ftr'))))","metadata":{"execution":{"iopub.status.busy":"2023-01-04T11:54:16.60647Z","iopub.execute_input":"2023-01-04T11:54:16.607846Z","iopub.status.idle":"2023-01-04T11:55:07.050663Z","shell.execute_reply.started":"2023-01-04T11:54:16.607804Z","shell.execute_reply":"2023-01-04T11:55:07.049236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_feather('train_fin_df.ftr')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:14:17.554733Z","iopub.execute_input":"2023-01-04T18:14:17.556003Z","iopub.status.idle":"2023-01-04T18:14:26.17462Z","shell.execute_reply.started":"2023-01-04T18:14:17.555957Z","shell.execute_reply":"2023-01-04T18:14:26.17339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2023-01-04T12:18:20.752068Z","iopub.execute_input":"2023-01-04T12:18:20.752471Z","iopub.status.idle":"2023-01-04T12:18:22.965892Z","shell.execute_reply.started":"2023-01-04T12:18:20.752438Z","shell.execute_reply":"2023-01-04T12:18:22.964565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_size(df):\n# Transform float64 columns to float32\n    print(\"reduce float data size\")\n    cols = list(df.dtypes[df.dtypes == 'float64'].index)\n    for col in tqdm_notebook(cols):\n        df[col] = df[col].astype(np.float32)\n    # Transform int64 columns to int32\n    print(\"reduce cat data size\")\n    cols = list(df.dtypes[df.dtypes == 'int64'].index)\n    for col in tqdm_notebook(cols):\n        df[col] = df[col].astype(np.int32)\n    return df\n        \ntrain = reduce_size(train)\ntest = reduce_size(test)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:35:55.05656Z","iopub.execute_input":"2023-01-04T19:35:55.057051Z","iopub.status.idle":"2023-01-04T19:35:56.964976Z","shell.execute_reply.started":"2023-01-04T19:35:55.057011Z","shell.execute_reply":"2023-01-04T19:35:56.963171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def missing_values_table(df):\n    # Total missing values by column\n    mis_val = df.isnull().sum()\n\n    # Percentage of missing values by column\n    mis_val_percent = 100 * df.isnull().sum() / len(df)\n\n    # build a table with the thw columns\n    mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n\n    # Rename the columns\n    mis_val_table_ren_columns = mis_val_table.rename(\n    columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n\n    # Sort the table by percentage of missing descending\n    mis_val_table_ren_columns = mis_val_table_ren_columns[\n        mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n    '% of Total Values', ascending=False).round(1)\n\n    # Print some summary information\n    print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n        \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n          \" columns that have missing values.\")\n\n    # Return the dataframe with missing information\n    return mis_val_table_ren_columns\n\n# Missing values for training data\nmissing_values_train = missing_values_table(test)\n#cm = sns.color_palette('Set2', as_cmap=True)\n#missing_values_train[:20]#.style.background_gradient(cmap=cm)\nTHRESHOLD = 80\nprint(test.shape)\ndrop_cols = missing_values_train[missing_values_train['% of Total Values']>THRESHOLD].index.to_list()\nprint(f\"Drop {len(drop_cols)} features with more than {THRESHOLD}% of missing values\")\n\ntest = test.drop(drop_cols,axis=1)\nprint(\"Training data shape after dropping highly missing values columns\", test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:38:38.090175Z","iopub.execute_input":"2023-01-04T19:38:38.090562Z","iopub.status.idle":"2023-01-04T19:38:50.173849Z","shell.execute_reply.started":"2023-01-04T19:38:38.090535Z","shell.execute_reply":"2023-01-04T19:38:50.172162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = train.corrwith(train['target'], axis=0)\ncorr = corr[corr.notna()].sort_values(key=abs, ascending=False)\nTHRESHOLD = 0.15\nCORR_SELECTION=True\nif CORR_SELECTION:\n    selected_feats = corr[corr.abs()>THRESHOLD].index\n    train = train[list(selected_feats)]\n    print(f\"Training data shape after dropping uncorrelated features\"\n          f\"(threshold Pearson correlation = {THRESHOLD})\", \n          train.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:17:15.47911Z","iopub.execute_input":"2023-01-04T18:17:15.4796Z","iopub.status.idle":"2023-01-04T18:17:25.092931Z","shell.execute_reply.started":"2023-01-04T18:17:15.479552Z","shell.execute_reply":"2023-01-04T18:17:25.091751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[list(selected_feats)]","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:55:46.405294Z","iopub.execute_input":"2023-01-04T19:55:46.405764Z","iopub.status.idle":"2023-01-04T19:55:46.429342Z","shell.execute_reply.started":"2023-01-04T19:55:46.405728Z","shell.execute_reply":"2023-01-04T19:55:46.42715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:17:28.168142Z","iopub.execute_input":"2023-01-04T18:17:28.168553Z","iopub.status.idle":"2023-01-04T18:17:28.50194Z","shell.execute_reply.started":"2023-01-04T18:17:28.168521Z","shell.execute_reply":"2023-01-04T18:17:28.500738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NaN_Val = np.array(train.isnull().sum())\nNaN_prec = np.array((train.isnull().sum() * 100 / len(train)).round(2))\nNaN_Col = pd.DataFrame([np.array(list(train.columns)).T,NaN_Val.T,NaN_prec.T,np.array(list(train.dtypes)).T], index=['Features','Num of Missing values','Percentage','DataType']\n).transpose()\npd.set_option('display.max_rows', None)\nNaN_Col","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:17:31.331295Z","iopub.execute_input":"2023-01-04T18:17:31.331756Z","iopub.status.idle":"2023-01-04T18:17:31.822963Z","shell.execute_reply.started":"2023-01-04T18:17:31.331718Z","shell.execute_reply":"2023-01-04T18:17:31.821535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in train.columns if col != \"customer_ID\"]\n","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:17:43.64772Z","iopub.execute_input":"2023-01-04T18:17:43.648127Z","iopub.status.idle":"2023-01-04T18:17:43.65376Z","shell.execute_reply.started":"2023-01-04T18:17:43.648097Z","shell.execute_reply":"2023-01-04T18:17:43.652793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in features:\n    train[col] = train[col].fillna(train[col].median())","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:17:46.66533Z","iopub.execute_input":"2023-01-04T18:17:46.666087Z","iopub.status.idle":"2023-01-04T18:17:49.979316Z","shell.execute_reply.started":"2023-01-04T18:17:46.666047Z","shell.execute_reply":"2023-01-04T18:17:49.977832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types = train.dtypes\ntarget_col = 'target'\n\ncat_cols = list(types[types.apply(lambda x:not(str(x).startswith('float')))].index)\ncat_cols = list(filter(lambda x:x!=target_col, cat_cols))\nfeatures = list(train.drop(target_col, axis=1).columns)\ngc.collect()\nprint('len cat_col', len(cat_cols))\nprint('len features', len(features))\n    \nwith open('features.pkl', 'wb') as f:\n    pickle.dump(features, f)\n\nwith open('cat_cols.pkl', 'wb') as f:\n    pickle.dump(cat_cols, f)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:18:23.767456Z","iopub.execute_input":"2023-01-04T18:18:23.767893Z","iopub.status.idle":"2023-01-04T18:18:24.288582Z","shell.execute_reply.started":"2023-01-04T18:18:23.767861Z","shell.execute_reply":"2023-01-04T18:18:24.287127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'] = train[\"target\"].astype('int')\nprint('Transform all String features to category.\\n')\nos.makedirs('label_encoders')\n\nfor usecol in tqdm_notebook(cat_cols):\n#    print(usecol)\n    train[usecol] = train[usecol].astype('str')\n#    test[usecol] = test[usecol].astype('str')\n\n    #Fit LabelEncoder\n    le = LabelEncoder().fit(\n            np.unique(train[usecol].unique().tolist()))#+\n#                      test[usecol].unique().tolist()))\n\n    #At the end 0 will be used for null values so we start at 1 \n    train[usecol] = le.transform(train[usecol])+1\n#    test[usecol]  = le.transform(test[usecol])+1\n\n    train[usecol] = train[usecol].replace(np.nan, 0).astype('int').astype('category')\n#    test[usecol]  = test[usecol].replace(np.nan, 0).astype('int').astype('category')\n\n    joblib.dump(le, f'label_encoders/{usecol}_label_encoder.pkl')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T18:18:56.245773Z","iopub.execute_input":"2023-01-04T18:18:56.24616Z","iopub.status.idle":"2023-01-04T18:18:56.276531Z","shell.execute_reply.started":"2023-01-04T18:18:56.246132Z","shell.execute_reply":"2023-01-04T18:18:56.274598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop(['target'],axis = 1)\ny = train['target']","metadata":{"execution":{"iopub.status.busy":"2023-01-04T13:06:34.499626Z","iopub.execute_input":"2023-01-04T13:06:34.500099Z","iopub.status.idle":"2023-01-04T13:06:35.145512Z","shell.execute_reply.started":"2023-01-04T13:06:34.50006Z","shell.execute_reply":"2023-01-04T13:06:35.144601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape , y.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-04T13:07:21.093856Z","iopub.execute_input":"2023-01-04T13:07:21.094354Z","iopub.status.idle":"2023-01-04T13:07:21.103177Z","shell.execute_reply.started":"2023-01-04T13:07:21.094309Z","shell.execute_reply":"2023-01-04T13:07:21.102021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-01-04T13:08:13.810744Z","iopub.execute_input":"2023-01-04T13:08:13.811175Z","iopub.status.idle":"2023-01-04T13:08:13.817452Z","shell.execute_reply.started":"2023-01-04T13:08:13.811146Z","shell.execute_reply":"2023-01-04T13:08:13.815842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,X_test,y_train,y_test = train_test_split(X, y, test_size=0.2, random_state=75, stratify=y)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T13:09:00.554128Z","iopub.execute_input":"2023-01-04T13:09:00.554574Z","iopub.status.idle":"2023-01-04T13:09:01.936757Z","shell.execute_reply.started":"2023-01-04T13:09:00.554545Z","shell.execute_reply":"2023-01-04T13:09:01.935192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMClassifier","metadata":{"execution":{"iopub.status.busy":"2023-01-04T13:17:31.083568Z","iopub.execute_input":"2023-01-04T13:17:31.084033Z","iopub.status.idle":"2023-01-04T13:17:31.090265Z","shell.execute_reply.started":"2023-01-04T13:17:31.084001Z","shell.execute_reply":"2023-01-04T13:17:31.088645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb = LGBMClassifier(objective= 'binary',\n        metric= 'binary_logloss',\n        boosting= 'dart',\n        seed= 75,\n        num_leaves= 100,\n        learning_rate= 0.01,\n        feature_fraction= 0.20,\n        bagging_freq= 10,\n        bagging_fraction= 0.50,\n        n_jobs= -1,\n        lambda_l2= 2,\n        min_data_in_leaf= 40,\n        n_estimators=15000)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T13:21:49.592748Z","iopub.execute_input":"2023-01-04T13:21:49.593264Z","iopub.status.idle":"2023-01-04T13:21:49.602164Z","shell.execute_reply.started":"2023-01-04T13:21:49.593227Z","shell.execute_reply":"2023-01-04T13:21:49.600377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb.fit(X_train, y_train, eval_metric='binary_logloss', eval_set=[(X_test, y_test)], early_stopping_rounds=100,\n        verbose=150)","metadata":{"execution":{"iopub.status.busy":"2023-01-04T13:23:34.951126Z","iopub.execute_input":"2023-01-04T13:23:34.951622Z","iopub.status.idle":"2023-01-04T17:20:02.358145Z","shell.execute_reply.started":"2023-01-04T13:23:34.951567Z","shell.execute_reply":"2023-01-04T17:20:02.35296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"joblib.dump(lgb, \"lgb_model_with_diff.sav\")","metadata":{"execution":{"iopub.status.busy":"2023-01-04T17:27:31.211086Z","iopub.execute_input":"2023-01-04T17:27:31.211472Z","iopub.status.idle":"2023-01-04T17:27:35.301531Z","shell.execute_reply.started":"2023-01-04T17:27:31.211445Z","shell.execute_reply":"2023-01-04T17:27:35.300289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T17:36:40.184921Z","iopub.execute_input":"2023-01-04T17:36:40.185341Z","iopub.status.idle":"2023-01-04T17:36:40.206449Z","shell.execute_reply.started":"2023-01-04T17:36:40.185312Z","shell.execute_reply":"2023-01-04T17:36:40.20466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import partial, reduce\n\nmodels_folder = '/kaggle/working/label_encoders/'\n\ndef get_sample(path):\n    test = pd.read_feather(path).set_index('customer_ID')\n\n    test = reduce_size(test)\n    \n    test = test[features]\n    \n    gc.collect()\n\n    for usecol in tqdm_notebook(cat_cols):\n        le = joblib.load(os.path.join(models_folder,f'{usecol}_label_encoder.pkl'))\n        test[usecol] = test[usecol].astype('str').apply(lambda x:x.split('.')[0])\n        test[usecol] = test[usecol].map(lambda s: '<unknown>' if s not in le.classes_ else s)\n        le.classes_ = np.append(le.classes_, '<unknown>')\n        #At the end 0 will be used for null values so we start at 1 \n        test[usecol]  = le.transform(test[usecol])+1\n        test[usecol] = test[usecol].replace(np.nan, 0).astype('int').astype('category')\n    return test\n\n\n\ndef get_batchs(df, batch_size, keep_features):\n    n_batchs = int(len(df)/batch_size)\n    cid = list(df.index)\n    for i in range(n_batchs):\n        idx = cid[i*batch_size:(i+1)*batch_size]\n        if i == n_batchs-1:\n            idx = idx + cid[(i+1)*batch_size:]\n        yield df.loc[idx, keep_features]\n        \ndef predict(sample_df, model):\n    y_pred = model.predict(sample_df)\n    sub = pd.Series(y_pred, index=sample_df.index, name='prediction')\n#    sub.to_frame().to_csv(outfile)\n    return sub","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:19:44.072952Z","iopub.execute_input":"2023-01-04T19:19:44.073396Z","iopub.status.idle":"2023-01-04T19:19:44.088784Z","shell.execute_reply.started":"2023-01-04T19:19:44.073359Z","shell.execute_reply":"2023-01-04T19:19:44.086817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_batchs=50\nsub_lg = pd.Series()\nfor path in glob.glob(\"test_*.ftr\"):\n    print(path)\n    test = get_sample(path)\n    samples_df = get_batchs(test,batch_size=n_batchs, keep_features=features)\n    sample_sub_lg = pd.concat(map(partial(predict, model=lgb), tqdm_notebook(samples_df)))\n    sub_lg = pd.concat([sub_lg, sample_sub_lg], ignore_index=True)\n    del test\n    gc.collect()\n    \nprint(len(sub_lg))\n\nsub_lg.to_frame().to_csv('submission_lgb_with_diff.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:19:44.596934Z","iopub.execute_input":"2023-01-04T19:19:44.597409Z","iopub.status.idle":"2023-01-04T19:19:47.623075Z","shell.execute_reply.started":"2023-01-04T19:19:44.597366Z","shell.execute_reply":"2023-01-04T19:19:47.620857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loaded_model = joblib.load('/kaggle/working/lgb_model_with_diff.sav')","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:53:11.359064Z","iopub.execute_input":"2023-01-04T19:53:11.359596Z","iopub.status.idle":"2023-01-04T19:53:13.794669Z","shell.execute_reply.started":"2023-01-04T19:53:11.359549Z","shell.execute_reply":"2023-01-04T19:53:13.792721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_pred = loaded_model.predict()","metadata":{"execution":{"iopub.status.busy":"2023-01-04T19:54:22.500788Z","iopub.execute_input":"2023-01-04T19:54:22.501237Z","iopub.status.idle":"2023-01-04T19:54:22.525716Z","shell.execute_reply.started":"2023-01-04T19:54:22.5012Z","shell.execute_reply":"2023-01-04T19:54:22.523724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_dataset = pd.read_csv('/kaggle/input/amex-default-prediction/sample_submission.csv')\noutput = pd.DataFrame({'customer_ID': sample_dataset.customer_ID, 'prediction': predictions})\noutput.to_csv('submission1.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}