{"cells":[{"metadata":{"_uuid":"f21a730ab133ba2817dff7278be1e174d61c9160"},"cell_type":"markdown","source":"# ELo Merchant Category Recommendation"},{"metadata":{"_uuid":"bea6817da1464dc00fc669e897da34953ade2740"},"cell_type":"markdown","source":"## File descriptions\n-  train.csv - the training set\n-  test.csv - the test set\n-  sample_submission.csv - a sample submission file in the correct format - contains all card_ids you are expected to predict for.\n-  historical_transactions.csv - up to 3 months' worth of historical transactions for each card_id\n-  merchants.csv - additional information about all merchants / merchant_ids in the dataset.\n-  new_merchant_transactions.csv - two months' worth of data for each card_id containing ALL purchases that card_id made at merchant_ids that were not visited in the historical data.\n"},{"metadata":{"_uuid":"46d712031202cebaf1431a4b53b7f8945647b409"},"cell_type":"markdown","source":"## What to predict?\nPredicting a loyalty score for each card_id represented in test.csv"},{"metadata":{"_uuid":"fa64ce60f0e4fa7d7dc4edd058038620957e4354"},"cell_type":"markdown","source":"## 1. Setting up the Environment and Loading Data"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# Import the necessary libraries\nimport numpy as np\nimport pandas as pd\nimport nltk\nimport matplotlib.pyplot as plt\nimport datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nimport time\nimport sys\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0416b8ef147c08bb00b0c767f67d151dfb3015e","trusted":true},"cell_type":"code","source":"#Reduce the memory usage\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Loading data\nnew_transactions = reduce_mem_usage(pd.read_csv('../input/new_merchant_transactions.csv',parse_dates=['purchase_date']))\nold_transactions = reduce_mem_usage(pd.read_csv('../input/historical_transactions.csv',parse_dates=['purchase_date']))\ntrain = reduce_mem_usage(pd.read_csv('../input/train.csv',parse_dates=[\"first_active_month\"]))\ntest = reduce_mem_usage(pd.read_csv('../input/test.csv',parse_dates=[\"first_active_month\"]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef2de6fabd32153c61398c38165d552daf7313dd","trusted":true},"cell_type":"code","source":"new_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c65f40218e43de492c69138165f9b15e0facd8e4","trusted":true},"cell_type":"code","source":"old_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3cbbb8b362c97b3499285926fb5d800c41454b6","trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"88290d5665ddc5bc04d0555641e88b91be2a1b54","scrolled":true,"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c42dbae5ec6555ced264b359c7f0ec312b5f1cbc","trusted":true},"cell_type":"code","source":"#Finding columns with null values\nold_transactions.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c10b8dec0cc4482ff1fb944fcf417a0917e4ff1d","trusted":true},"cell_type":"code","source":"#Replacing null values with the most frequent values in the column.\nold_transactions['category_2'].fillna(1.0,inplace = True)\nold_transactions['category_3'].fillna('B',inplace = True)\n\nnew_transactions['category_2'].fillna(1.0,inplace = True)\nnew_transactions['category_3'].fillna('B',inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffe182566f6a5969a6813883ec2d4466a4d78160","trusted":true},"cell_type":"code","source":"#Changing Categorical Columns to Boolean\nold_transactions['authorized_flag'] = old_transactions['authorized_flag'].map({'Y':1, 'N':0}).astype(bool)\nold_transactions['category_1'] = old_transactions['category_1'].map({'Y':1, 'N':0}).astype(bool)\nold_transactions['category_3'] = old_transactions['category_3'].map({'A':0, 'B':1, 'C':2}).astype('category')\n\nnew_transactions['authorized_flag'] = new_transactions['authorized_flag'].map({'Y':1, 'N':0}).astype(bool)\nnew_transactions['category_1'] = new_transactions['category_1'].map({'Y':1, 'N':0}).astype(bool)\nnew_transactions['category_3'] = new_transactions['category_3'].map({'A':0, 'B':1, 'C':2}).astype('category')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53c9639fff535c20a6b754018b7019394072c7d9"},"cell_type":"code","source":"old_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d0b060481e83f29a66b1122e3fc8d24163f9756"},"cell_type":"code","source":"new_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc8d9e9b6a5b7b2a215c765dc21d230bfcb16498","trusted":true},"cell_type":"code","source":"#Handling the Outliers\ntrain['outliers'] = 0\ntrain.loc[train['target'] < -30, 'outliers'] = 1\noutls = train['outliers'].value_counts()\nprint(\"Outliers:\\n{}\".format(outls))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ee4a69dfdc34b2a3c2011c9eedc1c64da24b0437"},"cell_type":"markdown","source":"## 2. Feature Engineering"},{"metadata":{"_uuid":"47b09faf5c5ca20ebb940884b19a0e45c4a395fe","trusted":true},"cell_type":"code","source":"#define function to name columns for aggregation\ndef create_new_columns(name,aggs):\n    return [name + '_' + k + '_' + agg for k in aggs.keys() for agg in aggs[k]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91d0d779e436aeef1094a1689c67c646b7dca787"},"cell_type":"code","source":"#Adding a few features using purchase_amount, purchase_date\naggs={}\naggs['purchase_amount'] = ['sum','max','min','mean','var','std']\naggs['purchase_date'] = ['max','min', 'nunique']\naggs['card_id'] = ['size', 'count']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27f3faac773b7070357ee315e2134ca60827f1db","trusted":true},"cell_type":"code","source":"for col in ['category_2','category_3']:\n    old_transactions[col+'_mean'] = old_transactions.groupby([col])['purchase_amount'].transform('mean')\n    old_transactions[col+'_min'] = old_transactions.groupby([col])['purchase_amount'].transform('min')\n    old_transactions[col+'_max'] = old_transactions.groupby([col])['purchase_amount'].transform('max')\n    old_transactions[col+'_sum'] = old_transactions.groupby([col])['purchase_amount'].transform('sum')\n    old_transactions[col+'_std'] = old_transactions.groupby([col])['purchase_amount'].transform('std')\n    aggs[col+'_mean'] = ['mean']    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"726a9408f9adfa224cf4039132fd524de1b178ba","trusted":true},"cell_type":"code","source":"new_columns = create_new_columns('old',aggs)\nhistorical_trans_group_df = old_transactions.groupby('card_id').agg(aggs)\nhistorical_trans_group_df.columns = new_columns\nhistorical_trans_group_df.reset_index(drop=False,inplace=True)\nhistorical_trans_group_df['old_purchase_date_diff'] = (historical_trans_group_df['old_purchase_date_max'] - historical_trans_group_df['old_purchase_date_min']).dt.days\nhistorical_trans_group_df['old_purchase_date_average'] = historical_trans_group_df['old_purchase_date_diff']/historical_trans_group_df['old_card_id_size']\nhistorical_trans_group_df['old_purchase_date_uptonow'] = (datetime.datetime.today() - historical_trans_group_df['old_purchase_date_max']).dt.days\nhistorical_trans_group_df['old_purchase_date_uptomin'] = (datetime.datetime.today() - historical_trans_group_df['old_purchase_date_min']).dt.days\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c416d8adde1b65daabead37095f59c9c9c97296"},"cell_type":"code","source":"aggs['purchase_amount'] = ['sum','max','min','mean','var','std']\naggs['purchase_date'] = ['max','min', 'nunique']\naggs['card_id'] = ['size', 'count']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1770661a85dd81d6f1fbbfb91e8758e538a95c77","trusted":true},"cell_type":"code","source":"for col in ['category_2','category_3']:\n    new_transactions[col+'_mean'] = new_transactions.groupby([col])['purchase_amount'].transform('mean')\n    new_transactions[col+'_min'] = new_transactions.groupby([col])['purchase_amount'].transform('min')\n    new_transactions[col+'_max'] = new_transactions.groupby([col])['purchase_amount'].transform('max')\n    new_transactions[col+'_sum'] = new_transactions.groupby([col])['purchase_amount'].transform('sum')\n    new_transactions[col+'_std'] = new_transactions.groupby([col])['purchase_amount'].transform('std')\n    aggs[col+'_mean'] = ['mean']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6853def23056c4e0e4befa3d77c988b62a85b9ce","trusted":true},"cell_type":"code","source":"new_columns = create_new_columns('new',aggs)\nnew_merchant_trans_group_df = new_transactions.groupby('card_id').agg(aggs)\nnew_merchant_trans_group_df.columns = new_columns\nnew_merchant_trans_group_df.reset_index(drop=False,inplace=True)\nnew_merchant_trans_group_df['new_purchase_date_diff'] = (new_merchant_trans_group_df['new_purchase_date_max'] - new_merchant_trans_group_df['new_purchase_date_min']).dt.days\nnew_merchant_trans_group_df['new_purchase_date_average'] = new_merchant_trans_group_df['new_purchase_date_diff']/new_merchant_trans_group_df['new_card_id_size']\nnew_merchant_trans_group_df['new_purchase_date_uptonow'] = (datetime.datetime.today() - new_merchant_trans_group_df['new_purchase_date_max']).dt.days\nnew_merchant_trans_group_df['new_purchase_date_uptomin'] = (datetime.datetime.today() - new_merchant_trans_group_df['new_purchase_date_min']).dt.days\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b14a75e91e94f55bde8f71ad242e43b8e8667520","trusted":true},"cell_type":"code","source":"#Creating Dummy Variables \nold_transactions = pd.get_dummies(old_transactions, columns=['category_2', 'category_3'])\nnew_transactions = pd.get_dummies(new_transactions, columns=['category_2', 'category_3'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fa0adfd8169b97ea1da062f49be50e8296704e9","trusted":true},"cell_type":"code","source":"#Adding a few features created using purchase_date.\nfor df in [old_transactions, new_transactions]:\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['dayofyear'] = df['purchase_date'].dt.dayofyear\n    df['quarter'] = df['purchase_date'].dt.quarter\n    df['is_month_start'] = df['purchase_date'].dt.is_month_start\n    df['purchase_month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93e6136bca50ce0207f67003c5b1a3f2e8675e68"},"cell_type":"code","source":"#Adding a few features created using first_active_month.\nfor df in [train, test]:\n    df['elapsed_time'] = (datetime.date(2018, 2, 1) - df['first_active_month'].dt.date).dt.days\n    df['dayofweek'] = df['first_active_month'].dt.dayofweek\n    df['weekofyear'] = df['first_active_month'].dt.weekofyear\n    df['dayofyear'] = df['first_active_month'].dt.dayofyear\n    df['quarter'] = df['first_active_month'].dt.quarter\n    df['is_month_start'] = df['first_active_month'].dt.is_month_start\n    df['month'] = df['first_active_month'].dt.month\n    df['weekend'] = (df.first_active_month.dt.weekday >=5).astype(int)\n    df['hour'] = df['first_active_month'].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"04f3a1c8dd169ab17660741b6abd5645d595c826","trusted":true},"cell_type":"code","source":"old_transactions.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c0d6fe483c37eca6ce71d61e3ba33c8ea2359cb","trusted":true},"cell_type":"code","source":"#Adding Aggregate Columns.\ndef aggregate_transactions(history):\n    \n    history.loc[:, 'purchase_date'] = pd.DatetimeIndex(history['purchase_date']).astype(np.int64) * 1e-9\n    agg_func = {\n    'year':['sum','mean','nunique'],\n    'weekend':['sum','mean','nunique'],\n    'dayofweek':['min','max','mean','nunique'],\n    'weekofyear':['min','max','mean','nunique'],\n    'hour':['min','max','mean','nunique'],\n    'category_1': ['sum', 'mean'],\n    'category_2_1.0': ['mean'],\n    'category_2_2.0': ['mean'],\n    'category_2_3.0': ['mean'],\n    'category_2_4.0': ['mean'],\n    'category_2_5.0': ['mean'],\n    'category_3_0': ['mean'],\n    'category_3_1': ['mean'],\n    'category_3_2': ['mean'],\n    'merchant_id': ['nunique'],\n    'merchant_category_id': ['nunique'],\n    'state_id': ['nunique'],\n    'city_id': ['nunique'],\n    'subsector_id': ['nunique'],\n    'installments': ['sum', 'mean', 'max', 'min', 'std'],\n    'purchase_month': ['mean', 'max', 'min', 'std'],\n    'month_lag': ['mean', 'max', 'min', 'std'],\n    'month_diff': ['mean']\n    }\n    \n    agg_history = history.groupby(['card_id']).agg(agg_func)\n    agg_history.columns = ['_'.join(col).strip() for col in agg_history.columns.values]\n    agg_history.reset_index(inplace=True)\n    \n    df = (history.groupby('card_id').size().reset_index(name='transactions_count'))\n    \n    agg_history = pd.merge(df, agg_history, on='card_id', how='left')\n    \n    return agg_history","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c0d6fe483c37eca6ce71d61e3ba33c8ea2359cb","trusted":true},"cell_type":"code","source":"def aggregate_per_month(history):\n    grouped = history.groupby(['card_id', 'month_lag'])\n\n    agg_func = {\n            'purchase_amount': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n            'installments': ['count', 'sum', 'mean', 'min', 'max', 'std'],\n            }\n\n    intermediate_group = grouped.agg(agg_func)\n    intermediate_group.columns = ['_'.join(col).strip() for col in intermediate_group.columns.values]\n    intermediate_group.reset_index(inplace=True)\n\n    final_group = intermediate_group.groupby('card_id').agg(['mean', 'std'])\n    final_group.columns = ['_'.join(col).strip() for col in final_group.columns.values]\n    final_group.reset_index(inplace=True)\n    \n    return final_group","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a8e46e6ac86f563d8436923e9c8fe4a701c316f","trusted":true},"cell_type":"code","source":"old = aggregate_transactions(old_transactions)\nold.columns = ['old_' + c if c != 'card_id' else c for c in old.columns]\nold[:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea616ab6bce9676c110c948f8e790f8dae5abbe5","trusted":true},"cell_type":"code","source":"new = aggregate_transactions(new_transactions)\nnew.columns = ['new_' + c if c != 'card_id' else c for c in new.columns]\nnew[:5]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b219febb8e6ae2cd6c121bd86218ec2b85db0ee2","trusted":true},"cell_type":"code","source":"final_group_old =  aggregate_per_month(old_transactions) \nfinal_group_old[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c57484aaac30bb30d6137a2ab438819e3e1fc2a"},"cell_type":"code","source":"final_group_new =  aggregate_per_month(new_transactions) \nfinal_group_new[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e0afee016fc7093ad2bf88ec14a5b8ae5389492"},"cell_type":"code","source":"#merge all created dataframes with train and test\nprint(\"...\")\ntrain = train.merge(historical_trans_group_df,on='card_id',how='left')\nprint(\"...\")\ntest = test.merge(historical_trans_group_df,on='card_id',how='left')\n\nprint(\"...\")\ntrain = train.merge(new_merchant_trans_group_df,on='card_id',how='left')\nprint(\"...\")\ntest = test.merge(new_merchant_trans_group_df,on='card_id',how='left')\n\nprint(\"...\")\ntrain = train.merge(old, on='card_id', how='left')\nprint(\"...\")\ntest = test.merge(old, on='card_id', how='left')\n\nprint(\"...\")\ntrain = train.merge(new, on='card_id', how='left')\nprint(\"...\")\ntest = test.merge(new, on='card_id', how='left')\n\nprint(\"...\")\ntrain = train.merge(final_group_old, on='card_id', how='left')\nprint(\"...\")\ntest = test.merge(final_group_old, on='card_id', how='left')\n\nprint(\"...\")\ntrain = train.merge(final_group_new, on='card_id', how='left')\nprint(\"...\")\ntest = test.merge(final_group_new, on='card_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a00cce0362e672d2e938b009105459096a44dfe"},"cell_type":"code","source":"#Adding a few more features\nfor df in [train, test]:\n    for f in ['old_purchase_date_max','old_purchase_date_min','new_purchase_date_max','new_purchase_date_min']:\n        df[f] = pd.to_datetime(df[f])\n    df['old_first_buy'] = (df['old_purchase_date_min'] - df['first_active_month']).dt.days\n    df['old_last_buy'] = (df['old_purchase_date_max'] - df['first_active_month']).dt.days\n    df['new_first_buy'] = (df['new_purchase_date_min'] - df['first_active_month']).dt.days\n    df['new_last_buy'] = (df['new_purchase_date_max'] - df['first_active_month']).dt.days\n    for f in ['old_purchase_date_max','old_purchase_date_min','new_purchase_date_max','new_purchase_date_min']:\n        df[f] = df[f].astype(np.int64) * 1e-9\n    df['card_id_total'] = df['new_card_id_size']+df['old_card_id_size']\n    df['card_id_cnt_total'] = df['new_card_id_count']+df['old_card_id_count']\n    df['purchase_amount_total'] = df['new_purchase_amount_sum']+df['old_purchase_amount_sum']\n    df['purchase_amount_mean'] = df['new_purchase_amount_mean']+df['old_purchase_amount_mean']\n    df['purchase_amount_max'] = df['new_purchase_amount_max']+df['old_purchase_amount_max']\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e11218e58cc49fb0a77c780f8cedcd3d7583c1a1","trusted":true},"cell_type":"code","source":"#Storing the new train and test externally\n#test.to_csv('test.csv')\n#train.to_csv('train.csv')\ntarget = train['target']\ndel train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e590c495af89e1c16f2ed505a5ede63ba8c0dbb6"},"cell_type":"markdown","source":"## Training"},{"metadata":{"_uuid":"879b6fc7c2bd8e98f53b6deb4ec1ceab144b6f05","trusted":true},"cell_type":"code","source":"features = [c for c in train.columns if c not in ['card_id','target', 'first_active_month','outliers']]\ncat_features = ['feature_2', 'feature_3']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e6f9dde5e489c0c3e667260031934f02f51b3fe","trusted":true},"cell_type":"code","source":"param = {'num_leaves': 111,\n         'min_data_in_leaf': 149, \n         'objective':'regression',\n         'max_depth': 9,\n         'learning_rate': 0.005,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.7522,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.7083 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.2634,\n         \"random_state\": 133,\n         \"verbosity\": -1}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af59a681abb7f4117649d043d3ae93caeb8440c7","trusted":true},"cell_type":"code","source":"#Applying KFolds\nfolds = KFold(n_splits=5, shuffle=True, random_state=15)\noof = np.zeros(len(train))\npredictions = np.zeros(len(test))\nstart = time.time()\nfeature_importance_df = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n    print(\"fold {}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][features],label=target.iloc[trn_idx])\n    val_data = lgb.Dataset(train.iloc[val_idx][features],label=target.iloc[val_idx])\n\n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 200)\n    oof[val_idx] = clf.predict(train.iloc[val_idx][features], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(test[features], num_iteration=clf.best_iteration) / folds.n_splits\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ce09019d81e4793a6c0eb2c5e4ad28191f415b1","trusted":true,"scrolled":true},"cell_type":"code","source":"#Applying RepeatedKFolds\nfrom sklearn.model_selection import RepeatedKFold\nfolds = RepeatedKFold(n_splits=5, n_repeats=2, random_state=4950)\noof_2 = np.zeros(len(train))\npredictions_2 = np.zeros(len(test))\nfeature_importance_df_2 = pd.DataFrame()\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds.split(train.values, target.values)):\n    print(\"fold {}\".format(fold_))\n    trn_data = lgb.Dataset(train.iloc[trn_idx][features], label=target.iloc[trn_idx], categorical_feature=cat_features)\n    val_data = lgb.Dataset(train.iloc[val_idx][features], label=target.iloc[val_idx], categorical_feature=cat_features)\n\n    num_round = 10000\n    clf_r = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=-1, early_stopping_rounds = 200)\n    oof_2[val_idx] = clf_r.predict(train.iloc[val_idx][features], num_iteration=clf_r.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"Feature\"] = features\n    fold_importance_df[\"importance\"] = clf_r.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df_2 = pd.concat([feature_importance_df_2, fold_importance_df], axis=0)\n    \n    predictions_2 += clf_r.predict(test[features], num_iteration=clf_r.best_iteration) / (5 * 2)\n\nprint(\"CV score: {:<8.5f}\".format(mean_squared_error(oof_2, target)**0.5))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3619e37dc95d68e1b5d755fd125c432746fa52cf","trusted":true},"cell_type":"code","source":"#Applying BayesianRidge\nfrom sklearn.linear_model import BayesianRidge\n\ntrain_stack = np.vstack([oof,oof_2]).transpose()\ntest_stack = np.vstack([predictions, predictions_2]).transpose()\n\nfolds_stack = RepeatedKFold(n_splits=5, n_repeats=1, random_state=4590)\noof_stack = np.zeros(train_stack.shape[0])\npredictions_3 = np.zeros(test_stack.shape[0])\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds_stack.split(train_stack,target)):\n    print(\"fold {}\".format(fold_))\n    trn_data, trn_y = train_stack[trn_idx], target.iloc[trn_idx].values\n    val_data, val_y = train_stack[val_idx], target.iloc[val_idx].values\n    \n    clf_3 = BayesianRidge()\n    clf_3.fit(trn_data, trn_y)\n    \n    oof_stack[val_idx] = clf_3.predict(val_data)\n    predictions_3 += clf_3.predict(test_stack) / 5\n    \nnp.sqrt(mean_squared_error(target.values, oof_stack))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f95cbb4c2b4815ab55e1dc68d7acef0557d17cb"},"cell_type":"markdown","source":"## Submission"},{"metadata":{"_uuid":"2e628562702b8ed308725c89fff8bece327cc029","trusted":true},"cell_type":"code","source":"sub_df = pd.DataFrame({\"card_id\":test[\"card_id\"].values})\nsub_df[\"target\"] = predictions_3\nsub_df.to_csv(\"submit_tour_de_force.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d3ff8642e3af0e22f8442a4e6a68dc1fc53348f","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}