{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport sklearn\nimport datetime\nimport gc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\nfrom scipy.stats import boxcox\nfrom scipy import stats\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"742fdea09225088c181106188fa8ce402a6add4b"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntrain = reduce_mem_usage(train)\ntest = pd.read_csv(\"../input/test.csv\")\ntest = reduce_mem_usage(test)\nprint(\"The deimesnsions for test are\", test.shape)\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90259f79e9ffa32c4d8583fd80fe67f3ae8efa05"},"cell_type":"code","source":"historical_transactions = pd.read_csv(\"../input/historical_transactions.csv\")\nhistorical_transactions = reduce_mem_usage(historical_transactions)\nhistorical_transactions = historical_transactions.sample(frac= 0.2, replace=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68e1356bc904c7611355c1bb0564310ff7eddf88"},"cell_type":"code","source":"historical_transactions['category_3'] = historical_transactions['category_3'].fillna(\n                                            historical_transactions['category_3'].mode()[0])\nhistorical_transactions['category_2'] = historical_transactions['category_2'].fillna(\n                                            historical_transactions['category_2'].mode()[0])\nhistorical_transactions['merchant_id'] = historical_transactions['merchant_id'].fillna(\n                                            historical_transactions['merchant_id'].mode()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49fe6cc4f38d5e8974a2346bf1569b34ec6fb0f8"},"cell_type":"code","source":"merchants = pd.read_csv(\"../input/merchants.csv\")\nmerchants = reduce_mem_usage(merchants)\nnew_merchant_transactions = pd.read_csv(\"../input/new_merchant_transactions.csv\")\nnew_merchant_transactions = reduce_mem_usage(new_merchant_transactions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2225f9a4e5741764e16c4fd520ce8d2304552359"},"cell_type":"code","source":"new_merchant_transactions['category_3'] = new_merchant_transactions['category_3'].fillna(\n                                            new_merchant_transactions['category_3'].mode()[0])\nnew_merchant_transactions['category_2'] = new_merchant_transactions['category_2'].fillna(\n                                            new_merchant_transactions['category_2'].mode()[0])\nnew_merchant_transactions['merchant_id'] = new_merchant_transactions['merchant_id'].fillna(\n                                            new_merchant_transactions['merchant_id'].mode()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30644ea29906a29d8cff0097bf0cf664369410a2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be5ab2d01706bb28844c4eb834e211bff7025d69"},"cell_type":"code","source":"from sympy import log\nplt.hist(historical_transactions['month_lag'], range=[-15, 0.025], align='mid')\ntrain1 =  historical_transactions[['purchase_amount', 'month_lag', 'installments']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf8b98f28d9532ae339aecaa675c26e04a9e20b9"},"cell_type":"code","source":"#['purchase_amount = **0.5', 'month_lag', 'installments']\ntrain1['purchase_amount'] = (train1['purchase_amount'])**0.5\nprint((train1['purchase_amount']).corr(train['target']))\nprint((historical_transactions['purchase_amount']).corr(train['target']))\n#plt.hist(train1['purchase_amount'], range=[0, 1], facecolor='gray', align='mid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6cd040c27d01f912c9f1a8f71570249f1d4e2fb"},"cell_type":"code","source":"for df in [historical_transactions,new_merchant_transactions]:\n    df['purchase_date'] = pd.to_datetime(df['purchase_date'])\n    df['year'] = df['purchase_date'].dt.year\n    df['weekofyear'] = df['purchase_date'].dt.weekofyear\n    df['month'] = df['purchase_date'].dt.month\n    df['dayofweek'] = df['purchase_date'].dt.dayofweek\n    df['weekend'] = (df.purchase_date.dt.weekday >=5).astype(int)\n    df['hour'] = df['purchase_date'].dt.hour\n    df['authorized_flag'] = df['authorized_flag'].map({'Y':1, 'N':0})\n    df['category_1'] = df['category_1'].map({'Y':1, 'N':0})\n    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n    df['month_diff'] += df['month_lag']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82d9a08c326b0cf48441378c142e711d2502b919"},"cell_type":"code","source":"def create_features(df1, df2, column1,variables, statistics):\n    start = 0\n    mydict_names= {\"card_id\":\"card_id\"}\n    mydict_stats = {}\n    if len(variables):\n        for i in statistics:\n            count = 1\n            mynames = list(df1)\n            if not variables[0] + '_' + i in mynames:\n                mydict_names[variables[0]] = variables[0] + '_' + i\n            else:\n                mydict_names[variables[0]] = variables[0] + '_' + i + str(count)\n            mydict_stats[variables[0]] = i\n            if i != \"mode\":\n                df3 = (df2.groupby('card_id', as_index=False).agg(mydict_stats).rename\n                       (columns=mydict_names))\n            else:\n                df3 = (df2.groupby('card_id',as_index=False).agg(\n                       lambda x: stats.mode(x)[0][0]).rename(columns=mydict_names))\n            df1 = pd.merge(df1, df3, on='card_id', how='left')\n        del variables[0]\n        return create_features(df1, df2, column1,variables, statistics)\n    else:\n        print(list(df1))\n        del statistics\n        return df1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9679c34ac89029b61c43e19a9fcae8a41038df8"},"cell_type":"code","source":"historical_transactions['purchase_amount'] = (historical_transactions['purchase_amount'])**0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03be5c0083397e4413e0c1b52732726aa47d1146"},"cell_type":"code","source":"variables = [\"purchase_amount\", \"month_lag\", \"installments\", 'month_diff']\nstatistics = [\"sum\", \"mean\", \"max\", \"min\", \"var\", \"median\"]\ntrain = create_features(train, historical_transactions, \"card_id\", variables, statistics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab169541991033c8fd5f27db8a4ce17cae2209f1"},"cell_type":"code","source":"variables = [\"purchase_amount\", \"month_lag\", \"installments\", 'month_diff']\nstatistics = [\"sum\", \"mean\", \"max\", \"min\", \"var\", \"median\"]\ntrain = create_features(train, new_merchant_transactions, \"card_id\", variables, statistics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bace257ee2432d3b3e429d7f6fae4649e842dcc0"},"cell_type":"code","source":"variables = [\"purchase_amount\", \"month_lag\", \"installments\", \"month_diff\"]\nstatistics = [\"sum\", \"mean\", \"max\", \"min\", \"var\", \"median\"]\ntest = create_features(test, historical_transactions, \"card_id\", variables, statistics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b557463cb92956eb45f467d1e67867ddab5d886"},"cell_type":"code","source":"variables = [\"purchase_amount\", \"month_lag\", \"installments\", \"month_diff\"]\nstatistics = [\"sum\", \"mean\", \"max\", \"min\", \"var\", \"median\"]\ntest = create_features(test, new_merchant_transactions, \"card_id\", variables, statistics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7bf9fa081bfa0bc1e81989e833ec14e08cb2e99"},"cell_type":"code","source":"def drop_columns(df, columns):\n    try:\n        for i in columns:\n            df.drop(str(i), axis=1, inplace=True)\n    except KeyError:\n        print(\"column named \", i, \" is missing in the data frame\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b27fed023bf685c32464739fa42da0a93defeb67"},"cell_type":"code","source":"variables = [\"purchase_date\"]\nstatistics = [\"max\", \"min\"]\ntrain = create_features(train, historical_transactions, \"card_id\", variables, statistics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ff063b0170d9d6417eba1e820f9aecde1f4e824"},"cell_type":"code","source":"variables = [\"purchase_date\"]\nstatistics = [\"max\", \"min\"]\ntest = create_features(test, historical_transactions, \"card_id\", variables, statistics)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"679a9f3a36e6caac21e9f90c5f6955c6cf18bbe2"},"cell_type":"code","source":"variables = [\"subsector_id\", \"city_id\", \"state_id\"]\ndef recode_variables(df, variables):\n    for var in variables:\n        df[var] = pd.factorize(df[var])[0] + 1\n        #df = pd.get_dummies(df, columns=[str(var)])\nrecode_variables(historical_transactions, variables)\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed6fb2976feee9bc5084fcd23c19d32477ca2abc"},"cell_type":"code","source":"def convert_dates(df, converts):\n    for i in converts:\n        df[i] = pd.to_datetime(df[i])\nconvert_dates(train, [\"first_active_month\"])\nconvert_dates(test, [\"first_active_month\"])\nconvert_dates(historical_transactions, [\"purchase_date\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ae3e28f711b0952c398e5bd09373ff180e92cad"},"cell_type":"code","source":"holidays = [\"1 Jan\", \"22 Feb\",\"23 Feb\",\"24 Feb\",\"25 Feb\",\"26 Feb\",\"20 Mar\",\"10 Apr\",\"12 Apr\",\n            \"21 Apr\",\"1 May\",\"11 Jun\",\"12 Jun\",\"20 Jun\",\"9 Aug\",\"7 Sep\",\"22 Sep\",\"12 Oct\",\n            \"15 Oct\",\"28 Oct\",\"2 Nov\",\"15 Nov\",\"20 Nov\",\"21 Dec\",\"24 Dec\",\"25 Dec\",\"31 Dec\"]\nholidays = [datetime.datetime.strptime(i,'%d %b') for i in holidays]\ndef create_date_features(df, columns, holidays=[]):\n    df[column + \"month\"] = df[column].dt.month\n    df[column + \"purchase_time\"]=pd.cut(df[column].dt.hour,[0,6,12,18,24],labels=['Night','Morning','Afternoon','Evening'])\n    df[column + \"week_day\"] = df[column].dt.day_name() #  Not important feature\n    print(\"done\")   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70fbefe8c6866e044dad71d603691668793e77cc"},"cell_type":"code","source":"convert_dates(train, [\"purchase_date_min\", \"purchase_date_max\"])\nconvert_dates(test, [\"purchase_date_min\", \"purchase_date_max\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9393cca5740f0010f5c400301c47b02edf934346"},"cell_type":"code","source":"def convert_deltas(df, converts):\n    for i in converts:\n        df[i] =  pd.to_datetime(df[i], format='%Y%d%b:%H:%M:%S.%f')\nconvert_deltas(train, [\"purchase_date_min\", \"purchase_date_max\"])\nconvert_deltas(test, [\"purchase_date_min\", \"purchase_date_max\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78d95d53f10c534962c65c726847da1d8b14a368"},"cell_type":"code","source":"train['max_date_diff'] = (train['purchase_date_max'] - train['first_active_month']).dt.days\ntrain['min_date_diff'] = (train['purchase_date_min'] - train['first_active_month']).dt.days\ntest['max_date_diff'] = (test['purchase_date_max'] - test['first_active_month']).dt.days\ntest['min_date_diff'] = (test['purchase_date_min'] - test['first_active_month']).dt.days","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eee0f0b6ecc268250ef715e2f40819c3ac00e0c3"},"cell_type":"code","source":"train['purchase_date_max'] = train['purchase_date_max'].astype(int)\ntrain['purchase_date_min'] = train['purchase_date_min'].astype(int)\ntest['purchase_date_max'] = test['purchase_date_max'].astype(int)\ntest['purchase_date_min'] = test['purchase_date_min'].astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e18b2027020f8c48dbb8eff67f1f7023981677f7"},"cell_type":"code","source":"avoid = ['month_lag_sum', 'month_lag_mean', 'month_lag_max', 'month_lag_min', 'month_lag_var',\n         'month_lag_median', 'month_diff_sum', 'month_diff_mean', 'month_diff_max', 'month_diff_min',\n         'month_diff_var', 'month_diff_median', 'month_lag_sum_1', 'month_lag_mean_1', \n         'month_lag_max_1', 'month_lag_min_1', 'month_lag_var_1','month_lag_median_1',\n         'month_diff_sum_1', 'month_diff_mean_1', 'month_diff_max_1', 'month_diff_min_1',\n         'month_diff_var_1', 'month_diff_median_1'     \n]\ndef create_categories(df,lengths = []):\n    for i in list(df):\n        if len(df[str(i)].value_counts()) < 100 and i not in avoid:    \n            lengths.append(i)\n    for col in lengths:\n        df.sort_values(by=[col])\n        df[col] = df[col].astype('category')\n    print(\"done\")   \n#create_categories(train)\n#create_categories(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c789742661a4bb66684721322bcc3d3f0aa76042"},"cell_type":"code","source":"def drop_correlated_feat(df):\n    corr_matrix = df.corr().abs()\n\n    # Select upper triangle of correlation matrix\n    upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))\n\n    # Find index of feature columns with correlation greater than 0.95\n    to_drop = [column for column in upper.columns if any(upper[column] > 0.95)]\n    return df\ntrain = drop_correlated_feat(train)\ntest = drop_correlated_feat(test)\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2a536551ada322d0959023c977f34558ca17bf0"},"cell_type":"code","source":"columns_to_drop = ['card_id', 'first_active_month']\ndrop_columns(train, columns_to_drop)\ndrop_columns(test, columns_to_drop)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e798709691caa3dae9cdd6082f33e745ee3be53"},"cell_type":"code","source":"target = train[\"target\"]\ntrain.drop('target',  axis=1, inplace=True)\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"423885657a5376fd12b2b39833df82c480751758"},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train,\n                                                    target,\n                                                    test_size=0.2,\n                                                    shuffle = True)\n#del train\n#del target\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56e8ff9a17bd24146eed15836a1f549ef1528d38"},"cell_type":"code","source":"import lightgbm as lgb\ncats = list(X_train.select_dtypes(['category']))\nd_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cats, free_raw_data=False)\nd_test = lgb.Dataset(X_test, label=y_test, categorical_feature=cats, free_raw_data=False)\nparams = {'num_leaves': round(0.6*(pow(2,9))),\n         'min_data_in_leaf': 149, \n         'objective':'regression',\n         'max_depth': 9,\n         'learning_rate': 0.005,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.7522,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.7083 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 0.2634,\n         \"random_state\": 1033,\n         \"seed\": 42,\n         \"verbosity\": -1}\nmodel = lgb.train(params, d_train, 1000, valid_sets=[d_test], early_stopping_rounds=100, verbose_eval=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5344c1b15a31c4c2362d28b156e0edeef7841fa1"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,10))\nlgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)\nax.grid(False)\nplt.title(\"LightGBM - Feature Importance\", fontsize=15)\nplt.show()\n[\"purchase_amount_max1\", \"month_diff_mean\", \"month_lag_var\", \"purchase_date_max\", \"min_date_diff\",\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cd2790d162d6b8d591ac1a3f8a4cbdf0000e84c"},"cell_type":"code","source":"y_pred = model.predict(test, num_iteration=model.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"218a7611f594d934e4fbc0ebcbb0408735024d70"},"cell_type":"code","source":"submission = pd.read_csv(\"../input/test.csv\")\nsubmission = pd.DataFrame({\"card_id\":submission[\"card_id\"].values})\nsubmission[\"target\"] = y_pred\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}