{"cells":[{"metadata":{"_uuid":"1fd9dd7b634898c46c633ebf92d4c29e96885275"},"cell_type":"markdown","source":"## Importing packages & data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import re\nimport itertools\nimport string\nimport gc\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport lightgbm as lgb\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\n\n%matplotlib inline\nplt.rcParams[\"figure.figsize\"] = (15,6)","execution_count":60,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false,"collapsed":true},"cell_type":"code","source":"# Importing training, test data\n# Setting item_id to index for easier handling of ids\ntrain_raw = pd.read_csv('../input/train.csv', index_col='item_id', parse_dates=['activation_date'])\nvalid_raw = pd.read_csv('../input/test.csv', index_col='item_id', parse_dates=['activation_date'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7dac16c30f343750b90cb3a6f795b4c7442e9191"},"cell_type":"code","source":"train_active_raw = pd.read_csv('../input/train_active.csv')\ntest_active_raw = pd.read_csv('../input/test_active.csv')","execution_count":46,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"da125a011a6ff3c1241e068887e9c56c42717094"},"cell_type":"code","source":"train_active_raw.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cf43b070f4de8778afd897a8d08ce82a92b66f7","collapsed":true},"cell_type":"code","source":"all_raw = train_raw.append(valid_raw, sort=True)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a3904e1484aa954b9a5a182995950d942f8b446"},"cell_type":"code","source":"all_fake = all_raw.sample(frac=0.01)","execution_count":42,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d919afde8bd38df78cd3188945239b05696daba"},"cell_type":"code","source":"all_samples = pd.concat([\n    train,\n    train_active,\n    test,\n    test_active\n]).reset_index(drop=True)\nall_samples.drop_duplicates(['item_id'], inplace=True)","execution_count":44,"outputs":[]},{"metadata":{"_uuid":"5291aab4b829e6d959877d46c8f16653748f8cd1"},"cell_type":"markdown","source":"## Data transformation"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"78dbe351ad8a75411b56a0f1b44592a698e00cd0"},"cell_type":"code","source":"TARGET_FEATURE = ['deal_probability']\nNUM_FEATURES = ['price', 'weekday', 'num_desc_punct']\nTEX_FEATURES = ['title', 'description']\nCAT_FEATURES = ['city', 'user_type', 'category_name', 'param_1', 'param_2', 'param_3', 'image_top_1']","execution_count":21,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c09a4ef60284ff2795333b0472be42ec253b9b59"},"cell_type":"code","source":"def remove_spec_chars(text):\n    try:\n        text = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', text)\n        text = ' '.join(text.split())\n        return text\n    except Exception as error:\n        print(error)\n        return 'error'","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0ab0ac0ad30fac83f2b93271d71d872933958a79"},"cell_type":"code","source":"def transform(df, cat_features, num_features, target_feature, tex_features):\n    # Filling null prices with median\n    df['price'] = df['price'].fillna(df.price.mean())\n    # Creating new location\n    df['city'] = df['city'].astype(str)\n    df['region'] = df['region'].astype(str)\n    df['city'] = df['region'] + '_' + df['city']\n    # Creating weekday feature\n    df['weekday'] = df['activation_date'].dt.weekday\n    # Cleaning up text features\n    for tex in tex_features:\n        df[tex] = df[tex].astype(str) \n        df[tex] = df[tex].astype(str).fillna('missing')\n        df[tex] = df[tex].str.lower() # Lowercase all text\n        df[tex] = df[tex].apply(lambda x: remove_spec_chars(x))\n        df[tex + '_num_words'] = df[tex].apply(lambda t: len(t.split())) # Count number of Words\n        df[tex + '_num_unique_words'] = df[tex].apply(lambda t: len(set(t.split()))) # Count unique words\n        df[tex + '_words_vs_unique'] = df[f'{tex}_num_unique_words'] / df[f'{tex}_num_words'] * 100 # Count unique words vs words ratio\n    df['num_desc_punct'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n    # Transforming categorical variables to numerical ones\n    le = LabelEncoder()\n    for col in CAT_FEATURES:\n        df[col] = df[col].astype('category')\n        df[col] = df[col].cat.codes\n    # Figuring out what columns to drop\n    new_tex = [''.join(t) for t in itertools.product(TEX_FEATURES, ['_num_words', '_num_unique_words', '_words_vs_unique'])]\n    to_out = set(df.columns) - set(cat_features) - set(num_features) - set(target_feature) - set(new_tex)\n    df = df.drop(to_out, axis=1) # dropping irrelevant features\n    return df","execution_count":31,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"be3806df181271e18e7b764a4750e1cfb6c529d3"},"cell_type":"code","source":"# Playing with fake data to test transform function\ndata = all_raw.sample(frac=0.01, replace=True).dropna().reset_index()","execution_count":25,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea32794373d2d85abc064d4c7dc3934e9a107bed"},"cell_type":"code","source":"all = transform(all_raw, CAT_FEATURES, NUM_FEATURES, TARGET_FEATURE, TEX_FEATURES)\n# Splitting training and validation set after transform\ntrain, valid = all[all.deal_probability.notna()], all[all.deal_probability.isnull()]\n# Creating testing set out of the training set\nX_train, y_train = train.drop(['deal_probability'], axis=1), train['deal_probability']\nX_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.33, random_state=42)\n# Dropping null probabilities for validation set\nX_valid = valid.drop(['deal_probability'], axis=1)\n# Generating codes for categorical features\nCAT_CODES = sorted([X_train.columns.get_loc(col) for col in CAT_FEATURES])","execution_count":32,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4374d49c315a60e3e3309709de19ea2cedb3f4a"},"cell_type":"code","source":"CAT_CODES","execution_count":33,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37aeaa54e0d60d906ea8465863275424629404d5"},"cell_type":"code","source":"X_train.head()","execution_count":34,"outputs":[]},{"metadata":{"_uuid":"4c35bfedc3ba53835b11aeafb2e4188964f701ba"},"cell_type":"markdown","source":"## Modeling"},{"metadata":{"trusted":true,"_uuid":"de5fd13ae6f94ef4be319ac814a1f59e6e802877"},"cell_type":"code","source":"lgb_train = lgb.Dataset(X_train, y_train)\nlgb_test = lgb.Dataset(X_test, y_test, reference=lgb_train)\n                       \nevals_result = {}  # to record eval results for plotting\n\nparams = {\n    # Num leaves\n    'num_leaves': 60,\n    # Increase accuracy\n    'learning_rate': 0.2,\n    'num_boost_round': 100,\n    # Reduce overfitting\n    'lambda_l2': 1.8,\n    # Bagging\n    \"feature_fraction\" : 0.6,\n    \"bagging_fraction\" : 0.8,\n    \"bagging_freq\" : 2,\n    # Algo settings\n    'metric': ('l2'),\n    'verbose_eval': 10,\n    'verbosity': -1\n}\n\n# Training gbm\ngbm = lgb.train(params,\n                lgb_train,\n                valid_sets=[lgb_train, lgb_test],\n                categorical_feature=CAT_CODES,\n                evals_result=evals_result,\n                verbose_eval=params['verbose_eval'])\n\nax = lgb.plot_metric(evals_result, metric='l2')\nplt.show()\nax = lgb.plot_importance(gbm, max_num_features=-1)\nplt.show()\n\ny_pred = gbm.predict(X_test, gbm.best_iteration)\nrmse = round(mean_squared_error(y_test, y_pred) ** 0.5, 5)\nprint(f'RMSE {rmse}')","execution_count":39,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ab328f352e93610e151788eb7af8af80808b106"},"cell_type":"code","source":"'''\nlgb_train = lgb.Dataset(X_train, y_train)\nlgb_test = lgb.Dataset(X_test, y_test, reference=lgb_train)\n                       \nevals_result = {}  # to record eval results for plotting\n\nparams = {\n    'num_leaves': 60,\n    'bagging_fraction': 0.5,\n    'bagging_freq': 1,\n    'min_data_in_leaf': 1000,\n    'learning_rate': 0.005,\n    'num_boost_round': 100,\n    'metric': ('l2'),\n    'verbose': 0,\n}\n    \nfor learning_rate in [0.1, 0.01, 0.005]:\n    for num_boost_round in [100, 500, 1000]:\n        # Setting hyper-parameters\n        params['learning_rate'] = learning_rate\n        # Training gbm\n        gbm = lgb.train(params,\n                        lgb_train,\n                        num_boost_round=1000,\n                        valid_sets=[lgb_train, lgb_test],\n                        categorical_feature=CAT_CODES,\n                        evals_result=evals_result,\n                        verbose_eval=0)\n        y_pred = gbm.predict(X_test, gbm.best_iteration)\n        rmse = round(mean_squared_error(y_test, y_pred) ** 0.5, 5)\n        train_test_diff = round(abs(gbm.best_score[\"training\"][\"l2\"]-gbm.best_score[\"valid_1\"][\"l2\"]), 5)\n        print(f'learning_rate {learning_rate} num_boost_round {num_boost_round} RMSE {rmse} train_test_diff {train_test_diff}')\n'''","execution_count":11,"outputs":[]},{"metadata":{"_uuid":"df46346b1a0dd4f01b521e361227f4a74cc3ffa1"},"cell_type":"markdown","source":"# Results"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"29ad1f3b3b7099b109e4643c344b14f35a152e11"},"cell_type":"code","source":"def plot(scores):\n    scores.set_index('parameters')\n    ax1 = scores.plot('parameters', 'training_error', 'line', xticks=scores.index)\n    scores.plot('parameters', 'testing_error', 'line', ax=ax1);\n    ax1.set_ylabel('Error')\n    ax1.legend(shadow=True)\n    ax2 = ax1.twinx()\n    scores.plot('parameters', 'kaggle', 'line', title='Training/Testing/Kaggle scores', style=['--'], colormap='summer', ax=ax2);\n    ax2.set_ylabel('Kaggle score')\n    ax2.legend(loc='lower left', shadow=True);","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f4f4aee4e4b1b2279019e20775e16624139f2f5"},"cell_type":"code","source":"scores = pd.DataFrame([\n    {'parameters': 'num_leaves=5', 'lmse': 0.2381, 'training_error': 0.056674, 'testing_error': 0.056648, 'kaggle': 0.244},\n    {'parameters': 'num_leaves=31', 'lmse': 0.2359, 'training_error': 0.054945, 'testing_error': 0.055673, 'kaggle': 0.2455},\n    {'parameters': 'bag_freq', 'lmse': 0.2359, 'training_error': 0.05466, 'testing_error': 0.05607, 'kaggle': 0.2419},\n    {'parameters': 'param_X', 'lmse': 0.2299, 'training_error': 0.05162, 'testing_error': 0.05285, 'kaggle': 0.2343},\n    {'parameters': 'min_data=1000', 'lmse': 0.2296, 'training_error': 0.051316, 'testing_error': 0.052732, 'kaggle': 0.2343},\n    {'parameters': 'rounds=500,lr=0.01', 'lmse': 0.22963, 'training_error': 0.0518084, 'testing_error': 0.0527287, 'kaggle': 0.2341},\n    {'parameters': 'image_top_1', 'lmse': 0.22642, 'training_error': 0.0501103, 'testing_error': 0.0512681, 'kaggle': 0.2305},\n    {'parameters': 'rounds=1000', 'lmse': 0.22538, 'training_error': 0.04856, 'testing_error': 0.0507949, 'kaggle': None},\n    {'parameters': 'NLP', 'lmse': 0.22428, 'training_error': 0.0472652, 'testing_error': 0.0503023, 'kaggle': None},\n    {'parameters': 'desc_punc', 'lmse': 0.22416, 'training_error': 0.0472601, 'testing_error': 0.0502457, 'kaggle': None}\n])\nplot(scores)","execution_count":19,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}