{"cells":[{"metadata":{"_uuid":"208338288dcff3afa0867b306720403185a0f482","_cell_guid":"79cde592-cafb-467a-9e13-13297afede14"},"cell_type":"markdown","source":"### This kernel is forked from https://www.kaggle.com/bminixhofer/aggregated-features-lightgbm . I have done some changes to get better score "},{"metadata":{"_uuid":"af7422df064b7f16934cca9e86e8213bc9c667e8","_cell_guid":"033852ed-2dd3-4bc3-bec1-ac3481ab175f","_kg_hide-input":true,"collapsed":true,"trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib_venn import venn2, venn2_circles\nimport string\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nimport nltk\nfrom nltk.corpus import stopwords\nimport scipy\nimport lightgbm as lgb\nfrom sklearn.metrics import mean_squared_error\nfrom math import sqrt\nfrom sklearn.linear_model import Ridge\nfrom sklearn.cross_validation import KFold\n\n\n\nsns.set()\n%matplotlib inline\n\nNFOLDS = 5\nSEED = 42\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d8e0a1a86fe5fbfa5162d3180d0b1ebea94f97d","_cell_guid":"94a03718-1718-4981-88ef-532efcb435ee"},"cell_type":"markdown","source":"# Data Loading\n"},{"metadata":{"_uuid":"02aad0b977053435e089bd189ea45e4a88731e87","_cell_guid":"bc719fcf-f418-4db5-a4fa-ebfa27716521","collapsed":true,"trusted":false},"cell_type":"code","source":"train = pd.read_csv('../input/avito-demand-prediction/train.csv')\ntest = pd.read_csv('../input/avito-demand-prediction/test.csv')\ngp = pd.read_csv('../input/aggregated-features-lightgbm/aggregated_features.csv') \ntrain = train.merge(gp, on='user_id', how='left')\ntest = test.merge(gp, on='user_id', how='left')\n\nagg_cols = list(gp.columns)[1:]\n\ndel gp\ngc.collect()\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be70118249f82d7e85cf7ee525303bc4d580932e","_cell_guid":"78cc3f11-d2c4-4ed8-a7b8-ba44f884c3d6"},"cell_type":"markdown","source":"# Feature Engineering\n### Text cleaning does not help So, I am commenting them"},{"metadata":{"_uuid":"138fe214a8f82385aa344f6c17e78faf352fb690","_cell_guid":"13d87e1c-0eed-47cb-926b-1b1c7d618900","collapsed":true,"trusted":false},"cell_type":"code","source":"# def cleanup(s):                      \n#     \"\"\"\n#     function to clean text data\n    \n#     \"\"\"\n#     s = str(s)\n#     s = s.lower()\n# #     s = re.sub('\\s\\W',' ',s)\n# #     s = re.sub(\"https\\S+\\w+\",\"\",s)\n# #     s=[word if word not in ss else \"\" for word in TweetTokenizer().tokenize(s)]\n# #     s = \" \".join(s)\n# #     s = re.sub('rt*.@\\w+',' ',s)\n# #     s = re.sub('@\\w+',' ',s)\n# #     s = re.sub('\\W,\\s',' ',s)\n# #     s = re.sub(r'[^\\w,]', ' ', s)\n#     s = re.sub(\"\\d+\", \"\", s)\n#     s = re.sub('\\s+',' ',s)\n#     s = re.sub('[!@#$_“”¨«»®´·º½¾¿¡§£₤‘’]', '', s)\n# #     s = s.replace(\".co\",\"\")\n# #     s = s.replace(\",\",\"\")\n# #     s = s.replace(\"[\\w*\",\" \")\n#     s = ''.join(''.join(a)[:2] for _, a in itertools.groupby(s))\n#     return s\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9c60409e1dc6c618f229e15c2b368660385c2ec1","_cell_guid":"41e1111d-674e-4c86-b232-c0086832ee6d","collapsed":true,"trusted":false},"cell_type":"code","source":"\n\nfor df in [train, test]:\n    df['description'].fillna('unknowndesc', inplace=True)\n    df['title'].fillna('unknowntitle', inplace=True)\n\n    df['weekday'] = pd.to_datetime(df['activation_date']).dt.day\n    \n    for col in ['description', 'title']:\n        df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))\n        df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))\n\n    df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100\n    df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100\n    \n    df['city'] = df['region'] + '_' + df['city']\n    df['num_desc_punct'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n    \n    for col in agg_cols:\n        df[col].fillna(-1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc558a03cbddc3954b01783636785cdee6cf954a","_cell_guid":"8d61ada6-8f3a-4b58-baad-199589122985","collapsed":true,"trusted":false},"cell_type":"code","source":"count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=2)\n\ntitle_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))\n\ntrain_title_counts = title_counts[:len(train)]\ntest_title_counts = title_counts[len(train):]\n\n\ncount_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'), \n                                        lowercase=True, ngram_range=(1, 2),\n                                        max_features=17000)\n\ndesc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))\n\ntrain_desc_counts = desc_counts[:len(train)]\ntest_desc_counts = desc_counts[len(train):]\n\ntrain_title_counts.shape, train_desc_counts.shape\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc6f15f981f05ad9b1e89ab7acb54ea2350278c6","_cell_guid":"56338e56-c266-4342-a8f3-10621db49dee","collapsed":true,"trusted":false},"cell_type":"code","source":"target = 'deal_probability'\npredictors = [\n    'num_desc_punct', \n    'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',\n    'avg_times_up_user', 'avg_days_up_user', 'n_user_items', \n    'price', 'item_seq_number'\n]\ncategorical = [\n    'image_top_1', 'param_1', 'param_2', 'param_3', \n    'city', 'region', 'category_name', 'parent_category_name', 'user_type'\n]\n\npredictors = predictors + categorical","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bcde9bb9b9b26347c7db7da47123c6fec40860e","_cell_guid":"7695c1a1-06cf-40e6-8792-77b62f48262a","collapsed":true,"trusted":false},"cell_type":"code","source":"for feature in categorical:\n    print(f'Transforming {feature}...')\n    encoder = LabelEncoder()\n    train[feature].fillna('unknown',inplace=True)\n    test[feature].fillna('unknown',inplace=True)\n    encoder.fit(train[feature].append(test[feature]).astype(str))\n    \n    train[feature] = encoder.transform(train[feature].astype(str))\n    test[feature] = encoder.transform(test[feature].astype(str))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a0f573f22ce96488b58b414b2e49c3d499d87a94","_cell_guid":"ee2f2491-7997-48eb-b022-ba13899e759e"},"cell_type":"markdown","source":"# Hyper Parameter Tuning\n\n### I did it on cloud so I m just commenting it out to save time"},{"metadata":{"_uuid":"e3b3d12868ab835e78eee689522ed7cd4a3303f0","_cell_guid":"fdd39943-2e02-4403-9413-aaac70f5511f","collapsed":true,"trusted":false},"cell_type":"code","source":"# def objective(space):\n#     mod = lgb.LGBMRegressor(n_estimators = 5000, \n#             num_leaves = int(space['num_leaves']),\n#             subsample = space['subsample'],min_child_weight = space['min_child_weight'],\n#             colsample_bytree=space['colsample_bytree'],\n#             learning_rate =space['learning_rate'],n_jobs=-1,\n#                 )\n# #     temp_train=copy.copy(newtrain)\n#     folds=KFold(5,random_state=100)\n#     fold_score=[]\n#     i=1\n#     st=time.time()\n#     print('=================*=================')\n#     print(space)\n#     for train_index,test_index in folds.split(X=X):\n#         mod.fit(X=X[train_index],y=y.values[train_index],eval_set=[ (X[test_index],y.values[test_index])],early_stopping_rounds=20,verbose=30,eval_metric='rmse')    \n#         score=mod.best_score_.get('valid_0').get('rmse')\n#         print('cv',i,': ', score)\n#         i=i+1\n#         fold_score.append(score)                \n#     print(\"SCORE:\") \n#     print(np.mean(fold_score))\n#     print('time',time.time()-st)\n#     return 1-np.mean(fold_score) \n\n# space ={\n#     #'max_depth':hp.quniform('max_depth',2,10,1),\n#     'num_leaves': hp.quniform('num_leaves', 200, 300, 4),\n#     'min_child_weight': hp.quniform ('min_child_weight', 1, 2, 1),\n#     'subsample': hp.quniform ('subsample', 0.8, .95,0.05),\n#     'learning_rate': hp.quniform('learning_rate', 0.01,0.2,.03),\n#    # A problem with max_depth casted to float instead of int with\n#    # the hp.quniform method.\n# #     'gamma': hp.quniform('gamma', 0, 0.6, 0.1),\n#     'colsample_bytree': hp.quniform('colsample_bytree', 0.7, .95, 0.05),\n#    }  \n# trials = Trials()\n# best = fmin(fn=objective,space=space,algo=tpe.suggest,max_evals=80)\n# print(best)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"26dc041ad0496392176d06882673d130e5219b63","_cell_guid":"0e3cd2be-1a46-4f66-8f7c-9fbb95511b62","collapsed":true,"scrolled":true,"trusted":false},"cell_type":"code","source":"train[\"price\"] = np.log(train[\"price\"]+0.001)\ntrain[\"price\"].fillna(-999,inplace=True)\ntrain[\"image_top_1\"].fillna(-999,inplace=True)\n\ntest[\"price\"] = np.log(test[\"price\"]+0.001)\ntest[\"price\"].fillna(-999,inplace=True)\ntest[\"image_top_1\"].fillna(-999,inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dc7dbed2a9943fa245d860de412cdef0e17a49d9","_cell_guid":"ee4b0666-f0be-4965-9dd9-6651d5c49b2b"},"cell_type":"markdown","source":"# LightGBM \n"},{"metadata":{"_uuid":"ac57e30cf4452005a2cb20db424d7bcf1f11d1df","_cell_guid":"7e94bcc4-ab56-4a38-bdcf-218791e95c2c","collapsed":true,"trusted":false},"cell_type":"code","source":"rounds = 20000\nearly_stop_rounds = 50\nlgbm_params = {\n    'objective' : 'regression',\n    'metric' : 'rmse',\n    'num_leaves' : 300,\n#     'max_depth': 15,\n    'learning_rate' : 0.021,\n    'feature_fraction' : 0.6,\n    'bagging_fraction' : .8,\n    'verbosity' : -1\n}\n\nfeature_names = np.hstack([\n    count_vectorizer_desc.get_feature_names(),\n    count_vectorizer_title.get_feature_names(),\n    predictors\n])\nprint('Number of features:', len(feature_names))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"96aa1864c7cc6491d139863f9a817895d16525df","_cell_guid":"bfa6795a-05a5-4372-88b3-531d947e0eee","collapsed":true,"trusted":false},"cell_type":"code","source":"VALID = False","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a50b3e34d76019fdbfd45c3566eec09556f24cb","_cell_guid":"9e910115-312f-4437-8ba3-9fb6f7426af8","collapsed":true,"scrolled":true,"trusted":false},"cell_type":"code","source":"x_test = scipy.sparse.hstack([\n    test_desc_counts,\n    test_title_counts,\n    test.loc[:, predictors]\n], format='csr')\n\nif VALID == True:\n    train_index, valid_index = train_test_split(np.arange(len(train)), test_size=0.1, random_state=42)\n\n    x_train = scipy.sparse.hstack([\n            train_desc_counts[train_index],\n            train_title_counts[train_index],\n            train.loc[train_index, predictors]\n    ], format='csr')\n    y_train = train.loc[train_index, target]\n\n    x_valid = scipy.sparse.hstack([\n        train_desc_counts[valid_index],\n        train_title_counts[valid_index],\n        train.loc[valid_index, predictors]\n    ], format='csr')\n    y_valid = train.loc[valid_index, target]\n\n     \n    # LGBM Dataset Formatting \n    lgtrain = lgb.Dataset(x_train, y_train,\n                    feature_name=list(feature_names),\n                    categorical_feature = categorical)\n    lgvalid = lgb.Dataset(x_valid, y_valid,\n                    feature_name=list(feature_names),\n                    categorical_feature = categorical)\n     \n    # Go Go Go\n    lgb_clf = lgb.train(\n        lgbm_params,\n        lgtrain,\n        num_boost_round=20000,\n        valid_sets=[lgtrain, lgvalid],\n        valid_names=['train','valid'],\n        early_stopping_rounds=50,\n        verbose_eval=100)\n    print(\"Model Evaluation Stage\")\n    del x_valid ; x_train; gc.collect()\n\nelse:\n    # LGBM Dataset Formatting \n    X = scipy.sparse.hstack([\n        train_desc_counts,\n        train_title_counts,\n        train.loc[: , predictors]\n    ], format='csr')\n    y = train.deal_probability\n    \n    lgtrain = lgb.Dataset(X, y.values,\n                    feature_name=list(feature_names),\n                    categorical_feature = categorical)\n     # Go Go Go\n    lgb_clf = lgb.train(\n        lgbm_params,\n        lgtrain,\n        num_boost_round=2200)\n    \n    del X; gc.collect()\n\ndel train,test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37cbdd3d10742d44e169a2db4386f7bb3c7b09bc","_cell_guid":"5827b4fb-0bfc-4ef2-ab43-5be682151de8","collapsed":true,"trusted":false},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 14))\nlgb.plot_importance(lgb_clf, max_num_features=50, ax=ax)\nplt.title(\"Light GBM Feature Importance\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c93c6232801704e05d2a94c38b7df77d7ed14f4","_cell_guid":"9d49a8cc-4b52-47f4-868a-f7b3006cd9f3","collapsed":true,"trusted":false},"cell_type":"code","source":"subm = pd.read_csv('../input/avito-demand-prediction/sample_submission.csv')\nsubm['deal_probability'] = np.clip(lgb_clf.predict(x_test), 0, 1)\nsubm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3603d98b15ca6fc0cc626239b990c33af2b429de","_cell_guid":"122b4ba2-afa8-4b97-9adf-bb049ebb3138","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}