{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\nimport nltk\nfrom nltk.corpus import stopwords\nimport string\nimport scipy\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Read data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"data = pd.read_csv('../input/train.csv')\ny = data.deal_probability.copy()\nX_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.20, random_state=23)\n\ndel data\ngc.collect()\n\nX_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Feature Engineering"},{"metadata":{},"cell_type":"markdown","source":"2.1 Add new features"},{"metadata":{"trusted":true},"cell_type":"code","source":"for df in [X_train, X_test]:\n    df['description'].fillna('unknowndesc', inplace=True)\n    df['title'].fillna('unknowntitle', inplace=True)\n    \n    for col in ['description', 'title']:\n        df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))\n        df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))\n\n    df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100\n    df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100\n\n    df['weekday'] = pd.to_datetime(df['activation_date']).dt.weekday\n    df['Day of Month'] = pd.to_datetime(df['activation_date']).dt.day\n    df['city'] = df['region'] + '_' + df['city']\n    df['num_desc_punct'] = df['description'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2.2 Count-vector and TF-IDF"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Countvectorizer\ncount_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True,\n                                        ngram_range=(1, 2), max_features=8000)\ntitle_counts = count_vectorizer_title.fit_transform(X_train['title'].append(X_test['title']))\ntrain_title_counts = title_counts[:len(X_train)]\ntest_title_counts = title_counts[len(X_train):]\n\n#TF-IDF\ncount_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'), lowercase=True,\n                                        ngram_range=(1, 2), max_features=17000)\ndesc_counts = count_vectorizer_desc.fit_transform(X_train['description'].append(X_test['description']))\n\ntrain_desc_counts = desc_counts[:len(X_train)]\ntest_desc_counts = desc_counts[len(X_train):]\n\ntrain_title_counts.shape, train_desc_counts.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2.3 Transforming categorical variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"#target = 'deal_probability'\npredictors = [\n    'num_desc_punct', 'num_words_title', 'words_vs_unique_title', 'num_unique_words_title',\n    'words_vs_unique_description', 'num_unique_words_description', 'num_words_description',\n    'price', 'item_seq_number', 'Day of Month', 'weekday'\n]\ncategorical = [\n    'image_top_1', 'param_1', 'param_2', 'param_3', \n    'city', 'region', 'category_name', 'parent_category_name', 'user_type'\n]\n\npredictors = predictors + categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for feature in categorical:\n    print(f'Transforming {feature}...')\n    encoder = LabelEncoder()\n    X_train[feature].fillna('unknown',inplace=True)\n    X_test[feature].fillna('unknown',inplace=True)\n    encoder.fit(X_train[feature].append(X_test[feature]).astype(str))\n    \n    X_train[feature] = encoder.transform(X_train[feature].astype(str))\n    X_test[feature] = encoder.transform(X_test[feature].astype(str))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2.4 Filling missing data"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train[\"price\"] = np.log(X_train[\"price\"]+0.001)\nX_train[\"price\"].fillna(-1,inplace=True)\nX_train[\"image_top_1\"].fillna(-1,inplace=True)\n\nX_test[\"price\"] = np.log(X_test[\"price\"]+0.001)\nX_test[\"price\"].fillna(-1,inplace=True)\nX_test[\"image_top_1\"].fillna(-1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# 3. Preparing training set and testing set"},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_names = np.hstack([\n    count_vectorizer_desc.get_feature_names(),\n    count_vectorizer_title.get_feature_names(),\n    predictors\n])\nprint('Number of features:', len(feature_names))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = scipy.sparse.hstack([\n    test_desc_counts,\n    test_title_counts,\n    X_test.loc[:, predictors]\n], format='csr')\n\ntrain = scipy.sparse.hstack([\n    train_desc_counts,\n    train_title_counts,\n    X_train.loc[: , predictors]\n], format='csr')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train为训练集的数据，test为测试集的数据，对应的y分别是y_train和y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\nlgbm_params = {\n    'objective' : 'regression',\n    'metric' : 'rmse',\n    'num_leaves' : 300,\n#     'max_depth': 15,\n    'learning_rate' : 0.02,\n    'feature_fraction' : 0.6,\n    'bagging_fraction' : .8,\n    'verbosity' : -1\n}\n\nlgtrain = lgb.Dataset(train, y_train,\n                feature_name=list(feature_names),\n                categorical_feature = categorical)\nlgvalid = lgb.Dataset(test, y_test,\n                feature_name=list(feature_names),\n                categorical_feature = categorical)\n\n# Go Go Go\nlgb_clf = lgb.train(\n    lgbm_params,\n    lgtrain,\n    num_boost_round=5000,\n    valid_sets=[lgtrain, lgvalid],\n    valid_names=['train','valid'],\n    early_stopping_rounds=50,\n    verbose_eval=100)\nprint(\"Model Evaluation Stage\")\nprint('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, lgb_clf.predict(test))))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}