{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":false,"_kg_hide-input":false,"scrolled":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport time\nimport re\nfrom __future__ import print_function\nfrom collections import defaultdict\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport seaborn as sns\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.pipeline import make_union, make_pipeline\nfrom sklearn.preprocessing import FunctionTransformer, StandardScaler, LabelEncoder, MinMaxScaler,  Imputer, LabelBinarizer, OneHotEncoder\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.linear_model import LogisticRegressionCV\nfrom sklearn.metrics import accuracy_score, mean_squared_error, r2_score\nfrom sklearn.model_selection import RandomizedSearchCV, train_test_split\n\nimport xgboost as xgb\nimport lightgbm as lgb\n\n%matplotlib inline\nplt.rcParams[\"figure.figsize\"] = (15, 8)\npd.options.display.float_format = '{:.2f}'.format\n\nimport os\n# print(os.listdir(\"../input\"))\n\ndef check_category(row):\n    \"\"\"\n    Function for fill empty category to None/None/None\n    \"\"\"\n    if isinstance(row.category_name, str) and '/' in row.category_name:\n        return row.category_name\n    else:\n        return 'None/None/None'\n\ndef check_brand(row):\n    \"\"\"\n    Function for fill empty brand to No brand\n    \"\"\"\n    if isinstance(row.brand_name, str) and row.brand_name:\n        return row.brand_name\n    else:\n        return 'No brand'\n\ndef change_tables(data, type_sample='train'):\n    \"\"\"\n    Function for split category_name to three columns. And drop empty values.\n    \"\"\"\n    data['category_name_1'] = data.apply(check_category, axis=1)\n    data['brand_name_1'] = data.apply(check_brand, axis=1)\n    data['category_1'] = data.apply(lambda row: row.category_name_1.split('/')[0], axis=1)\n    data['category_2'] = data.apply(lambda row: row.category_name_1.split('/')[1], axis=1)\n    data['category_3'] = data.apply(lambda row: row.category_name_1.split('/')[2], axis=1)\n    if 'price' not in data.columns:\n        data['price'] = 0\n    price = data['price']\n    data['brand_name'] = data['brand_name_1']\n    data_name_id = 'train_id'\n    if type_sample == 'test':\n        data_name_id = 'test_id'\n    data.drop(labels=[data_name_id, 'category_name', 'price', 'item_condition_id', 'shipping', 'category_name_1', 'brand_name_1'], axis=1, inplace=True)\n    data['price'] = price\n\ndf_train = pd.read_csv('../input/train.tsv', sep='\\t')  # test_stg2.tsv\nchange_tables(df_train)\nprint('df_train len:', df_train.shape)\ny_train = df_train['price']\n\ndf_test = pd.read_csv('../input/test.tsv', sep='\\t')\nchange_tables(df_test, type_sample='test')\nprint('df_test len:', df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00170dc30a2efd701fbaee710086697a0e5b71f4","scrolled":true},"cell_type":"code","source":"from sklearn.pipeline import make_union, make_pipeline\nfrom sklearn.preprocessing import FunctionTransformer, OrdinalEncoder, StandardScaler, LabelEncoder, MinMaxScaler, LabelBinarizer, OneHotEncoder\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.impute import SimpleImputer, MissingIndicator\n    \nclass LenDescription(BaseEstimator, TransformerMixin):\n    def fit(self, X, y=None):\n        return self\n        \n    def transform(self, X, y=None):\n        X['descr_length'] = X['item_description'].str.len()\n        return X['descr_length'].as_matrix().reshape(-1, 1)\n\n    def fit_transform(self, X, y=None):\n        return self.fit(X).transform(X)\n\n\nclass DictDescription(BaseEstimator, TransformerMixin):\n    def fit(self, X, y=None):\n        return self\n        \n    def transform(self, X, y=None):\n        return (row for _, row in X.iterrows())\n\n    def fit_transform(self, X, y=None):\n        return self.fit(X).transform(X)\n\n\nclass CountWords(BaseEstimator, TransformerMixin):\n    \n    def fit(self, X, y=None):\n        return self\n        \n    def transform(self, X, y=None):\n        X['splited_row'] = X.apply(lambda row: len(row.item_description.str.split(' ')), axis=1)\n        return X['splited_row'].values\n\n    def fit_transform(self, X, y=None):\n        return self.fit(X).transform(X)\n\n\ndef get_brand_cols(df):\n    return df[['brand_name']]\n\ndef get_category_cols(df):\n    return df[['category_name']]\n\ndef get_name_cols(df):\n    return df[['name']]\n\ndef get_descr_cols(df):\n    return df[['item_description']]\n\ndef get_all_cols(df):\n    return df[['name', 'brand_name']]\n\ndef get_cat_cols(df):\n    return df[['category_1', 'category_2']]\n\ndef get_last_cat_cols(df):\n    return df[['category_3']]\n\nvec = make_union(*[\n    make_pipeline(FunctionTransformer(get_cat_cols, validate=False), OneHotEncoder(sparse=False)),\n    make_pipeline(FunctionTransformer(get_brand_cols, validate=False), OrdinalEncoder()),\n    make_pipeline(FunctionTransformer(get_last_cat_cols, validate=False), OrdinalEncoder()),\n    make_pipeline(FunctionTransformer(get_descr_cols, validate=False), LenDescription()),\n#     make_pipeline(FunctionTransformer(get_descr_cols, validate=False), CountWords()),\n    make_pipeline(FunctionTransformer(get_descr_cols, validate=False), DictDescription(), DictVectorizer()),\n    make_pipeline(FunctionTransformer(get_name_cols, validate=False), OrdinalEncoder()),\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0f2334578ec1bc464e80b85582363ee6c01cbd88"},"cell_type":"code","source":"len_train = df_train.shape[0]\nlen_test = df_test.shape[0]\nX = df_train.append(df_test, ignore_index=True)\nx_transform = vec.fit_transform(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"2a7c2d7bf48196e460dff1d927062ccca36bf4c0"},"cell_type":"code","source":"Y = y_train.append(df_test['price'], ignore_index=True)\nindices = np.arange(len_train)\nX_train, X_test, Y_train, _ = train_test_split(x_transform, Y, train_size=len_train, shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-output":true,"scrolled":false},"cell_type":"code","source":"import scipy.stats as st\n\nparam_grid = {\n    \"nthread\": [-1],\n    'objective':['reg:linear'],\n    \"n_estimators\": [300, 500],\n    \"max_depth\": st.randint(3, 8),\n    \"learning_rate\": st.uniform(0.05, 0.5),\n    \"colsample_bytree\": st.beta(10, 1),\n    \"subsample\": st.beta(10, 1),\n    \"gamma\": st.uniform(0, 10),\n    'reg_alpha': st.expon(0, 50),\n    'min_child_weight': [4, 8, 16],\n}\n\nxg_reg = RandomizedSearchCV(xgb.XGBRegressor(), param_grid, n_jobs=5, cv=2, verbose=True)\nxg_reg.fit(X_train, Y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf004eb1bf2b5c475ead4895b613470e744deccc"},"cell_type":"code","source":"print(xg_reg.best_params_)\nprint(xg_reg.best_estimator_)\nprint(xg_reg.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c9889fb4799a177e7535ca1c88db666c0ac36125"},"cell_type":"code","source":"score = xg_reg.score(X_train, Y_train)\nprint('score:', score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"4c0582868d9d3500a6f7f67bba563b5fd4e10af3"},"cell_type":"code","source":"preds = xg_reg.predict(X_test)\n\ndf_res = pd.read_csv('../input/sample_submission.csv')\n\ng = df_res['price']\n# print(g.values, type(g.values))\nprint('mean_squared_error: ', mean_squared_error(g, preds))\nprint('r2_score: ', r2_score(g, preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63e04d5cc43dfab7d4311bd4e7949a8d7375acc5","scrolled":false},"cell_type":"code","source":"# X_test = pd.concat(X_test, pd.DataFrame({'price': preds}))\n# X_test['test_id'] = X_test['test_id'].astype(np.int)\n# X_test[['test_id', 'price']].to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"31c3b2c30aa7516d536aabccd499ba18589d1957"},"cell_type":"code","source":"import lightgbm as lgb\n\ndf_res = pd.read_csv('../input/sample_submission.csv')\ng = df_res['price']\n\nparams = {\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': {'l2', 'l1'},\n    'max_depth': 10, \n    'learning_rate': 0.01,\n    'verbose': 1, \n    'early_stopping_round': 20}\nn_estimators = 500\n\nd_train = lgb.Dataset(X_train, label=Y_train)\nd_valid = lgb.Dataset(X_test, label=g)\nwatchlist = [d_valid]\n\nmodel = lgb.train(params, d_train, n_estimators, watchlist, verbose_eval=1)\npreds = model.predict(X_test, num_iteration=model.best_iteration)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"a24361c9114f7be387e5746d56db17e5c3d86fac"},"cell_type":"code","source":"print('mean_squared_error: ', mean_squared_error(g, preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"eead7fe27d240318e5bc85ede147052bcf9fc6fb"},"cell_type":"code","source":"# data = X_test.tocoo(copy=False)\n# data_x = pd.DataFrame({'index': data.row, 'price': data.price}\n#                  )[['index', 'price']].reset_index(drop=True)\n# X = pd.concat([data_x, pd.DataFrame({'price': preds})])\n# X['test_id'] = X['test_id'].astype(np.int)\n# X[['test_id', 'price']].to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}