{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":true},"cell_type":"markdown","source":"Base line Code Using Tfidf-bigram/trigram"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import cross_val_score\nfrom lightgbm import LGBMRegressor\nfrom scipy import sparse\nfrom category_encoders.hashing import HashingEncoder\nimport os\nfrom scipy.sparse import hstack, csr_matrix\nimport tqdm\nprint(os.listdir(\"../input\"))","execution_count":43,"outputs":[]},{"metadata":{"_cell_guid":"ce47532c-b50f-4c96-a7c5-2cc1bd271c1a","_uuid":"ee4c968ddd55bfdc2be2da2a66423af6b24f5b26","trusted":true},"cell_type":"code","source":"%%time\n\ntrain = pd.read_csv('../input/train.csv',nrows =10000)\ntest = pd.read_csv('../input/test.csv',nrows=10000)","execution_count":44,"outputs":[]},{"metadata":{"_cell_guid":"5efa7529-661b-454a-8294-ac587cc69528","_uuid":"e07f7724c3ab2f06c11709963b6a4c2d8e26618c","collapsed":true,"trusted":true},"cell_type":"code","source":"cat_feats = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'user_type', 'image_top_1']\ntext_feats = ['title', 'description']\nnum_feats = ['price', 'item_seq_number']\nallcols = cat_feats + text_feats + num_feats","execution_count":45,"outputs":[]},{"metadata":{"_cell_guid":"4e4026da-69bd-4be9-9be9-438a3f56f0ae","_uuid":"87dfa80b77a0eb9c1d38a5f791f627c960635355","collapsed":true,"trusted":true},"cell_type":"code","source":"merged = pd.concat((train[allcols], test[allcols]), axis=0)\nmerged['price'] = merged['price'].apply(np.log1p)","execution_count":46,"outputs":[]},{"metadata":{"_cell_guid":"2787c5b6-accc-4d80-8bad-4615ebcc1063","_uuid":"62ce678fcffa5c859fde7791a683a5a92ba150c9","trusted":true},"cell_type":"code","source":"merged.head()","execution_count":47,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37cbe767e8b048e7f5d71e4f4cb454f3a6bb0184"},"cell_type":"code","source":"merged.isnull().sum()","execution_count":48,"outputs":[]},{"metadata":{"_cell_guid":"ec10cec2-4825-43ad-86d8-d95ce43d93d3","_uuid":"297b1d7307ffa0b9d9390ef95b083a98279360b3","collapsed":true,"trusted":true},"cell_type":"code","source":"word_vectorizer = TfidfVectorizer(\n    sublinear_tf=True,\n    stop_words = None,\n    encoding='KOI8-R',\n    analyzer='word',\n    token_pattern=r'\\w{1,}',\n    ngram_range=(1,1),\n    dtype=np.float32,\n    max_features=9000\n)\n# Character Stemmer\nchar_vectorizer = TfidfVectorizer(\n    sublinear_tf=True,\n    encoding='KOI8-R',\n    analyzer='char',\n    ngram_range=(1, 4),\n    dtype=np.float32,\n    max_features=5000\n)","execution_count":49,"outputs":[]},{"metadata":{"_cell_guid":"f09d6166-a44c-45ec-9a02-1368cc2bffa8","_uuid":"3c2c64dac8cbb4e847d443bcd3f5fed3a97cd3ef","trusted":true},"cell_type":"code","source":"%%time\ntfidf_matrices_1 = []\nfor feat in text_feats:\n    tfidf_matrices_1.append(word_vectorizer.fit_transform(merged[feat].fillna('').values))","execution_count":50,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50ab636578fc6787382f5d2b01d59bccc45ebcfb"},"cell_type":"code","source":"%%time\ntfidf_matrices_2 = []\nfor feat in text_feats:\n    tfidf_matrices_2.append(char_vectorizer.fit_transform(merged[feat].fillna('').values))","execution_count":51,"outputs":[]},{"metadata":{"_cell_guid":"9ed2a7ae-53c0-4804-81eb-727ea902a8c7","_uuid":"1911e2bf7df16495a38d4c6167b75a47210ff827","trusted":true},"cell_type":"code","source":"%%time\ntfidf_matrices = sparse.hstack(tfidf_matrices_1,format='csr')","execution_count":52,"outputs":[]},{"metadata":{"_cell_guid":"194a3167-61af-454b-b882-bc085be78064","_uuid":"42345897d93966817db87b8e73f550cd401e15b9","trusted":true},"cell_type":"code","source":"%%time\nhe = HashingEncoder()\ncat_df = he.fit_transform(merged[cat_feats].values)\n","execution_count":53,"outputs":[]},{"metadata":{"_cell_guid":"c762f2a9-1e93-4492-ac40-d0d686780002","_uuid":"87646b349af2e087e28da51f63872adba99e967a","collapsed":true,"trusted":true},"cell_type":"code","source":"full_matrix = sparse.hstack([cat_df.values, tfidf_matrices, merged[num_feats].fillna(-1).values], format='csr')\n","execution_count":54,"outputs":[]},{"metadata":{"_cell_guid":"945dadf8-b05f-4c0a-9739-7465ddf5870f","_uuid":"5cdf4b394095f0ed47054b7ac1d972812935e9f1","trusted":true},"cell_type":"code","source":"%%time\nmodel = LGBMRegressor(max_depth=4, learning_rate=0.3, n_estimators=500)\nres = cross_val_score(model, full_matrix[:train.shape[0]], train['deal_probability'].values, cv=5, scoring='neg_mean_squared_error')\nres = [np.sqrt(-r) for r in res]\nprint(np.mean(res), np.std(res))","execution_count":56,"outputs":[]},{"metadata":{"_cell_guid":"f75d79e3-213a-42c7-a94d-53505d22048f","_uuid":"4b2f4728a485a0593d745ca11d78fbe11cc6ee06","collapsed":true,"trusted":true},"cell_type":"code","source":"model.fit(full_matrix[:train.shape[0]], train['deal_probability'].values)\npreds = model.predict(full_matrix[train.shape[0]:])","execution_count":57,"outputs":[]},{"metadata":{"_cell_guid":"d57d0a79-645f-435f-a588-22de2b9edda0","_uuid":"a88f2eb8df45b103723910fb9beb6c01db368f15","collapsed":true,"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline","execution_count":58,"outputs":[]},{"metadata":{"_cell_guid":"556209a6-8ad4-4ba4-a544-64d75ac3ad95","_uuid":"3fcc34fb0d90195ec0d577a561c962a522772099","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 7))\nplt.hist(preds, bins=50);","execution_count":59,"outputs":[]},{"metadata":{"_cell_guid":"18ccd7c4-5b74-4d0f-a7d0-975565355203","_uuid":"6d4804647c0d8332819189ebeda0797e80227224","trusted":true},"cell_type":"code","source":"#sub = pd.read_csv('../input/sample_submission.csv')\n#sub['deal_probability'] = preds\n#sub['deal_probability'].clip(0.0, 1.0, inplace=True)\n#sub.to_csv('../input/first_attempt.csv', index=False)\n#sub.head()","execution_count":60,"outputs":[]},{"metadata":{"_cell_guid":"7b10d0e2-b881-4a3e-9dd8-9a1c7d81bd38","_uuid":"aa547cf5d50eac58f2c909b83de493593c4a8dfd","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}