{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false,"collapsed":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nprint(\"Data:\\n\",os.listdir(\"../input\"))\n\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import LabelEncoder\n\n# Text\nimport re\nfrom nltk.corpus import stopwords \nfrom nltk.stem.snowball import SnowballStemmer\n\n# Options\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true},"cell_type":"markdown","source":"Preprocessing like https://www.kaggle.com/nicapotato/bow-meta-text-and-dense-features-lgbm\n\nHowever, instead of Tf-idf encoding of text columns, I tried to convert to  **vw-format**.\n\nAfter that I can use default tricks from LinearModelsWorld like momentum/adaptive learning rate/hashing trick etc.\n\nMoreover, the addition of various interactions between the features will become available simply by specifying the necessary parameters.\n\n## I. Preprocessing"},{"metadata":{"_uuid":"56d012f8a91a7d1a949251d86a527f009b37c0bf","_cell_guid":"58465a7d-9554-40d9-879e-de657359df61","trusted":false,"collapsed":true},"cell_type":"code","source":"print(\"Data Load Stage\")\n%time training = pd.read_csv('../input/train.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])#.sample(1000)\ntrain_len = training.shape[0]\n%time testing = pd.read_csv('../input/test.csv', index_col = \"item_id\", parse_dates = [\"activation_date\"])#.sample(1000)\n\nprint('Train shape: {} Rows, {} Columns'.format(*training.shape))\nprint('Test shape: {} Rows, {} Columns'.format(*testing.shape))\n\nprint(\"Combine Train and Test\")\ndf = pd.concat([training,testing],axis=0)\ndel (training, testing); gc.collect()\nprint('All Data shape: {} Rows, {} Columns'.format(*df.shape))\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a650fae90e6cf16253e23b950a97e31f8ca5302","_cell_guid":"f34df850-0800-43ce-9db7-4d4fc3c8ef51","trusted":false,"collapsed":true},"cell_type":"code","source":"print(\"Feature Engineering\")\ndf[\"price\"] = np.log(df[\"price\"]+0.001)\ndf[\"price\"].fillna(-1000,inplace=True)\ndf[\"image_top_1\"].fillna(-1,inplace=True)\n\nprint(\"\\nCreate Time Variables\")\ndf[\"weekday\"] = df['activation_date'].dt.weekday\ndf[\"woy\"] = df['activation_date'].dt.week\ndf[\"dom\"] = df['activation_date'].dt.day\ndf.drop([\"activation_date\",\"image\"],axis=1,inplace=True)\n\nprint(\"\\nText Features\") \ndf['group'] = df.apply(lambda row: ' '.join([\n    str(row['param_1']), \n    str(row['param_2']), \n    str(row['param_3'])]),axis=1) # Group Param Features\ndf.drop([\"param_1\",\"param_2\",\"param_3\"],axis=1,inplace=True)\n\ndf.description.fillna('nan', inplace=True)\ndf.title.fillna('nan', inplace=True)\n\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"99bf904ab17d7e6b1f2886b64bc9a20213f553da","_cell_guid":"984a2f60-ba2c-440d-90da-9d34467c41fc","trusted":false,"collapsed":true},"cell_type":"code","source":"print('Rename columns')\ncolnames_mapper = {\n    'parent_category_name': 'cat_1',\n    'category_name': 'cat_2',\n    'description': 'desc',\n    'image_top_1': 'img_code',\n    'item_seq_number': 'item',\n    'region': 'reg',\n    'user_id': 'usr',\n    'user_type': 'usr_type',\n}\n\ncat_cols = ['usr', 'usr_type', 'reg', 'city', 'item', 'cat_1', 'cat_2', 'img_code', 'weekday']\ntext_cols = ['title', 'desc', 'group']\nother_cols = ['price',  'woy', 'dom']\ntarget_col = 'deal_probability'\n\ndf = df.rename(index=str, columns=colnames_mapper)[cat_cols + other_cols + text_cols + [target_col]]\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"636cdb3e184766f9e17caf118ae74f6f3132e23b","_cell_guid":"b4180c9c-7969-467b-80a4-3bc8c6faf195","trusted":false,"collapsed":true},"cell_type":"code","source":"print(\"\\nEncode Categorical Variables\")\n# Encoder:\nlbl = LabelEncoder()\nfor col in tqdm(cat_cols):\n    df[col] = lbl.fit_transform(df[col].astype(str))\n    col_max = df[col].max()\n    if col_max < 2**8 - 1:\n        df[col] = df[col].astype('uint8')\n    elif col_max < 2**16 - 1:\n        df[col] = df[col].astype('uint16')\n    elif col_max < 2**32 - 1:\n        df[col] = df[col].astype('uint32')\n    \ndel(col_max)     \ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86ee47bfad40024345eaee3f141e16a408cd2595","_cell_guid":"e4c1b66a-09ae-4968-95f5-69e1c4c46d01"},"cell_type":"markdown","source":"## II. Feature Extractor"},{"metadata":{"_uuid":"118c0d2c3089c65b047aa318353679890e419857","_cell_guid":"011de7cd-7dd2-4576-bacd-4ac728a671f3"},"cell_type":"markdown","source":"I defined several namespaces:\n    - categorical features with ohe\n    - words after lemming from text columns\n    - statistics from text columns (number of unique words, number of exclamation mark, number of capital symbols in current text field)\n    - other fields from data\n    \n   Function **vw extractor** converts a row from dataframe to vw format row. "},{"metadata":{"_uuid":"dac78658aec510c67fa1bcae4bd49e874ebbd411","_cell_guid":"d56fe0b6-4716-4919-8782-205e7b361b33","trusted":false,"collapsed":true},"cell_type":"code","source":"russian_stop = set(stopwords.words('russian'))\nsnowball_stemmer = SnowballStemmer(\"russian\")\n\ndef vw_extractor(row):\n\n    output_line = '|cat '\n    for col in cat_cols:\n        output_line += '{0}_{1} '.format(col, row[col])\n\n    n_upper = {}\n    n_exclamation = {}\n    n_uniq_words = {}\n    for col in text_cols:\n        n_exclamation[col] = row[col].count('!')\n        n_upper[col] = sum(1 for ch in row[col] if ch.isupper())\n\n        text = row[col].lower()\n        words = set(re.findall('\\w+', text)).difference(russian_stop)\n        n_uniq_words[col] = len(words)\n\n        stemmed_words = {snowball_stemmer.stem(word) for word in words}\n        output_line += '|{} '.format(col)\n        output_line += ' '.join(stemmed_words) + ' '\n\n    output_line += '|stat_text '\n    output_line += 'n_up_tit:{0} n_up_desc:{1} '.format(n_upper['title'], n_upper['desc'])\n    output_line += 'n_exc_tit:{0} n_exc_desc:{1} '.format(n_exclamation['title'], n_exclamation['desc'])\n    output_line += 'n_uniq_tit:{0} n_uniq_desc:{1} '.format(n_uniq_words['title'], n_uniq_words['desc'])\n\n    output_line += '|other '\n    for col in other_cols:\n        output_line += '{0}:{1:.6} '.format(col, float(row[col]))\n\n    return output_line\n\n# just check our vw format before transformation\nfor row in tqdm(df.head().iterrows(), total=df.head().shape[0], miniters=1000):\n    print(vw_extractor(row[1]), '\\n')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9c76f928884eaef80bab87071a156b8d6add5411","_cell_guid":"e0f05a69-84a2-4913-a214-1c17f7d5151c"},"cell_type":"markdown","source":"## Build vw train and validation files"},{"metadata":{"_uuid":"3d25d622cf5bba87f57a6280d356afe75f0ed58c","_cell_guid":"66ea8f08-91bc-40ff-97e2-b80beef46e31","trusted":false,"collapsed":true},"cell_type":"code","source":"def train2vw(data, features_extractor, valid_rate=0, train_output='train', valid_output='valid', yvalid_output='yvalid'):\n    writer_train = open(train_output, 'w')\n    writer_val = open(valid_output, 'w')\n    writer_yval = open(yvalid_output, 'w')\n    \n    for row in tqdm(data.iterrows(), total=data.shape[0], miniters=2500):\n        label = row[1][target_col]\n        features = features_extractor(row[1])\n        output_line = '{0:.6} {1}\\n'.format(label, features)\n        \n        if np.random.rand() > valid_rate:\n            writer_train.write(output_line)\n        else:\n            writer_val.write(output_line)\n            writer_yval.write('%s\\n' % label)\n            \n    writer_train.close()\n    writer_val.close()\n    writer_yval.close()\n    \ntrain2vw(df[:train_len], vw_extractor, 0.2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7eb89a694f23f7a166a75217e09fb9c41faf708a","_cell_guid":"6010ff7d-5788-4072-90d2-a0a06960a515"},"cell_type":"markdown","source":"## Validation\n\n Function for calculation rmse without loading to memory"},{"metadata":{"_uuid":"018d0ac6cdf31c137a24ac7decd5fba8cc7945a8","_cell_guid":"149ae56a-ffdc-473f-b48a-dea230bcb355","collapsed":true,"trusted":false},"cell_type":"code","source":"def get_rmse(ytest_input='ytest', pred_input='pred'):\n    n, loss = 0, 0\n    reader_ytest = open(ytest_input, 'r')\n    reader_pred = open(pred_input, 'r')\n\n    for label, pred in tqdm(zip(reader_ytest, reader_pred)):    \n        n+=1\n        true_score = float(label)\n        pred_score = float(pred)\n        loss += np.square(pred_score - true_score)\n    reader_ytest.close()\n    reader_pred.close()\n    return np.sqrt(loss / n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d868901968bfbd7c6e281d0736c7513adefa059a","_cell_guid":"25212e57-f8a5-469b-bf0b-734793138d79"},"cell_type":"markdown","source":"Unfortunately, kaggle server does not have vw, so I commented on all its calls\n\nTraining process lasted no more than 10 minutes on my local machine with 8 Gb RAM"},{"metadata":{"_uuid":"a75eff71b91aa303f611832edfe86bfc25c71d60","_cell_guid":"91688c7c-cdf2-4e3b-a802-92aacb8e8680","trusted":false,"collapsed":true},"cell_type":"code","source":"# ! vw -d train --loss_function squared -f model -b 16 --passes 10 --cache_file cache --quiet\n# ! vw -i model -t valid -r pred --quiet\n# print('Validation RMSE: {:.5}'.format(get_rmse('yvalid', 'pred')))\n\nprint('Validation RMSE: 0.24855')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97dd5c260e62637292cada191113043920a13d76","_cell_guid":"fb673bdc-bc32-439b-a278-26ad0e1b7cc7","trusted":false,"collapsed":true},"cell_type":"code","source":"# ! vw -d train --loss_function squared --learning_rate 0.01 -f model -b 20 --passes 10 --cache_file cache --quiet\n# ! vw -i model -t valid -r pred --quiet\n# print('Validation RMSE: {:.5}'.format(get_scores('yvalid', 'pred')))\n\nprint('Validation RMSE: 0.22743')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69f6df7fb49c527b224a0d80e32b5c9632f2bd70","_cell_guid":"160f274a-016d-4fb9-bd70-79c266d47bf5","trusted":false,"collapsed":true},"cell_type":"code","source":"# ! vw -d train --loss_function squared --learning_rate 0.01 -f model -b 26 --passes 20 --cache_file cache --quiet\n# ! vw -i model -t valid -r pred --quiet\n# print('Validation RMSE: {:.5}'.format(get_rmse('yvalid', 'pred')))\n\nprint('Validation RMSE: 0.22686')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f439eca68fdfe36aa2ece1bf2ceed4c074d2f866","_cell_guid":"9bf5e809-0797-4ce7-b5f4-1bb2195df45a"},"cell_type":"markdown","source":"## Submission"},{"metadata":{"_uuid":"b0e9161ddffbce47d65d3088742934e13de42078","_cell_guid":"4e251f1a-5294-402a-a78c-5dbf5cbdb0f4","collapsed":true,"trusted":false},"cell_type":"code","source":"def test2vw(data, features_extractor, test_output='test'):\n    writer_test = open(test_output, 'w')\n    \n    for row in tqdm(data.iterrows(), total=data.shape[0], miniters=2500):\n        features = features_extractor(row[1])\n        output_line = '{0}\\n'.format(features)\n        writer_test.write(output_line)\n            \n    writer_test.close()\n\n# need more hard memory on server, so I comment next line    \n# test2vw(df[train_len:], vw_extractor)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bba02b23910fc0f707dda2ec0ea68e1c5719612a","_cell_guid":"e0c9e7cb-4a87-4cdf-8b4d-7eca35c10fbb"},"cell_type":"markdown","source":"Concat train and validation files for training final model"},{"metadata":{"_uuid":"0895ff5d87ea26f348aa3b18bbab611da88f12ad","_cell_guid":"3221e2dd-f43b-4511-b604-42b1a388eac2","collapsed":true,"trusted":false},"cell_type":"code","source":"# !cp train full_train\n# !cat valid >> full_train\n\n# ! vw -i model -t test -r pred --quiet\n\n# sub = pd.read_csv('pred', header=None)\n# sub.index = df[train_len:].index\n# sub.columns = [target_col]\n# sub.reset_index(inplace=True)\n# sub[target_col].clip(0.0, 1.0, inplace=True) \n# sub.to_csv(\"vw_sub.csv\",index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca7cec901701ee0d9d99a7cc782518d434f0b70d","_cell_guid":"0d882444-bc12-4b4d-8dae-b9de1b18814c"},"cell_type":"markdown","source":"Private score of the best model is 0.2321 and there is a high potential for improvement.\nBut this result is already enough for blending))\n\nThis is my first kernel and I hope my code is clearer than my comments\n\n* **LibFM Is Coming...**"},{"metadata":{"_uuid":"2ca7890745424040fc9e099072fd88743f208b8f","_cell_guid":"780533d6-a807-4e71-a14c-1d604c91fdc4","trusted":false,"collapsed":true},"cell_type":"code","source":"print('end')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"32615b3d439c7b1fc7fc002906efdde52b3af469","_cell_guid":"cf14c91f-3693-4ec0-b53c-a2c9d9dd088e","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}