{"cells":[{"metadata":{"_cell_guid":"97521ef5-0845-407d-a115-0691fa73da87","_uuid":"d6b5ce86b28e0c40683796252360150beca57e81"},"cell_type":"markdown","source":"## Intro"},{"metadata":{"_cell_guid":"48dfe719-0ea0-4016-8a3e-3cdf7a5872a2","_uuid":"79d874b1196fbcf0c6b646e2a73499a252ca2395"},"cell_type":"markdown","source":"In this script I use word embeddings and RNN to predict the deal_probability directly.  <br>\nThe embeddings are learng as part of the training process. <br>\nUsiging FastText's pre train vectores in Russian, better results can be achieved<br>\nPre-traiened FastText: https://s3-us-west-1.amazonaws.com/fasttext-vectors/word-vectors-v2/cc.ru.300.vec.gz"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\nfrom keras.preprocessing import sequence, text\nfrom keras.models import Sequential\nimport keras.layers as layer \nfrom keras import regularizers\nfrom keras.callbacks import EarlyStopping\nfrom keras import backend as K\n\nfrom sklearn.model_selection import train_test_split","execution_count":1,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')\ndf_all = pd.concat([df_train, df_test])\ndel(df_train, df_test)","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"9cbe51e2-3a7f-4c9a-ad2f-44ac561c5703","_uuid":"0323fd79b64a4a48b49b13c54eb7b545934d5a4b"},"cell_type":"markdown","source":"## Setting up the text feature"},{"metadata":{"collapsed":true,"_cell_guid":"0fddeb07-b995-4a35-809b-f988fbaa819f","_uuid":"cdd3fe2a01a34df20f957e4d34e3259cc6ad02e6","trusted":true},"cell_type":"code","source":"text_features = [ 'title', 'category_name', 'parent_category_name', 'description', 'param_1', 'param_2', 'param_3',]","execution_count":3,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"f87e670d-82a1-4eba-a07e-017b72a5d486","_uuid":"f888cc84747961d46b2102f47dd95ed8363e6155","trusted":true},"cell_type":"code","source":"df_all = df_all[text_features + ['deal_probability']]","execution_count":4,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"737a1c01-e7f1-4d82-870d-89731d7fd36a","_uuid":"42ce585d478b7695bd4ba07c41510bccfa3bc49c","trusted":true},"cell_type":"code","source":"df_all['text'] = \"\"\nfor text_col in text_features:\n    df_all['text'] += \" \" + df_all[text_col].fillna(\"\")\n    \npattern = re.compile('[^(?u)\\w\\s]+')\ndf_all['text'] =df_all['text'].apply(lambda x: re.sub(pattern, \"\", x).lower())","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"239f5762-bdc6-47e3-aa57-d293d26d9655","_uuid":"8e760f118d977eef7ec5d77762709b62cb58483b"},"cell_type":"markdown","source":"## Tokenizing"},{"metadata":{"collapsed":true,"_cell_guid":"c6793428-c4e7-4e52-aea4-dabfdba9fe9b","_uuid":"80c52938d897800b7cbd777ab40cdaf0f0827d81","trusted":true},"cell_type":"code","source":"max_len = 30\ntk = text.Tokenizer(num_words=50000)\ntk.fit_on_texts(df_all['text'].str.lower().tolist())\nX = tk.texts_to_sequences(df_all['text'].str.lower().values)\nX = sequence.pad_sequences(X, maxlen=max_len)","execution_count":6,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"375403f4-b007-4051-8271-896abc5e7bbb","_uuid":"917f9ad4a07fabd8e3770f22580ac4f261d76fa0","trusted":true},"cell_type":"code","source":"df_all.drop(text_features, axis=1, inplace=True)","execution_count":7,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"1f6d74ab-3edc-484b-ac6a-38666d2b8ad0","_uuid":"e7ee41808f4d44fdab06c8b5ad71f0d9b03e9bb8","trusted":true},"cell_type":"code","source":"word_index = tk.word_index","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"4894bb0c-884d-4cef-adbe-39f2a2b3d898","_uuid":"da026d060a8d065d2b8429662e63916ba7d2149c"},"cell_type":"markdown","source":"## Train test split"},{"metadata":{"collapsed":true,"_cell_guid":"3b86658c-d93a-4ca3-ae00-5ea5533e5add","_uuid":"83f94d20d1981e60478c88c764d6068627b82289","trusted":true},"cell_type":"code","source":"df_train = df_all[df_all['deal_probability'].notnull()]\nX_train, X_val, y_train, y_val  = train_test_split(X[:len(df_train)], df_train['deal_probability'].values, test_size=0.01)","execution_count":27,"outputs":[]},{"metadata":{"_cell_guid":"611e9f33-6b1f-49dd-8d21-dc8d13b7055a","_uuid":"28ea1adb5ca5d1c5a16169cd9ffc4b9431367fb1"},"cell_type":"markdown","source":"## Build keras model"},{"metadata":{"collapsed":true,"_cell_guid":"1fd29cb4-d064-4872-a671-0f0c03d98577","_uuid":"326982e2f6c93a57387e80643625da7c85196599","trusted":true},"cell_type":"code","source":"def rmse(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))","execution_count":10,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"fa36191ff4b049a872cec9779445032208e81c9f","_cell_guid":"868c73b6-6069-4295-a031-cc9224bfe250","trusted":true},"cell_type":"code","source":"def get_model():\n\n    model = Sequential()\n    model.add(layer.Embedding(len(word_index) + 1, 30, input_shape=(max_len,)))\n    model.add(layer.LSTM(30, recurrent_dropout=0.2, dropout=0.2, kernel_regularizer=regularizers.l2(2e-5),\n                activity_regularizer=regularizers.l1(2e-5)))\n    \n    model.add(layer.Dense(32,  kernel_regularizer=regularizers.l2(2e-5),\n                activity_regularizer=regularizers.l1(2e-5)))\n    model.add(layer.PReLU())\n    model.add(layer.Dropout(0.2))\n    model.add(layer.BatchNormalization())\n    \n    model.add(layer.Dense(32, kernel_regularizer=regularizers.l2(2e-5),\n                activity_regularizer=regularizers.l1(2e-5)))\n    model.add(layer.PReLU())\n    model.add(layer.Dropout(0.2))\n    model.add(layer.BatchNormalization())\n\n    \n    model.add(layer.Dense(1))\n    model.add(layer.Activation('sigmoid'))\n    model.compile(loss='mean_squared_error', optimizer='nadam')\n    \n    \n    \n    return model  ","execution_count":48,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"63767cfa-4a37-482e-a7c2-e7c28fc9d38c","_uuid":"6a211d498235607f80f5cf4b36a6eb4c06e63b28","trusted":true},"cell_type":"code","source":"early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=4, mode='auto')","execution_count":49,"outputs":[]},{"metadata":{"_cell_guid":"dbd1c8e0-f284-45a3-a5b0-1937ef3cafd4","_uuid":"163ee4d9cb2609281493c832dcb6d2e6fc8275a8","trusted":true},"cell_type":"code","source":"model = get_model()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"66058ed9-60ed-4f77-94e2-c6e4513615ee","_uuid":"e3cf8f7a998a8a066a9099bbb4d433ec10f88bca","trusted":true},"cell_type":"code","source":"model.fit(X_train, y=y_train, \n                     validation_data = (X_val, y_val),\n                     batch_size=4096, epochs=10000,\n                     verbose=1, shuffle=True, callbacks=[early_stopping])","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"4c4ce102-1ae9-439d-9d14-2e330e0ddd93","_uuid":"becf9a3e6c37de823e14df1b13d0eace4f89694b","trusted":false},"cell_type":"code","source":"y_pred = model.predict(X[len(df_train):])\ndf_test = pd.read_csv('../input/test.csv')\ndf_test['deal_probability'] = y_pred.T[0]\ndf_test = df_test[['item_id','deal_probability']]","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_cell_guid":"91774760-a99e-4b57-98a7-2bdf91bdc110","_uuid":"da6dec0e783685db4fcfcba7c1ece7157f44384b","trusted":false},"cell_type":"code","source":"df_test.to_csv('word2score.csv', index=None)","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}