{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport gc\nfrom keras import backend as K\nfrom keras.layers import Dense,Input,Bidirectional,Activation,Conv1D,GRU\nfrom keras.callbacks import Callback\nfrom keras.layers import Dropout,Embedding,GlobalMaxPooling1D, MaxPooling1D, Add, Flatten\nfrom keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D\nfrom keras import initializers, regularizers, constraints, optimizers, layers, callbacks\nfrom keras.callbacks import EarlyStopping,ModelCheckpoint\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\nimport re\nfrom unidecode import unidecode\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib\n%matplotlib inline\nimport matplotlib.pyplot as plt # for plotting\n# Any results you write to the current directory are saved as output.","execution_count":34,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/fasttext-russian-2m/wiki.ru.vec'\n\ntrain = pd.read_csv('../input/avito-demand-prediction/train.csv', usecols=['description', 'deal_probability'])\ntest = pd.read_csv('../input/avito-demand-prediction/test.csv', usecols=['description'])","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b52c8b421d7bbbd3b432f5fd7fa64f26796455f7"},"cell_type":"code","source":"special_character_removal = re.compile(r'[^A-Za-z\\.\\-\\?\\!\\,\\#\\@\\% ]',re.IGNORECASE)\ndef clean_text(x):\n    x_ascii = unidecode(x)\n    x_clean = special_character_removal.sub('',x_ascii)\n    return x_clean\n\ntrain['description'] = train['description'].apply(lambda x: clean_text(str(x)))\ntest['description'] = test['description'].apply(lambda x: clean_text(str(x)))","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"e8bd3575-f711-4ca6-a653-8ec1c74c0204","_uuid":"cf43ac37cbd14d8baa088648c2275123550135d6","collapsed":true,"trusted":true},"cell_type":"code","source":"train['description'] = train['description'].astype(str)\ntest[\"description\"] = test['description'].astype(str)\ntrain[\"description\"].fillna(\"что нибудь\")\ntest[\"description\"].fillna(\"что нибудь\")\n\ncols = \"description\"\ntrain[cols + '_num_words'] = train[cols].apply(lambda comment: len(comment.split())) # Count number of Words\ntrain[cols + '_num_unique_words'] = train[cols].apply(lambda comment: len(set(w for w in comment.split())))\ntrain[cols + '_words_vs_unique'] = train[cols+'_num_unique_words'] / train[cols+'_num_words'] * 100\ntrain[cols + '_num_chars'] = train[cols].apply(len)\n\ntest[cols + '_num_words'] = test[cols].apply(lambda comment: len(comment.split())) # Count number of Words\ntest[cols + '_num_unique_words'] = test[cols].apply(lambda comment: len(set(w for w in comment.split())))\ntest[cols + '_words_vs_unique'] = test[cols+'_num_unique_words'] / test[cols+'_num_words'] * 100\ntest[cols + '_num_chars'] = test[cols].apply(len)\n","execution_count":42,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff6fbea5d9aa8879450936dc471bb3ea6c8b1db8"},"cell_type":"code","source":"features = train[['description_num_chars', 'description_words_vs_unique','description_num_unique_words']]\ntest_features = test[['description_num_chars', 'description_words_vs_unique','description_num_unique_words']]\nfeatures = features.fillna(0)\ntest_features = test_features.fillna(0)\nss = StandardScaler()\nss.fit(np.vstack([features, test_features]))\nfeatures = ss.transform(features)\ntest_features = ss.transform(test_features)","execution_count":43,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea30ffe63a4d93215706464e06c26425e9870667"},"cell_type":"code","source":"X_train = train[\"description\"]\ny_train = train[[\"deal_probability\"]].values\n\ndel train\ngc.collect()\n\nX_test = test[\"description\"]\ndel test\ngc.collect()","execution_count":44,"outputs":[]},{"metadata":{"_cell_guid":"7abce6cf-fcab-45fa-a7a8-38180d14dbb9","_uuid":"49a48d7986cd280684860cb3ae694a696963ec90"},"cell_type":"markdown","source":"Max_features - Max number of words used in the tokenizer, the more features the better I asssume\n\nMaxlen - The cutoff point of word length in a description\n\nEmbed_size - Since we are using pretrained fasttext embeddings, it is already trained for 300 dimensions"},{"metadata":{"_cell_guid":"da409613-3688-4d2e-a072-f67dee02617b","_uuid":"efad6a0ecd758a759f14287a69bfd9cafa8c8fb2","collapsed":true,"trusted":true},"cell_type":"code","source":"max_features=20000\nmaxlen=100\nembed_size=300","execution_count":45,"outputs":[]},{"metadata":{"_cell_guid":"7a665c08-b4a9-4792-b40b-481b3da907e5","_uuid":"b07e998ccedaf3aaaf4b4e67b207ad5490eb24f7","trusted":true},"cell_type":"code","source":"from keras.preprocessing import text, sequence\ntok=text.Tokenizer(num_words=max_features)\ntok.fit_on_texts(X_train)\nX_train=tok.texts_to_sequences(X_train)\nX_test=tok.texts_to_sequences(X_test)\nx_train=sequence.pad_sequences(X_train,maxlen=maxlen)\ndel X_train\ngc.collect()\nx_test=sequence.pad_sequences(X_test,maxlen=maxlen)\ndel X_test\ngc.collect()","execution_count":46,"outputs":[]},{"metadata":{"_cell_guid":"9e57a7cb-c061-4361-bbe2-05c0486a3f18","_uuid":"9488bc9d68dfd1fde1f99d23a9f1ed7b30ceb87f","collapsed":true,"trusted":true},"cell_type":"code","source":"embeddings_index = {}\nwith open(EMBEDDING_FILE,encoding='utf8') as f:\n    for line in f:\n        values = line.rstrip().rsplit(' ')\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype='float16')\n        embeddings_index[word] = coefs\n","execution_count":47,"outputs":[]},{"metadata":{"_cell_guid":"e2490100-fc9c-4e46-ae84-7dfa65fcddba","_uuid":"d56ad119931a971b2588355deb726a045764c9ad","collapsed":true,"trusted":true},"cell_type":"code","source":"word_index = tok.word_index\n#prepare embedding matrix\nnum_words = min(max_features, len(word_index) + 1)\nembedding_matrix = np.zeros((num_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features:\n        continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        # words not found in embedding index will be all-zeros.\n        embedding_matrix[i] = embedding_vector","execution_count":48,"outputs":[]},{"metadata":{"_cell_guid":"105a6e06-e2a7-4c00-87c9-6c0ae3e6ce5e","_uuid":"af6c76de3f9f97858c998df69fb0b5bbd26248f0","trusted":true},"cell_type":"code","source":"del tok\ngc.collect()\n\n","execution_count":49,"outputs":[]},{"metadata":{"_cell_guid":"1ec148f1-2ce1-4197-a698-4daaac9a6872","_uuid":"a5d18366fadd3c71dac1b274a67708f84cef5fb8"},"cell_type":"markdown","source":"Please change the size of GRU and CONV1D to 128 and 64 preferably, used a smaller size to get quick outputs"},{"metadata":{"_cell_guid":"1a4a1cf3-7faf-4ee4-a72e-a258169778a5","_uuid":"560d3faac051bbb95dae6f1bf7013d52b404533c","trusted":true},"cell_type":"code","source":"#def root_mean_squared_error(y_true, y_pred):\n#    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) \ndef root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true))) \nfeatures_input = Input(shape=(features.shape[1],))\nsequence_input = Input(shape=(maxlen, ))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False)(sequence_input)\nx = SpatialDropout1D(0.5)(x)\n#x = Bidirectional(LSTM(32, return_sequences=True,dropout=0.1,recurrent_dropout=0.1, return_state = True))(x)\nx, x_h, x_c = Bidirectional(GRU(32, return_sequences=True,dropout=0.1,recurrent_dropout=0.1, return_state = True))(x)\n#x = Bidirectional(GRU(32, return_sequences=True,dropout=0.1,recurrent_dropout=0.1, return_state = True))(x)\n#x = Conv1D(16, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(x)\navg_pool = GlobalAveragePooling1D()(x)\nmax_pool = GlobalMaxPooling1D()(x)\n\nx = concatenate([avg_pool, max_pool, x_h, features_input]) \npreds = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model([sequence_input, features_input], preds)\nmodel.compile(loss='MSE',optimizer=Adam(lr=1e-3),metrics=['accuracy',root_mean_squared_error])","execution_count":51,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"076f742be55cb5cce0e668aac9bdc3cb0da17033"},"cell_type":"code","source":"length_train = len(x_train)\nX_tra = x_train[int(length_train/10):]\nfeatures_tra = features[int(length_train/10):]\nX_val = x_train[:int(length_train/10)]\nfeatures_val = features[:int(length_train/10)]\ny_tra = y_train[int(length_train/10):]\ny_val = y_train[:int(length_train/10)]","execution_count":52,"outputs":[]},{"metadata":{"_cell_guid":"e1962822-5dfb-4249-a714-ce95346150d4","_uuid":"7956b05d34604689b7a10d56a61640091559eda2","collapsed":true,"trusted":true},"cell_type":"code","source":"# filepath=\"../input/best-model/best.hdf5\"\nbatch_size = 1000\nepochs = 2\nfilepath=\"weights_base.best.hdf5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\nearly = EarlyStopping(monitor=\"val_acc\", mode=\"max\", patience=5)\ncallbacks_list = [checkpoint, early]","execution_count":53,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b24a8b06360b6be5af44bfbb0890942be0ace765"},"cell_type":"code","source":"del x_train\ngc.collect()\ndel y_train\ngc.collect()","execution_count":54,"outputs":[]},{"metadata":{"_cell_guid":"265115a8-296e-4b67-a6fc-02d0a584b501","scrolled":false,"_uuid":"f7377e50952ffb6cc14bab3b34442788864eed68","collapsed":true,"trusted":true},"cell_type":"code","source":"model.fit([X_tra, features_tra], y_tra, batch_size=batch_size, epochs=epochs, validation_data=([X_val, features_val], y_val),callbacks = callbacks_list,verbose=1)\n#Loading model weights\nmodel.load_weights(filepath)\nprint('Predicting....')\ny_pred = model.predict([x_test, test_features],batch_size=1024,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5c703574-cf76-495f-b800-1fc6c9384ded","_uuid":"ddc5afff4b22841fb184e32cc4b19a2225dec451","collapsed":true,"trusted":false},"cell_type":"code","source":"sub = pd.read_csv('../input/avito-demand-prediction/sample_submission.csv')\nsub['deal_probability'] = y_pred\nsub['deal_probability'].clip(0.0, 1.0, inplace=True)\nsub.to_csv('gru_cnn_description.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"78c64614-5215-4d6b-bbb4-7298b7307494","_uuid":"d0e23473f83ceee9ec3c5ad009bc618ec860ceac","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}