{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nimport gc\nfrom keras import backend as K\nfrom keras.layers import Dense,Input,Bidirectional,Activation,Conv1D,GRU\nfrom keras.callbacks import Callback\nfrom keras.layers import Dropout,Embedding,GlobalMaxPooling1D, MaxPooling1D, Add, Flatten\nfrom keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D\nfrom keras import initializers, regularizers, constraints, optimizers, layers, callbacks\nfrom keras.callbacks import EarlyStopping,ModelCheckpoint\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\n# Any results you write to the current directory are saved as output.","execution_count":22,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/fasttext-russian-2m/wiki.ru.vec'\n\ntrain = pd.read_csv('../input/avito-demand-prediction/train.csv')\ntest = pd.read_csv('../input/avito-demand-prediction/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"eb5ffb5e35b107578faaaee0d9d987cd7e051361"},"cell_type":"code","source":"train['description'] = train['description'].astype(str)\ntest[\"description\"] = test['description'].astype(str)\n","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"5590d891fcf00a0e149244913c6592db2ce10338"},"cell_type":"code","source":"train[\"description\"] = train[\"description\"].fillna(\"fillna\")\ntest[\"description\"] = test[\"description\"].fillna(\"fillna\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d9643c725b444fefb0ac3a39a8d4bca0a3cad7db"},"cell_type":"code","source":"train['description'] = (train['description'].fillna('') + ' ' + train['title'].fillna('') + ' ' + train['parent_category_name'].fillna('') + ' ' + train['category_name'].fillna('') + ' ' + train['param_1'].fillna(''))\ntest['description'] = (test['description'].fillna('') + ' ' + test['title'].fillna('') + ' ' + test['parent_category_name'].fillna('') + ' ' + test['category_name'].fillna('') + ' ' + test['param_1'].fillna(''))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e8bd3575-f711-4ca6-a653-8ec1c74c0204","_uuid":"cf43ac37cbd14d8baa088648c2275123550135d6","trusted":true},"cell_type":"code","source":"X_train = train[\"description\"]\ny_train = train[[\"deal_probability\"]].values\ndel train\ngc.collect()\n\nX_test = test[\"description\"]\ndel test\ngc.collect()\n\n","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"da409613-3688-4d2e-a072-f67dee02617b","_uuid":"efad6a0ecd758a759f14287a69bfd9cafa8c8fb2","collapsed":true,"trusted":true},"cell_type":"code","source":"max_features=50000\nmaxlen=100\nembed_size=300","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"7a665c08-b4a9-4792-b40b-481b3da907e5","_uuid":"b07e998ccedaf3aaaf4b4e67b207ad5490eb24f7","trusted":true},"cell_type":"code","source":"from keras.preprocessing import text, sequence\ntok=text.Tokenizer(num_words=max_features)\ntok.fit_on_texts(X_train)\nX_train=tok.texts_to_sequences(X_train)\nX_test=tok.texts_to_sequences(X_test)\nx_train=sequence.pad_sequences(X_train,maxlen=maxlen)\ndel X_train\ngc.collect()\nx_test=sequence.pad_sequences(X_test,maxlen=maxlen)\ndel X_test\ngc.collect()","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"9e57a7cb-c061-4361-bbe2-05c0486a3f18","_uuid":"9488bc9d68dfd1fde1f99d23a9f1ed7b30ceb87f","collapsed":true,"trusted":true},"cell_type":"code","source":"embeddings_index = {}\nwith open(EMBEDDING_FILE,encoding='utf8') as f:\n    for line in f:\n        values = line.rstrip().rsplit(' ')\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype='float16')\n        embeddings_index[word] = coefs\n","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"e2490100-fc9c-4e46-ae84-7dfa65fcddba","_uuid":"d56ad119931a971b2588355deb726a045764c9ad","collapsed":true,"trusted":true},"cell_type":"code","source":"word_index = tok.word_index\n#prepare embedding matrix\nnum_words = min(max_features, len(word_index) + 1)\nembedding_matrix = np.zeros((num_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features:\n        continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        # words not found in embedding index will be all-zeros.\n        embedding_matrix[i] = embedding_vector","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"105a6e06-e2a7-4c00-87c9-6c0ae3e6ce5e","_uuid":"af6c76de3f9f97858c998df69fb0b5bbd26248f0","trusted":true},"cell_type":"code","source":"del tok\ngc.collect()\n\n","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"1ec148f1-2ce1-4197-a698-4daaac9a6872","_uuid":"a5d18366fadd3c71dac1b274a67708f84cef5fb8"},"cell_type":"markdown","source":"Defining Capsule Network"},{"metadata":{"_cell_guid":"7c3133c2-cc51-426e-9637-764d3c6b1e5b","_uuid":"370e345e2fb12cf5f271069cd2d13216a108490c","collapsed":true,"trusted":true},"cell_type":"code","source":"from keras.layers import K, Activation\nfrom keras.engine import Layer\nfrom keras.layers import Dense, Input, Embedding, Dropout, Bidirectional, GRU, Flatten, SpatialDropout1D, CuDNNLSTM\n\ndef squash(x, axis=-1):\n    # s_squared_norm is really small\n    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n    # return scale * x\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\n\n# A Capsule Implement with Pure Keras\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = K.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"1a4a1cf3-7faf-4ee4-a72e-a258169778a5","_uuid":"560d3faac051bbb95dae6f1bf7013d52b404533c","collapsed":true,"trusted":true},"cell_type":"code","source":"Routings = 6\nNum_capsule = 10\nDim_capsule = 16\nrate_drop_dense = 0.35\n\n#def root_mean_squared_error(y_true, y_pred):\n#    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) \ndef root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true))) \nsequence_input = Input(shape=(maxlen, ))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False)(sequence_input)\nx = SpatialDropout1D(0.2)(x)\nx = Bidirectional(GRU(32, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)\ncapsule = Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings,\n                      share_weights=True)(x)\ncapsule = Flatten()(capsule)\ncapsule = Dropout(0.4)(capsule)\npreds = Dense(1, activation=\"sigmoid\")(capsule)\nmodel = Model(sequence_input, preds)\nmodel.compile(loss='MSE',optimizer=Adam(lr=1e-3),metrics=['accuracy', root_mean_squared_error])","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"46df26aa-adcd-4b2c-8644-76a1e51df2bc","_uuid":"19975febdf6a0bd3077d8a92da13bb433085ce80","trusted":true},"cell_type":"code","source":"batch_size = 1000\nepochs = 3\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.9, random_state=233)\ndel x_train\ngc.collect()\ndel y_train\ngc.collect()","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"e1962822-5dfb-4249-a714-ce95346150d4","_uuid":"7956b05d34604689b7a10d56a61640091559eda2","collapsed":true,"trusted":true},"cell_type":"code","source":"# filepath=\"../input/best-model/best.hdf5\"\nfilepath=\"weights_base.best.hdf5\"\ncheckpoint = ModelCheckpoint(filepath, monitor='val_root_mean_squared_error', verbose=1, save_best_only=True, mode='min')\nearly = EarlyStopping(monitor=\"val_root_mean_squared_error\", mode=\"min\", patience=5)\ncallbacks_list = [checkpoint, early]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"265115a8-296e-4b67-a6fc-02d0a584b501","_uuid":"f7377e50952ffb6cc14bab3b34442788864eed68","scrolled":false,"trusted":true},"cell_type":"code","source":"model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)\n#Loading model weights\nmodel.load_weights(filepath)\nprint('Predicting....')\ny_pred = model.predict(x_test,batch_size=1024,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5c703574-cf76-495f-b800-1fc6c9384ded","_uuid":"ddc5afff4b22841fb184e32cc4b19a2225dec451","collapsed":true,"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/avito-demand-prediction/sample_submission.csv')\nsub['deal_probability'] = y_pred\nsub['deal_probability'].clip(0.0, 1.0, inplace=True)\nsub.to_csv('gru_capsule_description.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"391004b0-7159-4567-9bb9-a364f92527be","_uuid":"821b5bcf03ad665022b69b8da6a166e0c2c789a3","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"78c64614-5215-4d6b-bbb4-7298b7307494","_uuid":"d0e23473f83ceee9ec3c5ad009bc618ec860ceac","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}