{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfrom random import shuffle\n%matplotlib inline\n\nfrom nltk.tokenize import TweetTokenizer\nimport datetime\nimport lightgbm as lgb\nfrom scipy import stats\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom wordcloud import WordCloud\nfrom collections import Counter\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import LinearSVC\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.metrics import accuracy_score\nfrom xgboost import XGBClassifier\nimport time\npd.set_option('max_colwidth',400)\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D, GRU, CuDNNGRU, CuDNNLSTM, BatchNormalization\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, MaxPooling1D, Add, Flatten, Masking\nfrom keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D\nfrom keras.models import Model, load_model\nfrom keras import initializers, regularizers, constraints, optimizers, layers, callbacks\nfrom keras import backend as K\nfrom keras.engine import InputSpec, Layer\nfrom keras.optimizers import Adam\n\nfrom keras.callbacks import ModelCheckpoint, TensorBoard, Callback, EarlyStopping, ReduceLROnPlateau\nfrom sklearn.preprocessing import OneHotEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b77f6a1c831c98851143feb25c9903cb1154bf2","_kg_hide-input":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/finaltrain/newTRAIN.csv\")\n#train = pd.read_csv(\"../input/ndsczw/train.csv\")\nval = pd.read_csv(\"../input/finaltrain/newVAL.csv\")\ntest = pd.read_csv(\"../input/ndsczw/test.csv\")\n\ndef dataset_split(data):\n    df_beauty=data[data[\"Category\"]<=16]\n    df_fashion=data[data['Category'].between(17, 30, inclusive=True)]\n    df_mobile=data[data['Category'].between(31, 57, inclusive=True)]\n    return df_beauty\n\ndef shuffle_data(df):\n    def shuffle_string (string):\n        listString = string.split(\" \")\n        shuffle(listString)\n        return \" \".join(listString)\n    \n    trainAppend = df.copy()\n    trainAppend.title = df.title.apply(shuffle_string)\n    expandedDf = pd.concat([df,trainAppend],ignore_index=True)\n    return expandedDf\n\n\ntrain = dataset_split(train)\ntrain = shuffle_data(train)\nval = dataset_split(val)\ntest = test[:76545]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d081b2f0d46faf01a943c309568c27f92462f94"},"cell_type":"code","source":"max_features = 90000\ntk = Tokenizer(lower = True, filters='', num_words=max_features)\nfull_text = list(train['title'].values) + list(test[\"title\"].values)\ntk.fit_on_texts(full_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33929a60e1872b73e40424daa1178a3d8fbf8f5a"},"cell_type":"code","source":"train_tokenized = tk.texts_to_sequences(train['title'].fillna('missing'))\ntest_tokenized = tk.texts_to_sequences(test['title'].fillna('missing'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4330f6c01064b5fda4ca9661dc4f1cefb439cf75"},"cell_type":"code","source":"train['title'].apply(lambda x: len(x.split())).plot(kind='hist');\nplt.yscale('log');\nplt.title('Distribution of question text length in characters')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a3f7fc48edb7d8d4aec66042dfb45e5af225c44"},"cell_type":"code","source":"max_len = 70\nX_train = pad_sequences(train_tokenized, maxlen = max_len)\nX_val = pad_sequences(tk.texts_to_sequences(val.title.fillna('missing')), maxlen = max_len)\nX_test = pad_sequences(test_tokenized, maxlen = max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"744ee4a1fbc66cb47aae6a18f829dea284b860c9"},"cell_type":"code","source":"embedding_path = \"../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt\"\n#embedding_path = \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cd86c2400db5ea1511b107250c8aa8c98ea909b"},"cell_type":"code","source":"embed_size = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2948d63cafb76b446c5e167a6dce7915ca43da6d"},"cell_type":"code","source":"def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(embedding_path, encoding='utf-8', errors='ignore'))\nall_embs = np.stack(embedding_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\n\nword_index = tk.word_index\nnb_words = max(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words + 1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9011f3f1e1affbe0795e2f7a98d4f0e98a8c4925"},"cell_type":"code","source":"ohe = OneHotEncoder(sparse=False)\ny_ohe = ohe.fit_transform(train['Category'].values.reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55bc04ce8d311a8176886a211217d4e4459d4700"},"cell_type":"code","source":"def build_model(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, kernel_size1=3, kernel_size2=2, dense_units=128, dr=0.1, conv_size=32, epochs=20):\n    file_path = \"best_model.hdf5\"\n    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 1,\n                                  save_best_only = True, mode = \"min\")\n    early_stop = EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 3)\n\n    inp = Input(shape = (max_len,))\n    x = Embedding(max_features + 1, embed_size, weights=[embedding_matrix],trainable = False)(inp)\n    x1 = SpatialDropout1D(spatial_dr)(x)\n\n    x_gru = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)\n    x_lstm = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)\n    \n    x_conv1 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(x_gru)\n    avg_pool1_gru = GlobalAveragePooling1D()(x_conv1)\n    max_pool1_gru = GlobalMaxPooling1D()(x_conv1)\n    \n    x_conv2 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(x_gru)\n    avg_pool2_gru = GlobalAveragePooling1D()(x_conv2)\n    max_pool2_gru = GlobalMaxPooling1D()(x_conv2)\n    \n    \n    x_conv3 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(x_lstm)\n    avg_pool1_lstm = GlobalAveragePooling1D()(x_conv3)\n    max_pool1_lstm = GlobalMaxPooling1D()(x_conv3)\n    \n    x_conv4 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(x_lstm)\n    avg_pool2_lstm = GlobalAveragePooling1D()(x_conv4)\n    max_pool2_lstm = GlobalMaxPooling1D()(x_conv4)\n    \n    \n    x = concatenate([avg_pool1_gru, max_pool1_gru, avg_pool2_gru, max_pool2_gru,\n                    avg_pool1_lstm, max_pool1_lstm, avg_pool2_lstm, max_pool2_lstm])\n    x = BatchNormalization()(x)\n    x = Dropout(dr)(Dense(dense_units, activation='relu') (x))\n    x = BatchNormalization()(x)\n    x = Dropout(dr)(Dense(int(dense_units / 2), activation='relu') (x))\n    x = Dense(17, activation = \"sigmoid\")(x)\n    model = Model(inputs = inp, outputs = x)\n    model.compile(loss = \"binary_crossentropy\", optimizer = Adam(lr = lr, decay = lr_d), metrics = [\"accuracy\"])\n    model.summary()\n    history = model.fit(X_train, y_ohe, batch_size = 512, epochs = epochs, validation_split=0.3, \n                        verbose = 1, callbacks = [check_point, early_stop])\n    model = load_model(file_path)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90ac92a570fe3c180252dfcf1f22c5f98a08e615"},"cell_type":"code","source":"\n%%time\nmodel1 = build_model(lr = 1e-4, lr_d = 0, units = 128, spatial_dr = 0.5, kernel_size1=4, kernel_size2=3, dense_units=58*2, dr=0.1, conv_size=16, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b206256803deeb5ba630578da1175b3cc01f555"},"cell_type":"code","source":"pred = model1.predict(X_test, batch_size = 1024, verbose = 1)\npredictions = np.round(np.argmax(pred, axis=1)).astype(int)\nprint(np.unique(predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"754211d82f222f50cbb19712aca94984b2191588"},"cell_type":"code","source":"def build_model1(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, kernel_size1=3, kernel_size2=2, dense_units=128, dr=0.1, conv_size=32, epochs=20):\n    file_path = \"best_model.hdf5\"\n    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 1,\n                                  save_best_only = True, mode = \"min\")\n    early_stop = EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 3)\n\n    inp = Input(shape = (max_len,))\n    x = Embedding(max_features + 1, embed_size, trainable = False)(inp)\n    x1 = SpatialDropout1D(spatial_dr)(x)\n\n    x_gru = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)\n    \n    x_conv1 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(x_gru)\n    avg_pool1_gru = GlobalAveragePooling1D()(x_conv1)\n    max_pool1_gru = GlobalMaxPooling1D()(x_conv1)\n    \n    x_conv2 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(x_gru)\n    avg_pool2_gru = GlobalAveragePooling1D()(x_conv2)\n    max_pool2_gru = GlobalMaxPooling1D()(x_conv2)\n\n    \n    \n    x = concatenate([avg_pool1_gru, max_pool1_gru, avg_pool2_gru, max_pool2_gru])\n    x = BatchNormalization()(x)\n    x = Dropout(dr)(Dense(dense_units, activation='relu') (x))\n    x = BatchNormalization()(x)\n    x = Dropout(dr)(Dense(int(dense_units / 2), activation='relu') (x))\n    x = Dense(17, activation = \"sigmoid\")(x)\n    model = Model(inputs = inp, outputs = x)\n    model.compile(loss = \"binary_crossentropy\", optimizer = Adam(lr = lr, decay = lr_d), metrics = [\"accuracy\"])\n    model.summary()\n    history = model.fit(X_train, y_ohe, batch_size = 512, epochs = epochs, validation_split=0.1, \n                        verbose = 1, callbacks = [check_point, early_stop])\n    model = load_model(file_path)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"836e8eec34c09dcca660d35f39ddd5ec4801147a"},"cell_type":"code","source":"def build_model2(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, kernel_size1=3, kernel_size2=2, dense_units=128, dr=0.1, conv_size=32, epochs=20):\n    file_path = \"best_model.hdf5\"\n    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 1,\n                                  save_best_only = True, mode = \"min\")\n    early_stop = EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 3)\n\n    inp = Input(shape = (max_len,))\n    x = Embedding(max_features + 1, embed_size, trainable = False)(inp)\n    x1 = SpatialDropout1D(spatial_dr)(x)\n\n    x_gru = Bidirectional(CuDNNGRU(units * 2, return_sequences = True))(x1)\n    x_gru = Bidirectional(CuDNNGRU(units, return_sequences = True))(x_gru)\n    \n    x_conv1 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(x_gru)\n    avg_pool1_gru = GlobalAveragePooling1D()(x_conv1)\n    max_pool1_gru = GlobalMaxPooling1D()(x_conv1)\n    \n    x_conv2 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(x_gru)\n    avg_pool2_gru = GlobalAveragePooling1D()(x_conv2)\n    max_pool2_gru = GlobalMaxPooling1D()(x_conv2)\n    \n    x = concatenate([avg_pool1_gru, max_pool1_gru, avg_pool2_gru, max_pool2_gru])\n    x = BatchNormalization()(x)\n    x = Dropout(dr)(Dense(dense_units, activation='relu') (x))\n    x = BatchNormalization()(x)\n    x = Dropout(dr)(Dense(int(dense_units / 2), activation='relu') (x))\n    x = Dense(17, activation = \"sigmoid\")(x)\n    model = Model(inputs = inp, outputs = x)\n    model.compile(loss = \"binary_crossentropy\", optimizer = Adam(lr = lr, decay = lr_d), metrics = [\"accuracy\"])\n    model.summary()\n    history = model.fit(X_train, y_ohe, batch_size = 512, epochs = epochs, validation_split=0.1, \n                        verbose = 1, callbacks = [check_point, early_stop])\n    model = load_model(file_path)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f6b13c224c17bf70eac09f9bc3846d8332fa8d4","scrolled":true},"cell_type":"code","source":"%%time\nmodel2 = build_model2(lr = 1e-4, lr_d = 1e-7, units = 64, spatial_dr = 0.3, kernel_size1=4, kernel_size2=3, dense_units = 64, dr=0.1, conv_size=8, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"582cadcbd9fdd70e4c6da7160d743057fd9a487e","scrolled":true},"cell_type":"code","source":"model3 = build_model1(lr = 1e-4, lr_d = 1e-7, units = 256, spatial_dr = 0.1, kernel_size1=4, kernel_size2=3, dense_units = 64, dr=0.1, conv_size=16, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"141e3c63b1bc57f9cfd131dddbe562d556afac8e"},"cell_type":"code","source":"pred = model3.predict(X_test, batch_size = 1024, verbose = 1)\npredictions = np.round(np.argmax(pred, axis=1)).astype(int)\nprint(np.unique(predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c5f7ad040c9433f5565720990a0993e72292018"},"cell_type":"code","source":"pred = model2.predict(X_test, batch_size = 1024, verbose = 1)\npredictions = np.round(np.argmax(pred, axis=1)).astype(int)\nprint(np.unique(predictions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0571d9edafb014eb480950a8c67fb892a4780240"},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        \"\"\"\n        Keras Layer that implements an Attention mechanism for temporal data.\n        Supports Masking.\n        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n        # Input shape\n            3D tensor with shape: `(samples, steps, features)`.\n        # Output shape\n            2D tensor with shape: `(samples, features)`.\n        :param kwargs:\n        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n        The dimensions are inferred based on the output shape of the RNN.\n        Example:\n            model.add(LSTM(64, return_sequences=True))\n            model.add(Attention())\n        \"\"\"\n        self.supports_masking = True\n        #self.init = initializations.get('glorot_uniform')\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        # do not pass the mask to the next layers\n        return None\n\n    def call(self, x, mask=None):\n        # eij = K.dot(x, self.W) TF backend doesn't support it\n\n        # features_dim = self.W.shape[0]\n        # step_dim = x._keras_shape[1]\n\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        # apply mask after the exp. will be re-normalized next\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            a *= K.cast(mask, K.floatx())\n\n        # in some cases especially in the early stages of training the sum may be almost zero\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n    #print weigthted_input.shape\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        #return input_shape[0], input_shape[-1]\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19030bd2b6277b9e5dbc6933ca0abcf68fc7b3f9"},"cell_type":"code","source":"def build_model3(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, dense_units=128, dr=0.1, use_attention=True):\n    inp = Input(shape = (max_len,))\n    x = Embedding(max_features + 1, embed_size, trainable = False)(inp)\n    x1 = SpatialDropout1D(spatial_dr)(x)\n\n    x_gru = Bidirectional(CuDNNGRU(units * 2, return_sequences = True))(x1)\n    if use_attention:\n        x_att = Attention(max_len)(x_gru)\n        x = Dropout(dr)(Dense(dense_units, activation='relu') (x_att))\n    else:\n        x_att = Flatten() (x_gru)\n        x = Dropout(dr)(Dense(dense_units, activation='relu') (x_att))\n\n    x = BatchNormalization()(x)\n    #x = Dropout(dr)(Dense(int(dense_units / 2), activation='relu') (x))\n    x = Dense(17, activation = \"sigmoid\")(x)\n    model = Model(inputs = inp, outputs = x)\n    model.compile(loss = \"binary_crossentropy\", optimizer = Adam(lr = lr, decay = lr_d), metrics = [\"accuracy\"])\n    #model.summary()\n    #history = model.fit(X_train, y_ohe, batch_size = 512, epochs = epochs, validation_split=0.1, \n    #                    verbose = 1, callbacks = [check_point, early_stop])\n    #model = load_model(file_path)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e7072209b383437e44560d062c1614418a1607a"},"cell_type":"code","source":"%%time\nfile_path = \"best_model.hdf5\"\ncheck_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 1,\n                              save_best_only = True, mode = \"min\")\nearly_stop = EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 3)\nmodel4 = build_model3(lr = 1e-3, lr_d = 1e-7, units = 128, spatial_dr = 0.3, dense_units=58*2, dr=0.1, use_attention=True)\nhistory = model4.fit(X_train, y_ohe, batch_size = 512, epochs = 10, validation_split=0.1, \n                    verbose = 1, callbacks = [check_point, early_stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fecf3405fccd307de507cea500df6b559e70cc95"},"cell_type":"code","source":"def build_model4(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, kernel_size1=3, kernel_size2=2, dense_units=128, dr=0.1, conv_size=32, epochs=20):\n    file_path = \"best_model.hdf5\"\n    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 1,\n                                  save_best_only = True, mode = \"min\")\n    early_stop = EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 3)\n\n    inp = Input(shape = (max_len,))\n    x = Embedding(max_features + 1, embed_size, trainable = False)(inp)\n    x1 = SpatialDropout1D(spatial_dr)(x)\n\n    x_gru = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)\n    \n    x_conv1 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(x_gru)\n    avg_pool1_gru = GlobalAveragePooling1D()(x_conv1)\n    max_pool1_gru = GlobalMaxPooling1D()(x_conv1)\n       \n    x = concatenate([avg_pool1_gru, max_pool1_gru])\n    x = BatchNormalization()(x)\n    x = Dropout(dr)(Dense(dense_units, activation='relu') (x))\n    x = BatchNormalization()(x)\n    #x = Dropout(dr)(Dense(int(dense_units / 2), activation='relu') (x))\n    x = Dense(17, activation = \"sigmoid\")(x)\n    model = Model(inputs = inp, outputs = x)\n    model.compile(loss = \"binary_crossentropy\", optimizer = Adam(lr = lr, decay = lr_d), metrics = [\"accuracy\"])\n    model.summary()\n    history = model.fit(X_train, y_ohe, batch_size = 512, epochs = epochs, validation_split=0.1, \n                        verbose = 1, callbacks = [check_point, early_stop])\n    model = load_model(file_path)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29fcf5b25f20763deb75612eac29d58add377fee"},"cell_type":"code","source":"%%time\nmodel5 = build_model4(lr = 1e-4, lr_d = 1e-7, units = 128, spatial_dr = 0.3, kernel_size1=3, dense_units=58*2, dr=0.8, conv_size=8, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b21c1b068660aaca49e9f1d1c8259bd5c97ad56"},"cell_type":"code","source":"model6 = build_model4(lr = 1e-4, lr_d = 1e-7, units = 256, spatial_dr = 0.3, kernel_size1=4, dense_units=58*2, dr=0.1, conv_size=8, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcae5f568eac0945c349eb2253df8010d03faa97"},"cell_type":"code","source":"def build_model5(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, kernel_size1=3, kernel_size2=2, dense_units=128, dr=0.1, conv_size=32, epochs=20):\n    file_path = \"best_model.hdf5\"\n    check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 1,\n                                  save_best_only = True, mode = \"min\")\n    early_stop = EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 3)\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1,\n                              patience=2, min_lr=0.001)\n\n    inp = Input(shape = (max_len,))\n    x = Embedding(max_features + 1, embed_size, trainable = False)(inp)\n    x1 = SpatialDropout1D(spatial_dr)(x)\n    x_m = Masking()(x1)\n    x_gru = LSTM(units)(x_m)\n\n    x = BatchNormalization()(x_gru)\n    x = Dropout(dr)(Dense(dense_units, activation='relu') (x))\n    x = BatchNormalization()(x)\n    #x = Dropout(dr)(Dense(int(dense_units / 2), activation='relu') (x))\n    x = Dense(17, activation = \"sigmoid\")(x)\n    model = Model(inputs = inp, outputs = x)\n    model.compile(loss = \"binary_crossentropy\", optimizer = Adam(lr = lr, decay = lr_d), metrics = [\"accuracy\"])\n    model.summary()\n    history = model.fit(X_train, y_ohe, batch_size = 512, epochs = epochs, validation_split=0.3, \n                        verbose = 1, callbacks = [check_point, early_stop, reduce_lr])\n    model = load_model(file_path)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b98e6c57a2f16b29b8ae7265919eb309e36e770"},"cell_type":"code","source":"model7 = build_model5(lr = 1e-4, lr_d = 1e-7, units = 128, spatial_dr = 0.3, kernel_size1=4, dense_units=64, dr=0.8, conv_size=8, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b361ee40a154a00f8864e529da151ab2c5fd94e"},"cell_type":"code","source":"model8 = build_model5(lr = 1e-4, lr_d = 1e-7, units = 256, spatial_dr = 0.3, kernel_size1=4, dense_units=64, dr=1.2, conv_size=8, epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d061f5fc4342f0f93ef09fa2f4e897fddcfbc33"},"cell_type":"code","source":"from nltk import word_tokenize\ntfv = TfidfVectorizer(min_df=3,  max_features=None, \n            strip_accents='unicode', analyzer='word',token_pattern=r'\\w{3,}',\n            tokenizer=word_tokenize,  ngram_range=(1, 3), use_idf=1,\n            smooth_idf=1,sublinear_tf=1, stop_words = 'english')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a9b7eb68b8f928be597631c45716dec9bd0bf85"},"cell_type":"code","source":"tfv.fit(list(train.title.values) + list(val.title.values) + list(test.title.values))\nxtrain = tfv.transform(train.title.values)\nxval = tfv.transform(val.title.values)\nxtest = tfv.transform(test.title.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd7e8c144b7f3c909e892c77d85e8b3076b4e373"},"cell_type":"code","source":"clf = xgb.XGBClassifier(max_depth=12, n_estimators=2000, colsample_bytree=0.8, random_state = 123, objective='multi:softmax',  num_class = 58, subsample=0.8, n_jobs=-1, learning_rate=0.1, silent = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"060d1be8a0e17d9ae53ee622afc4f53029019639"},"cell_type":"code","source":"ytrain, yval = train.Category, val.Category\neval_set_beauty  = [(xtrain,ytrain), (xval,yval)]\nclf.fit(xtrain, ytrain, eval_set = eval_set_beauty, eval_metric=['merror'], early_stopping_rounds=40)\nprint('Beauty Accuracy:', accuracy_score(ytrain, clf.predict(xtrain)))\nprint('Beauty Cross Validation Accuracy:', accuracy_score(yval, clf.predict(xval)))\n#beauty_preds = clf_beauty.predict(test_beauty_tfv, ntree_limit=1000)\nxgproba = clf.predict_proba(xtrain, ntree_limit=1000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"771278144d9fe02f509ee4e08b7a97d3e69b8283"},"cell_type":"code","source":"pred1 = model1.predict(X_train, batch_size = 1024, verbose = 1)\npred2 = model2.predict(X_train, batch_size = 1024, verbose = 1)1\npred3 = model3.predict(X_train, batch_size = 1024, verbose = 1)\npred4 = model4.predict(X_train, batch_size = 1024, verbose = 1)\npred5 = model5.predict(X_train, batch_size = 1024, verbose = 1)\npred6 = model6.predict(X_train, batch_size = 1024, verbose = 1)\npred7 = model7.predict(X_train, batch_size = 1024, verbose = 1)\npred8 = model8.predict(X_train, batch_size = 1024, verbose = 1)\nxgpred = clf.predict(X_train)\nohe = OneHotEncoder(sparse=False)\ny_ohe = ohe.fit_transform(xgpred.values.reshape(-1, 1))\n#pred = pred / 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98f76f940467b64ecbf3f14167236f431288295c"},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nimport xgboost as xgb\n\n\npredictions = np.round(np.argmax(pred, axis=1)).astype(int)\nx_ensemble = np.concatenate((pred1,pred2,pred3,pred4,pred5,pred6,pred7,pred8,y_ohe),axis=1)\ngbm = xgb.XGBClassifier(n_estimators= 2000, max_depth= 4, min_child_weight= 2, gamma=0.9, subsample=0.8, colsample_bytree=0.8, objective= 'multi:softmax', nthread= -1, scale_pos_weight=1)\ngbm.fit(x_ensemble, train.Category)\nensemble_predictions = gbm.predict(x_ensemble)\nprint(accuracy_score(ensemble_predictions, train.Category))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c08a4c46799e1068685ed1ee3ecf0bdf59518f2f"},"cell_type":"code","source":"pred1 = model1.predict(X_val, batch_size = 1024, verbose = 1)\npred = pred1\npred2 = model2.predict(X_val, batch_size = 1024, verbose = 1)\npred += pred2\npred3 = model3.predict(X_val, batch_size = 1024, verbose = 1)\npred += pred3\npred4 = model4.predict(X_val, batch_size = 1024, verbose = 1)\npred += pred4\npred5 = model5.predict(X_val, batch_size = 1024, verbose = 1)\npred += pred5\npred6 = model6.predict(X_val, batch_size = 1024, verbose = 1)\npred += pred6\npred7 = model7.predict(X_val, batch_size = 1024, verbose = 1)\npred += pred7\npred8 = model8.predict(X_val, batch_size = 1024, verbose = 1)\npred += pred8\nxgpred = clf.predict(X_val)\nohe = OneHotEncoder(sparse=False)\ny_ohe = ohe.fit_transform(xgpred.values.reshape(-1, 1))\n#pred = pred / 8\npred_ensemble = np.concatenate(pred1,pred2,pred3,pred4,pred5,pred6,pred7,pred8)\npredictions = np.round(np.argmax(pred,axis=1)).astype(int)\nprint(accuracy_score(predictions, val.Category))\nx_ensemble = np.concatenate((pred1,pred2,pred3,pred4,pred5,pred6,pred7,pred8,y_ohe),axis=1)\nensemble_predictions = gbm.predict(x_ensemble)\nprint(accuracy_score(ensemble_predictions, val.Category))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0db64fe6362402e4402a1a0baadb7888bc9cbc9"},"cell_type":"code","source":"pred1 = model1.predict(X_test, batch_size = 1024, verbose = 1)\npred = pred1\npred2 = model2.predict(X_test, batch_size = 1024, verbose = 1)\npred += pred2\npred3 = model3.predict(X_test, batch_size = 1024, verbose = 1)\npred += pred3\npred4 = model4.predict(X_test, batch_size = 1024, verbose = 1)\npred += pred4\npred5 = model5.predict(X_test, batch_size = 1024, verbose = 1)\npred += pred5\npred6 = model6.predict(X_test, batch_size = 1024, verbose = 1)\npred += pred6\npred7 = model7.predict(X_test, batch_size = 1024, verbose = 1)\npred += pred7\npred8 = model8.predict(X_test, batch_size = 1024, verbose = 1)\npred += pred8\nxgpred = clf.predict(X_val)\nohe = OneHotEncoder(sparse=False)\ny_ohe = ohe.fit_transform(xgpred.values.reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a85b44803f22516a660806b60d41230dfb7db04a"},"cell_type":"code","source":"x_ensemble = np.concatenate((pred1,pred2,pred3,pred4,pred5,pred6,pred7,pred8,y_ohe),axis=1)\nensemble_predictions = gbm.predict(x_ensemble)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"168293f1b010688eea232b402009b12c209ddc0b"},"cell_type":"code","source":"pd.DataFrame(ensemble_predictions).to_csv(\"lstm_cnn_submission.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}