{"cells":[{"metadata":{"_uuid":"7e05ae8a9068ee927dc1fb9ce389d6439cdce646"},"cell_type":"markdown","source":"Maybe we don't need some fashion models (such as Capsule, Attention, Transformer) to reach a higher score. \nSeems GRU + Pooling works also very well :)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\n\nif not os.path.exists('obj'):\n    os.mkdir('obj')\nif not os.path.exists('models'):\n    os.mkdir('models')\n    \nprint(os.listdir('./models/'))\nprint(os.listdir(\"../input\"))\nprint(os.listdir('../input/embeddings'))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e2a7ca4c7be97e43fcdbfa6aec115003acb75f0"},"cell_type":"markdown","source":"Let's do some basic preprocessing. From https://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\nimport re\nimport sys\nimport pickle\n\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom tqdm import tqdm\ntqdm.pandas()\n\nmax_features = 99999\nMAX_TEXT_LENGTH = 70\n\ndef clean_text(x):\n    puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~',\n              '@', '£', '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à',\n              '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',\n              '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',\n              '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n        \"can't\" : \"cannot\",\n        \"couldn't\" : \"could not\",\n        \"didn't\" : \"did not\",\n        \"doesn't\" : \"does not\",\n        \"don't\" : \"do not\",\n        \"hadn't\" : \"had not\",\n        \"hasn't\" : \"has not\",\n        \"haven't\" : \"have not\",\n        \"he'd\" : \"he would\",\n        \"he'll\" : \"he will\",\n        \"he's\" : \"he is\",\n        \"i'd\" : \"I would\",\n        \"i'd\" : \"I had\",\n        \"i'll\" : \"I will\",\n        \"i'm\" : \"I am\",\n        \"isn't\" : \"is not\",\n        \"it's\" : \"it is\",\n        \"it'll\":\"it will\",\n        \"i've\" : \"I have\",\n        \"let's\" : \"let us\",\n        \"mightn't\" : \"might not\",\n        \"mustn't\" : \"must not\",\n        \"shan't\" : \"shall not\",\n        \"she'd\" : \"she would\",\n        \"she'll\" : \"she will\",\n        \"she's\" : \"she is\",\n        \"shouldn't\" : \"should not\",\n        \"that's\" : \"that is\",\n        \"there's\" : \"there is\",\n        \"they'd\" : \"they would\",\n        \"they'll\" : \"they will\",\n        \"they're\" : \"they are\",\n        \"they've\" : \"they have\",\n        \"we'd\" : \"we would\",\n        \"we're\" : \"we are\",\n        \"weren't\" : \"were not\",\n        \"we've\" : \"we have\",\n        \"what'll\" : \"what will\",\n        \"what're\" : \"what are\",\n        \"what's\" : \"what is\",\n        \"what've\" : \"what have\",\n        \"where's\" : \"where is\",\n        \"who'd\" : \"who would\",\n        \"who'll\" : \"who will\",\n        \"who're\" : \"who are\",\n        \"who's\" : \"who is\",\n        \"who've\" : \"who have\",\n        \"won't\" : \"will not\",\n        \"wouldn't\" : \"would not\",\n        \"you'd\" : \"you would\",\n        \"you'll\" : \"you will\",\n        \"you're\" : \"you are\",\n        \"you've\" : \"you have\",\n        \"'re\": \" are\",\n        \"wasn't\": \"was not\",\n        \"we'll\":\" will\",\n        \"didn't\": \"did not\",\n        \"tryin'\": \"trying\" }\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\n    \ndef save_obj(obj, name): \n    with open('obj/' + name + '.pkl', 'wb') as file:\n        pickle.dump(obj, file, pickle.HIGHEST_PROTOCOL)\n        \ndef load_obj(name):\n    with open('obj/' + name + '.pkl', 'rb') as file:\n        return pickle.load(file)\n\ndef load_data(data_path):\n    data = pd.read_csv(data_path)\n    data['question_text'] = data['question_text'].progress_apply(lambda x: x.lower())\n    data['question_text'] = data['question_text'].progress_apply(lambda x: clean_text(x))\n    data['question_text'] = data['question_text'].progress_apply(lambda x: clean_numbers(x))\n    data['question_text'] = data['question_text'].progress_apply(lambda x: replace_typical_misspell(x))\n    data['question_text'] = data['question_text'].fillna(\"_##_\").values\n    return data\n\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    \n    all_embs = np.stack(list(embeddings_index.values()))\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    # Why random embedding for OOV? what if use mean?\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    #embedding_matrix = np.random.normal(emb_mean, 0, (nb_words, embed_size)) # std 0\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(list(embeddings_index.values()))\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    #embedding_matrix = np.random.normal(emb_mean, 0, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(list(embeddings_index.values()))\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    #embedding_matrix = np.random.normal(emb_mean, 0, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix\n\ndata = load_data('../input/train.csv')\ndata = data.sample(frac = 1.0).reset_index(drop = True)\n\ntokenizer = Tokenizer(num_words = max_features)\ntokenizer.fit_on_texts(data['question_text'])\n\ntrain_df, val_df = train_test_split(data, test_size = 0.1, stratify = data['target'])\n\ntrain_x = train_df['question_text']\ntrain_y = train_df['target'].values\nval_x = val_df['question_text']\nval_y = val_df['target'].values\n\ntrain_x = tokenizer.texts_to_sequences(train_x)\nval_x = tokenizer.texts_to_sequences(val_x)\n\ntrain_x = pad_sequences(train_x, MAX_TEXT_LENGTH)\nval_x = pad_sequences(val_x, MAX_TEXT_LENGTH)\n\nembedding_glove = load_glove(tokenizer.word_index)\nembedding_fasttext = load_fasttext(tokenizer.word_index)\nembedding_para = load_para(tokenizer.word_index)\n\nsave_obj(embedding_glove, 'glove')\nsave_obj(embedding_fasttext, 'fasttext')\nsave_obj(embedding_para, 'para')\nsave_obj(tokenizer, 'tokenizer')\nsave_obj(train_x, 'train_x')\nsave_obj(val_x, 'val_x')\nsave_obj(train_y, 'train_y')\nsave_obj(val_y, 'val_y')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"245fd2420cc4cce305c2bbc96654b8d5f29de6f0"},"cell_type":"code","source":"os.listdir('obj/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c96ae2ec98dec8969c509bf739bd6a77ef150968"},"cell_type":"code","source":"import os\nimport time\nimport math\nimport keras\nimport argparse\n\nimport tensorflow as tf\n\nfrom keras.layers import Dense, Bidirectional, Dropout\nfrom keras.layers import Activation, CuDNNGRU, Conv1D, Input, Conv2D\nfrom keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D, MaxPooling2D\nfrom keras.layers import Concatenate, concatenate, Embedding, Flatten\nfrom keras.layers import SpatialDropout1D, Reshape, MaxPool2D, AveragePooling2D\nfrom keras.optimizers import SGD, RMSprop, Adam\nfrom keras.callbacks import Callback\nfrom keras.models import load_model, Model\nfrom keras.utils import Sequence\nfrom keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom keras.callbacks import LearningRateScheduler\nfrom keras import backend as K\n\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2af634ce211ec1bb8cc2e26043ed06ecef91883"},"cell_type":"code","source":"EMBEDDING_LENGTH = 300\nMAX_TEXT_LENGTH = 70\nBATCH_SIZE = 256\n\nMODEL_PATH = './models/'\n\nMAX_EPOCHS = 4","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6737d60efc365aec54a4a65290983139e8a3dcb"},"cell_type":"markdown","source":"Only use basic exponential learning rate decay"},{"metadata":{"trusted":true,"_uuid":"cc6d6eb032f4ca76462c49c41adb45c81a9e672f"},"cell_type":"code","source":"def step_decay_func(args):\n    def step_decay(epoch):\n        initial_lrate = args.lr\n        decay = 0.9\n        lrate = initial_lrate * pow(decay, epoch)\n        print('epoch:', epoch, ' lr:', lrate)\n        return lrate\n    return step_decay\n\ndef precision(y_true, y_pred):\n    \"\"\"Precision metric.\n    Only computes a batch-wise average of precision.\n    Computes the precision, a metric for multi-label classification of\n    how many selected items are relevant.\n    \"\"\"\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef recall(y_true, y_pred):\n    \"\"\"Recall metric.\n    Only computes a batch-wise average of recall.\n    Computes the recall, a metric for multi-label classification of\n    how many relevant items are selected.\n    \"\"\"\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef fbeta_score(y_true, y_pred, beta=1):\n    \"\"\"Computes the F score.\n    The F score is the weighted harmonic mean of precision and recall.\n    Here it is only computed as a batch-wise average, not globally.\n    This is useful for multi-label classification, where input samples can be\n    classified as sets of labels. By only using accuracy (precision) a model\n    would achieve a perfect score by simply assigning every class to every\n    input. In order to avoid this, a metric should penalize incorrect class\n    assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)\n    computes this, as a weighted mean of the proportion of correct class\n    assignments vs. the proportion of incorrect class assignments.\n    With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning\n    correct classes becomes more important, and with beta > 1 the metric is\n    instead weighted towards penalizing incorrect class assignments.\n    \"\"\" \n    if beta < 0:\n        raise ValueError('The lowest choosable beta is zero (only precision).')\n        \n    # If there are no true positives, fix the F score at 0 like sklearn.\n    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:\n        return 0\n\n    p = precision(y_true, y_pred)\n    r = recall(y_true, y_pred)\n    bb = beta ** 2\n    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n    return fbeta_score\n\ndef fmeasure(y_true, y_pred):\n    return fbeta_score(y_true, y_pred, beta = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab0debdd33ac74d36a78113f7c62e05e219299d3"},"cell_type":"code","source":"class CheckpointCallback(Callback):\n    def __init__(self, path):\n        self.path = path\n\n    def on_epoch_end(self, epoch, logs = None):\n        self.model.save(self.path + 'quora-' + str(epoch) + '.h5')\n        \nclass DataSequence(Sequence):\n    def __init__(self, train_x, train_y):\n        pid = os.getpid()\n        now = time.time()\n        now = (now - int(now)) # Get fractional-part\n        now = int(now * 1000)\n        np.random.seed(pid * now)\n        trn_idx = np.random.permutation(len(train_x))\n        self.train_x = train_x[trn_idx]\n        self.train_y = train_y[trn_idx]\n\n    def __len__(self):\n        n_batches = math.ceil(len(self.train_x) / BATCH_SIZE)\n        return n_batches\n\n    def __getitem__(self, index):\n        texts = self.train_x[index * BATCH_SIZE : (index + 1) * BATCH_SIZE]\n        return texts, self.train_y[index * BATCH_SIZE : (index + 1) * BATCH_SIZE]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19469fb12895715976185297d5bdeb7775b74845"},"cell_type":"markdown","source":"This model only use GRU and Pooling (MaxPooling and AveragePooling)"},{"metadata":{"trusted":true,"_uuid":"c9c7b01bd35e659d5386009a363666614015a131"},"cell_type":"code","source":"def model_gru_pool(args, embedding_matrix):\n    regular = keras.regularizers.l1(args.l1)\n\n    inp = Input(shape = (MAX_TEXT_LENGTH,))\n\n    net = Embedding(embedding_matrix.shape[0], EMBEDDING_LENGTH, weights=[embedding_matrix], trainable = True)(inp)\n    net = SpatialDropout1D(args.dropout)(net)\n\n    net = Bidirectional(CuDNNGRU(128, return_sequences = True, kernel_regularizer = regular, recurrent_regularizer = regular))(net)\n    net = SpatialDropout1D(args.dropout)(net)\n    avg_pool1 = GlobalAveragePooling1D(data_format = 'channels_last')(net)\n    max_pool1 = GlobalMaxPooling1D(data_format = 'channels_last')(net)\n\n    net = Bidirectional(CuDNNGRU(96, return_sequences = True, kernel_regularizer = regular, recurrent_regularizer = regular))(net)\n    net = SpatialDropout1D(args.dropout)(net)\n    avg_pool2 = GlobalAveragePooling1D(data_format = 'channels_last')(net)\n    max_pool2 = GlobalMaxPooling1D(data_format = 'channels_last')(net)\n\n    net = Bidirectional(CuDNNGRU(64, return_sequences = True, kernel_regularizer = regular, recurrent_regularizer = regular))(net)\n    net = SpatialDropout1D(args.dropout)(net)\n    avg_pool3 = GlobalAveragePooling1D(data_format = 'channels_last')(net)\n    max_pool3 = GlobalMaxPooling1D(data_format = 'channels_last')(net)\n\n    conc = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2, avg_pool3, max_pool3])\n    conc = Dropout(args.dropout)(conc)\n\n    conc = Dense(512, activation = 'relu')(conc)\n    conc = Dropout(args.dropout)(conc)\n    \n    outp = Dense(1, activation = 'sigmoid')(conc)\n\n    model = Model(inputs = inp, outputs = outp)\n    return model\n\ndef build_model(args, embedding_matrix):\n    model = model_gru_pool(args, embedding_matrix)\n    #optimizer = SGD(momentum=0.0, nesterov=True)\n    optimizer = Adam()\n    model.compile(loss = 'binary_crossentropy',\n                  optimizer = optimizer,\n                  metrics = [fmeasure])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c68167d52cdac13f0ad47db961186ec752dca1bb"},"cell_type":"markdown","source":"Train the model"},{"metadata":{"trusted":true,"_uuid":"06f0f2cfbc7a354b38a391022ec508d39a59db33"},"cell_type":"code","source":"from collections import namedtuple\n\ndef main():\n    glove_embeddings = load_obj('glove')\n    paragram_embeddings = load_obj('para')\n    fasttext_embeddings = load_obj('fasttext')\n\n    embedding_matrix = np.mean([glove_embeddings, paragram_embeddings, fasttext_embeddings], axis=0)\n\n    train_x = load_obj('train_x')\n    val_x = load_obj('val_x')\n    train_y = load_obj('train_y')\n    val_y = load_obj('val_y')\n\n    mg = DataSequence(train_x, train_y)\n    args = namedtuple('Argument', 'lr l1 dropout')\n    args.lr = 0.001\n    args.l1 = 1e-6\n    args.dropout = 0.33\n    lr_callback = LearningRateScheduler(step_decay_func(args))\n    ckpt_callback = CheckpointCallback('./models/')\n    model = build_model(args, embedding_matrix)\n    model.fit_generator(mg, epochs = MAX_EPOCHS, steps_per_epoch = len(train_x) / BATCH_SIZE,\n                        validation_data = (val_x, val_y),\n                        callbacks = [lr_callback, ckpt_callback],\n                        verbose = True, max_queue_size = 64)\n    \nmain()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe440cd309f681c2905652a9d5e40c28161935d3"},"cell_type":"code","source":"def predict():\n    data = load_data('../input/test.csv')\n    test_x = data['question_text']\n\n    tokenizer = load_obj('tokenizer')\n    test_x = tokenizer.texts_to_sequences(test_x)\n    test_x = pad_sequences(test_x, MAX_TEXT_LENGTH)\n    print('Preprocessing completed')\n    \n    model = load_model('./models/quora-3.h5', custom_objects = {'fmeasure': fmeasure})\n    print('Load model completed')\n    \n    output = model.predict(test_x)\n    print('Predict completed')\n\n    submission = data[['qid']].copy()\n    submission['prediction'] = (output > 0.4).astype(int)\n    submission.to_csv('submission.csv', index = False)\n\npredict()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0dd94bb07bc89b7000d9a630a465ae13b8247e2"},"cell_type":"code","source":"!head submission.csv","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}