{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# TODO: any Emoji or smileys in texts?\nimport re\nPATTERN = re.compile(r\"[\\w\\']+|[\\.\\!\\?\\:\\(\\)\\\"]+\")\nWORD_PATTERN = re.compile(r\"[\\w\\']+\")\nDIGIT_PATTERN = re.compile(r'\\d')\nMATH_PATTERN = re.compile(r'\\[\\/?math\\]|\\\\\\w+')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv')\ntest_data = pd.read_csv('../input/test.csv')\ny_train = train_data.target.values.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a0d7200e734a940dea7e1310bc480888c1f1057"},"cell_type":"code","source":"for corpus in (train_data, test_data):\n    for _, row in corpus.iterrows():\n        row['question_text'] = MATH_PATTERN.sub('', row.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3db3b121d0d1827c3aeda9fbfef310269d53e0f9"},"cell_type":"code","source":"print('train_data.shape = {}, test_data.shape = {}'.format(train_data.shape, test_data.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0c0b08eb22e53491fd0bc409d2848dfd1d46744"},"cell_type":"code","source":"import gensim\n\nw2v = gensim.models.KeyedVectors.load_word2vec_format(\n    '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',\n    binary=True\n)\n\nfrom nltk.stem.snowball import SnowballStemmer\nstemmer = SnowballStemmer('english')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b06d9616eba8e93aefc3ddf89b8a2bf2751bb9f"},"cell_type":"code","source":"W2V_SLICE = slice(0, 300)\nEND_INDEX = 300\nSTART_INDEX = 301\nUNKNOWN_INDEX = 302  # XXX: may be a bad idea, also try skipping them altogether\nNUM_1_TO_5_INDEX = 303\nNUM_9_TO_18_INDEX = 304\nNUM_YEAR_INDEX = 305\nNUM_OTHER_INDEX = 306\nJEW_OPEN_INDEX = 307\nJEW_CLOSE_INDEX = 308\nQUESTION_INDEX = 309\nEXCLAMATION_INDEX = 310\nELLIPSIS_INDEX = 311\nPERIOD_INDEX = 312\nCOLON_INDEX = 313\n\nNUM_FEATURES = 314\n\nEND_ID = 0\nSTART_ID = 1\nUNKNOWN_ID = 2\nNUM_1_TO_5_ID = 3\nNUM_9_TO_18_ID = 4\nNUM_YEAR_ID = 5\nNUM_OTHER_ID = 6\nJEW_OPEN_ID = 7\nJEW_CLOSE_ID = 8\nQUESTION_ID = 9\nEXCLAMATION_ID = 10\nELLIPSIS_ID = 11\nPERIOD_ID = 12\nCOLON_ID = 13\n\nNUM_SPECIAL_IDS = 14\n\nWORD_TO_ID = dict()\n\n# TODO: test this crap\ndef lookup_word(words_list, passthrough_tokens):\n    \"\"\"Returns a tuple: (word, size)\n    \"\"\"\n    word = words_list[0]\n    if len(words_list) > 1:\n        pair = word + '_' + words_list[1]\n        if pair in w2v:\n            return pair, 2\n        \n    if word.endswith(\"'s\"):\n        word = word[:-2]\n        \n    if word in w2v:\n        return word, 1\n    \n    word = word.lower()\n    if word in w2v:\n        return word, 1\n    \n    return (word if passthrough_tokens else None), 1\n    \ndef lookup_words(words_list, passthrough_tokens):\n    while words_list:\n        word, size = lookup_word(words_list, passthrough_tokens)\n        if word is not None:\n            yield word\n        words_list = words_list[size:]\n\n# TODO: check for double quotes as they may be air quotes\nfor dataset in (train_data, test_data):\n    for _, row in dataset.iterrows():\n        tokens = [\n            match.group(0).strip(\"'\")\n            for match in PATTERN.finditer(row.question_text)\n        ]\n        for word in lookup_words(tokens, passthrough_tokens=False):\n            if word is None:\n                continue  # skip non-word tokens at this time\n            word_id = WORD_TO_ID.get(word)\n            if word_id is None:\n                word_id = len(WORD_TO_ID) + NUM_SPECIAL_IDS\n                if word in w2v:\n                    WORD_TO_ID[word] = word_id\n\nNUM_IDS = len(WORD_TO_ID) + NUM_SPECIAL_IDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b81e2a603180ec7dc509ddff858631890f34e79d"},"cell_type":"code","source":"ID_TO_VEC = np.zeros((NUM_IDS, NUM_FEATURES), dtype='float32')\nID_TO_VEC[END_ID, END_INDEX] = 1\nID_TO_VEC[START_ID, START_INDEX] = 1\nID_TO_VEC[UNKNOWN_ID, UNKNOWN_INDEX] = 1\nID_TO_VEC[NUM_1_TO_5_ID, NUM_1_TO_5_INDEX] = 1\nID_TO_VEC[NUM_9_TO_18_ID, NUM_9_TO_18_INDEX] = 1\nID_TO_VEC[NUM_YEAR_ID, NUM_YEAR_INDEX] = 1\nID_TO_VEC[NUM_OTHER_ID, NUM_OTHER_INDEX] = 1\nID_TO_VEC[JEW_OPEN_ID, JEW_OPEN_INDEX] = 1\nID_TO_VEC[JEW_CLOSE_ID, JEW_CLOSE_INDEX] = 1\nID_TO_VEC[QUESTION_ID, QUESTION_INDEX] = 1\nID_TO_VEC[EXCLAMATION_ID, EXCLAMATION_INDEX] = 1\nID_TO_VEC[ELLIPSIS_ID, ELLIPSIS_INDEX] = 1\nID_TO_VEC[PERIOD_ID, PERIOD_INDEX] = 1\nID_TO_VEC[COLON_ID, COLON_INDEX] = 1\n\nfor word, word_id in sorted(WORD_TO_ID.items(), key=lambda kv: kv[1]):\n    ID_TO_VEC[word_id, W2V_SLICE] = w2v[word]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43a3cb7140ccd350ffd8813c99ffa09cc0be1581"},"cell_type":"code","source":"print(len(WORD_TO_ID), ID_TO_VEC.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fea8ad8e054f218b5953b09442b05207a287c8fe"},"cell_type":"code","source":"def word_to_id(word):\n    if WORD_PATTERN.match(word):\n        word_id = WORD_TO_ID.get(word)\n        if word_id is not None:\n            return word_id\n        else:\n            # TODO: store magnitude for each number?\n            match = DIGIT_PATTERN.search(word)  # TODO: make reals their own class\n            if match:\n                num = int(match.group(0))\n                if 1 <= num <= 5:\n                    return NUM_1_TO_5_ID\n                elif 9 <= num <= 18:\n                    return NUM_9_TO_18_ID\n                elif 1200 <= num <= 2500:\n                    return NUM_YEAR_ID\n                else:\n                    return NUM_OTHER_ID\n            else:\n                return UNKNOWN_ID\n    elif word.startswith('((('):\n        return JEW_OPEN_ID\n    elif word.startswith(')))'):\n        return JEW_CLOSE_ID\n    elif word.startswith('?'):\n        return QUESTION_ID\n    elif word.startswith('!'):\n        return EXCLAMATION_ID\n    elif word == '...':\n        return ELLIPSIS_ID\n    elif word.startswith('.'):\n        return PERIOD_ID\n    elif word == ':':\n        return COLON_ID\n    else:\n        return None  # to be filtered out\n\ndef text_to_ids(text):\n    tokens = [\n        match.group(0).strip(\"'\")\n        for match in PATTERN.finditer(text)\n    ]\n    word_ids = (\n        word_to_id(word)\n        for word in lookup_words(tokens, passthrough_tokens=True)\n    )\n    return [START_ID] + [\n        word_id\n        for word_id in word_ids\n        if word_id is not None\n    ] + [END_ID]\n\nX_train = [\n    text_to_ids(text)\n    for text in train_data.question_text\n]\n\nX_test = [\n    text_to_ids(text)\n    for text in test_data.question_text\n]\n\ny_train = train_data.target.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99bc45e0e4ac576647b638fa7ccfbd833d0ca121"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train_train, X_train_val, \\\ny_train_train, y_train_val = train_test_split(\n    X_train, y_train, test_size=0.1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8af0902ef8ebbf86f370426b4f3e3935df3525fa"},"cell_type":"code","source":"import keras\n\ndef div_up(a, b):\n    return (a + b - 1) // b\n\ndef batch_slice(index, size):\n    return slice(index * size, (index + 1) * size)\n\ndef pad_samples(list_list_ids):\n    num_words = 8 * div_up(max(map(len, list_list_ids)), 8)\n    samples = np.zeros((len(list_list_ids), num_words), dtype='int32')  # END_ID = 0\n    for i, list_ids in enumerate(list_list_ids):\n        samples[i, 0:len(list_ids)] = list_ids\n    return samples\n\n# TODO: generate more data by splicing together more offensive texts\n# TODO: mini-epochs for more precise early stopping\nclass DataGenerator(keras.utils.Sequence):\n    def __init__(\n        self,\n        list_list_ids,\n        labels=None,\n        batch_size=64,\n        num_mini_epochs=1,\n        shuffle=True,\n    ):\n        self.labels = None if labels is None else labels.astype('float32')\n        self.list_list_ids = list_list_ids\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.indices = list(range(len(list_list_ids)))\n        self.num_mini_epochs = num_mini_epochs\n        self.mini_epoch = -1  # see on_epoch_end\n        self.on_epoch_end()\n        \n    def __len__(self):\n        return div_up(len(self.mini_epoch_indices), self.batch_size)\n    \n    def __getitem__(self, index):\n        batch_indices = self.mini_epoch_indices[batch_slice(index, self.batch_size)]\n        \n        samples = pad_samples([self.list_list_ids[i] for i in batch_indices])\n        if self.labels is None:\n            return samples\n        else:\n            return samples, self.labels[batch_indices]\n    \n    def on_epoch_end(self):\n        self.mini_epoch = (self.mini_epoch + 1) % self.num_mini_epochs\n        if self.mini_epoch == 0 and self.shuffle:\n            np.random.shuffle(self.indices)\n            \n        mini_epoch_size = div_up(len(self.indices), self.num_mini_epochs)\n        self.mini_epoch_indices = self.indices[batch_slice(self.mini_epoch, mini_epoch_size)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8be06e8941b5a8214a57b2fb0a1a9028532ac2f"},"cell_type":"code","source":"import keras.layers as K\nfrom keras.models import Model, Sequential\n\ndef combine_stuff(input_layer, *layer_specs, pool=2, unpool=False):\n    layer = input_layer\n    for size, filters in layer_specs:\n        layer = K.Conv1D(filters, size, padding='same')(layer)\n        layer = K.BatchNormalization()(layer)\n        layer = K.LeakyReLU(0.01)(layer)\n\n    layer = K.MaxPooling1D(pool)(layer)\n    if unpool:\n        layer = K.UpSampling1D(pool)(layer)\n        \n    return layer\n\n\n# TODO: compress the model a bit\n# TODO: only use the tuning layer for subject matter, not linkage?\ndef build_model():\n    input_layer = K.Input((None,), dtype='int32')\n    \n    manual_embed_layer = K.Embedding(\n        ID_TO_VEC.shape[0],\n        NUM_FEATURES - 300,\n        weights=[ID_TO_VEC[:, 300:]],\n        trainable=False,\n    )(input_layer)\n    \n    pretrained_embed_layer = K.Embedding(\n        ID_TO_VEC.shape[0],\n        300,\n        weights=[ID_TO_VEC[:, :300]],\n        trainable=False,  # trainable=True baseline: val_loss = 0.1053\n    )(input_layer)\n    \n    tuning_embed_layer = K.Embedding(\n        ID_TO_VEC.shape[0],\n        100,\n        embeddings_initializer='lecun_normal',\n    )(input_layer)\n    \n    embedded_tuning_embed_layer = K.Conv1D(300, 1)(tuning_embed_layer)\n    \n    embed_layer = K.concatenate([\n        manual_embed_layer,\n        K.add([pretrained_embed_layer, embedded_tuning_embed_layer])\n    ])\n\n    basic_layer = combine_stuff(embed_layer, (2, 200), (2, 300), pool=4, unpool=True)\n    e_basic_layer = K.concatenate([manual_embed_layer, pretrained_embed_layer, basic_layer])\n    \n    linkage_layer_4 = combine_stuff(e_basic_layer, (4, 400), (4, 500), pool=8)\n    linkage_layer_6 = combine_stuff(e_basic_layer, (6, 400), (6, 500), pool=8)\n    \n    linkage_layer = K.add([linkage_layer_4, linkage_layer_6])\n    \n    sentence_layer = combine_stuff(linkage_layer, (2, 500))\n    layer = K.GlobalMaxPooling1D()(sentence_layer)\n    \n    layer = K.Dense(256)(layer)\n    layer = K.BatchNormalization()(layer)\n    layer = K.LeakyReLU(0.01)(layer)\n    #layer = K.Dropout(0.5)(layer)\n\n    layer = K.Dense(1, activation='sigmoid')(layer)\n    return Model(input_layer, layer)\n\nkeras_model = build_model()\nkeras_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ac89411a0dc80a360e7f651fce69d8e35f4df5c","scrolled":false},"cell_type":"code","source":"# TODO: add cross-validation\n# TODO: proper custom-initialized embedding for fine-tuning?\nkeras_model.compile(\n    optimizer='adam',\n    loss='binary_crossentropy',\n)\n\n# TODO: filter the training set, drop noisy samples (like the ones with zero words)\nnum_mini_epochs = 5\ntrain_generator = DataGenerator(X_train_train, y_train_train, num_mini_epochs=num_mini_epochs)\nval_generator = DataGenerator(X_train_val, y_train_val, batch_size=256)\n\nkeras_model.fit_generator(\n    train_generator,\n    validation_data=val_generator,\n    epochs=2 * num_mini_epochs,\n    use_multiprocessing=True,\n    workers=2,\n    callbacks=[keras.callbacks.EarlyStopping(patience=2, restore_best_weights=True)],\n    verbose=2,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbd94dc4082a08196d1314165961618b7439f4f9"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\neval_val_generator = DataGenerator(X_train_val, y_train_val, shuffle=False, batch_size=256)\nprob_val = keras_model.predict_generator(\n    eval_val_generator,\n    use_multiprocessing=True,\n    workers=2,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74a0093c1823d3a8be128739f8ed1950f0e7cf5e"},"cell_type":"code","source":"def calc_f1(thresh):\n    return f1_score(y_train_val, (prob_val > thresh).astype('int32'))\n\nthresh_options = np.linspace(0.01, 0.5, 100)\nf1_values = np.array([calc_f1(thresh) for thresh in thresh_options])\nbest_thresh_index = np.argmax(f1_values)\nbest_thresh = thresh_options[best_thresh_index]\nprint('best_thresh = {}, f1_score = {}'.format(best_thresh, f1_values[best_thresh_index]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90b37d294447c13309426b0726874aa0991b8dc4"},"cell_type":"code","source":"if False:  # Kaggle doesn't like multiple outputs in this comp\n    train_error_generator = DataGenerator(X_train, shuffle=False, batch_size=256)\n    prob_train = keras_model.predict_generator(\n        train_error_generator,\n        use_multiprocessing=True,\n        workers=2,\n    )\n\n    mispred_train = y_train != (prob_train > best_thresh).ravel()\n    print(mispred_train.mean(), mispred_train.sum())\n    train_data[mispred_train.astype('bool')].to_csv('mispreditions.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ce763ec00da8b383d286c49b3e3411ac448e510"},"cell_type":"code","source":"test_generator = DataGenerator(X_test, shuffle=False, batch_size=256)\nprob_test = keras_model.predict_generator(\n    test_generator,\n    use_multiprocessing=True,\n    workers=2 ,\n)\n\npred_test = (prob_test > best_thresh).astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb48e53add392eb706fd896e2e4334ebf7e80f04"},"cell_type":"code","source":"submission = pd.DataFrame({'qid': test_data.qid, 'prediction': pred_test.ravel()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfbfe3fc1de353cb8302feb235676a99b1485639"},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d426205f789b6e153e32f162ea6b6b799e6e4e2b"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88b345a4f4763e3f975f694d49a372a81e6dbe34"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}