{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc,os,sys\nimport operator \n\nfrom sklearn.metrics import f1_score, roc_auc_score\nfrom keras import initializers, regularizers, constraints, optimizers, layers, callbacks\nfrom keras import backend as K\nfrom keras.engine import InputSpec, Layer\nfrom keras.callbacks import ModelCheckpoint, Callback, EarlyStopping, ReduceLROnPlateau\nfrom keras.preprocessing.text import Tokenizer, text_to_word_sequence\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D, GRU, CuDNNGRU, CuDNNLSTM, BatchNormalization\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, MaxPooling1D, Add, Flatten, Masking\nfrom keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D\nfrom keras.models import Model, load_model\nfrom keras.optimizers import Adam\n\npd.set_option(\"display.max_colwidth\", 200)\n\nsns.set_style('darkgrid')\npd.options.display.float_format = '{:,.3f}'.format","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%%time\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\n\nprint(train.shape, test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                #if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                #    df[col] = df[col].astype(np.float16)\n                #el\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        #else:\n            #df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB --> {:.2f} MB (Decreased by {:.1f}%)'.format(\n        start_mem, end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = reduce_mem_usage(train)\ntest = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data analysis"},{"metadata":{},"cell_type":"markdown","source":"## target"},{"metadata":{"trusted":true},"cell_type":"code","source":"valcnt = train['target'].value_counts().to_frame()\nvalcnt.plot.bar()\nvalcnt.T / len(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.sort_values(['target'], ascending=False).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.sort_values(['target']).head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## question_text"},{"metadata":{"trusted":true},"cell_type":"code","source":"# word-count histgram\nword_counts = train['question_text'].apply(lambda x: len(x.split()))\nword_counts.hist(bins=50, figsize=(10,3))\n\nprint('max words: ', max(word_counts))\nprint('sum words: ', sum(word_counts))\ndel word_counts","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }\n\nZEN = \"\".join(chr(0xff01 + i) for i in range(94))\nHAN = \"\".join(chr(0x21 + i) for i in range(94))\nZEN2HAN = str.maketrans(ZEN, HAN)\n\ndef preprocess(data):\n    def clean_special_chars(text):\n        punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~`\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&…'\n        for p in punct:\n            text = text.replace(p, ' ')\n        for p in punct_mapping:\n            text = text.replace(p, punct_mapping[p])\n        #for p in '0123456789':\n        #    text = text.replace(p, ' ')\n        text = text.translate(ZEN2HAN)\n        return text\n\n    data = data.astype(str).apply(lambda x: clean_special_chars(x))\n    return data\n\ntrain['question_text'] = preprocess(train['question_text'])\ntest['question_text'] = preprocess(test['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['question_text'].values[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"text_to_word_sequence(train['question_text'].values[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train.drop(['qid','target'], axis=1)\nY_train = train['target']\nX_test  = test.drop(['qid'], axis=1)\n#train_id  = train['qid']\n#test_id  = test['qid']\ndel train, test\n\nprint(X_train.shape, X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## tokenize text"},{"metadata":{"trusted":true},"cell_type":"code","source":"TARGET_COLUMN = 'target'\nTEXT_COLUMN = 'question_text'\nMAX_NUM_WORDS = 300000\nTOKENIZER_FILTER = '\\r\\t\\n'\n\n# Create a text tokenizer.\ntokenizer = Tokenizer(num_words=MAX_NUM_WORDS, filters=TOKENIZER_FILTER)\ntokenizer.fit_on_texts(list(X_train[TEXT_COLUMN]) + list(X_test[TEXT_COLUMN]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## all words"},{"metadata":{"trusted":true},"cell_type":"code","source":"counter = sorted(dict(tokenizer.word_docs).items(), key=lambda x:x[1], reverse=True)\nwordcount = pd.Series([x[1] for x in counter], [x[0] for x in counter])\ndel counter\n\nwordcount[:30].plot.bar(color='navy', width=0.7, figsize=(12,3))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## toxic words"},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer_tx = Tokenizer(num_words=MAX_NUM_WORDS, filters=TOKENIZER_FILTER)\ntokenizer_tx.fit_on_texts(list(X_train.loc[Y_train == 1, TEXT_COLUMN]))\n\ncounter = sorted(dict(tokenizer_tx.word_docs).items(), key=lambda x:x[1], reverse=True)\nwordcount_tx = pd.Series([x[1] for x in counter], [x[0] for x in counter])\nwordcount_stats = pd.concat([wordcount, wordcount_tx], axis=1, keys=['all', 'toxic'], sort=False)\n\n# word count of contains toxic text is over 80%\nwordcount_tx = wordcount_stats[wordcount_stats['all'] * 0.8 <= wordcount_stats['toxic']].copy()\nwordcount_tx[wordcount_tx['toxic'] > 0]\nwordcount_tx.sort_values(by='toxic', ascending=False, inplace=True)\n\nprint(len(wordcount_tx))\nwordcount_tx['toxic'][:30].plot.bar(color='red', width=0.7, figsize=(12,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# word list - prioritize toxic words\nwordcount = pd.concat([wordcount_tx['all'], wordcount]).to_frame().reset_index()\nwordcount.drop_duplicates(keep='first', inplace=True)\nwordcount = wordcount.set_index('index')[0]\ndel counter, wordcount_tx, wordcount_stats","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## vocabulary size"},{"metadata":{"trusted":true},"cell_type":"code","source":"wordsum = wordcount.sum()\nn_words = len(wordcount)\nprint('words:', n_words)\n\ncumsum_rate = wordcount.cumsum() / wordsum\ncover_rate = {}\nfor i in range(100, 90, -1):\n    p = i / 100\n    cover_rate[str(i)+'%'] = n_words - len(cumsum_rate[cumsum_rate > p])\n#del cumsum_rate\n\npd.Series(cover_rate).plot.barh(color='navy', figsize=(12, 3), title='vocab-size by coverage-rate')\npd.Series(cover_rate).to_frame().T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VOCAB_SIZE = 50000\n\nprint('covered until', wordcount[VOCAB_SIZE], 'times word')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## embedding matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nEMBEDDINGS_DIMENSION = 300\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n#EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n\ndef get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype='float32')\n\ndef load_embeddings(path):\n    with open(path) as f:\n        return dict(get_coefs(*line.strip().split(' ')) for line in f)\n\ndef build_matrix(path):\n    embedding_index = load_embeddings(path)\n    embedding_matrix = np.zeros((VOCAB_SIZE + 1, EMBEDDINGS_DIMENSION))\n    unknown_words = []\n    for i in range(VOCAB_SIZE):\n        try:\n            word = wordcount.index[i]\n            embedding_matrix[i] = embedding_index[word]\n        except KeyError:\n            unknown_words.append(word)\n    return embedding_matrix, unknown_words\n\n#crawl_matrix, unknown_words_crawl = build_matrix(CRAWL_EMBEDDING_PATH)\nglove_matrix, unknown_words_glove = build_matrix(EMBEDDING_FILE)\n\nword2index = dict((wordcount.index[i], i) for i in range(VOCAB_SIZE))\n\n#embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)\nembedding_matrix = glove_matrix\nembedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"words_count = len(unknown_words_glove)\nprint('n unknown words (glove):', words_count, ', {:.3%} of all words'.format(words_count / n_words))\nprint('unknown words (glove):', unknown_words_glove)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## word-id matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 128\n\ndef word_index(word):\n    try:\n        return word2index[word]\n    except KeyError:\n        return VOCAB_SIZE\n\n# All comments must be truncated or padded to be the same length.\ndef pad_text(texts, tokenizer):\n    matrix = [list(map(word_index, text_to_word_sequence(t, filters=TOKENIZER_FILTER))) for t in texts]\n    return pad_sequences(matrix, maxlen=MAX_SEQUENCE_LENGTH)\n\ntrain_text = pad_text(X_train[TEXT_COLUMN], tokenizer)\ntest_text = pad_text(X_test[TEXT_COLUMN], tokenizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del (X_train, X_test)\ngc.collect()\n\nprint(pd.DataFrame([[val for val in dir()], [sys.getsizeof(eval(val)) for val in dir()]],\n                   index=['name','size']).T.sort_values('size', ascending=False).reset_index(drop=True)[:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\nclass F1Callback(Callback):\n    def __init__(self):\n        self.f1s = []\n\n    def on_epoch_end(self, epoch, logs):\n        eps = np.finfo(np.float32).eps\n        recall = logs[\"val_true_positives\"] / (logs[\"val_possible_positives\"] + eps)\n        precision = logs[\"val_true_positives\"] / (logs[\"val_predicted_positives\"] + eps)\n        f1 = 2*precision*recall / (precision+recall+eps)\n        print(\"f1_val (from log) =\", f1)\n        self.f1s.append(f1)\n\ndef true_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n\ndef possible_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_true, 0, 1)))\n\ndef predicted_positives(y_true, y_pred):\n    return K.sum(K.round(K.clip(y_pred, 0, 1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(lr=0.0, lr_d=0.0, units=64, spatial_dr=0.0, \n                dense_units=0, dr=0.1, conv_size=32, epochs=20):\n    \n    file_path = \"best_model.hdf5\"\n    check_point = ModelCheckpoint(file_path, monitor=\"val_loss\", verbose=1, save_best_only=True, mode=\"min\")\n    early_stop = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=3)\n\n    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n    embedding_layer = Embedding(*embedding_matrix.shape,\n                            weights=[embedding_matrix],\n                            input_length=MAX_SEQUENCE_LENGTH,\n                            trainable=True)\n    \n    x = embedding_layer(sequence_input)\n    x = SpatialDropout1D(spatial_dr)(x)\n    x = Bidirectional(CuDNNGRU(units, return_sequences=True))(x)   \n    x = Conv1D(conv_size, 2, padding=\"valid\", kernel_initializer=\"he_uniform\")(x)\n  \n    #att = Attention(MAX_SEQUENCE_LENGTH)(x)\n    avg_pool1 = GlobalAveragePooling1D()(x)\n    max_pool1 = GlobalMaxPooling1D()(x)     \n    \n    #x = concatenate([att, avg_pool1, max_pool1])\n    x = concatenate([avg_pool1, max_pool1])\n    x = BatchNormalization()(x)\n    x = Dense(int(dense_units / 2), activation='relu')(x)\n    x = Dropout(dr)(x)\n    \n    preds = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=sequence_input, outputs=preds)\n    model.compile(loss=\"binary_crossentropy\", optimizer=Adam(lr=lr, decay=lr_d), \n                  metrics=['acc', true_positives, possible_positives, predicted_positives])\n    model.summary()\n    history = model.fit(train_text, Y_train, batch_size=1024, epochs=epochs, validation_split=0.1, \n                        verbose=1, callbacks=[check_point, early_stop, F1Callback()])\n\n    #model = load_model(file_path, custom_objects={'F1Callback':F1Callback, 'Attention':Attention})\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(lr=1e-3, lr_d=1e-7, units=64, spatial_dr=0.2, dense_units=64, dr=0.1, conv_size=64, epochs=10)\npred = model.predict(test_text, batch_size=1024)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n\n#train_meta = model.predict(train_text, batch_size=1024)\n#threshold_search(Y_train, train_meta)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = (pred[:,0] > 0.5).astype(np.int)\n\nsubmission = pd.read_csv('../input/sample_submission.csv', index_col='qid')\nsubmission['prediction'] = pred\nsubmission.reset_index(drop=False, inplace=True)\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(pred).value_counts().to_frame().T / len(pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}