{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3edb4c52b9865693ad805b14254d39ac39440e11"},"cell_type":"code","source":"pd.options.display.max_colwidth = -1\npd.options.display.max_columns = 15","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"class Data:\n    \"\"\" Loads and preprocesses data \"\"\"\n    def __init__(self, id_col='id', text_col='comment_text'):\n        self.train_df, self.test_df = self.load_data()\n        self.text_col = 'comment_text'\n        self.id_col = 'id'\n        \n    def preprocessing(self):\n        \"\"\" Clean the text in some way \"\"\"\n        return\n\n    def load_data(self):\n        train_path = '../input/jigsaw-toxic-comment-classification-challenge/train.csv'\n        test_path = '../input/jigsaw-toxic-comment-classification-challenge/test.csv'\n\n        train_df = pd.read_csv(train_path)\n        test_df = pd.read_csv(test_path)\n        \n        return train_df, test_df\n    \n    def get_comments(self, subset='train'):\n        if subset == 'train':\n            data = list(self.train_df[self.text_col])\n        if subset == 'test':\n            data = list(self.test_df[self.text_col])\n        if subset == 'all':\n            data = list(self.train_df[self.text_col]) + list(self.test_df[self.text_col])\n        return data\n    \n    def get_training_labels(self):\n        labels_columns = self.train_df.columns.difference([self.text_col, self.id_col])\n        labels = self.train_df.loc[:, labels_columns].values\n        return labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2609fecadfabcaa197777bcc8fc1661699a412a7"},"cell_type":"code","source":"# explore data\ndata = Data()\ntrain_df = data.train_df\ntrain_df.sample(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc210a1d9fb80a8db07b338aa314fb42d2074822"},"cell_type":"code","source":"# At Bruce K's last presentation, Matt E asks \"What the base rates are for each toxicity\" (16:04)\ntrain_df.loc[:, train_df.columns.difference(['id', 'comment_text'])].sum()/len(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"927a4bf37f9f104eb254833ae992f9f00eeb5530"},"cell_type":"code","source":"# At Bruce K's last presentation, Matt E asks \"If a comment is severely toxic, does that mean that it's toxic as well?\" (3:40)\nprint(train_df.loc[train_df['severe_toxic'] == 1].sample(3))\ntrain_df.loc[train_df['severe_toxic'] == 1, train_df.columns.difference(['id', 'comment_text'])].sum(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a905daa3f5dc8d463e2097bbc6c9a574063b44c3"},"cell_type":"code","source":"# Any other questions about the data?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cba3bf47c434ead061ed4674d82242079860c31"},"cell_type":"code","source":"import spacy\n\nfrom collections import defaultdict\n\n\nclass TextMapper:\n    \"\"\" Maps text into model input format \"\"\"\n    PADDING_SYMBOL = \"<PAD>\"\n    UNKNOWN_SYMBOL = \"<UNK>\"\n    BASE_ALPHABET = [PADDING_SYMBOL, UNKNOWN_SYMBOL]\n\n    def __init__(self, comment_texts, max_sent_len=400, threshold=20, lowercase=False):\n        self.lowercase = lowercase\n        self.max_sent_len = max_sent_len\n\n        self.word_to_ix = dict()  # maps words to index values\n        self.ix_to_word = dict()  # maps index values to words\n        self.corpus_info = dict()  # contains infomation about corpus\n\n        self.nlp = spacy.load('en', disable=['parser', 'tagger', 'ner'])\n        self.calc_corpus_info(comment_texts)\n        self.init_mappings(threshold)\n\n    def init_mappings(self, threshold=20, check_coverage=True):\n        # what information are we losing about the words?\n        word_counts = self.corpus_info['word_counts']\n        word_counts = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)\n        vocab = [word for word, count in word_counts if count > threshold]\n        vocab = self.BASE_ALPHABET + vocab\n\n        # can add more criteria to select/normalize words (ignore punctuation, normalize numbers,.. etc)\n        \n        self.word_to_ix = {word: ix for ix, word in enumerate(vocab)}\n        self.ix_to_word = {ix: word for ix, word in enumerate(vocab)}\n        if check_coverage:\n            self.print_coverage_statistics()\n    \n    def calc_corpus_info(self, comment_list):\n        self.corpus_info['word_counts'] = defaultdict(int)\n        self.corpus_info['sent_lengths'] = []\n        self.corpus_info['word_lengths'] = []\n        for comment in comment_list:\n            if self.lowercase:\n                comment = comment.lower()\n            tokenized_comment = self.nlp(comment)\n            self.corpus_info['sent_lengths'].append(len(tokenized_comment))\n            for token in tokenized_comment:\n                text = token.text\n                self.corpus_info['word_counts'][text] += 1\n                self.corpus_info['word_lengths'].append(len(text))\n                \n    def text_to_x(self, text):\n        x = np.zeros(self.max_sent_len)\n\n        if self.lowercase:\n            text = text.lower()\n        tokenized_comment = self.nlp(text)\n        for ind, token in enumerate(tokenized_comment[:self.max_sent_len]):\n            word = token.text\n            x[ind] = self.get_word_index(word)\n        return x\n\n    def get_word_index(self, word):\n        try:\n            num = self.word_to_ix[word]\n        except KeyError:\n            num = self.word_to_ix[self.UNKNOWN_SYMBOL]\n        return num\n\n    def x_to_text(self, x):\n        words = [self.ix_to_word[int(i)] for i in x]\n        comment_text = \" \".join(words)\n\n        # remove padding\n        comment_text = comment_text.split(self.PADDING_SYMBOL)[0]\n        return comment_text\n\n    def get_texts_x(self, texts):\n        x_rep = np.array([self.text_to_x(text) for text in texts])\n        return x_rep\n    \n    def set_max_sent_length(self, sent_len):\n        self.max_sent_len = sent_len\n    \n    def print_coverage_statistics(self):\n        word_mappings = self.word_to_ix.keys()\n        print(\"Number of unique words: {}\".format(len(word_mappings)))\n        total_tokens = 0\n        mapped_tokens = 0\n        word_counts = self.corpus_info['word_counts']\n        for word, count in word_counts.items():\n            total_tokens += count\n            if word in word_mappings:\n                mapped_tokens += count\n        print(\"Percent of unique words mapped: {}%\".format(100*len(word_mappings)/len(word_counts)))\n        print(\"Percent of total tokens mapped: {}%\".format(100*mapped_tokens/total_tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f79d8b8f84fa01235f5ac8a7926a39630f321b3"},"cell_type":"code","source":"# text_mapper = TextMapper(data.get_comments('train'))\n# # what are the consequences of using testing data here?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a36f2906a31ce1b4f1e08350e4fc2dd292f8ee7"},"cell_type":"code","source":"# # tune text mapper word threshold\n\n# text_mapper.init_mappings(threshold=4)\n# # other methods of measuring coverage are more accurate but take more time\n# # consequence of setting too high / too low thresholds?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"856c622561e690ee7898132e270c8fe3a7daa9b1"},"cell_type":"code","source":"# # tune text mapper sentence length\n# from scipy import stats\n\n# sentence_lengths = text_mapper.corpus_info['sent_lengths']\n# stats.describe(sentence_lengths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"492352a21cd0777691cf2378ca4c43ca4aee6cc6"},"cell_type":"code","source":"# import matplotlib.pyplot as plt\n\n# plt.hist(sentence_lengths, bins=np.arange(0, 1000, 25), cumulative=True, normed=1)\n# plt.hlines(0.975, 0, 1000, colors='red')\n# # consequence of setting too high / too low sentence lengths?","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d459ebe73813124aa03a0e6578fe80ca8b3977c"},"cell_type":"code","source":"# text_mapper.set_max_sent_length(400)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d774555ab8e65a05cfd0322c558f6807a82b292"},"cell_type":"code","source":"# # initialize pretrained word embeddings\n\n# def load_glove(word_to_ix):\n#     EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\n#     def get_coefs(word,*arr):\n#         return word, np.asarray(arr, dtype='float32')\n\n#     print(\"Loading embeddings\")\n#     embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n#     print(\"Loaded {} embeddings\".format(len(embeddings_index)))\n    \n#     # calculate statistics on distributions in embeddings\n#     all_embs = np.stack(embeddings_index.values())\n#     emb_mean, emb_std = all_embs.mean(), all_embs.std()\n#     embed_size = all_embs.shape[1]\n    \n#     # question about embeddings: what values are in these vectors?\n#     print(\"Embedding mean: {}\\nEmbedding std: {}\".format(emb_mean, emb_std))\n\n#     # create embeddings matrix for words in our corpus (word_to_ix)\n#     nb_words = len(word_to_ix)\n#     embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n#     matched_words = 0\n#     for word, i in word_to_ix.items():\n#         embedding_vector = embeddings_index.get(word)\n#         if embedding_vector is not None:\n#             embedding_matrix[i] = embedding_vector\n#             matched_words += 1\n#     print(\"Percent of words with pretrained embedding {}%\".format(100*matched_words/nb_words))\n#     return embedding_matrix ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"301b30d843676ec55813e837cdb749bcd24046a1","scrolled":false},"cell_type":"code","source":"# embedding_matrix = load_glove(text_mapper.word_to_ix)\n# embedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"3d9648cb5113b7bbab11e8b7499937764f52219b"},"cell_type":"code","source":"# full_train_x = text_mapper.get_texts_x(data.get_comments('train'))\n# full_train_y = data.get_training_labels()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a501bf9debfd66343bdc9135c8505060e2f416b"},"cell_type":"code","source":"# # sanity check\n# first_n_words = 50\n\n# # randomize for fun\n# import random\n# i = int(random.random()*10000)\n# random_comment = data.get_comments('train')[i]\n# print(\"Random comment \\n\\n{}\\n\\n\".format(random_comment))\n# model_input = text_mapper.get_texts_x([random_comment])[0][:first_n_words]\n# print(\"Model input \\n\\n{}\\n\\n\".format(model_input))\n# model_input_to_text = text_mapper.x_to_text(model_input)\n# print(\"Translated model input \\n\\n{}\\n\\n\".format(model_input_to_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e67f34e9fa69af577bf4f865d9ee291e9ccd9fc0"},"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# train_x, test_x, train_y, test_y = train_test_split(full_train_x, full_train_y, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"996b50c8fa0027a87342ff5822c5ca5f4356d4fa","scrolled":true},"cell_type":"code","source":"# from keras.models import Model\n# from keras.layers import Input, Dense, Conv1D, Activation, Embedding, MaxPooling1D, Flatten, Dropout, Bidirectional, GlobalMaxPooling1D, LSTM, SpatialDropout1D\n# from keras.layers import CuDNNLSTM, Concatenate, GlobalAveragePooling1D, CuDNNGRU","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6d92232ce2ee1ab9b1eebf5ed14390aaed71c83","scrolled":true},"cell_type":"code","source":"# sent_len = text_mapper.max_sent_len\n# unique_tokens = len(text_mapper.word_to_ix)\n# embedding_size = embedding_matrix.shape[1]\n# spacial_dropout = 0.5\n# lstm_kernel_size = 40\n# pred_size = 6\n\n# def simple_model():\n#     inp = Input(shape=(sent_len, ), name='word_ixs')\n#     embedding = Embedding(input_dim=unique_tokens,\n#                           output_dim=embedding_size,\n#                           input_length=sent_len,\n#                           weights=[embedding_matrix],\n#                           name=\"word_embedding\")(inp)\n\n#     spatial_drop_layer = SpatialDropout1D(spacial_dropout)(embedding)\n#     lstm = Bidirectional(CuDNNLSTM(lstm_kernel_size, return_sequences=True, name=\"lstm\"))(spatial_drop_layer)\n#     gru, h_r, h_l = Bidirectional(CuDNNGRU(lstm_kernel_size, return_sequences=True, name=\"gru\", return_state=True))(lstm)\n#     max_pool = GlobalMaxPooling1D(name=\"global_max_pool\")(gru)\n#     avg_pool = GlobalAveragePooling1D(name=\"global_avg_pool\")(gru)\n#     concat_features = Concatenate()([h_r, max_pool, avg_pool])\n#     preds = Dense(pred_size, activation='sigmoid')(concat_features)\n#     model = Model(inputs=inp, outputs=preds)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cea0318643415d5b5c9e4cc2530d5b6de82f0cf5","scrolled":false},"cell_type":"code","source":"# from keras.optimizers import Adam\n# from keras.callbacks import EarlyStopping, ModelCheckpoint\n\n# model_save_path = \"simple_model.h5\"\n# learning_rate = 0.003\n\n# early_stopping = EarlyStopping(monitor='val_acc', patience=1, verbose=1)\n# checkpointer = ModelCheckpoint(filepath=model_save_path, monitor='val_acc', save_best_only=True, verbose=1)\n# callbacks = [early_stopping, checkpointer]\n\n# simple_model = simple_model()\n# simple_model.compile(optimizer=Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['acc'])\n# simple_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd3e406389f01785656ec357a258a4c1d8fc8e2d"},"cell_type":"code","source":"# simple_model.fit(x=train_x, y=train_y, validation_data=(test_x, test_y), shuffle=True, epochs=1, callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96e6083d3a16d653a78f56e1a56edb3536545568"},"cell_type":"code","source":"# how can we add more information to the network? Think of all the information the network doesn't know...","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"437384f0a62a901841252bf113fdaf8aecc6a778"},"cell_type":"code","source":"# def add_features(df):\n#     \"\"\" stolen from https://www.kaggle.com/larryfreeman/toxic-comments-code-for-alexander-s-9872-model \"\"\"\n#     df['comment_text'] = df['comment_text'].apply(lambda x:str(x))\n#     df['total_length'] = df['comment_text'].apply(len)\n#     df['capitals'] = df['comment_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))\n#     df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']),\n#                                 axis=1)\n#     df['num_words'] = df.comment_text.str.count('\\S+')\n#     df['num_unique_words'] = df['comment_text'].apply(lambda comment: len(set(w for w in comment.split())))\n#     df['words_vs_unique'] = df['num_unique_words'] / df['num_words']  \n\n#     return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f182ecbf5a5045df2dbcf5e9082ca9910804ad0d"},"cell_type":"code","source":"# train_df = add_features(train_df)\n# train_df.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0d32fa48ba3bf39af5168bb625160fe95ebc9fa"},"cell_type":"code","source":"# # let's add the feature 'caps_vs_length' to our model\n# full_add_feature = np.array([[x] for x in train_df['caps_vs_length'].values])\n# full_add_feature[0:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d598216c53cbc8684344c120573cf1e5879b8b5b"},"cell_type":"code","source":"# train_text_x, test_text_x, train_add_x, test_add_x, train_labels, test_labels = train_test_split(full_train_x, full_add_feature, full_train_y, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8bfeaf9bad4baf85935249748a6edddf44becb2c"},"cell_type":"code","source":"# sent_len = text_mapper.max_sent_len\n# unique_tokens = len(text_mapper.word_to_ix)\n# embedding_size = embedding_matrix.shape[1]\n# spacial_dropout = 0.5\n# lstm_kernel_size = 40\n# pred_size = 6\n\n# def nice_model():\n#     inputs = []\n#     text_inp = Input(shape=(sent_len, ), name='word_ixs')\n#     inputs.append(text_inp)\n#     add_inp = Input(shape=(1, ), name='add_feature_inp')\n#     inputs.append(add_inp)\n#     embedding = Embedding(input_dim=unique_tokens,\n#                           output_dim=embedding_size,\n#                           input_length=sent_len,\n#                           weights=[embedding_matrix],\n#                           name=\"word_embedding\")(text_inp)\n\n#     spatial_drop_layer = SpatialDropout1D(spacial_dropout)(embedding)\n#     lstm = Bidirectional(CuDNNLSTM(lstm_kernel_size, return_sequences=True, name=\"lstm\"))(spatial_drop_layer)\n#     gru, h_r, h_l = Bidirectional(CuDNNGRU(lstm_kernel_size, return_sequences=True, name=\"gru\", return_state=True))(lstm)\n#     max_pool = GlobalMaxPooling1D(name=\"global_max_pool\")(gru)\n#     avg_pool = GlobalAveragePooling1D(name=\"global_avg_pool\")(gru)\n#     concat_features = Concatenate()([h_l, h_r, max_pool, avg_pool, add_inp])\n#     preds = Dense(pred_size, activation='sigmoid', name='preds')(concat_features)\n#     model = Model(inputs=inputs, outputs=preds)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"039faf666af697cd4730a550880422a984c6c913"},"cell_type":"code","source":"# model_save_path = \"nice_model.h5\"\n# learning_rate = 0.003\n\n# early_stopping = EarlyStopping(monitor='val_acc', patience=1, verbose=1)\n# checkpointer = ModelCheckpoint(filepath=model_save_path, monitor='val_acc', save_best_only=True, verbose=1)\n# callbacks = [early_stopping, checkpointer]\n\n# nice_model = nice_model()\n# nice_model.compile(optimizer=Adam(lr=learning_rate), loss='binary_crossentropy', metrics=['acc'])\n# nice_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b984ed8489c65b23f25faf8a25f67bdeeb5a383"},"cell_type":"code","source":"# train_x = {}\n# train_y = {}\n# test_x = {}\n# test_y = {}\n\n# train_x['word_ixs'] = train_text_x\n# test_x['word_ixs'] = test_text_x\n# train_x['add_feature_inp'] = train_add_x\n# test_x['add_feature_inp'] = test_add_x\n\n# train_y['preds'] = train_labels\n# test_y['preds'] = test_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"e051c14387a6477e1f4d0a017a75552409a07468"},"cell_type":"code","source":"# nice_model.fit(x=train_x, y=train_y, validation_data=(test_x, test_y), shuffle=True, epochs=3, callbacks=callbacks)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}