{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport nltk\nfrom nltk.corpus import wordnet\nfrom nltk import word_tokenize\nfrom nltk import pos_tag\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np\nnp.random.seed(0)\nfrom keras.models import Model\nfrom keras.layers import Dense, Input, Dropout, LSTM, Activation\nfrom keras.layers.embeddings import Embedding\nfrom keras.preprocessing import sequence\nfrom keras.initializers import glorot_uniform\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom tqdm import tqdm\ntqdm.pandas()\nnp.random.seed(1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"984588ee5ec87f72f9d9853369048352bbd54c72","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1617e87e2af399e3f84c315f92fc6b7d0c1f51fb","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4138e99ce3cc9e3be8a26eb8a6056a3e896c00ba","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e361398bf359710e7e868341d1902a4f7f7c6fc2","trusted":true},"cell_type":"code","source":"def clean_text(x):\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"28f499c7f0389bed82b70eb12debc2c44614fbdd","trusted":true},"cell_type":"code","source":"train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"85bd90bf6caf34b69efef7e7310aa7403b6dec75","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"80a1d2e9845ad8066dfeccc69299064d424ca34a","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08af3e6ad6bfddd81192e4ffe37195fefbc63db8","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72ed05a7b259c6e04241d4316dc4a97ace3577f7","collapsed":true,"trusted":false},"cell_type":"code","source":"# train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n# sentences = train_df[\"question_text\"].apply(lambda x: x.split())\n# vocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffe4f5b479fca997b5f87ecfd9cdcadf81146dcc","collapsed":true,"trusted":false},"cell_type":"code","source":"#oov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"606b306496ff3c8602b821de8c30d4c581dc6e73","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c44ff25674dd14d3b5b4c48f8ef8d38c1de7dcd1","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7521ff995777f2dd25760c50d095a1c658281abc","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4eb0720b2a512c17a2306b19f1277a8e122cc2b5","trusted":true},"cell_type":"code","source":"train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c58348ca8dbd8d5d61e6261a560c32a8de7e66d3"},"cell_type":"code","source":"def get_synonyms(word, pos=None):\n    wordnet_pos = {\n        \"NN\": wordnet.NOUN,\n        \"VB\": wordnet.VERB,\n        \"VBD\": wordnet.VERB,\n        \"VBG\": wordnet.VERB,\n        \"VBN\": wordnet.VERB,\n        \"VBP\": wordnet.VERB,\n        \"JJ\": wordnet.ADJ,\n        \"RB\": wordnet.ADV,\n        \"RBR\": wordnet.ADV,\n        \"RBS\": wordnet.ADV,\n    }\n    if pos:\n        if pos in list(wordnet_pos.keys()):\n            synsets = wordnet.synsets(word, pos=wordnet_pos[pos])\n            synonyms = []\n            for synset in synsets:\n                synonyms += [str(lemma.name()) for lemma in synset.lemmas()]\n            synonyms = [synonym.replace(\"_\", \" \") for synonym in synonyms]\n            synonyms = list(set(synonyms))\n            synonyms = [synonym for synonym in synonyms if synonym != word]\n            if synonyms:\n                return synonyms[0]\n    return ''\n\n\ndef get_syn_sentence(text):\n    words = text.split()\n    words_with_pos_tag = pos_tag(words)\n    words_with_pos_tag\n    new_sentence_words = []\n    for word, pos in words_with_pos_tag:\n        synonym = get_synonyms(word, pos)\n        if synonym:\n            new_sentence_words.append(synonym)\n        else:\n            new_sentence_words.append(word)\n    synonym_sentence = ' '.join(new_sentence_words)\n    return synonym_sentence","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e7d537e41860246b722b026330ec6aa0029d452"},"cell_type":"code","source":"#df_obscene = train_df.loc[train_df['column_name'] == some_value]\ndf_obscene = train_df[train_df['target'] == 1]\ndf_obscene = df_obscene.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77de424a19dafb30d3bb50410c68a988de17024f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"677d9dfd4c02c03c80008b4e329c38ade479c812"},"cell_type":"code","source":"sentence_with_synonyms = []\nfor idx, row in df_obscene.iterrows():\n    sentence_with_synonyms.append(get_syn_sentence(row['question_text']))\n                             \ndf_obscene['question_text'] = sentence_with_synonyms\n\n\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed62796e4dbaad4b98fb0acdc7d651abe0e4947e"},"cell_type":"code","source":"#df_obscene.head()\ntrain_df = train_df.append(df_obscene, ignore_index=True, sort = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e99fe69e46ff2f06d85995336638f4d329566f17"},"cell_type":"code","source":"#train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"840a73963eb3d350c54ec27d800a82354f83834b","trusted":true},"cell_type":"code","source":"## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd00b18004d444cc317142c5440ab5a69e3e93e7","collapsed":true,"trusted":false},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f61e3619a1b40093595eda4b6dd41fe2fc29a803","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fc34add0fb621f6f6a31c5130bf3cbe45a23327","collapsed":true,"trusted":false},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = LSTM(128, return_sequences = True)(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3bca746ef74a637d091d9e6b892f850c9a789026","collapsed":true,"trusted":false},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a7348fdc91a2e6bf86f582b56c898563a049082","collapsed":true,"trusted":false},"cell_type":"code","source":"pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"622235b5be71b77ed74ac5740c7d256675cfc8d9","collapsed":true,"trusted":false},"cell_type":"code","source":"pred_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec127f3934623004032c98090ffdccdd894ce510","collapsed":true,"trusted":false},"cell_type":"code","source":"pred_test_y = (pred_test_y>0.35).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}