{"cells":[{"metadata":{"trusted":true,"_uuid":"6d5c02c7188cc8386983f6ac25b999e21feb13a2"},"cell_type":"code","source":"import re\nimport time\nimport gc\nimport random\nimport os\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\ndef seed_torch(seed=1029):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nembed_size = 300 # how big is each word vector\nmax_features = 95000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 72 # max number of words in a question to use\n\nbatch_size = 1536\ntrain_epochs = 8\n\nSEED = 1029\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\ndef load_and_prec():\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n\n    # lower\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: x.lower())\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: x.lower())\n\n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n\n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n\n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    ## fill up the missing values\n    train_df[\"question_text\"]  = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_df[\"question_text\"]  = test_df[\"question_text\"].fillna(\"_##_\").values\n    return train_df,test_df\n\n\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix\nfrom tqdm import tqdm\ntqdm.pandas()\n\nstart_time = time.time()\n\ntrain_df,test_df = load_and_prec()\n# embedding_matrix_1 = load_glove(word_index)\n# embedding_matrix_2 = load_para(word_index)\n\n# total_time = (time.time() - start_time) / 60\n# print(\"Took {:.2f} minutes\".format(total_time))\n\n# embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2], axis=0)\n# # embedding_matrix = np.concatenate((embedding_matrix_1, embedding_matrix_2), axis=1)\n# print(np.shape(embedding_matrix))\n\n# del embedding_matrix_1, embedding_matrix_2\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60877a2262362a8029fd98d48812bebc1bbcf5c9"},"cell_type":"code","source":"def print_score(clf, X_train, y_train, train=True):\n    '''\n    print the accuracy score, classification report and confusion matrix of classifier\n    '''\n    if train:\n        '''\n        training performance\n        '''\n        print(\"Train Result:\\n\")\n        y_train_predict=clf.predict(X_train);\n        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_train, y_train_predict)))\n        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_train,y_train_predict)))\n        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_train, y_train_predict)))\n\n        res = cross_val_score(clf, X_train, y_train, cv=10, scoring='accuracy')\n        print(\"Average Accuracy: \\t {0:.4f}\".format(np.mean(res)))\n        print(\"Accuracy SD: \\t\\t {0:.4f}\".format(np.std(res)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fe379ecdba9a5bcc8a6d969f0a356af31c78d1a"},"cell_type":"code","source":"train_X=train_df[\"question_text\"]\ntrain_y=train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"349e59afe225795aa42dc56b6d8b77f9716faa5a"},"cell_type":"markdown","source":"# Naive Bayes classifier for multinomial models"},{"metadata":{"trusted":true,"_uuid":"4c99ce4153cc6287fab6fd6ea29068aabf36bcd2"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.model_selection import cross_val_score, cross_val_predict\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nfrom sklearn.ensemble import RandomForestClassifier\nimport logging\nimport pandas as pd\nimport numpy as np\nfrom numpy import random\nimport gensim\nimport nltk\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nnb = Pipeline([('vect', CountVectorizer()),\n               ('tfidf', TfidfTransformer()),\n               ('clf', MultinomialNB()),\n              ])\nnb.fit(train_X, train_y);\nprint_score(nb, train_X, train_y, train=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4aa22d254de5a04c8995cbbc361236a14222189b"},"cell_type":"code","source":"from sklearn.metrics import classification_report","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3bf427cc0f2f87f80e8adcde7fde999d7d77788"},"cell_type":"markdown","source":"# Linear support vector machine"},{"metadata":{"trusted":true,"_uuid":"f2cf7714c3b7916ec06f09a90a4bc2f7db8c02d2"},"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\n\nsgd = Pipeline([('vect', CountVectorizer()),\n               ('tfidf', TfidfTransformer()),\n                ('clf', SGDClassifier(loss='hinge', penalty='l2',alpha=1e-3, random_state=42, max_iter=5, tol=None)),\n               ])\nsgd.fit(train_X, train_y);\nprint_score(sgd, train_X, train_y, train=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ee4a6de7ef881aaa16e61be4c24771e46be0266"},"cell_type":"markdown","source":"# BOW with Keras"},{"metadata":{"trusted":true,"_uuid":"144f238e2407418995716a290f3f1cf45123b1c6"},"cell_type":"code","source":"import itertools\nimport os\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nfrom sklearn.preprocessing import LabelBinarizer, LabelEncoder\nfrom sklearn.metrics import confusion_matrix\n\nfrom tensorflow import keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout\nfrom keras.preprocessing import text, sequence\nfrom keras import utils\n\ntrain_size = int(len(train_df) * .7)\ntrain_posts = train_df['question_text'][:train_size]\ntrain_tags = train_df['target'][:train_size]\n\ntest_posts = train_df['question_text'][train_size:]\ntest_tags = train_df['target'][train_size:]\n\nmax_words = 1000\ntokenize = text.Tokenizer(num_words=max_words, char_level=False)\ntokenize.fit_on_texts(train_posts) # only fit on train\n\nx_train = tokenize.texts_to_matrix(train_posts)\nx_test = tokenize.texts_to_matrix(test_posts)\n\nencoder = LabelEncoder()\nencoder.fit(train_tags)\ny_train = encoder.transform(train_tags)\ny_test = encoder.transform(test_tags)\n\nnum_classes = np.max(y_train) + 1\ny_train = utils.to_categorical(y_train, num_classes)\ny_test = utils.to_categorical(y_test, num_classes)\n\nbatch_size = 32\nepochs = 2\n\n# Build the model\nmodel = Sequential()\nmodel.add(Dense(512, input_shape=(max_words,)))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(num_classes))\nmodel.add(Activation('softmax'))\n\nmodel.compile(loss='categorical_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])\n              \nhistory = model.fit(x_train, y_train,\n                    batch_size=batch_size,\n                    epochs=epochs,\n                    verbose=1,\n                    validation_split=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f76daf1a268f80e68748587f7d836ebac5a9ec4"},"cell_type":"code","source":"score = model.evaluate(x_test, y_test,\n                       batch_size=batch_size, verbose=1)\nprint('Test accuracy:', score[1])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}