{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Pre-processing"},{"metadata":{},"cell_type":"markdown","source":"1. Attempt to preserve more information\n2. Attempt to achieve higher overlapping with embeddings "},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\n#nltk.download('popular')\n\nimport re\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer\n\n# text cleaning & tokenization\ndef tokenize(text, stop_set = None, lemmatizer = None):\n    \n    # clean text\n    text = text.encode('ascii', 'ignore').decode('ascii')\n    #text = text.lower()\n    \n    text = re.sub(r'\\b(?:(?:https?|ftp)://)?\\w[\\w-]*(?:\\.[\\w-]+)+\\S*', ' ', text) # remove hyperlink,subs charact in the brackets\n    text = re.sub(\"[\\r\\n]\", ' ', text) # remove new line characters\n    #text = re.sub(r'[^\\w\\s]','',text)\n    text = text.strip()\n    \n    #tokens = word_tokenize(text)\n    # use TweetTokenizer instead of word_tokenize -> to prevent splitting at apostrophies\n    tknzr = TweetTokenizer()\n    tokens = tknzr.tokenize(text)\n    \n    # retain tokens with at least two words\n    tokens = [token for token in tokens if re.match(r'.*[a-z]{1,}.*', token)]\n    \n    # remove stopwords - optional\n    # removing stopwords lost important information\n    if stop_set != None:\n        tokens = [token for token in tokens if token not in stop_set]\n    \n    # lemmmatization - optional\n    if lemmatizer != None:\n        tokens = [lemmatizer.lemmatize(token) for token in tokens]\n    return tokens\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#stop_set = set(stopwords.words('english'))\n#lemmatizer = WordNetLemmatizer()\n\n# with lemmatization\n#train['tokens'] = train['question_text'].map(lambda x: tokenize(x, stop_set, lemmatizer))\n\n# without lemmatization\ntrain['tokens'] = train['question_text'].map(lambda x: tokenize(x))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(token_col):\n    \n    vocab = {}\n    for tokens in token_col:\n        for token in tokens:\n            vocab[token] = vocab.get(token, 0) + 1\n\n    return vocab\n\ntrain_vocab = build_vocab(train['tokens'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Word Embedding"},{"metadata":{},"cell_type":"markdown","source":"#### Google News"},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\n\ndef check_coverage(vocab,embedding):\n    \n    oov = {}\n    k = 0\n    i = 0\n    \n    for word in vocab:\n        if word in embedding:\n            k += vocab[word]\n        else:\n            oov[word] = vocab[word]\n            i += vocab[word]\n\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x\n  \n\nnot_found_vocab = check_coverage(train_vocab, embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#'Aadhar' in embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Google News Embeddings\n# replace not found words\nto_remove = ['to','of','and', 'a']\n\n#replace_dict = {}\n#replace_dict = {'quora':'Quora', 'i\\'ve':'I\\'ve', 'instagram':'Instagram', 'upsc':'UPSC', 'bitcoin':'Bitcoin', 'trump\\'s':'Trump',\n#               'mbbs':'MBBS', 'whatsapp':'WhatsApp', 'favourite':'favorite', 'ece':'ECE', 'aiims':'AIIMS', 'colour':'color',\n#               'doesnt':'doesn\\'t','centre':'center','sbi':'SBI','cgl':'CGL','iim':'IIM','btech':'BTech'}\n\nreplace_dict = {'favourite':'favorite', 'bitcoin':'Bitcoin', 'colour':'color', 'doesnt':'doesn\\'t', 'centre':'center', 'Quorans':'Quora',\n               'travelling':'traveling', 'counselling':'counseling', 'didnt':'didn\\'t', 'btech':'BTech','isnt':'isn\\'t',\n               'Shouldn\\'t':'shouldn\\'t', 'programme':'program', 'realise':'realize', 'Wouldn\\'t':'wouldn\\'t', 'defence':'defense',\n               'Aren\\'t':'aren\\'t', 'organisation':'organization', 'How\\'s':'how\\'s', 'e-commerce':'ecommerce', 'grey':'gray',\n               'bitcoins':'Bitcoin', 'honours':'honors', 'learnt':'learned', 'licence':'license', 'mtech':'MTech', 'colours':'colors',\n               'e-mail':'email', 't-shirt':'tshirt', 'Whatis':'What\\'s', 'theatre':'theater', 'labour':'labor', 'Isnt':'Isn\\'t',\n               'behaviour':'behavior','aadhar':'Aadhar', 'Qoura':'Quora', 'aluminium':'aluminum'}\n\ndef clean_token(tokens, remove_list, re_dict, embedding):\n    \n    c_tokens = []\n    for token in tokens:\n        if token not in remove_list:\n            token2 = token\n            if token2 in embedding:\n                c_tokens.append(token2)\n            elif token2 in re_dict:\n                token2 = re_dict[token2]\n                c_tokens.append(token2)\n            else:    \n                # apostrophe\n                if token2.endswith('\\'s'):\n                    token2 = token2[:-2]\n                    \n                if (token2.endswith('s')) & (token2[:-1] in embedding):\n                    token2 = token2[:-1]\n                    \n                # break dash\n                if \"-\" in token2:\n                    token2 = token2.split('-')\n                    c_tokens += token2\n                else:\n                    c_tokens.append(token2)\n        \n\n    return c_tokens\n\ntrain['clean_tokens'] = train['tokens'].map(lambda x: clean_token(x, to_remove, replace_dict, embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_vocab = build_vocab(train['clean_tokens'])\nnot_found_vocab = check_coverage(train_vocab, embeddings_index)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create X & y"},{"metadata":{"trusted":true},"cell_type":"code","source":"def doc_mean(tokens, embedding):\n    \n    e_values = []\n    e_values = [embedding[token] for token in tokens if token in embedding]\n    \n    if len(e_values) > 0:\n        return np.mean(np.array(e_values), axis=0)\n        #return np.sum(np.array(e_values), axis=0)\n    else:\n        #return np.ones(300)*-999\n        return np.zeros(300)\n      \nX = np.vstack(train['clean_tokens'].apply(lambda x: doc_mean(x, embeddings_index)))\n#X = np.vstack(train['tokens'].apply(lambda x: doc_mean(x, embeddings_index)))\n\ny = train['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# free up RAM\nimport gc\n\ndel not_found_vocab\ndel embeddings_index\ndel train_vocab\ndel train\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Machine Learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import linear_model, tree, ensemble, metrics, model_selection, exceptions\n\n\ndef print_score(y_true, y_pred):\n    print(' accuracy : ', metrics.accuracy_score(y_true, y_pred))\n    print('precision : ', metrics.precision_score(y_true, y_pred))\n    print('   recall : ', metrics.recall_score(y_true, y_pred))\n    print('       F1 : ', metrics.f1_score(y_true, y_pred))\n\n    \n# train-test split\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size = 0.2, random_state = 2019)\n#X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size = 0.2, train_size = 0.2, random_state = 2019)\nnp.random.seed(2019)\n\n# biased sampling\n# def select_train(X, y):\n#     pos_index = np.where(y == 1)[0]\n#     neg_index = np.where(y == 0)[0]\n#     size_select = min(len(pos_index), len(neg_index)) // 2\n#     return np.sort(np.append(np.random.choice(pos_index, size_select, replace = False), np.random.choice(neg_index, size_select, replace = False)))\n\n# train_index = select_train(X_train, y_train)\n# val_index = np.setdiff1d(range(len(X_train)), train_index)\n# X_trt, y_trt, X_trv, y_trv = [0, 0], [0, 0], [0, 0], [0, 0]\n# X_trt[1], y_trt[1] = X_train[train_index,:], y_train[train_index]\n# X_trv[1], y_trv[1] = X_train[val_index,:], y_train[val_index]\n\n# X_trt[0], X_trv[0], y_trt[0], y_trv[0] = model_selection.train_test_split(X_train, y_train, test_size = len(X_trv[1]), random_state = 2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# free up RAM\nimport gc\n\n#del not_found_vocab\n#del embeddings_index\n#del train_vocab\n#del train\n\ndel X\ndel y\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 1 - Logistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"# use full training dataset with cross validation\nlr = linear_model.LogisticRegression(solver = 'liblinear')\n\ncv_score = model_selection.cross_val_score(lr, X_train, y_train, cv = 5)\nprint(\"Cross validation score:\")\nprint(cv_score)\n\n \nlr.fit(X_train, y_train)\ny_pred_lr = lr.predict(X_test)\ny_pred_proba = lr.predict_proba(X_test)[:,1]\n\nprint_score(y_test, y_pred_lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# threshold search\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.91, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_test, (y_pred_proba > thresh).astype(int))\n    thresholds.append([thresh, res])\n    # print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n\nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\nprint(\"Best F1: \", thresholds[0][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # With biased sampling\n\n# lr = [linear_model.LogisticRegression(solver = 'liblinear') for _ in range(2)]\n\n# y_val_lr, y_pred_lr = [0, 0], [0, 0]\n# for i in range(2):\n#     lr[i].fit(X_trt[i], y_trt[i])\n#     y_val_lr[i] = lr[i].predict(X_trv[i])\n#     y_pred_lr[i] = lr[i].predict(X_test)\n    \n# print('-- validation result comparison --')\n# for i in range(2):\n#     print('- with' + ('' if i else 'out') + ' biased sampling -')\n#     print_score(y_trv[i], y_val_lr[i])\n# print('-- test result comparison --')\n# for i in range(2):\n#     print('- with' + ('' if i else 'out') + ' biased sampling -')\n#     print_score(y_test, y_pred_lr[i])    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 2 - Naive Bayes"},{"metadata":{"trusted":true},"cell_type":"code","source":"# probably need more thoughts on if to use Gaussian or to use multinominal with TF-IDF\n\nfrom sklearn import naive_bayes\n\nnb = naive_bayes.GaussianNB()\n\ncv_score = model_selection.cross_val_score(nb, X_train, y_train, cv = 5)\nprint(\"Cross validation score:\")\nprint(cv_score)\n\n\nnb.fit(X_train, y_train)\ny_pred_nb = nb.predict(X_test)\ny_pred_proba = nb.predict_proba(X_test)[:,1]\nprint_score(y_test, y_pred_nb)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# threshold search\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.91, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_test, (y_pred_proba > thresh).astype(int))\n    thresholds.append([thresh, res])\n    # print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n\nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\nprint(\"Best F1: \", thresholds[0][1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 3 - Random Forest"},{"metadata":{"trusted":true},"cell_type":"code","source":"rf = ensemble.RandomForestClassifier(n_estimators = 80, random_state = 2019, max_depth = 100)\n\ncv_score = model_selection.cross_val_score(rf, X_train, y_train, cv = 5)\nprint(\"Cross validation score:\")\nprint(cv_score)\n\n\nrf.fit(X_train, y_train)\ny_pred_rf = rf.predict(X_test)\ny_pred_proba = rf.predict_proba(X_test)[:,1]\nprint_score(y_test, y_pred_rf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# threshold search\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.91, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_test, (y_pred_proba > thresh).astype(int))\n    thresholds.append([thresh, res])\n    # print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n\nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\nprint(\"Best F1: \", thresholds[0][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# rf = [ensemble.RandomForestClassifier(n_estimators = 500, random_state = 2019, max_depth = 200) for _ in range(2)]\n\n# y_val_rf, y_pred_rf = [0, 0], [0, 0]\n# for i in range(2):\n#     rf[i].fit(X_trt[i], y_trt[i])\n#     y_val_rf[i] = rf[i].predict(X_trv[i])\n#     y_pred_rf[i] = rf[i].predict(X_test)\n    \n    \n# print('-- validation result comparison --')\n# for i in range(2):\n#     print('- with' + ('' if i else 'out') + ' biased sampling -')\n#     print_score(y_trv[i], y_val_rf[i])\n# print('-- test result comparison --')\n# for i in range(2):\n#     print('- with' + ('' if i else 'out') + ' biased sampling -')\n#     print_score(y_test, y_pred_rf[i])    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 4 - Boosting"},{"metadata":{},"cell_type":"markdown","source":"1. 'dart' without setting scale_pos_weight"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# lgb_c = lgb.LGBMClassifier(learning_rate = 0.04,n_estimators = 3000)\n\n# lgb_c.fit(X_train, y_train,\n#           eval_set = [(X_test, y_test)],\n#           early_stopping_rounds = 5,\n#           eval_metric = 'auc',\n#           verbose = 50)\n\n\n# y_pred = lgb_c.predict(X_test, num_iteration=lgb_c.best_iteration_)\n# print_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # threshold search\n\n# y_pred_proba = lgb_c.predict_proba(X_test,num_iteration=lgb_c.best_iteration_)[:,1]\n\n# thresholds = []\n# for thresh in np.arange(0.1, 0.91, 0.01):\n#     thresh = np.round(thresh, 2)\n#     res = metrics.f1_score(y_test, (y_pred_proba > thresh).astype(int))\n#     thresholds.append([thresh, res])\n#     # print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n\n# thresholds.sort(key=lambda x: x[1], reverse=True)\n# best_thresh = thresholds[0][0]\n# print(\"Best threshold: \", best_thresh)\n# print(\"Best F1: \", thresholds[0][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# lgb_c = lgb.LGBMClassifier(learning_rate = 0.04,n_estimators = 3000, boosting_type = 'dart', scale_pos_weight = 3.5)\n\n# lgb_c.fit(X_train, y_train,\n#           eval_set = [(X_test, y_test)],\n#           early_stopping_rounds = 5,\n#           eval_metric = 'auc',\n#           verbose = 50)\n\n\n# y_pred = lgb_c.predict(X_test, num_iteration=lgb_c.best_iteration_)\n# print_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# lgb_c = lgb.LGBMClassifier(learning_rate = 0.04,n_estimators = 2000)\n\n# lgb_c.fit(X_train, y_train,\n#           eval_set = [(X_test, y_test)],\n#           early_stopping_rounds = 5,\n#           eval_metric = 'auc',\n#           verbose = 50)\n\n\n# y_pred = lgb_c.predict(X_test, num_iteration=lgb_c.best_iteration_)\n# print_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# lgb_c = lgb.LGBMClassifier(learning_rate = 0.04,n_estimators = 2000, scale_pos_weight = 3.5)\n\n# lgb_c.fit(X_train, y_train,\n#           eval_set = [(X_test, y_test)],\n#           early_stopping_rounds = 5,\n#           eval_metric = 'auc',\n#           verbose = 50)\n\n\n# y_pred = lgb_c.predict(X_test, num_iteration=lgb_c.best_iteration_)\n# print_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# gridParams = {\n#     'num_leaves':[20, 60],\n#     'max_depth': [-1, 50, 200],\n#     'reg_alpha' : [0, 1,1.2],\n#     }\n\n# lgb_c = lgb.LGBMClassifier(n_estimators = 3000, boosting_type = 'dart')\n# grid_lgb = model_selection.RandomizedSearchCV(lgb_c, gridParams, scoring='accuracy', cv = 3)\n\n\n# grid_lgb.fit(X_train, y_train)\n\n# print(grid_lgb.best_params_)\n# print(grid_lgb.best_score_)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 5 - Deep learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import tensorflow as tf\n\n# model = tf.keras.models.Sequential([\n#     tf.keras.layers.Dense(12, activation=tf.nn.relu, input_shape=(300,)),\n#     tf.keras.layers.Dense(100, activation=tf.nn.relu),\n#     #tf.keras.layers.Dense(100, activation=tf.nn.relu),\n#     tf.keras.layers.Dropout(0.2),\n#     tf.keras.layers.Dense(1, activation=tf.nn.sigmoid)\n# ])\n\n# model.compile(optimizer=tf.train.AdamOptimizer(0.001),\n#               loss='binary_crossentropy',\n#               metrics=['accuracy'])\n\n# model.fit(X_train, y_train, epochs=10, validation_data=(X_test, y_test))\n# y_pred = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print_score(y_test, np.round(y_pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prediction for test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# test = pd.read_csv('../input/test.csv')\n# test['tokens'] = test['question_text'].map(lambda x: tokenize(x))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}