{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Pre-processing"},{"metadata":{},"cell_type":"markdown","source":"1. Attempt to preserve more information\n2. Attempt to achieve higher overlapping with embeddings "},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\n#nltk.download('popular')\n\nimport re\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer\n\n# text cleaning & tokenization\ndef tokenize(text, stop_set = None, lemmatizer = None):\n    \n    # clean text\n    text = text.encode('ascii', 'ignore').decode('ascii')\n    text = text.lower()\n    \n    text = re.sub(r'\\b(?:(?:https?|ftp)://)?\\w[\\w-]*(?:\\.[\\w-]+)+\\S*', ' ', text) # remove hyperlink,subs charact in the brackets\n    text = re.sub(\"[\\r\\n]\", ' ', text) # remove new line characters\n    #text = re.sub(r'[^\\w\\s]','',text)\n    text = text.strip() ## convert to lowercase split indv words\n    \n    #tokens = word_tokenize(text)\n    # use TweetTokenizer instead of word_tokenize -> to prevent splitting at apostrophies\n    tknzr = TweetTokenizer()\n    tokens = tknzr.tokenize(text)\n    \n    # retain tokens with at least two words\n    tokens = [token for token in tokens if re.match(r'.*[a-z]{2,}.*', token)]\n    \n    # remove stopwords - optional\n    # removing stopwords lost important information\n    if stop_set != None:\n        tokens = [token for token in tokens if token not in stop_set]\n    \n    # lemmmatization - optional\n    if lemmatizer != None:\n        tokens = [lemmatizer.lemmatize(token) for token in tokens]\n    return tokens\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#stop_set = set(stopwords.words('english'))\n#lemmatizer = WordNetLemmatizer()\n\n# with lemmatization\n#train['tokens'] = train['clean_text'].map(lambda x: tokenize(x, stop_set, lemmatizer))\n\n# without lemmatization\ntrain['tokens'] = train['question_text'].map(lambda x: tokenize(x))\ntest['tokens'] = test['question_text'].map(lambda x: tokenize(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(token_col):\n    \n    vocab = {}\n    for tokens in token_col:\n        for token in tokens:\n            vocab[token] = vocab.get(token, 0) + 1\n\n    return vocab\n\ntrain_vocab = build_vocab(train['tokens'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Word Embedding"},{"metadata":{},"cell_type":"markdown","source":"#### Google News"},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\n\ndef check_coverage(vocab,embedding):\n    \n    oov = {}\n    k = 0\n    i = 0\n    \n    for word in vocab:\n        if word in embedding:\n            k += vocab[word]\n        else:\n            oov[word] = vocab[word]\n            i += vocab[word]\n\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x\n  \n\nnot_found_vocab = check_coverage(train_vocab, embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_found_vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Google News Embeddings\n# replace not found words\nto_remove = ['to','of','and']\n\nreplace_dict = {'quora':'Quora', 'i\\'ve':'I\\'ve', 'instagram':'Instagram', 'upsc':'UPSC', 'bitcoin':'Bitcoin', 'trump\\'s':'Trump',\n               'mbbs':'MBBS', 'whatsapp':'WhatsApp', 'favourite':'favorite', 'ece':'ECE', 'aiims':'AIIMS', 'colour':'color',\n               'doesnt':'doesn\\'t','centre':'center','sbi':'SBI','cgl':'CGL','iim':'IIM','btech':'BTech'}\n\ndef clean_token(tokens, remove_list, re_dict):\n    tokens = [token for token in tokens if token not in remove_list]\n    tokens = [re_dict[token] if token in re_dict else token for token in tokens]\n    return tokens\n\ntrain['clean_tokens'] = train['tokens'].map(lambda x: clean_token(x, to_remove, replace_dict))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_vocab = build_vocab(train['clean_tokens'])\nnot_found_vocab = check_coverage(train_vocab, embeddings_index)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create X & y"},{"metadata":{"trusted":true},"cell_type":"code","source":"def doc_mean(tokens, embedding):\n    \n    e_values = []\n    e_values = [embedding[token] for token in tokens if token in embedding]\n    \n    if len(e_values) > 0:\n        return np.mean(np.array(e_values), axis=0)\n    else:\n        return np.zeros(300)\n      \nX = np.vstack(train['clean_tokens'].apply(lambda x: doc_mean(x, embeddings_index)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['target'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Machine Learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import linear_model, tree, ensemble, metrics, model_selection, exceptions\n\n\ndef print_score(y_true, y_pred):\n    print(' accuracy : ', metrics.accuracy_score(y_true, y_pred))\n    print('precision : ', metrics.precision_score(y_true, y_pred))\n    print('   recall : ', metrics.recall_score(y_true, y_pred))\n    print('       F1 : ', metrics.f1_score(y_true, y_pred))\n\n    \n# train-test split\nX_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size = 0.8, random_state = 2019)\n\nnp.random.seed(2019)\n\n# biased sampling\ndef select_train(X, y):\n    pos_index = np.where(y == 1)[0]\n    neg_index = np.where(y == 0)[0]\n    size_select = min(len(pos_index), len(neg_index)) // 2\n    return np.sort(np.append(np.random.choice(pos_index, size_select, replace = False), np.random.choice(neg_index, size_select, replace = False)))\n\ntrain_index = select_train(X_train, y_train)\nval_index = np.setdiff1d(range(len(X_train)), train_index)\nX_trt, y_trt, X_trv, y_trv = [0, 0], [0, 0], [0, 0], [0, 0]\nX_trt[1], y_trt[1] = X_train[train_index,:], y_train[train_index]\nX_trv[1], y_trv[1] = X_train[val_index,:], y_train[val_index]\n\nX_trt[0], X_trv[0], y_trt[0], y_trv[0] = model_selection.train_test_split(X_train, y_train, test_size = len(X_trv[1]), random_state = 2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# free up RAM\nimport gc\n\ndel not_found_vocab\ndel embeddings_index\ndel train_vocab\ndel train\n\ndel X\ndel y\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 1 - Logistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"# use full training dataset with cross validation\nlr = linear_model.LogisticRegression(solver = 'liblinear')\n\ncv_score = model_selection.cross_val_score(lr, X_train, y_train, cv = 5)\nprint(\"Cross validation score:\")\nprint(cv_score)\n\n\nlr.fit(X_train, y_train)\ny_pred_lr = lr.predict(X_test)\nprint_score(y_test, y_pred_lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# With biased sampling\n\nlr = [linear_model.LogisticRegression(solver = 'liblinear') for _ in range(2)]\n\ny_val_lr, y_pred_lr = [0, 0], [0, 0]\nfor i in range(2):\n    lr[i].fit(X_trt[i], y_trt[i])\n    y_val_lr[i] = lr[i].predict(X_trv[i])\n    y_pred_lr[i] = lr[i].predict(X_test)\n    \nprint('-- validation result comparison --')\nfor i in range(2):\n    print('- with' + ('' if i else 'out') + ' biased sampling -')\n    print_score(y_trv[i], y_val_lr[i])\nprint('-- test result comparison --')\nfor i in range(2):\n    print('- with' + ('' if i else 'out') + ' biased sampling -')\n    print_score(y_test, y_pred_lr[i])    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\ndel y_pred_lr\ndel lr\ndel y_val_lr\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 2 - Naive Bayes"},{"metadata":{"trusted":true},"cell_type":"code","source":"# probably need more thoughts on if to use Gaussian or to use multinominal with TF-IDF\n\nfrom sklearn import naive_bayes\n\nnb = naive_bayes.GaussianNB()\n\ncv_score = model_selection.cross_val_score(nb, X_train, y_train, cv = 5)\nprint(\"Cross validation score:\")\nprint(cv_score)\n\n\nnb.fit(X_train, y_train)\ny_pred_nb = nb.predict(X_test)\nprint_score(y_test, y_pred_nb)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 3 - Random Forest"},{"metadata":{"trusted":true},"cell_type":"code","source":"rf = ensemble.RandomForestClassifier(n_estimators = 200, random_state = 2019)\n\ncv_score = model_selection.cross_val_score(rf, X_train, y_train, cv = 5)\nprint(\"Cross validation score:\")\nprint(cv_score)\n\n\nrf.fit(X_train, y_train)\ny_pred_rf = rf.predict(X_test)\nprint_score(y_test, y_pred_rf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rf = [ensemble.RandomForestClassifier(n_estimators = 200, random_state = 2019) for _ in range(2)]\n\ny_val_rf, y_pred_rf = [0, 0], [0, 0]\nfor i in range(2):\n    rf[i].fit(X_trt[i], y_trt[i])\n    y_val_rf[i] = rf[i].predict(X_trv[i])\n    y_pred_rf[i] = rf[i].predict(X_test)\n    \n    \nprint('-- validation result comparison --')\nfor i in range(2):\n    print('- with' + ('' if i else 'out') + ' biased sampling -')\n    print_score(y_trv[i], y_val_rf[i])\nprint('-- test result comparison --')\nfor i in range(2):\n    print('- with' + ('' if i else 'out') + ' biased sampling -')\n    print_score(y_test, y_pred_rf[i])    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Attempt 4 - Boosting"},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\nlgb_c = lgb.LGBMClassifier(learning_rate = 0.08, metric = ['auc', 'logloss'], n_estimators = 350, num_leaves = 38)\n\nlgb_c.fit(X_train, y_train,\n          eval_set = [(X_test, y_test)],\n          early_stopping_rounds = 5,\n          verbose = 20)\n\n\ny_pred = lgb_c.predict(X_test, num_iteration=lgb_c.best_iteration_)\nprint_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# With biased sampling\n\nlgb_c = [lgb.LGBMClassifier(learning_rate = 0.08, metric = ['auc', 'logloss'], n_estimators = 350, num_leaves = 38) for _ in range(2)]\n\ny_val_lgb_c, y_pred_lgb_c = [0, 0], [0, 0]\nfor i in range(2):\n    lgb_c[i].fit(X_trt[i], y_trt[i],\n                eval_set = [(X_trv[i], y_trv[i])],\n                 early_stopping_rounds = 5,\n                 verbose = 20)\n    \n    y_pred_lgb_c[i] = lgb_c[i].predict(X_test)\n    \nprint('-- test result comparison --')\nfor i in range(2):\n    print('- with' + ('' if i else 'out') + ' biased sampling -')\n    print_score(y_test, y_pred_lgb_c[i])  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Attempt 5: Neural Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.neural_network import MLPClassifier\n\nclf = MLPClassifier(hidden_layer_sizes=(100,100,100), max_iter=100, alpha=0.0001,solver='sgd', verbose=10,  random_state=21,tol=0.001)\nclf.fit(X_train, y_train)\n\ny_pred = clf.predict(X_test)\nprint_score(y_test, y_pred)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}