{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport nltk\nfrom nltk.corpus import stopwords\nimport string\nimport xgboost as xgb\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, HashingVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn import ensemble, metrics, model_selection, naive_bayes\nfrom nltk.tokenize import RegexpTokenizer\nimport nltk.stem as stm\nfrom nltk import WordNetLemmatizer, word_tokenize\nfrom gensim.models import Word2Vec\nfrom sklearn.linear_model import LogisticRegression\nfrom tqdm import tqdm\nimport re\nimport string\nfrom time import time\n\nimport spacy\nfrom spacy.lang.en.stop_words import STOP_WORDS\nfrom spacy.lang.en import English\n#import textstat\n\nfrom sklearn.ensemble import VotingClassifier\nfrom sklearn.naive_bayes import BernoulliNB, MultinomialNB\nfrom sklearn.linear_model import SGDClassifier, LogisticRegression\nfrom sklearn.calibration import CalibratedClassifierCV\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2700c1ec02ff283029587c904a1d960af5432bae"},"cell_type":"code","source":"start_time = time()\ncolor = sns.color_palette()\ntqdm.pandas()\nalphabet = 'abcdefghijklmnopqrstuvwxyz'\n_punctuation = ['.', '..', '...', ',', ':', ';', '-', '*', '\"', '!', '?']\n#embeddings_index = {}\n#f = open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\n#for line in tqdm(f):\n#    values = line.split()\n#    word = values[0]\n#    coefs = np.asarray(values[1:], dtype='float32')\n#    embeddings_index[word] = coefs\n#f.close()\n\n#print('Found %s word vectors.' % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a79c80fd4fc6511cdc4d12c028fd926751a6e1ce"},"cell_type":"code","source":"def clean(text):\n    \n    ## Remove puncuation\n    text = text.translate(string.punctuation)\n    \n    ## Convert words to lower case and split them\n    text = text.lower()\n    \n    ## Remove stop words\n    #text = text.split()\n    #stops = set(stopwords.words(\"english\"))\n    #text = [w for w in text if not w in stops and len(w) >= 3]\n    \n    #text = \" \".join(text)\n\n    # Clean the text\n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = re.sub(r\"what's\", \"what is \", text)\n    text = re.sub(r\"\\'s\", \" \", text)\n    text = re.sub(r\"\\'ve\", \" have \", text)\n    text = re.sub(r\"n't\", \" not \", text)\n    text = re.sub(r\"i'm\", \"i am \", text)\n    text = re.sub(r\"\\'re\", \" are \", text)\n    text = re.sub(r\"\\'d\", \" would \", text)\n    text = re.sub(r\"\\'ll\", \" will \", text)\n    text = re.sub(r\",\", \" \", text)\n    text = re.sub(r\"\\.\", \" \", text)\n    text = re.sub(r\"!\", \" ! \", text)\n    text = re.sub(r\"\\/\", \" \", text)\n    text = re.sub(r\"\\^\", \" ^ \", text)\n    text = re.sub(r\"\\+\", \" + \", text)\n    text = re.sub(r\"\\-\", \" - \", text)\n    text = re.sub(r\"\\=\", \" = \", text)\n    text = re.sub(r\"'\", \" \", text)\n    text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n    text = re.sub(r\":\", \" : \", text)\n    text = re.sub(r\" e g \", \" eg \", text)\n    text = re.sub(r\" b g \", \" bg \", text)\n    text = re.sub(r\" u s \", \" american \", text)\n    text = re.sub(r\"\\0s\", \"0\", text)\n    text = re.sub(r\" 9 11 \", \"911\", text)\n    text = re.sub(r\"e - mail\", \"email\", text)\n    text = re.sub(r\"j k\", \"jk\", text)\n    text = re.sub(r\"\\s{2,}\", \" \", text)\n    text = re.sub('[^a-zA-Z]',' ', text)\n    text = re.sub('  +',' ',text)\n    \n    #text = text.split()\n    #stemmer = SnowballStemmer('english')\n    #stemmed_words = [stemmer.stem(word) for word in text]\n    #text = \" \".join(stemmed_words)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"01a9399fa4e91e43243ae184fd2a762261958889"},"cell_type":"code","source":"def runMNB(train_X, train_y, test_X, test_y, test_X2):\n    model = naive_bayes.MultinomialNB()\n    model.fit(train_X, train_y)\n    pred_test_y = model.predict_proba(test_X)\n    pred_test_y2 = model.predict_proba(test_X2)\n    return pred_test_y, pred_test_y2, model\n\ndef runBNB(train_X, train_y, test_X, test_y, test_X2):\n    model = BernoulliNB()\n    model.fit(train_X,train_y)\n    pred_test_y = model.predict_proba(test_X)\n    pred_test_y2 = model.predict_proba(test_X2)\n    return pred_test_y, pred_test_y2, model\n\ndef runLogistic(train_X, train_y, test_X, test_y, test_X2):\n    model = LogisticRegression()\n    model.fit(train_X,train_y)\n    pred_test_y = model.predict_proba(test_X)\n    pred_test_y2 = model.predict_proba(test_X2)\n    return pred_test_y, pred_test_y2, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5c2b692ab11ccedea3c0c683b83086df79a0ed2"},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')\n#train_X = train_df[\"clean_text\"].fillna(\"_##_\").values\n#test_X = test_df[\"clean_text\"].fillna(\"_##_\").values\n#print(\"Number of rows in train dataset : \",train_df.shape[0])\n#print(\"Number of rows in test dataset : \",test_df.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6660033cda38a05d0ac63e2f728be02577ad409f"},"cell_type":"code","source":"train_df['clean_text'] = train_df['question_text'].apply(clean)\ntest_df['clean_text'] = test_df['question_text'].apply(clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b7723e423dead48dbce9a7aa79619582665fe83"},"cell_type":"code","source":"import numpy as np\ndef load_glove():\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    return embeddings_index\n\ndef load_para():\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    return embeddings_index\n    \nembeddings_index_1 = load_glove()\n#embeddings_index_2 = load_para()\nprint('embedding matrix is loaded.')\ndef str2embeddingGlove(text,embeddings_index):\n    word = text.split()\n    vec = 0\n    count = 0\n    for w in word:\n        embedding_vector = embeddings_index.get(w)\n        if embedding_vector is not None:\n            vec += embedding_vector\n            count += 1\n    return vec/(count+1)\n\ndef str2embeddingMean(text):\n    word = text.split()\n    vec = 0\n    count = 0\n    for w in word:\n        embedding_vector = embeddings_index_1.get(w)\n        embedding_vector1 = embeddings_index_2.get(w)\n        if embedding_vector is not None:\n            vec = vec + embedding_vector+embedding_vector1\n            count += 2\n    return vec/(count+1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ef1d230f9fce350a7c5f48e474f6b9c5030d1ab"},"cell_type":"markdown","source":"## **Embedding+LR**\nWord embedding vector is 300 dimension. We use LR model to  generate stacking features with 5 folds prediction."},{"metadata":{"trusted":true,"_uuid":"1d186eacaba61d2862b3e4aff5ac7cff8315a484"},"cell_type":"code","source":"#glove embedding+lr stacking features\nimport gc\ntrain_y = train_df['target']\ntrain = np.zeros((len(train_df),300),dtype=np.float)\ntest = np.zeros((len(test_df),300),dtype=np.float)\nfor i in range(len(train_df)):\n    train[i] = str2embeddingGlove(train_df['clean_text'][i],embeddings_index_1)\nfor i in range(len(test_df)):\n    test[i] = str2embeddingGlove(test_df['clean_text'][i],embeddings_index_1)\ndel embeddings_index_1\ngc.collect()\nprint('start to generate embedding features...')\nk_fold = 5\ncv_scores = []\nLOG_pred_full_test = 0\nLOG_pred_train = np.zeros((train_df.shape[0],2))\nkf = model_selection.KFold(n_splits=k_fold, shuffle=True, random_state=2017)\nfold = 1\nfor dev_index, val_index in kf.split(train):\n    print(str(fold)+\"-th fold is going on...\")\n    fold += 1\n    dev_X, val_X = train[dev_index], train[val_index]\n    dev_y, val_y = train_y[dev_index], train_y[val_index]\n    LOG_pred_val_y, LOG_pred_test_y, LOG_model = runLogistic(dev_X, dev_y, val_X, val_y, test)\n    LOG_pred_full_test = LOG_pred_full_test + LOG_pred_test_y\n    LOG_pred_train[val_index,:] = LOG_pred_val_y\n    cv_scores.append(metrics.log_loss(val_y, LOG_pred_val_y[:,1]))\n    del dev_X,dev_y,val_X,val_y\n    gc.collect()\nprint(\"Mean cv score : \", np.mean(cv_scores))\nLOG_pred_full_test = LOG_pred_full_test / k_fold\ntrain_df['glove_lr_socre'] = LOG_pred_train[:,1]\ntest_df['glove_lr_socre'] = LOG_pred_full_test[:,1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9ad7f7b8df0baeafca3d3fb667802ee2744c94b"},"cell_type":"markdown","source":"## **Meta Features**\nBasic statistical features sometimes make sense in model accuracy. We can call them meta features."},{"metadata":{"trusted":true,"_uuid":"c36a1ab1a216dbb19b3c6ab632343eba8a5f1d4d"},"cell_type":"code","source":"eng_stopwords = set(stopwords.words(\"english\"))   \ntrain_df[\"num_words\"] = train_df[\"question_text\"].apply(lambda x: len(str(x).split()))\ntest_df[\"num_words\"] = test_df[\"question_text\"].apply(lambda x: len(str(x).split()))\n\n## Number of unique words in the text ##\ntrain_df[\"num_unique_words\"] = train_df[\"question_text\"].apply(lambda x: len(set(str(x).split())))\ntest_df[\"num_unique_words\"] = test_df[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n\n## Number of characters in the text ##\ntrain_df[\"num_chars\"] = train_df[\"question_text\"].apply(lambda x: len(str(x)))\ntest_df[\"num_chars\"] = test_df[\"question_text\"].apply(lambda x: len(str(x)))\n\n## Number of stopwords in the text ##\ntrain_df[\"num_stopwords\"] = train_df[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\ntest_df[\"num_stopwords\"] = test_df[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n\n## Number of punctuations in the text ##\ntrain_df[\"num_punctuations\"] =train_df['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\ntest_df[\"num_punctuations\"] =test_df['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n\n## Number of title case words in the text ##\ntrain_df[\"num_words_upper\"] = train_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\ntest_df[\"num_words_upper\"] = test_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n\n## Number of title case words in the text ##\ntrain_df[\"num_words_title\"] = train_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\ntest_df[\"num_words_title\"] = test_df[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n\n## Average length of the words in the text ##\ntrain_df[\"mean_word_len\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\ntest_df[\"mean_word_len\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n\ntrain_df[\",\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\",\")]))\ntest_df[\",\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\",\")]))\n\ntrain_df[\";\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\";\")]))\ntest_df[\";\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\";\")]))\n\ntrain_df['\\\"'] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split('\\\"')]))\ntest_df['\\\"'] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split('\\\"')]))\n\ntrain_df[\"...\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"...\")]))\ntest_df[\"...\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"...\")]))\n\ntrain_df[\"?\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"?\")]))\ntest_df[\"?\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"?\")]))\n\ntrain_df[\"!\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"!\")]))\ntest_df[\"!\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"!\")]))\n\ntrain_df[\".\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\".\")]))\ntest_df[\".\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\".\")]))\n\ntrain_df[\":\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\":\")]))\ntest_df[\":\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\":\")]))\n\ntrain_df[\"*\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"*\")]))\ntest_df[\"*\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"*\")]))\n\ntrain_df[\"-\"] = train_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"-\")]))\ntest_df[\"-\"] = test_df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split(\"-\")]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"954441550beedfc4bd3b048c809c5c4ce55f462a"},"cell_type":"markdown","source":"## **Statistical Features**\nSome words with high frequency are statisticed behind. "},{"metadata":{"trusted":true,"_uuid":"219f2c37c3a74be19511f722a872a4a27dce18e3"},"cell_type":"code","source":"#train_df['syllable_analysis'] = train_df['question_text'].apply(textstat.syllable_count)\n#test_df['syllable_analysis'] = test_df['question_text'].apply(textstat.syllable_count)\n\n#train_df['reading_ease'] = train_df['question_text'].apply(textstat.flesch_reading_ease)\n#test_df['reading_ease'] = test_df['question_text'].apply(textstat.flesch_reading_ease)\n\n#train_df['flesch_level'] = train_df['question_text'].apply(textstat.flesch_kincaid_grade)\n#test_df['flesch_level'] = test_df['question_text'].apply(textstat.flesch_kincaid_grade)\n\n#train_df['fog_scale'] = train_df['question_text'].apply(textstat.gunning_fog)\n#test_df['fog_scale'] = test_df['question_text'].apply(textstat.gunning_fog)\n\n#train_df['auto_index'] = train_df['question_text'].apply(textstat.automated_readability_index)\n#test_df['auto_index'] = test_df['question_text'].apply(textstat.automated_readability_index)\n\n#train_df['coleman_index'] = train_df['question_text'].apply(textstat.coleman_liau_index)\n#test_df['coleman_index'] = test_df['question_text'].apply(textstat.coleman_liau_index)\n\n#train_df['linsear_formula'] = train_df['question_text'].apply(textstat.linsear_write_formula)\n#test_df['linsear_formula'] = test_df['question_text'].apply(textstat.linsear_write_formula)\n\n#def consensus_all(text):\n#    return textstat.text_standard(text,float_output=True)\n\n#train_df['consensus'] = train_df['question_text'].apply(consensus_all)\n#test_df['consensus'] = test_df['question_text'].apply(consensus_all)\nfreq_words = ['best','will','good','people','way','time','year','make','sex','indian','india','women','american',\n              'muslim','men','girl','black','white','think','quora','trump','chinese','many', '']\n\ntrain_df['why'] = train_df['clean_text'].apply(lambda x:0 if len(str(x).split())==0 or str(x).split()[0]!='why' else 1)\ntest_df['why'] = test_df['clean_text'].apply(lambda x:0 if len(str(x).split())==0 or str(x).split()[0]!='why' else 1)\n\nfor word in freq_words:\n    train_df[word] = train_df[\"clean_text\"].str.count(word)\n    test_df[word] = test_df[\"clean_text\"].str.count(word)\n\nfreq_grams = ['year old','doe mean','look like','united states','feel like','high school',\n             'computer science','donald trump','white people','black people','president trump',\n             'trump supporter','social medium','long doe','mechanical engineer','north korea',\n             'north indian','african american','hillary clinton']\nfor gram in freq_grams:\n    train_df[gram] = train_df[\"clean_text\"].str.count(gram)\n    test_df[gram] = test_df[\"clean_text\"].str.count(gram)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"702b910dfe45eaed500bc84415ff3e87606f99fc"},"cell_type":"markdown","source":"## **TFIDF features+stacking**"},{"metadata":{"trusted":true,"_uuid":"4db061f85289b624423ccc043b76a183b0b4500a"},"cell_type":"code","source":"tfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1,3))\nfull_tfidf = tfidf_vec.fit_transform(train_df['clean_text'].values.tolist() + test_df['clean_text'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['clean_text'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['clean_text'].values.tolist())\ntrain_y = train_df['target']\n\nk_fold = 5\ncv_scores = []\nMNB_pred_full_test = 0\nBNB_pred_full_test = 0\nLOG_pred_full_test = 0\nMNB_pred_train = np.zeros((train_df.shape[0],2))\nBNB_pred_train = np.zeros((train_df.shape[0],2))\nLOG_pred_train = np.zeros((train_df.shape[0],2))\nkf = model_selection.KFold(n_splits=k_fold, shuffle=True, random_state=2017)\nfold = 1\nfor dev_index, val_index in kf.split(train_tfidf):\n    print(str(fold)+\"-th fold is going on...\")\n    fold += 1\n    dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]\n    dev_y, val_y = train_y[dev_index], train_y[val_index]\n    MNB_pred_val_y, MNB_pred_test_y, MNB_model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)\n    BNB_pred_val_y, BNB_pred_test_y, BNB_model = runBNB(dev_X, dev_y, val_X, val_y, test_tfidf)\n    LOG_pred_val_y, LOG_pred_test_y, LOG_model = runLogistic(dev_X, dev_y, val_X, val_y, test_tfidf)\n    MNB_pred_full_test = MNB_pred_full_test + MNB_pred_test_y\n    BNB_pred_full_test = BNB_pred_full_test + BNB_pred_test_y\n    LOG_pred_full_test = LOG_pred_full_test + LOG_pred_test_y\n    MNB_pred_train[val_index,:] = MNB_pred_val_y\n    BNB_pred_train[val_index,:] = BNB_pred_val_y\n    LOG_pred_train[val_index,:] = LOG_pred_val_y\n    cv_scores.append(metrics.log_loss(val_y, MNB_pred_val_y[:,1]))\nprint(\"Mean cv score : \", np.mean(cv_scores))\nMNB_pred_full_test = MNB_pred_full_test / k_fold\nBNB_pred_full_test = BNB_pred_full_test / k_fold\nLOG_pred_full_test = LOG_pred_full_test / k_fold\ntrain_df['tf_nb_socre'] = MNB_pred_train[:,1]\ntest_df['tf_nb_socre'] = MNB_pred_full_test[:,1]\ntrain_df['tf_bb_socre'] = BNB_pred_train[:,1]\ntest_df['tf_bb_socre'] = BNB_pred_full_test[:,1]\ntrain_df['tf_lr_socre'] = LOG_pred_train[:,1]\ntest_df['tf_lr_socre'] = LOG_pred_full_test[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a7314b6eab62da60b67c37b5152f04171343832"},"cell_type":"code","source":"import gc\ndel tfidf_vec,full_tfidf,train_tfidf,test_tfidf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"17eb4b43b6ea6e6928e56e1dbf4a34b29fe6fac8"},"cell_type":"code","source":"tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,3))\ntfidf_vec.fit(train_df['clean_text'].values.tolist() + test_df['clean_text'].values.tolist())\ntrain_tfidf = tfidf_vec.transform(train_df['clean_text'].values.tolist())\ntest_tfidf = tfidf_vec.transform(test_df['clean_text'].values.tolist())\n\ntrain_y = train_df['target']\n\nk_fold = 5\ncv_scores = []\nMNB_pred_full_test = 0\nBNB_pred_full_test = 0\nLOG_pred_full_test = 0\nMNB_pred_train = np.zeros((train_df.shape[0],2))\nBNB_pred_train = np.zeros((train_df.shape[0],2))\nLOG_pred_train = np.zeros((train_df.shape[0],2))\nkf = model_selection.KFold(n_splits=k_fold, shuffle=True, random_state=2017)\nfold = 1\nfor dev_index, val_index in kf.split(train_tfidf):\n    print(str(fold)+\"-th fold is going on...\")\n    fold += 1\n    dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]\n    dev_y, val_y = train_y[dev_index], train_y[val_index]\n    MNB_pred_val_y, MNB_pred_test_y, MNB_model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)\n    BNB_pred_val_y, BNB_pred_test_y, BNB_model = runBNB(dev_X, dev_y, val_X, val_y, test_tfidf)\n    LOG_pred_val_y, LOG_pred_test_y, LOG_model = runLogistic(dev_X, dev_y, val_X, val_y, test_tfidf)\n    MNB_pred_full_test = MNB_pred_full_test + MNB_pred_test_y\n    BNB_pred_full_test = BNB_pred_full_test + BNB_pred_test_y\n    LOG_pred_full_test = LOG_pred_full_test + LOG_pred_test_y\n    MNB_pred_train[val_index,:] = MNB_pred_val_y\n    BNB_pred_train[val_index,:] = BNB_pred_val_y\n    LOG_pred_train[val_index,:] = LOG_pred_val_y\n    cv_scores.append(metrics.log_loss(val_y, MNB_pred_val_y[:,1]))\nprint(\"Mean cv score : \", np.mean(cv_scores))\nMNB_pred_full_test = MNB_pred_full_test / k_fold\nBNB_pred_full_test = BNB_pred_full_test / k_fold\nLOG_pred_full_test = LOG_pred_full_test / k_fold\ntrain_df['cv_nb_socre'] = MNB_pred_train[:,1]\ntest_df['cv_nb_socre'] = MNB_pred_full_test[:,1]\ntrain_df['cv_bb_socre'] = BNB_pred_train[:,1]\ntest_df['cv_bb_socre'] = BNB_pred_full_test[:,1]\ntrain_df['cv_lr_socre'] = LOG_pred_train[:,1]\ntest_df['cv_lr_socre'] = LOG_pred_full_test[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f430354065c8b1c7bd144ef903df13d44ca31406"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nimport scipy.sparse as sp\nfrom sklearn.model_selection import train_test_split\n\ntest_df['target']=np.nan\nall_text = pd.concat([train_df['clean_text'],test_df['clean_text']],axis=0)\nword_vect = TfidfVectorizer(\n            sublinear_tf=True,\n            strip_accents='unicode',\n            analyzer='word',\n            token_pattern=r'\\w{1,}',\n            stop_words='english',\n            ngram_range=(1, 2),\n            max_features=20000)\n\nword_vect.fit(all_text)\ntf_train = word_vect.transform(train_df['clean_text'])\ntf_test = word_vect.transform(test_df['clean_text'])\n\nprint('TFIDF features transformation is finished!')\n\ncols_to_drop = ['qid','question_text','clean_text']\ntrain_X = train_df.drop(cols_to_drop+['target'],axis=1).values\ntrain_y = train_df['target']\ntest_X = test_df.drop(cols_to_drop+['target'],axis=1).values\n\ntrain_X = sp.hstack((train_X,tf_train,train))\ntest_X = sp.hstack((test_X,tf_test,test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f629ccc780eb9b4d74fb93a0c1b513c9322c91f"},"cell_type":"code","source":"import lightgbm as lgb\nimport numpy as np\n\nclf = lgb.LGBMClassifier(learning_rate=0.03,objective='binary',reg_alpha=0.002,\n                             subsample=0.8,colsample_bytree=0.8,n_estimators=3000,\n                             early_stopping_round=100,silent=-1)\nclf.fit(train_X,train_y,eval_set=[(val_X,val_y)],eval_metric='binary_logloss',verbose=100,early_stopping_rounds=100)\npred_val_y = clf.predict_proba(val_X,num_iteration=clf.best_iteration_)[:,1]\npred_test_y = clf.predict_proba(test_X,num_iteration=clf.best_iteration_)[:,1]\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\n\npred_test_y = (pred_test_y > best_thresh).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89c0e311ed5668b53ec703878917ea51baa41152"},"cell_type":"code","source":"len(val_X.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"890b224fa67cc5b11c7252c35e75a7803e8e0583"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a366624fc13839e9041bd4316e16fc8abc3a0b4c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}