{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"################# GENERAL IMPORTS\nimport os\nimport string\nfrom pprint import pprint\nfrom operator import itemgetter\nimport matplotlib.pyplot as plt\nfrom sklearn.svm import LinearSVC\nfrom sklearn.preprocessing import MultiLabelBinarizer\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.metrics import f1_score, precision_score, recall_score,accuracy_score\n##################################### NLP SPECIFIC IMPORTS\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk import ngrams\nfrom nltk.stem import PorterStemmer\nfrom nltk.corpus import reuters\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom wordcloud import WordCloud\nfrom collections import Counter\nreuters.fileids()\nstopwords.words('english')\n\nimport numpy as np\nimport pandas as pd\nimport warnings\nfrom sys import modules\n\nwarnings.filterwarnings('ignore')\n%matplotlib inline\n\nfrom gensim.models import word2vec\nimport logging\n\nfrom keras.preprocessing import sequence\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Activation, Flatten\nfrom keras.layers.wrappers import TimeDistributed\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.recurrent import LSTM\nfrom keras.layers import Dropout\n\nimport seaborn as sns\nsns.set(style = 'darkgrid')\nprint(os.listdir(\"../input\"))\nimport re\npd.set_option('max_colwidth', 800)\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split\n\nnp.random.seed(1234)\n%matplotlib inline\n\nprint('all set')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"quora_train=pd.read_csv(\"../input/train.csv\")\nquora_test=pd.read_csv(\"../input/test.csv\")\nprint(\"Train size =\" ,quora_train.shape)\nprint(\"Test size =\" ,quora_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f29d589f9b24f3ab088fa3bc42172625abe806f9","trusted":false},"cell_type":"code","source":"# quora_train=quora_train[0:100000]\n# quora_test=quora_test[0:20000]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"148e5b19e85246cfa1076f3eac282720c88b7b30","trusted":false},"cell_type":"code","source":"quora_train['words'] = quora_train.question_text.apply(lambda x: len(x.split()))\nquora_train['characters'] = quora_train.question_text.apply(lambda x: len(x))\nquora_test['words'] = quora_test.question_text.apply(lambda x: len(x.split()))\nquora_test['characters'] = quora_test.question_text.apply(lambda x: len(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b1261dbb7912293aada5adeb02b5e07a22d23cd","trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(18, 7))\n\nplt.subplot(1, 2, 1)\nquora_train.groupby('target')['words'].mean().plot(kind='bar', ylim=(0,20), title= 'Average word count by target')\n\nplt.subplot(1, 2, 2)\nquora_train.groupby('target')['characters'].mean().plot(kind='bar', ylim=(0,105), title= 'Average character count by target')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f594202be25e8accbc6ea2f938f9ce8f385e907c","trusted":false},"cell_type":"code","source":"from nltk import pos_tag\n\ndef verb_count(text):\n    token_text= word_tokenize(text)\n    tagged_text = pos_tag(token_text)\n    counter=0\n    for w,t in tagged_text:\n        t = t[:2]\n        if t in ['VB']:\n            counter+=1\n    return counter\n\ndef noun_count(text):\n    token_text= word_tokenize(text)\n    tagged_text = pos_tag(token_text)\n    counter=0\n    for w,t in tagged_text:\n        t = t[:2]\n        if t in ['NN']:\n            counter+=1\n    return counter\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe35cd81e8610dff21290a5a07d7b0019f365b75","trusted":false},"cell_type":"code","source":"quora_train['question_text_prep'] = quora_train['question_text'].apply(lambda x: x.lower())\nquora_test['question_text_prep'] = quora_test['question_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad661a44375702c7e28907f59f04f0b316bd4c72","trusted":false},"cell_type":"code","source":"def pad_punctuation_w_space(string):\n    s = re.sub('([:;\"*.,!?()/\\=-])', r' \\1 ', string)\n    s=re.sub('[^a-zA-Z]',' ',s)\n    s = re.sub('\\s{2,}', ' ', s)\n    s =  re.sub(r\"\\b[a-zA-Z]\\b\", \"\", s) #code for removing single characters\n    return s\nquora_train['question_text_prep'] = quora_train['question_text_prep'].apply(lambda x: pad_punctuation_w_space(x))\nquora_test['question_text_prep'] = quora_test['question_text_prep'].apply(lambda x: pad_punctuation_w_space(x))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3430ffd41776b0e98a59cfc649a40c953ac358df","trusted":false},"cell_type":"code","source":"quora_train['question_text_prep'] = quora_train['question_text_prep'].apply(lambda x: x.split())\nquora_test['question_text_prep'] = quora_test['question_text_prep'].apply(lambda x: x.split())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad79ee4e17051f9a706295138ce29bc853962a4e","trusted":false},"cell_type":"code","source":"stop_list = stopwords.words('english') + list(string.punctuation)\nquora_train['question_text_prep'] = quora_train['question_text_prep'].apply(lambda x: [i for i in x if i not in stop_list])\nquora_test['question_text_prep'] = quora_test['question_text_prep'].apply(lambda x: [i for i in x if i not in stop_list]) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be47361638396bcd1378354d38f6f5045ae5d394","trusted":false},"cell_type":"code","source":"quora_train['question_text_prep_string'] = quora_train['question_text_prep'].str.join(\" \")\nquora_test['question_text_prep_string'] = quora_test['question_text_prep'].str.join(\" \")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"168eb82e592517530d74b943de2cf78e08730d8d","trusted":false},"cell_type":"code","source":"sents = list(quora_train.question_text_prep.values) \nsents[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3c8383fc124ff3b3c339e261e8de37d647291cd","trusted":false},"cell_type":"code","source":"min_num = 3 # minimum number of occurrences in text\nEMBEDDING_FILE= \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34f62f1018bfcade38032df9be24fecdae5a44b4","trusted":false},"cell_type":"code","source":"import numpy as np\ndef loadGloveModel(gloveFile):\n    print (\"Loading Glove Model\")\n    f = open(gloveFile,'r', encoding='utf8')\n    model = {}\n    for line in f:\n        splitLine = line.split(' ')\n        word = splitLine[0]\n        embedding = np.asarray(splitLine[1:], dtype='float32')\n        model[word] = embedding\n    print (\"Done.\",len(model),\" words loaded!\")\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af52f38ab512f62fd73120cd21460659656c86b3","trusted":false},"cell_type":"code","source":"word_model= loadGloveModel(EMBEDDING_FILE)   \n# print (word_model['hello']) # if we want to see an example for a vector","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"905b8b47a9f3bd2d7fc52c3db96e70b9468f5396","trusted":false},"cell_type":"code","source":"print('Loaded %s word vectors.' % len(word_model))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17b4dd02ce2ce6e0f628e4781e1464711aa25321","trusted":false},"cell_type":"code","source":"unknown_words = []\nfor question in quora_train.question_text_prep:\n    for word in question:\n        if word not in word_model:\n            unknown_words.append(word)\n        else: pass","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fba103a1e9c63bcdedbfd03a6d2321ed7e573b0","trusted":false},"cell_type":"code","source":"total_term_frequency = Counter(unknown_words)\n\nfor word, freq in total_term_frequency.most_common(20):\n    print(\"{}\\t{}\".format(word, freq))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65a8cfaea59788bc2c6942c2f295afb7b7b5b902","trusted":false},"cell_type":"code","source":"def get_vector(DataFrame):\n    vec_X = []\n    i = 0\n    for item in DataFrame.question_text_prep_string: \n        \n        sentence = pad_punctuation_w_space(item)\n        s = np.array([])\n        s = []\n        if len(sentence)==0:\n            s = np.array(word_model['UNK'])\n            vec_X.append(s) \n            i += 1\n        else:\n                for word in sentence.split():\n                    if len(s) == 0:\n                        try:\n                            s = np.array(word_model[word])\n                        except: \n                            s = np.array(word_model['UNK'])\n                    else:\n                        try:\n                            s += np.array(word_model[word])\n                        except: \n                            s += np.array(word_model['UNK'])         \n                vec_X.append(s) \n                i += 1\n\n    return vec_X","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79df43ff9db1aa875a15ae504a4c7d8e04777895","trusted":false},"cell_type":"code","source":"vec_X_train=get_vector(quora_train)\nvec_X_test=get_vector(quora_test)\nquora_train[\"vector\"]=vec_X_train\nquora_test[\"vector\"]=vec_X_test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e39551966653f02c12e4fffd29adc529628e69d","trusted":false},"cell_type":"code","source":"from imblearn.under_sampling import RandomUnderSampler","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6c5b02ad6b83f1cf43af134e0c59a7356b35f65","trusted":false},"cell_type":"code","source":"X = quora_train[['words','characters','vector']] #,'noun_count'\ny = quora_train['target']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a203ec87a35e9ec3b3a86a2f601a0f46a96cbed1","trusted":false},"cell_type":"code","source":"rus = RandomUnderSampler(return_indices=True, ratio = 0.42)\nX_rus, y_rus, id_rus = rus.fit_sample(X, y)\n\nprint('indexes:', id_rus)\nprint(len(id_rus))\nprint(quora_train.target.value_counts())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ce1eebfbbb9430dada1a318379edd49254d6391","trusted":false},"cell_type":"code","source":"quora_undr=quora_train.loc[id_rus]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"affa97836a3e0f79ba35b5129fd0d506f49d4969","trusted":false},"cell_type":"code","source":"quora_undr['target'].value_counts(ascending=True).plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d96ec4ca10e899199ddbb1a88a4cac7e1556136","trusted":false},"cell_type":"code","source":"quora_undr['target'].value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"708db25b47d6b066b1ec7b32149e0b6adb826865","trusted":false},"cell_type":"code","source":"quora_under_prep = quora_undr","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c6e9884133f67b1db58ba9c01eab01000cfd889","trusted":false},"cell_type":"code","source":"quora_under_prep['noun_count'] = quora_under_prep.question_text.apply(lambda x: noun_count(x))\nquora_test['noun_count'] = quora_test.question_text.apply(lambda x: noun_count(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa78323700654dbdcfb5533852513f7c67c7fbb1","trusted":false},"cell_type":"code","source":"quora_under_prep['vector_length']= quora_under_prep['vector'].apply(lambda x: len(x))\nquora_test['vector_length']= quora_test['vector'].apply(lambda x: len(x))\nquora_test['vector_length'].describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c231127654e570e9a040822d0d01b51d54e377df","trusted":false},"cell_type":"code","source":"quora_best=quora_under_prep","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65356cbc1d652675dc5395cdd08325c2d57f4b03","trusted":false},"cell_type":"code","source":"import numpy as np\nquora_best[\"joinvector\"]=[np.concatenate((np.array([quora_best[\"characters\"].iloc[i]]),quora_best[\"vector\"].iloc[i]), axis=None) for i in range(len(quora_best))]\nquora_best[\"joinvector_2\"]=[np.concatenate((np.array([quora_best[\"words\"].iloc[i]]),quora_best[\"joinvector\"].iloc[i]), axis=None) for i in range(len(quora_best))]\nquora_best[\"joinvector_all\"]=[np.concatenate((np.array([quora_best[\"noun_count\"].iloc[i]]),quora_best[\"joinvector_2\"].iloc[i]), axis=None) for i in range(len(quora_best))]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"525204885567de760c488e36f3da28cfc644831f","trusted":false},"cell_type":"code","source":"quora_test[\"joinvector\"]=[np.concatenate((np.array([quora_test[\"characters\"].iloc[i]]),quora_test[\"vector\"].iloc[i]), axis=None) for i in range(len(quora_test))]\nquora_test[\"joinvector_2\"]=[np.concatenate((np.array([quora_test[\"words\"].iloc[i]]),quora_test[\"joinvector\"].iloc[i]), axis=None) for i in range(len(quora_test))]\nquora_test[\"joinvector_all\"]=[np.concatenate((np.array([quora_test[\"noun_count\"].iloc[i]]),quora_test[\"joinvector_2\"].iloc[i]), axis=None) for i in range(len(quora_test))]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4666c42c6b7b684033a5f479ba839cdace380e7","trusted":false},"cell_type":"code","source":"X_joinvec=quora_best[\"joinvector_all\"].tolist()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c8b821d2a82f7eb96def4fa885f06b43cb965ef","trusted":false},"cell_type":"code","source":"Features = quora_best['joinvector_all']\n# Features2=quora_best['vector']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29a4433aeb8bbbc823482003111bd5278ccfad84","trusted":false},"cell_type":"code","source":"from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n\n\nX_train, X_val, y_train, y_val = train_test_split(Features,quora_best['target'],\n                                                    train_size=0.7, random_state = 143, stratify=quora_best['target'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8db6b8de75f7b844a49848f2f0897ea93e764145","trusted":false},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import GradientBoostingClassifier\n\n#evaluators:\nfrom sklearn.metrics import confusion_matrix, classification_report, accuracy_score, f1_score\nfrom sklearn.model_selection import cross_val_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c084ae33c9d718d7f515346072f9289c6bd44d55","trusted":false},"cell_type":"code","source":"X_grid5 = X_train.tolist()\ny_grid5 = y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1b9c78aabbc829b1fe0bb7186a7ba9c1984908de"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nRfr_clf = RandomForestClassifier()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cea722f5372c33f19c90574018b6ba9317d8522f"},"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV\n\nparam_grid = {'n_estimators':[10,50,100,200],\n              'criterion' : ['gini', 'entropy'],\n              'class_weight' : [{0:9,1:1},{0:1,1:1},{0:66,1:33}],\n              'max_depth' : range(2,10),\n              'min_samples_split': range(2,10),\n              'min_samples_leaf' : range(2,10),\n              'bootstrap': [True,False] }\n\nRFR_clf_gs = RandomizedSearchCV(estimator=Rfr_clf, param_distributions=param_grid, cv=3,\n                                verbose=0, n_jobs=-1,scoring='f1')\n\nRFR_clf_gs.fit(X_grid5, y_grid5)\nRFR_best=RFR_clf_gs.best_estimator_ \nprint(\"Randomized search process ended\")   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5b73f0a5dcd72d953e62240285664256d6725108"},"cell_type":"code","source":"y_pred= RFR_best.predict(X_grid5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3a15d5491c0601563b967be5d8e70fb825979b85"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67dbbffc4dc5ee70b3f5043912aefe7a95a0d898","trusted":false},"cell_type":"code","source":"# param_grid ={\"learning_rate\":(0.1,0.5,0.8),\n#              'max_depth' : range(2,53,10),\n#              'min_samples_split': range(2,53,10),\n#              'min_samples_leaf' : range(2,53,10),\n#              'n_estimators' : (100,200,300,400) ,\n#              'max_features': range(2,303,30),\n#              #'random_state': (143),\n#              'subsample': (0.1,0.5,0.8,1,2)} ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f900cdbac302654a1858f9a6ec1f3b0a826ff22","trusted":false},"cell_type":"code","source":"# GB_clf = GradientBoostingClassifier()\n# gs= GridSearchCV(estimator=GB_clf, param_grid=param_grid, cv=3,scoring='f1') # verbose=15, n_jobs=-1\n# gs.fit(X_grid5, y_grid5)\n# best_model=gs.best_estimator_ ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59c78f019a85f763c667574a950642ef2d0a7833","trusted":false},"cell_type":"code","source":"# y_pred= best_model.predict(X_grid5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"229a82970d3104e464d1fcb5cf7d574c3f614346","trusted":false},"cell_type":"code","source":"test_Features = quora_test['joinvector_all']\nX_test_original=test_Features.tolist()\n# y_test_pred= best_model.predict(X_test_original)\ny_test_pred= RFR_best.predict(X_test_original)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3cd53489b124b14def05c74a5cff4c44a4a8ca6c","trusted":false},"cell_type":"code","source":"quora_test_tmp=quora_test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b4eea90559a797c3a0de555ebad742f3d8576c3","trusted":false},"cell_type":"code","source":"quora_test_tmp[\"pred\"]=y_test_pred #(y_test_pred > delta).astype(int) \nquora_test_tmp1 = quora_test_tmp[['qid','question_text','pred']]\nquora_test_tmp1[quora_test_tmp1['pred']==1].sample(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3f0cd7e79808fc6aba26d5581d241deb932ae1a","trusted":false},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nout_df = pd.DataFrame({\"qid\":sub[\"qid\"].values})\nout_df['prediction'] = y_test_pred\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3030731d927fb5b2564a1d0be5b4628fbda2657a","trusted":false},"cell_type":"code","source":"round(out_df['prediction'].value_counts(normalize =True),3)*100","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"472897658b958146c4da7b0a2726aeb80cf588b8","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d53a51cef4cb6006578b2ed7bc1fe1a184cbbffa","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d2dd5d55797f972360f616e0f867251ff1c1eaa","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}