{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#imports\nimport re\n\nimport pandas as pd\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport wordcloud\nfrom wordcloud import WordCloud,STOPWORDS\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nnltk.download('stopwords')\nimport string\n\nimport sklearn\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\n\nfrom collections import Counter\nfrom collections import defaultdict\nfrom plotly import tools\nfrom plotly.subplots import make_subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Reading Data\ndf_train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ndf_test = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Meta features Creation**\nBased on this SRK's EDA [kernal](https://www.kaggle.com/sudalairajkumar/simple-feature-engg-notebook-spooky-author), creating following meta-features:-\n1. Length of test\n2. Number of words in the text\n3. Number of unique words in the text\n4. Number of characters in the text\n5. Number of stopwords\n6. Number of punctuations\n7. Number of upper case words\n8. Number of title case words\n9. Number of numericals in the text\n10. Average length of the words\n11. Check for urls\n\nThe idea behind creating above features is to check if it helps in identifing distinction b/w negatives and positives"},{"metadata":{"trusted":true},"cell_type":"code","source":"class metaFeatures:\n    \n    def __init__(self,df):\n        self.df = df\n    \n    #Number of words in the text\n    @staticmethod\n    def num_of_words(ech_row):\n        rtn = len(ech_row.split())\n        return rtn\n    \n    #Number of unique words in the text\n    @staticmethod\n    def num_of_unqwords(ech_row):\n        rtn = len(set(ech_row.split()))\n        return rtn\n    \n    #Number of characters in the text\n    @staticmethod\n    def num_of_chars(ech_row):\n        rtn = len(set(ech_row))\n        return rtn\n    \n    #Number of stopwords\n    @staticmethod\n    def num_of_stopwords(ech_row):\n        stopwords_lst = list(stopwords.words('english'))\n        rtn = len([s for s in str(ech_row).lower().split() if s in stopwords_lst])\n        return rtn\n    \n    #Number of punctuations\n    @staticmethod\n    def num_of_punctuations(ech_row):\n        punc_lst = list(string.punctuation)\n        rtn = len([p for p in str(ech_row).lower().split() if p in punc_lst])\n        return rtn\n    \n    #Number of upper case words\n    @staticmethod\n    def num_of_uppercase(ech_row):\n        rtn = len([p for p in str(ech_row).split() if p.isupper()])\n        return rtn\n    \n    #Number of title case words\n    @staticmethod\n    def num_of_titlecase(ech_row):\n        rtn = len([p for p in str(ech_row).split() if p.istitle()])\n        return rtn\n    \n    #Number of numericals in the text\n    @staticmethod\n    def num_of_numericals(ech_row):\n        numer_lst = ['0','1','2','3','4','5','6','7','8','9']\n        rtn = len([p for p in numer_lst if p in ech_row])\n        return rtn\n    \n    #Average length of the words\n    @staticmethod\n    def words_avglen(ech_row):\n        rtn = np.round( np.mean([len(p) for p in str(ech_row).split()]) ,2)\n        return rtn\n    \n    # URLs Check\n    @staticmethod\n    def urls_count(ech_row):\n        rtn = len([h for h in str(ech_row).lower().split() if 'http' in h or 'https' in h])\n        return rtn\n    \n    #final calculations\n    def calc(self):\n        \n        self.df['text_len'] = self.df['question_text'].apply(lambda x: len(x))\n        self.df['num_of_words'] = self.df['question_text'].apply(lambda x: self.num_of_words(x))\n        self.df['num_of_unqwords'] = self.df['question_text'].apply(lambda x: self.num_of_unqwords(x))\n        self.df['num_of_chars'] = self.df['question_text'].apply(lambda x: self.num_of_chars(x))\n        self.df['num_of_stopwords'] = self.df['question_text'].apply(lambda x: self.num_of_stopwords(x))\n        self.df['num_of_punctuations'] = self.df['question_text'].apply(lambda x: self.num_of_punctuations(x))\n        self.df['num_of_uppercase'] = self.df['question_text'].apply(lambda x: self.num_of_uppercase(x))\n        self.df['num_of_titlecase'] = self.df['question_text'].apply(lambda x: self.num_of_titlecase(x))\n        self.df['num_of_numericals'] = self.df['question_text'].apply(lambda x: self.num_of_numericals(x))\n        self.df['words_avglen'] = self.df['question_text'].apply(lambda x: self.words_avglen(x))\n        self.df['urls_count'] = self.df['question_text'].apply(lambda x: self.urls_count(x))\n        \n        return self.df\n\nmetafeatures = metaFeatures(df_train)\ndf_train_feat = metafeatures.calc()\n\n#positive and negative reviews\ndf_train_pos = df_train_feat[df_train_feat.target==1]\ndf_train_neg = df_train_feat[df_train_feat.target==0]\n\ndf_train_feat.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Data Cleaning**"},{"metadata":{"trusted":true},"cell_type":"code","source":"nltk.download('wordnet')\nclass CleanIt:\n    \n    def __init__(self,df):\n        self.df = df\n    \n    #Removes Punctuations\n    @staticmethod\n    def remove_punctuations(ech_row):\n        punct_tag = re.compile(r'[^\\w\\s]')\n        rtn = punct_tag.sub(r'',ech_row)\n        return rtn\n    \n    #Removes HTML syntaxes\n    @staticmethod\n    def remove_html(ech_row):\n        html_tag=re.compile(r'<.*?>')\n        rtn=html_tag.sub(r'',ech_row)\n        return rtn\n    \n    #Removes URL data\n    @staticmethod\n    def remove_url(ech_row):\n        url_clean= re.compile(r\"https://\\S+|www\\.\\S+\")\n        rtn=url_clean.sub(r'',ech_row)\n        return rtn\n    \n    #Removes Emojis\n    @staticmethod\n    def remove_emoji(ech_row):\n        emoji_clean= re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               u\"\\U00002702-\\U000027B0\"\n                               u\"\\U000024C2-\\U0001F251\"\n                               \"]+\", flags=re.UNICODE)\n        rtn=emoji_clean.sub(r'',ech_row)\n        url_clean= re.compile(r\"https://\\S+|www\\.\\S+\")\n        rtn=url_clean.sub(r'',rtn)\n        return rtn\n    \n    #Lower Case\n    @staticmethod\n    def make_it_lower(ech_row):\n        rtn=ech_row.lower()\n        return rtn\n    \n    #Remove extra spaces\n    @staticmethod\n    def remove_extraSpace(ech_row):\n        rtn=' '.join(ech_row.split())\n        return rtn\n    \n    #Replace abbreviated pronouns with full forms\n    @staticmethod\n    def remove_abb(data):\n        data = re.sub(r\"he's\", \"he is\", data)\n        data = re.sub(r\"there's\", \"there is\", data)\n        data = re.sub(r\"We're\", \"We are\", data)\n        data = re.sub(r\"That's\", \"That is\", data)\n        data = re.sub(r\"won't\", \"will not\", data)\n        data = re.sub(r\"they're\", \"they are\", data)\n        data = re.sub(r\"Can't\", \"Cannot\", data)\n        data = re.sub(r\"wasn't\", \"was not\", data)\n        data = re.sub(r\"don\\x89Ûªt\", \"do not\", data)\n        data= re.sub(r\"aren't\", \"are not\", data)\n        data = re.sub(r\"isn't\", \"is not\", data)\n        data = re.sub(r\"What's\", \"What is\", data)\n        data = re.sub(r\"haven't\", \"have not\", data)\n        data = re.sub(r\"hasn't\", \"has not\", data)\n        data = re.sub(r\"There's\", \"There is\", data)\n        data = re.sub(r\"He's\", \"He is\", data)\n        data = re.sub(r\"It's\", \"It is\", data)\n        data = re.sub(r\"You're\", \"You are\", data)\n        data = re.sub(r\"I'M\", \"I am\", data)\n        data = re.sub(r\"shouldn't\", \"should not\", data)\n        data = re.sub(r\"wouldn't\", \"would not\", data)\n        data = re.sub(r\"i'm\", \"I am\", data)\n        data = re.sub(r\"I\\x89Ûªm\", \"I am\", data)\n        data = re.sub(r\"I'm\", \"I am\", data)\n        data = re.sub(r\"Isn't\", \"is not\", data)\n        data = re.sub(r\"Here's\", \"Here is\", data)\n        data = re.sub(r\"you've\", \"you have\", data)\n        data = re.sub(r\"you\\x89Ûªve\", \"you have\", data)\n        data = re.sub(r\"we're\", \"we are\", data)\n        data = re.sub(r\"what's\", \"what is\", data)\n        data = re.sub(r\"couldn't\", \"could not\", data)\n        data = re.sub(r\"we've\", \"we have\", data)\n        data = re.sub(r\"it\\x89Ûªs\", \"it is\", data)\n        data = re.sub(r\"doesn\\x89Ûªt\", \"does not\", data)\n        data = re.sub(r\"It\\x89Ûªs\", \"It is\", data)\n        data = re.sub(r\"Here\\x89Ûªs\", \"Here is\", data)\n        data = re.sub(r\"who's\", \"who is\", data)\n        data = re.sub(r\"I\\x89Ûªve\", \"I have\", data)\n        data = re.sub(r\"y'all\", \"you all\", data)\n        data = re.sub(r\"can\\x89Ûªt\", \"cannot\", data)\n        data = re.sub(r\"would've\", \"would have\", data)\n        data = re.sub(r\"it'll\", \"it will\", data)\n        data = re.sub(r\"we'll\", \"we will\", data)\n        data = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", data)\n        data = re.sub(r\"We've\", \"We have\", data)\n        data = re.sub(r\"he'll\", \"he will\", data)\n        data = re.sub(r\"Y'all\", \"You all\", data)\n        data = re.sub(r\"Weren't\", \"Were not\", data)\n        data = re.sub(r\"Didn't\", \"Did not\", data)\n        data = re.sub(r\"they'll\", \"they will\", data)\n        data = re.sub(r\"they'd\", \"they would\", data)\n        data = re.sub(r\"DON'T\", \"DO NOT\", data)\n        data = re.sub(r\"That\\x89Ûªs\", \"That is\", data)\n        data = re.sub(r\"they've\", \"they have\", data)\n        data = re.sub(r\"i'd\", \"I would\", data)\n        data = re.sub(r\"should've\", \"should have\", data)\n        data = re.sub(r\"You\\x89Ûªre\", \"You are\", data)\n        data = re.sub(r\"where's\", \"where is\", data)\n        data = re.sub(r\"Don\\x89Ûªt\", \"Do not\", data)\n        data = re.sub(r\"we'd\", \"we would\", data)\n        data = re.sub(r\"i'll\", \"I will\", data)\n        data = re.sub(r\"weren't\", \"were not\", data)\n        data = re.sub(r\"They're\", \"They are\", data)\n        data = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", data)\n        data = re.sub(r\"you\\x89Ûªll\", \"you will\", data)\n        data = re.sub(r\"I\\x89Ûªd\", \"I would\", data)\n        data = re.sub(r\"let's\", \"let us\", data)\n        data = re.sub(r\"it's\", \"it is\", data)\n        data = re.sub(r\"can't\", \"cannot\", data)\n        data = re.sub(r\"dont\", \"do not\", data)\n        data = re.sub(r\"don't\", \"do not\", data)\n        data = re.sub(r\"you're\", \"you are\", data)\n        data = re.sub(r\"i've\", \"I have\", data)\n        data = re.sub(r\"that's\", \"that is\", data)\n        data = re.sub(r\"i'll\", \"I will\", data)\n        data = re.sub(r\"doesn't\", \"does not\",data)\n        data = re.sub(r\"i'd\", \"I would\", data)\n        data = re.sub(r\"didn't\", \"did not\", data)\n        data = re.sub(r\"ain't\", \"am not\", data)\n        data = re.sub(r\"you'll\", \"you will\", data)\n        data = re.sub(r\"I've\", \"I have\", data)\n        data = re.sub(r\"Don't\", \"do not\", data)\n        data = re.sub(r\"I'll\", \"I will\", data)\n        data = re.sub(r\"I'd\", \"I would\", data)\n        data = re.sub(r\"Let's\", \"Let us\", data)\n        data = re.sub(r\"you'd\", \"You would\", data)\n        data = re.sub(r\"It's\", \"It is\", data)\n        data = re.sub(r\"Ain't\", \"am not\", data)\n        data = re.sub(r\"Haven't\", \"Have not\", data)\n        data = re.sub(r\"Could've\", \"Could have\", data)\n        data = re.sub(r\"youve\", \"you have\", data)  \n        data = re.sub(r\"donå«t\", \"do not\", data)\n        return data\n    \n    # NLTK lemmatizer\n    @staticmethod\n    def lemma_traincorpus(ech_row):\n        lemmatizer=WordNetLemmatizer()\n        out_data=\"\"\n        for words in ech_row:\n            out_data+= lemmatizer.lemmatize(words)\n        return out_data\n\n    #final calculations\n    def calc(self):\n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.remove_extraSpace(x))\n        \n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.remove_punctuations(x))\n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.remove_html(x))\n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.remove_url(x))\n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.remove_emoji(x))\n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.remove_abb(x))\n        \n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.make_it_lower(x))\n        self.df['question_text'] = self.df['question_text'].apply(lambda x: self.lemma_traincorpus(x))\n        return self.df\n\n# import re\n# data_clean = CleanIt(df_train)\n# df_train_clean_feat = data_clean.calc()\n\n# _lst = list(df_train_clean_feat['question_text'].sample(10))\n# for x in _lst:\n#     print(x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **TF-IDF**\n\nIdea here is to represent text using TF-IDF tokens "},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom keras.preprocessing.text import Tokenizer\nfrom sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import ADASYN,SMOTE\nimport sklearn\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer\n\n## split to train and val\n_train_df, val_df = train_test_split(df_train_clean_feat, test_size=0.2, random_state=202)\n\nsample_train_df = _train_df\n\n#tfidf\ntfidf_vect=TfidfVectorizer(stop_words='english',ngram_range=(1, 2), lowercase=True, max_features=150000)\ntrain_tfidf=tfidf_vect.fit_transform(sample_train_df['question_text'])\nval_tfidf=tfidf_vect.transform(val_df['question_text'])\n\n\n#Balancing The Sample for TFIDF Baseline\n#SMOTE oversampling\nval_x = val_tfidf\nval_y = val_df['target']\n\nsmote=SMOTE(random_state=42)\nsmote_train_x,smote_train_y=smote.fit_sample(train_tfidf,sample_train_df['target'])\nsmote_train_x.shape,smote_train_y.shape\nprint(smote_train_x.shape,smote_train_y.shape,val_x.shape,val_y.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **ML Models**\n\nFor baseline score we are using logistic, xgboost and lightGBM"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport numpy as np\nfrom sklearn.model_selection import train_test_split,cross_val_score,GridSearchCV,cross_val_predict\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nimport xgboost\nfrom imblearn.metrics import classification_report_imbalanced\nfrom sklearn.metrics import classification_report,roc_auc_score,roc_curve,r2_score,recall_score,confusion_matrix,precision_recall_curve\nfrom collections import Counter\nfrom sklearn.model_selection import StratifiedKFold,KFold,StratifiedShuffleSplit\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom sklearn.ensemble import RandomForestRegressor,GradientBoostingClassifier,RandomForestClassifier\n\nclass Run_models:\n    #imports\n\n    def __init__(self,train_x,train_y,val_x,val_y):\n        self.train_x = train_x\n        self.train_y = train_y\n        self.val_x = val_x\n        self.val_y = val_y\n    \n    def kFold(self,mdl):\n\n        kfold=StratifiedKFold(n_splits=5, shuffle=True, random_state=17)\n        rtn1=list(np.round(cross_val_score(mdl,self.train_x,self.train_y,cv=kfold,scoring='roc_auc'),3))\n        pred_val_y=cross_val_predict(mdl,self.val_x,self.val_y)\n        #pred_val_y = mdl.predict(val_x)\n        rtn2= np.round(roc_auc_score(self.val_y, pred_val_y),3)\n        rtn = {'on_train':rtn1,'on_val':rtn2}\n        return rtn\n        \n    def logisticRegression(self):\n        mdl = LogisticRegression(max_iter=1000,penalty='l2')\n        rtn = self.kFold(mdl)\n        return rtn\n    \n    \n    def gradientBoost(self):\n        mdl = GradientBoostingClassifier(learning_rate=0.02, loss='deviance',n_estimators=100)\n        rtn = self.kFold(mdl)\n        return rtn\n    \n    def xgBoost(self):\n        mdl = XGBClassifier(n_estimators=50,random_state=42)\n        rtn = self.kFold(mdl)\n        return rtn\n                            \n    def lightGBM(self):\n        mdl = LGBMClassifier(n_estimators=100,random_state=42)\n        rtn = self.kFold(mdl)\n        return rtn\n    \n    def apply_all_models(self):\n        cv_rtn = {}\n        #cv_rtn['LogisticRregression'] = self.logisticRegression()\n        cv_rtn['gradientBoost'] = self.gradientBoost()\n        #cv_rtn['xgBoost'] = self.xgBoost()\n        cv_rtn['lightGBM'] = self.lightGBM()\n        return cv_rtn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n\n# models_all = Run_models(smote_train_x,smote_train_y,val_x,val_y)\n# ml_tfidf_result = models_all.apply_all_models()\n# print(ml_tfidf_result)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **ML Model Results**"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodels_all = Run_models(smote_train_x,smote_train_y,val_x,val_y)\nml_tfidf_result = models_all.apply_all_models()\n\nfor x in ml_tfidf_result.keys():\n    print(x,ml_tfidf_result[x])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`Looks like model is over fitting train data as it performs badly on validation set`"},{"metadata":{},"cell_type":"markdown","source":"**Using Embeddings**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#imports\nimport re\n\nimport pandas as pd\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport wordcloud\nfrom wordcloud import WordCloud,STOPWORDS\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nnltk.download('stopwords')\nimport string\n\nimport sklearn\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfTransformer\n\nfrom collections import Counter\nfrom collections import defaultdict\nfrom plotly import tools\nfrom plotly.subplots import make_subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\n#Reading Data\ndf_train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ndf_test = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Refered how to unzip embeddings file from [here](https://www.kaggle.com/c/quora-insincere-questions-classification/discussion/140385)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\n\nDataset = \"embeddings\"\nwith zipfile.ZipFile(\"../input/quora-insincere-questions-classification/\"+Dataset+\".zip\",\"r\") as z:\n     z.extractall(\".\")\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \".\"]).decode(\"utf8\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom gensim.models import Word2Vec,KeyedVectors\nfrom gensim.scripts.glove2word2vec import glove2word2vec\n\ngoogle_loaded='./GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\ngoogle_model = KeyedVectors.load_word2vec_format(google_loaded, binary=True)\nplt.plot(google_model['reviews'])\nplt.plot(google_model['injustice'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\ndata_clean = CleanIt(df_train)\ndf_train_clean_feat = data_clean.calc()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sklearn\nfrom keras.preprocessing.text import Tokenizer\nfrom sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import ADASYN,SMOTE\nfrom tensorflow.keras.preprocessing.text import Tokenizer\n\n## split to train and val\ntoken_train_df, token_val_df = train_test_split(df_train_clean_feat, test_size=0.1, random_state=202)\n\nnum_words = None # how many unique words to use \nfltrs='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n' # characters that will be filtered from the texts\n\n# Tokenize the sentences\ntokenizer = Tokenizer(num_words = None,filters=fltrs,lower=True)\ntokenizer.fit_on_texts(list(df_train.question_text) + list(df_test.question_text))\n\n\nX = tokenizer.texts_to_sequences(token_train_df.question_text)\nX_test = tokenizer.texts_to_sequences(token_val_df.question_text)\n\n# Create vocabulary from all words\nvocabulary = tokenizer.word_index\n\ndel tokenizer\nimport gc; gc.collect() #manual garbage collection\n\n\nprint(f\"Tokenized sample text: {X[0]}\\n\")\nprint({word: vocabulary[word.lower()] for word in (token_train_df.iloc[0].question_text).split(' ')[:-1]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.sequence import pad_sequences\n\nMAX_LENGTH = 50 # Max. number of words in sentence\npad = 'pre' # Where to add padding\n\nX_embed = pad_sequences(X, maxlen=MAX_LENGTH, padding=pad)\nX_test_embed = pad_sequences(X_test,maxlen=MAX_LENGTH, padding=pad)\n\nprint(f\"Padded sample text: \\n{X[0]}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"google_in_vocab = {}\ngoogle_out_vocab = {}\n\nfor w in vocabulary.keys():\n    if w in google_model:\n        w = w.lower()\n        google_in_vocab[w] = google_model[w]\n    else:\n        google_out_vocab[w]=0\n\n\ncoverage = np.round(len(google_in_vocab)*100/len(vocabulary),2)\nprint(f'% coverage og google embeddings is : {coverage} %')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Delete unused embeddings\ndel google_model\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **ML. Models on Google News Embeddings**\n\nFrom google news embeddings we are trying to aggrigate word embeddings of each sentence in `question_text`"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"def convert_sentence_embeddings(data,_vocab):\n    vocab=[w.lower() for w in data.split(' ') if w.lower() in _vocab.keys()]\n    if len(vocab)>0:\n        avg_pool=np.mean([_vocab[x] for x in vocab],axis=0)\n    else:\n        avg_pool = np.zeros(shape=(300,))\n    return avg_pool\n\n#_train_df, val_df, sampeling only 10000 due to memory issues\ntrain_google_df = token_train_df\nval_google_df = token_val_df\ntrain_google_df['google_news_vec']=token_train_df['question_text'].apply(lambda z: convert_sentence_embeddings(z,google_in_vocab) )\nval_google_df['google_news_vec']= token_val_df['question_text'].apply(lambda z: convert_sentence_embeddings(z,google_in_vocab) )\n\ngoogle_train_df = train_google_df #[train_google_df.need_col != np.nan]\ngoogle_val_df = val_google_df #[val_google_df.need_col != np.nan]\n\n\n\n\n# #SMOTE oversampling\n# val_google_x = google_val_df['google_news_vec']\n# val_google_y = google_val_df['target']\n\n# smote=SMOTE(random_state=42)\n# smote_google_train_x,smote_google_train_y=smote.fit_sample(google_train_df['google_news_vec'],google_train_df['target'])\n# print(smote_google_train_x.shape,smote_google_train_y.shape,val_google_x.shape,val_google_y.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **ML Model Results**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_google_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list(train_google_df['google_news_vec'].head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n#X\ntrain_google_x = list(train_google_df['google_news_vec'])\nval_google_x = list(val_google_df['google_news_vec'])\n#Y\ntrain_google_y = list(train_google_df['target'])\nval_google_y = list(val_google_df['target'])\n\nmodels_all = Run_models(train_google_x,train_google_y,val_google_x,val_google_y)\nml_tfidf_result = models_all.apply_all_models()\n\nfor x in ml_tfidf_result.keys():\n    print(x,ml_tfidf_result[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n#X\ntrain_google_x = list(train_google_df['google_news_vec'])\nval_google_x = list(val_google_df['google_news_vec'])\n#Y\ntrain_google_y = list(train_google_df['target'])\nval_google_y = list(val_google_df['target'])\n\nmodels_all = Run_models(train_google_x,train_google_y,val_google_x,val_google_y)\nml_tfidf_result = models_all.apply_all_models()\n\nfor x in ml_tfidf_result.keys():\n    print(x,ml_tfidf_result[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}