{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense,Flatten,Embedding,Activation, Dropout\nfrom tensorflow.keras.layers import Conv1D, MaxPooling1D, GlobalMaxPooling1D , LSTM ,Bidirectional\nfrom tensorflow.keras.optimizers import Adam\nfrom keras import optimizers, callbacks \nfrom sklearn.metrics import log_loss\nfrom keras.layers import Flatten\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntrain_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['question_text'].str.split().map(lambda x : len(x)).hist(bins=64)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"all question with max length is 45"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_length = 45\nembedding_glove = '../input/glove840b300dtxt/glove.840B.300d.txt'\n#embedding_fasttext = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n#embedding_para = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n#embedding_w2v = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\ncontractions = { \n\"ain't\": \"am not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how does\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so is\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\" u \": \" you \",\n\" ur \": \" your \",\n\" n \": \" and \"}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"puncts = '\\'!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\",\n                 \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '”': '\"', '“': '\"', \"£\": \"e\",\n                 '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta',\n                 '∅': '', '³': '3', 'π': 'pi', '\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}\nfor p in puncts:\n    punct_mapping[p] = ' %s ' % p\n\np = re.compile('(\\[ math \\]).+(\\[ / math \\])')\np_space = re.compile(r'[^\\x20-\\x7e]')\nprint(punct_mapping[\"∞\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_text = ' '.join(train_df['question_text'])\nall_text = all_text.split()\nfrequence  = pd.Series(all_text).value_counts()\none_word = frequence[frequence.values == 1]\none_word[5:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_question(x):\n    if type(x) is str:\n        x = x.lower() # transformer to lower \n        for p in punct_mapping:\n            x = x.replace(p,punct_mapping[p])\n        for key in contractions:\n            value = contractions[key]\n            x = x.replace(key, value)\n            \n        x = re.sub(r'([a-zA-Z0-9+._-]+@[a-zA-Z0-9._-]+\\.[a-zA-Z0-9_-]+)', '', x) # regex to remove to emails\n       \n        x = re.sub(r'(http|ftp|https)://([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])?', '', x)   #regex to remove URLs     \n        x = re.sub( u\"\\s+\", u\" \", x ).strip() # remove multiple  espace and back line\n        x = ' '.join([t for t in x.split() if t not in one_word])  #combining all the text excluding rare words.\n        return x\n    else:\n        return x\n\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: clean_question(x))        \ntrain_df['question_text'] = train_df['question_text'].tolist()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[200:210]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"preparing embedding matrix "},{"metadata":{"trusted":true},"cell_type":"code","source":"def pre_embe(data ):\n    token = Tokenizer()\n    token.fit_on_texts(data)\n    vocab_size  = len(token.word_index) + 1\n    print(\" vocabolury size :  \" ,vocab_size)\n    encoded_text = token.texts_to_sequences(train_df['question_text'])\n    X = pad_sequences(encoded_text, maxlen=max_length, padding='post')\n    print(\" exemple of fisrt question with encoding index \",X[1])\n    return X , vocab_size\n\nX , vac = pre_embe(train_df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}