{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntest_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\ntrain_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train_df['question_text']\nX_test = test_df['question_text']\nY_train = train_df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_vocav(data):\n    voc = {}\n    for sen in data:\n        sen = sen.split()\n        for word in sen:\n            try:\n                voc[word] += 1\n            except KeyError:\n                voc[word] = 1\n    return voc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"voc = prepare_vocav(X_train.values)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gensim.downloader as api\npath = api.load(\"word2vec-google-news-300\", return_path=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\nmodel = KeyedVectors.load_word2vec_format(path, binary = True);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator \n\ndef check_coverage(voc,model):\n    present = {}\n    not_present = {}\n    k = 0\n    i = 0\n    for word in (voc):\n        try:\n            present[word] = model[word]\n            k += voc[word]\n        except:\n            not_present[word] = voc[word]\n            i += voc[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(present) / len(voc)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(not_present.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"not_present = check_coverage(voc,model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(x):\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = [clean_text(line) for line in (X_train)]\nvoc = prepare_vocav(X_train)\nnot_present = check_coverage(voc,model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = [clean_text(line) for line in (X_test)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.preprocessing.text import Tokenizer\nfrom tensorflow.python.keras.preprocessing.sequence import pad_sequences\n\nVALIDATION_SPLIT = 0.2\ntokenizer_obj = Tokenizer()\ntokenizer_obj.fit_on_texts(X_train)\nX_token_train = tokenizer_obj.texts_to_sequences(X_train)\nX_token_test = tokenizer_obj.texts_to_sequences(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer_obj.word_index\nprint(len(word_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_length = 50\nX_train_pad = pad_sequences(X_token_train, maxlen=max_length)\nX_test_pad = pad_sequences(X_token_test, maxlen=max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train_pad, X_val_pad, Y_train, Y_val = train_test_split(X_train_pad, Y_train, test_size=0.2 , random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_DIM =300\nnum_words = len(word_index) + 1\nembedding_matrix = np.zeros((num_words, EMBEDDING_DIM))\n\nfor word, i in word_index.items():\n    if i > num_words:\n        continue\n    try:\n        embedding_vector = model[word]\n        embedding_matrix[i] = embedding_vector\n    except:\n        print(\"Not Present\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense, Embedding, Flatten,Dropout,LSTM\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers.convolutional import MaxPooling1D\nfrom keras.initializers import Constant\n\nmodel = Sequential()\nembedding_layer = Embedding(num_words,\n                            EMBEDDING_DIM,\n                            embeddings_initializer=Constant(embedding_matrix),\n                            input_length=max_length,\n                            trainable=False)\n\n\nmodel.add(embedding_layer)\nmodel.add(Dropout(0.3))\nmodel.add(LSTM(100))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1,activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\nprint(model.summary())\n\n\n# fit the model\nmodel.fit(X_train_pad, Y_train, batch_size=128, epochs=8, validation_data=(X_val_pad,Y_val), verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_pred=model.predict_classes(X_test_pad)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id1 = test_df['qid']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"id1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(Y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas\ndf = pandas.DataFrame(data={\"id\": id1, \"target\": Y_pred.reshape(-1)})\ndf.to_csv(\"./submission.csv\", sep=',',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}