{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport re\nimport string\nimport pandas as pd\nfrom nltk.corpus import stopwords\n\n\ndef DataCleaning(text):\n    text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n    text = text.lower().split()\n    stops = set(stopwords.words(\"english\"))\n    text = [w for w in text if not w in stops]\n    text = \" \".join(text)\n    return (text)\n\n\ndef Clean(text):\n    text = DataCleaning(text)\n    text = text.translate(str.maketrans(\"\", \"\", string.punctuation))\n    return text\n\n\ndef clean_data():\n    path = '../input/fakenewsdata/train.csv'\n    vector_dimension=300\n\n    data = pd.read_csv(path)\n\n    missing_rows = []\n    for i in range(len(data)):\n        if data.loc[i, 'text'] != data.loc[i, 'text']:\n            missing_rows.append(i)\n    data = data.drop(missing_rows).reset_index().drop(['index','id'],axis=1)\n\n    for i in range(len(data)):\n        data.loc[i, 'text'] = Clean(data.loc[i,'text'])\n\n    data = data.sample(frac=1).reset_index(drop=True)\n\n    x = data.loc[:,'text'].values\n    y = data.loc[:,'label'].values\n\n    return x,y\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e44d52931e3b180c7f3a66ffe7a65b832b90b38a"},"cell_type":"code","source":"xtrain,ytrain =clean_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd7376f0204f0a9b1f89f7ccdc4a09a7cbf7efb1"},"cell_type":"code","source":"path = '../input/fakenewsdata/train.csv'\ndata = pd.read_csv(path)\ndata1=pd.DataFrame()\ndata1['word_count'] = data['text'].apply(lambda x: len(str(x).split(\" \")))\ndata1['Text']=data['text']\ndata1[['Text','word_count']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e76352577e8c19ecda846adbb832ad64190dcc4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc4508c16bb74804d57f4d1d69d76597873c4c46"},"cell_type":"code","source":"import seaborn as sns\nwh1 = data[['title','author',\n            'text','label']] #Subsetting the data\ncor = wh1.corr() #Calculate the correlation of the above variables\nsns.heatmap(cor, square = True) #Plot the correlation as heat map","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from keras.utils import to_categorical\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nMAX_NB_WORDS=12000\n\ntokenizer = Tokenizer(nb_words=MAX_NB_WORDS)\ntokenizer.fit_on_texts(xtrain)\nsequences = tokenizer.texts_to_sequences(xtrain)\n\nword_index = tokenizer.word_index\n\n\nx_traindata = pad_sequences(sequences, maxlen=200)\n\ny_labels = to_categorical(np.asarray(ytrain))\n\nvocab_size = MAX_NB_WORDS\n\nwordindex=tokenizer.word_index\n\ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    max_features=MAX_NB_WORDS\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\nembedding_matrix=load_fasttext(wordindex)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cd1e1286e3b6773ea5b5bbed87cc84c128fcb21"},"cell_type":"code","source":"#data = pd.DataFrame(x_traindata, columns=['x', 'y'])\n\n\nplt.hist(x_traindata, normed=True, alpha=0.5)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd7847be13914328ebe2e5697b268b0c1928a31b"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.layers import Flatten\nfrom keras.layers import Embedding\nfrom keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation\nmodel = Sequential()\ne = Embedding(vocab_size, 300, weights=[embedding_matrix], input_length=200, trainable=False)\nmodel.add(e)\nmodel.add(Dropout(0.2))\nmodel.add(Conv1D(64, 5, activation='relu'))\nmodel.add(MaxPooling1D(pool_size=4))\nmodel.add(LSTM(150))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a21252da227de0a2304fbb0422be47fce1743a1f"},"cell_type":"code","source":"\n  \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab0a03552e4978c80b5b6d3e14a3f51f3b91a80b"},"cell_type":"code","source":"\nimport matplotlib.pyplot as plt\n\nhistory = model.fit(x_traindata, ytrain, epochs=2, verbose=1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f4207b6e7655214ab09a68ea96750822d2e2342"},"cell_type":"code","source":"plt.plot(history.history['acc'])\n\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()\n\nplt.plot(history.history['loss'])\n\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"694438d8d8dc2bc348daa416872c52da23e87105"},"cell_type":"code","source":"from IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\n\nSVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b96f6114d8e3fc0736b8c8b910cd924902af633"},"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(model, to_file='model.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdcdb6ce7048e19a065393af8e14ef8dd9384cfe"},"cell_type":"code","source":"import pandas as pd\nimport nltk\nfrom nltk.probability import FreqDist\nimport seaborn as sb\nfrom matplotlib import pyplot as plt\npath = '../input/fakenewsdata/train.csv'\n\ndata = pd.read_csv(path)\ntext=data['text']\nlabels=data['label']\nfrom matplotlib import pyplot as plt\nfreqdist = nltk.FreqDist(text)\nplt.figure(figsize=(16,5))\nfreqdist.plot(50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82612b0e3950ca5960aa156774851362d11db612"},"cell_type":"code","source":"inputnews=\"For individuals like Margaret Thome Bekema, finishing high school was a dream she didnt think would ever come true. Instead of graduating from Grand Rapids Catholic \"\ntestd=inputnews.lower()\n\ntokenizer2 = Tokenizer(nb_words=MAX_NB_WORDS)\ntokenizer2.fit_on_texts(xtrain)\nsequences = tokenizer2.texts_to_sequences(testd)\n    \nnewsequences=np.asarray(sequences)\nsequences=np.reshape(newsequences,(len(newsequences),))\nn =[]\nfor i in sequences:\n    n= n+i\n    \nnewsequences=np.asarray(n)    \nx_testdata = pad_sequences([newsequences], maxlen=200)\nprediction = model.predict_classes(x_testdata)\nprediction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cef8a92e9c4e66d468ff8623e8f2edd63605710"},"cell_type":"code","source":"\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5278ee86d538e3857e460f5e144e5f7e1a1415c7"},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72e0e98f30502b3940dc50e554ad7a1da63bde3c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}