{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f1fbda9a1af88e811e2f1bbdc3b5387ca84c0be"},"cell_type":"markdown","source":"![](http://www.marketing-professionnel.fr/wp-content/uploads/2011/03/quora-marketing.png)\n"},{"metadata":{"_uuid":"e2dd9b2a91a73cf4f87b6ab22f720a1089ce26fa"},"cell_type":"markdown","source":"**Quora**  is a question-and-answer website where questions are asked, answered, edited, and organized by its community of users in the form of opinions."},{"metadata":{"_uuid":"82129229e52f07397a36cacdfe21dedc7fd2c7f1"},"cell_type":"markdown","source":"Quora, people can ask questions and connect with others who contribute unique insights and quality answers. A key challenge is to weed out **insincere questions** -- those founded upon false premises, or that intend to make a statement rather than look for helpful answers.\nIn this competition,we  will develop models that identify and flag insincere questions. "},{"metadata":{"_uuid":"71e3b24e41c661cc847704e93c2dd048956f6b7e"},"cell_type":"markdown","source":"**Importing libraries : **"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport random\nfrom collections import Counter\nimport pprint\nimport time\nimport tensorflow as tf  # deep learning library. Tensors are just multi-dimensional arrays\nimport sys\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7af67449e0848eda348ed6e2d5948ea1f60413e"},"cell_type":"markdown","source":"**Data loading :**"},{"metadata":{"trusted":true,"_uuid":"bc4ff8b1ff0afd0a3b7b71fa05b053d47a769643"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8616f0a899a14e6e9969afad3e50e85925abfc88"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2acfd4e339ab6f6e106fd56fe30dd3ac026112c4"},"cell_type":"code","source":"train.tail()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"69e6ffe4acac417afb000eb911e8a5478cb42cd0"},"cell_type":"markdown","source":"**Cheking NaN values**"},{"metadata":{"trusted":true,"_uuid":"78ded98d05934f227ea6582c299913c8a79473af"},"cell_type":"code","source":"train.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12d69731292e6babdf82d20e47eb9c2ce345394f"},"cell_type":"code","source":"print('Shapes')\nprint(\"TRAIN :\",train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85eb37012b045fad5b88ea905a4be9c2beac4554"},"cell_type":"code","source":"print(len(train[train[\"target\"]==0]), \" :  sincere questions\")\nprint(len(train[train[\"target\"]==1]), \" :  insincere questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73e43204873c9cb18bab8a4ae9b6a84237bd5ce4"},"cell_type":"code","source":"SAMPLE_SIZE = 80000\ndf_0 = train[train[\"target\"]==0].sample(SAMPLE_SIZE, random_state = 101)\n    # filter out class 1\ndf_1 = train[train[\"target\"]==1].sample(SAMPLE_SIZE, random_state = 101)\n\ndf = pd.concat([df_0, df_1], axis=0).reset_index(drop=True)\n   # shuffle\ndf= shuffle(df)\ndf_data, df_test = train_test_split(df, test_size=0.2)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf190b98b0781e384ec1f7f18a627a91301533a9"},"cell_type":"code","source":"dictio = {0:\"sincere question  \",1:\"insincere question\"}\ndef pretty_print_text_and_label(i):\n    print(dictio[df_data.iloc[i][2]] + \"\\t:\\t\" + train.iloc[i][1][:80] + \"...\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11641e8229d41e3cc1a50de0a0ebc2780dd234a6"},"cell_type":"code","source":"print(\"type \\t : \\t question\\n\")\n# choose  a random spam set to analyse\n# random.randrange(start, stop, step)\npretty_print_text_and_label(random.randrange(0,4572))\npretty_print_text_and_label(random.randrange(0,4572,4))\npretty_print_text_and_label(random.randrange(0,4572,50))\npretty_print_text_and_label(random.randrange(0,4572,100))\npretty_print_text_and_label(random.randrange(0,4572,200))\npretty_print_text_and_label(random.randrange(0,4572,500))\npretty_print_text_and_label(random.randrange(0,4572,800))\npretty_print_text_and_label(random.randrange(0,4572,1000))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa415db017ae2a430156ecc45a63cdb5c6b4aa17"},"cell_type":"code","source":"Sincere_counts = Counter()\nInsencere_counts = Counter()\nTotal_counts = Counter()\npp = pprint.PrettyPrinter(indent=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8dc6d97ca43655592977176420e81553ddad6371"},"cell_type":"code","source":"for i in range(len(df_data)):  #range(len(train)):\n    if(df_data.iloc[i][2] == 0):\n        for word in df_data.iloc[i][1].split(\" \"):\n            Sincere_counts[word] += 1\n            Total_counts[word] += 1\n    else:\n        for word in df_data.iloc[i][1].split(\" \"):\n            Insencere_counts[word] += 1\n            Total_counts[word] += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"304ff76ccab247ab4c69c229e49ee06f13096787"},"cell_type":"code","source":"print(\"the most used word in insencere questions\")\npp.pprint(Insencere_counts.most_common()[0:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3cee49591512cb106cb3c4b8990748e2e5302bc"},"cell_type":"code","source":"print(\"the most used word in sencere questions\")\npp.pprint(Sincere_counts.most_common()[0:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18a65b383abc8b32a93eeb5c54c5f24c018945b7"},"cell_type":"code","source":"print(\"the most used word in all questions\")\npp.pprint(Sincere_counts.most_common()[0:20])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91a02d666ab5a738b110b1eeb66328bc7f2be45d"},"cell_type":"markdown","source":"**Transform Text into Numbers**\n"},{"metadata":{"_uuid":"1bef0b73639b2d092dc76c987e411d2a45b5b038"},"cell_type":"markdown","source":"Neural Networks only understand numbers hence we have to find a way to represent our text inputs in a way it can understand"},{"metadata":{"trusted":true,"_uuid":"a987dd12a2303020601804d90e37f955a152595d"},"cell_type":"code","source":"vocab = set(Total_counts.keys())\nvocab_size = len(vocab)\nprint(vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"472d8215c2ef5de82be5048af8cdd498852901f7"},"cell_type":"markdown","source":"\nWe can see that from all our dataset, we have a total of 13874 unique words. Use this to build up our vocabulary vector containing columns of all these words.\n\nBecause, 139026, can be a large size in memory (a matrix of size 139026 by 8000), we will take in to account just the 100 most used words"},{"metadata":{"trusted":true,"_uuid":"949352c04547e38b5bc55daadeaed2a210da4592"},"cell_type":"code","source":"vocab_vector = np.zeros((1, 200)) # np.zeros((1, vocab_size))\npp.pprint(vocab_vector.shape)\npp.pprint(vocab_vector)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e335c977f9589a956369903542310d2fb0dfc84"},"cell_type":"markdown","source":"Now, let's create a dictionary that allows us to look at every word in our vocabulary and map it to the vocab_vector column."},{"metadata":{"trusted":true,"_uuid":"a5bf4e09d776c79a1536923405ddb3ee1f933f7a"},"cell_type":"code","source":"word_column_dict = {}\np=0\nfor word,count in list(Total_counts.most_common()[0:200]):\n    # {key: value} is {word: column}\n    word_column_dict[word] = p\n    p+=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fae3222950e584b6ca882b4af74a4845f8c351bc"},"cell_type":"code","source":"def update_input_layer(text):\n    \n    global vocab_vector\n    \n    # clear out previous state, reset the vector to be all 0s\n    vocab_vector *= 0\n    for word in text.split(\" \"):\n        if word in word_column_dict:\n            vocab_vector[0][word_column_dict[word]] += 1\n        \n    return vocab_vector.tolist()[0]\n\n# example \nprint(update_input_layer(\"the the\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76b52fbce5bbf2a1d9406929b2e81016e63abc40"},"cell_type":"markdown","source":"**Build the SpamClassificationNeuralNetwork**"},{"metadata":{"trusted":true,"_uuid":"a42118630690b4474172b3ef41465f2dc63cefa3"},"cell_type":"code","source":"X_train = [update_input_layer(df_data.iloc[i][1]) for i in range(len(df_data))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0a8e8551dcd0887c59dfa87d3b8a1f278ebbc51"},"cell_type":"code","source":"y_train = [df_data.iloc[i][2] for i in range(len(df_data))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bfd986f4fa3fea30afaaaa73cd1cd8961501878b"},"cell_type":"code","source":"from keras import Sequential\nfrom keras.layers import Dense\nmodel = Sequential()  # a basic feed-forward model\n#model.add(tf.keras.layers.Flatten())  # takes our 28x28 and makes it 1x784\n\nmodel.add(Dense(128,input_dim=200, activation=tf.nn.relu))  # a simple fully-connected layer, 128 units, relu activation\nmodel.add(Dense(1, activation=tf.nn.sigmoid))  # our output layer. 10 units for 10 classes. Softmax for probability distribution\n\nmodel.compile(optimizer='adam',  # Good default optimizer to start with\n              loss='binary_crossentropy',  # how will we calculate our \"error.\" Neural network aims to minimize loss.\n              metrics=['accuracy'])  # what to track","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"033d7e355b58430b1d55fb9448bc69005de8e45f"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"04816a0f6575e114250832e6cb6c96cf77a685fc"},"cell_type":"code","source":"history = model.fit(np.asarray(X_train), np.asarray(y_train), epochs = 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e67c5d0c7660bc6158f6c4eb45a4f3a8fcbbbddb"},"cell_type":"code","source":"# Plot training & validation accuracy values\nplt.plot(history.history['acc'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84e8b7b3947d597a118119a31c4870ba4832e32d"},"cell_type":"code","source":"# Plot training & validation loss values\nplt.plot(history.history['loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b694bd5f46dc4afed8ad3d4fde59c14004ac6922"},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8849a10792bf5c36d3d1523a43e25b2d42d636a6"},"cell_type":"code","source":"X_test = [update_input_layer(df_test.iloc[i][1]) for i in range(len(df_test))]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1715bb9e90f1ec1d6022458546dd9937dafe555d"},"cell_type":"code","source":"predictions = model.predict(np.asarray(X_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e9d490501ee5c60b5a2eb37e6bc8e85019e4229"},"cell_type":"code","source":"k=0\nfor i in range(len(df_test)):\n    if(predictions[i][0]>0.5 and df_test.iloc[i,2]==1):\n        k+=1\n    elif(predictions[i][0]<0.5 and df_test.iloc[i,2]==0):\n        k+=1\n  \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d6478620543c09eeec44c0a5d03edd998881f44"},"cell_type":"code","source":"print(k/len(df_test)*100, \"% of testing data are well predicted\")  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"455e458c21eaf9c48dc772e05fa8a80e8c473e8a"},"cell_type":"code","source":"print(\"type \\t : \\t question\\n\")\nfor d in range(5):\n    r = random.randrange(0,len(df_test))\n    s = \"sincere question  :\"\n    q = \"===> predicted ===> sincere question \"\n    if df_test.iloc[r,2]==1:\n        s = \"insincere question  :\"\n    if predictions[r][0]>0.5 : \n        q = \"===> predicted   ===> insincere question \"\n        \n    print(s+df_test.iloc[r,1]+q)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}