{"cells":[{"metadata":{"_uuid":"b0ab2994eac925ee58c39fbc016a135ae6d86c61"},"cell_type":"markdown","source":"In this notebook, I will do some processing and some analysis on Quora Insincere Questions Classification dataset.\n\nI will :\n* Get number and percentage of toxic questions.\n* Perform some processing on the data.\n* Get the number of unique words after each processing step\n* Tokenize the data\n* Get the most common words and their counts\n* Plot some graphs to show the common words\n* Do some analysis on the toxic samples\n* Get the toxic words\n\nI hope you get some good ideas after reading this notebook\n\n* If you found this notebook helful, please give it an upvote.\n* If you have any quistion, put it in the comments."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.metrics import f1_score, roc_auc_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sys import getsizeof\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nimport string\nfrom string import digits\nimport re\nimport operator \nplt.style.use('seaborn-darkgrid')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6887f796fd25f035f743f48df852e1e5f7ab1615"},"cell_type":"code","source":"# https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing\ndef load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a7bc7d16919621e5a5de5202ad2c909020bbd56"},"cell_type":"code","source":"# https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing\ndef build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afe122f23e7c2d882beb658808454e924ae0386d"},"cell_type":"code","source":"# https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing\ndef check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6aa96c0cf4da1466dcad7a810b7ec49a60f3199"},"cell_type":"code","source":"def plot_learning_curve(history,model_info):\n    # summarize history for loss\n    plt.figure(figsize=(9,7))\n    plt.plot(history['loss'])\n    plt.plot(history['val_loss'])\n    plt.title(f'Model {model_info} Loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'validation'], loc='upper left')\n    plt.savefig(f'Model_{model_info}_Loss.png')\n    \n    # summarize history for accuracy\n    plt.figure(figsize=(9,7))\n    plt.plot(history['acc'])\n    plt.plot(history['val_acc'])\n    plt.title(f'Model {model_info} Accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'validation'], loc='upper left')\n    plt.savefig(f'Model_{model_info}_Acc.png')\n    print(f'Model_{model_info}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"135576054ab96b6f67c8df2b00c37b19cac0e3c9"},"cell_type":"code","source":"def load_combined():\n    train_data = pd.read_csv(\"../input/train.csv\")\n    test_data = pd.read_csv(\"../input/test.csv\")\n    \n    # Get the number of samples in training data :\n    train_size = train_data.shape[0]\n    test_size =  test_data.shape[0]\n    \n    # Combine training and test data :\n    combined = train_data.append(test_data, ignore_index  = True, sort = False)\n    \n    ## fill in nans :\n    combined[\"question_text\"].fillna('NAN', inplace = True)\n    \n    return combined, train_size, test_size\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45e3941e268c7d7b9838267de659ace683981d9b"},"cell_type":"code","source":"combined, train_size, test_size = load_combined()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d77d57b01efde0f1a849c6aefca5d421ae27f271"},"cell_type":"code","source":"question_text = combined[\"question_text\"].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"002f0280b6de9c8b5901b81601f22370b3ce3c7c"},"cell_type":"code","source":"%%time\n# Get Words :\ndef get_words(samples):\n    words = set()\n    for sample in samples.values:\n        for word in sample.split():\n            words.add(word)\n    num_words = len(words)\n    print(f'Number of unique words : {num_words}')\n    return words","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b5d171bc11e0cedd8c741d364600d47a9fb51b9b"},"cell_type":"markdown","source":"# Toxic VS Nontoxic"},{"metadata":{"trusted":true,"_uuid":"28157eea7e6000274b49dd74ffe32cd10c383004"},"cell_type":"code","source":"num_toxic    = combined[combined['target'] == 1 ].count()[0]\nnum_nontoxic = combined[combined['target'] == 0 ].count()[0]\nprint(f'Number of toxic samples    : {num_toxic}')\nprint(f'Number of nontoxic samples : {num_nontoxic}')\n\nprint(f'{round((num_toxic/train_size)*100,2)}% of the samples in the training data is Toxic')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe939f24c2884689c3a0265b10eaa77e6af6e7d1"},"cell_type":"markdown","source":"# Some Processing :"},{"metadata":{"trusted":true,"_uuid":"afeef02bb4fc929c518e427f0c66ffc5466d3767"},"cell_type":"code","source":"# Number of words without processing the dataset :\n_ = get_words(samples = combined[\"question_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d64c38484f098f6e345d8c78c84376ba053be0fb"},"cell_type":"code","source":"# Convert text to lowercase :\ncombined[\"question_text\"] = combined[\"question_text\"].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4072a20b3948f2018554813c27e4aa2230a77489"},"cell_type":"code","source":"# Number of words after converting the samples to lower case :\n_ = get_words(samples = combined[\"question_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f55da1abbd5710716c456f68c0365ac0d85ed705"},"cell_type":"code","source":"%%time\n# Process commas :\ncombined[\"question_text\"] = combined[\"question_text\"].apply(lambda x: re.sub(\"'\", '', x)).apply(lambda x: re.sub(\",\", ' COMMA', x))\n\n# Getting rid of punctuation\nexclude = set(string.punctuation)\ncombined[\"question_text\"] = combined[\"question_text\"].apply(lambda x: ''.join(ch for ch in x if ch not in exclude))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57e5bdde692213b47a5e41d7fdd25392aacf55c2"},"cell_type":"code","source":"# Number of words after dealing with punctuation :\n_ = get_words(samples = combined[\"question_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3871d9b202e56c060179bd433ccba51ca643a26"},"cell_type":"code","source":"# Getting rid of digits\nremove_digits = str.maketrans('', '', digits)\ncombined[\"question_text\"] = combined[\"question_text\"].apply(lambda x: x.translate(remove_digits))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"449771a6d01fa359f51623197163335148de2136"},"cell_type":"code","source":"# Number of words without digits :\nwords = get_words(samples = combined[\"question_text\"])\nmax_features = len(words)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"440d0791ea974db573ff315d0ab14686e76c0869"},"cell_type":"markdown","source":"# Tokenization :"},{"metadata":{"trusted":true,"_uuid":"e48a0fe3fd289ec99c0e5017b0e4ff387e3a83a3"},"cell_type":"code","source":"# prepare tokenizer\ntokenizer = Tokenizer(num_words = max_features)\ntokenizer.fit_on_texts(combined['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e866a31e96396c4b37e27510bc4f6c96b282e63c"},"cell_type":"code","source":"# integer encode the documents\nencoded_samples = tokenizer.texts_to_sequences(combined['question_text'])\ncombined['encoded_samples'] = encoded_samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afaf7dffb67c485273a20c1327ea6062b55dfc70"},"cell_type":"code","source":"# Get samples length :\ncombined['sample_len'] = combined['encoded_samples'].apply(len)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7b2eae6eec301774e84cf94723337720f7b50bf"},"cell_type":"markdown","source":"# Some Investigation : "},{"metadata":{"trusted":true,"_uuid":"288a7fc38a0e975422748d5a5fad18233cd53ae7"},"cell_type":"code","source":"max_len = combined['sample_len'].max()\nmin_len = combined['sample_len'].min()\navg_len = int(combined['sample_len'].mean())\n\nprint(f'The longest  sample has {max_len} words')\nprint(f'The shortest sample has {min_len} words')\nprint(f'The average number of words in samples is {avg_len}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b4cf27979438760c680c0a4866b3ee60f5d3df9"},"cell_type":"markdown","source":"Wait a minute, is it possible that there is a sample that doesn't have any words?\n\nLet's invistegate that ."},{"metadata":{"trusted":true,"_uuid":"7be02ab82c52ecd9575ca9e768508082678a52e0"},"cell_type":"code","source":"empty_samples_idx = combined[combined['sample_len'] <= 0].index\nquestion_text[empty_samples_idx]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db732a7e5e84cf990059c5d8abaea159603dee85"},"cell_type":"markdown","source":"So, there is one sample that is empty, What about samples with only one word ?"},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"f11322463ea376d061ce2d0f3d63ac80c1828de3"},"cell_type":"code","source":"oneWord_samples_idx = combined[combined['sample_len'] == 1].index\ntemp = pd.DataFrame({'question_text':question_text[oneWord_samples_idx],\n              'target':combined['target'][oneWord_samples_idx]})\ntemp","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"918f515f15c6e8c34418d229912503a7504bdb3b"},"cell_type":"markdown","source":"Wow! \n\nThis is not good at all, if we used this samples to train our model, it will make mistakes in predicting test samples.\n\nWe need to delete these samples, but first, we will make some calculations :"},{"metadata":{"trusted":true,"_uuid":"2e0c0cd8e30d5c0d1d246088ec24f8eeb63d08f1"},"cell_type":"code","source":"# Percentage of toxic samples in the dataset :\ntoxic_perc = round((combined['target'].sum()/train_size)*100,2)\nprint(f'Percentage of toxic samples in the dataset : {toxic_perc}%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f51e74b798a70a6a8d66b450144d2f21c66c9794"},"cell_type":"code","source":"t = round((temp['target'].sum()/combined['target'].sum())*100,4)\nprint(f'Percentage of toxic samples with one word only over the other toxic samples : {t}%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18a34ef4b9d34a153f2ca616af24db0dd931575e"},"cell_type":"code","source":"def get_count(df,col,min_len,max_len):\n    return df[(df[col]>=min_len) & (df[col]<=max_len)].count()[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1c2768f3d4c1ad0031f97cabc86c198c14ef975"},"cell_type":"code","source":"def plot_sample_len(df,col,title = 'Lengths of Samples',sp=10000):\n    # Get the range of lengths and number of samples for each range (for test data)\n    ranges = [(0,5), (6,10), (11,15), (16,20), (21,30), (31,40),\n              (41,50), (51,65), (66,80), (81,100), (101,150) ]\n    range_name = []\n    num_samps_in_range = []\n    for r in ranges:\n        num_samps_in_range.append(\n            get_count(df = df, col = col, min_len=r[0], max_len=r[1]))\n        range_name.append(f'{r[0]} -> {r[1]}')\n\n        # Plot range of lengths and number of samples for each range :\n    fig, ax = plt.subplots(figsize = (18, 10),)\n    ax.set(title = title,\n           xlabel = ' Length (# words)', ylabel = '# Samples')\n\n    r1 = ax.bar(range_name,\n                num_samps_in_range,\n                alpha = 0.9,\n                label = '# Samples')\n    for idx in range(len(ranges)) : \n        if(num_samps_in_range[idx] > 10000):\n            ax.text(range_name[idx],\n                    num_samps_in_range[idx]+(sp),\n                    num_samps_in_range[idx],\n                    horizontalalignment='center',\n                    size='large')\n        else :\n            ax.text(range_name[idx],\n                num_samps_in_range[idx]+100,\n                num_samps_in_range[idx],\n                horizontalalignment='center',\n                size='large')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a141c505d6163429db7dd1e5f73a12b7976d70b"},"cell_type":"code","source":"# ranges of sample length for all combined data :\nplot_sample_len(df = combined,col = 'sample_len',\n                title = 'Lengths of All Samples',sp=10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e219dcef3ecae268f0530fbdf7d0d109e8cd67f"},"cell_type":"code","source":"# ranges of sample length for training data :\nplot_sample_len(df = combined[:train_size],col = 'sample_len',\n                title = 'Lengths of Training Samples', sp=10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c407b603aff9d0ea55d52810d9e9c45cd5896ba0"},"cell_type":"code","source":"# ranges of sample length for test data :\nplot_sample_len(df = combined[train_size:],col = 'sample_len',\n                title = 'Lengths of Test Samples', sp=500)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dd3c920ff89bbbaa80587d6ca752265a817041fc"},"cell_type":"markdown","source":"This plot tells us that we don't need to set the padding length more than 65, but it's better if we set the max padding length to 70 just to be in the safe zone. "},{"metadata":{"_uuid":"e4bc98af1bf7f5ef264c44d1ceac01b3c8a1ddfa"},"cell_type":"markdown","source":"# Embeddings :"},{"metadata":{"trusted":true,"_uuid":"a73756d011a376fb11732b22ad921cda7086e9be"},"cell_type":"code","source":"glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nparagram =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\nwiki_news = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26e9dd373c304b7b712fc66be45b27866205df0e"},"cell_type":"code","source":"%%time\nprint(\"Extracting GloVe embedding ...\")\nembed_glove = load_embed(glove)\nprint(\"Extracting Paragram embedding ...\")\nembed_paragram = load_embed(paragram)\nprint(\"Extracting FastText embedding ...\")\nembed_fasttext = load_embed(wiki_news)\nprint('Done!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2d7c624d8a2e370820b2270e656ab87e33f128c"},"cell_type":"code","source":"vocab = build_vocab(combined['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8bbfd4e7283a41a0ef8a7c8fa195a07b586a277"},"cell_type":"code","source":"print(\"Glove : \")\noov_glove = check_coverage(vocab, embed_glove)\nprint(\"Paragram : \")\noov_paragram = check_coverage(vocab, embed_paragram)\nprint(\"FastText : \")\noov_fasttext = check_coverage(vocab, embed_fasttext)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86b8911496dad83efe2b088e8819113144ced729"},"cell_type":"markdown","source":"## Common and Rare Words :"},{"metadata":{"_uuid":"6405435f02b3c2d326214da592e1157f6a06082e"},"cell_type":"markdown","source":"We will find the most common word and thier count, we will also try to find any words that appeared only once ."},{"metadata":{"trusted":true,"_uuid":"b53646876db0f78c7b0adc5b7971348eee67f254"},"cell_type":"code","source":"# Get : words and thier count and words that appeard only once :\nword_freq  = []\nall_words  = []\nrare_words = []\nfor i,w in enumerate(tokenizer.word_counts):\n    word_freq.append(tokenizer.word_counts[w])\n    all_words.append(w)\n    if(tokenizer.word_counts[w] <= 1):\n        rare_words.append(w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95e3479539321eaf3c0ee5d4f71d27996caf4320"},"cell_type":"code","source":"num_all_words  = len(all_words)\nnum_rare_words = len(rare_words)\nprint(f'Total number of words                     : {num_all_words}')\nprint(f'Number of words that appeared only once    : {num_rare_words}')\nprint(f'{round((num_rare_words/num_all_words)*100,2)}% of the words appeared only once')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad6a78d0d3a47d6a12aa618e5dcb62d0072aeb52"},"cell_type":"code","source":"# Sort the words by their frequency :\ncommon_words       = []\ncommon_words_count = []\nfor y,x in sorted(zip(word_freq, all_words),reverse = True):\n    common_words.append(x)\n    common_words_count.append(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e933a051a06a1e60055cfd67c9e14ac01012f39","scrolled":true},"cell_type":"code","source":"# Plot the most 10 common words :\nfig, ax = plt.subplots(figsize = (18, 10),)\nax.set(title = 'Common Words',\n       xlabel = ' Word', ylabel = 'Count')\n\nr1 = ax.bar(common_words[:10],\n            common_words_count[:10],\n            alpha = 0.9,\n            label = '# Count')\nfor idx in range(10) : \n    ax.text(common_words[idx],\n            common_words_count[idx]+10000,\n            common_words_count[idx],\n            horizontalalignment='center',\n            size='small')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33dafe44d8713f6d793f306a98ce965651222c4e"},"cell_type":"code","source":"# The most 100 common words :\ntop100Words = pd.DataFrame({'Word':common_words[:100], 'Count':common_words_count[:100]})\nprint('The Top 10 Words :')\ntop100Words.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"964d26ed5bdc6c70ab6413b361e781df6868c6d8"},"cell_type":"markdown","source":"## Toxic Words :"},{"metadata":{"_uuid":"45e41d722e08f50b3ede3b36a4b51cf546096fc7"},"cell_type":"markdown","source":"We will find the words that appeared in the toxic samples :"},{"metadata":{"trusted":true,"_uuid":"5bb599056c138031103f80ba956ac6b5906a9375"},"cell_type":"code","source":"# Get a list of the words in the toxic samples:\ntoxic_samples = list(combined['encoded_samples'][combined['target'] == 1].values)\ntoxic_samples_words = [item for sublist in toxic_samples for item in sublist]\ntoxic_words = list(set(toxic_samples_words))\nprint(f'There are {len(toxic_words)} unique words in the toxic samples')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7534464bb90a4c413e69f005642ff306e5d672b3"},"cell_type":"code","source":"# Extract non-toxic samples :\nnontoxic_samples = list(combined['encoded_samples'][combined['target'] == 0].values)\n\n# Get all the words in the non-toxic samples : \nnontoxic_samples_words = [item for sublist in nontoxic_samples for item in sublist]\nnontoxic_samples_words = list(set(nontoxic_samples_words))\n\n# Get words that only appeared in toxic samples :\ntoxic_only_words = list(set(toxic_words) - set(nontoxic_samples_words))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"139308e9763d1f4f73a048e503f74591785980da"},"cell_type":"code","source":"print(f'Words that apeared only in toxic samples are {len(toxic_only_words)} words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"443c369f16ac1d67dac8c9694907aee411ef8a7e"},"cell_type":"code","source":"# Create a dictionary to convert word index into word :\nidx2word = {}\nfor k, v in tokenizer.word_index.items():\n    idx2word[v] = k\n\n# Convert indecies to words :\ntoxic_only_words_idx = toxic_only_words.copy()\ntoxic_only_words = [idx2word[x] for x in toxic_only_words_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f9f08b5d1b75162dd7872df5e6625492088c1f2"},"cell_type":"code","source":"# print some of the words that appeared only in the toxic samples :  \ntemp = pd.DataFrame({ '1-20' :toxic_only_words[:20],\n                      '20-40':toxic_only_words[20:40]})\nprint('Some of the words that appeared only in the toxic samples :')\ntemp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d94137715e2a791206b5e3a24a3e29988a6d600"},"cell_type":"markdown","source":"Let's try deleting words that appeared only one time :"},{"metadata":{"trusted":true,"_uuid":"355ac7acd3dff61fefc02dee6318c7d87edcb945"},"cell_type":"code","source":"# Deleting words that appeared only one time\ntoxic_only_words = list(set(toxic_only_words) - set(rare_words))\nprint(f'Number of toxic words after deleting the rare words : {len(toxic_only_words)} words')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38d00ec8c8f3c75dd49ba3eb8a0d58d6924e72a1"},"cell_type":"markdown","source":"### Top 10 Toxic Words :"},{"metadata":{"trusted":true,"_uuid":"b4a8eafff01088bf55b32707197fc738ea2ae74b"},"cell_type":"code","source":"# Get top 10 toxic words by their count :\ntemp = pd.DataFrame({'toxic word':toxic_only_words})\ntemp['count'] = temp['toxic word'].apply(lambda x: tokenizer.word_counts[x])\ntemp.sort_values(by=['count'], ascending = False, inplace = True)\ntemp = temp.reset_index(drop = True)[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f12a8c75da342bec37f66d842052256c33b7434"},"cell_type":"code","source":"# Plot the most 10 common toxic words :\nfig, ax = plt.subplots(figsize = (12, 10),)\nax.set(title = 'Common Toxic Words',\n       xlabel = 'Count', ylabel = 'Toxic Word')\n\nr1 = ax.barh(temp['toxic word'][:10],\n            temp['count'][:10],\n            alpha = 0.7,\n            color = '#EF4A6D')\nax.invert_yaxis()\nfor idx in range(10) : \n    ax.text(temp['count'][idx]-0.2,\n            idx,\n            temp['count'][idx],\n            horizontalalignment='right',\n            size='medium')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"634dec26be09bd1f3e7a2a766811f5cbf84771e2"},"cell_type":"markdown","source":"# Conclusion :"},{"metadata":{"trusted":true,"_uuid":"4779cfb0808dbaa17dfc071c53e00d5cf2e1df38"},"cell_type":"markdown","source":"Now after you got to know more about the data, you can process it the right way so you get better score in this competition."},{"metadata":{"trusted":true,"_uuid":"99d6158da28cd38e969ca1e0c313778a065232d8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}