{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport matplotlib\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\nimport seaborn as sns\nsns.set(style='whitegrid')\n\nfrom wordcloud import WordCloud\n\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA, TruncatedSVD\nfrom sklearn.metrics import classification_report,confusion_matrix\n\nfrom collections import defaultdict\nfrom collections import Counter\n\nimport re\nimport gensim\nimport string\n\nfrom tqdm import tqdm\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import Embedding, LSTM,Dense, SpatialDropout1D, Dropout\nfrom keras.initializers import Constant\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"id":"GbA39eB6Y5aW","execution":{"iopub.status.busy":"2022-07-28T07:55:46.157023Z","iopub.execute_input":"2022-07-28T07:55:46.157790Z","iopub.status.idle":"2022-07-28T07:55:53.725093Z","shell.execute_reply.started":"2022-07-28T07:55:46.157660Z","shell.execute_reply":"2022-07-28T07:55:53.724221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nnltk.download('all')","metadata":{"id":"pnMJIQn7ZE_H","outputId":"43775e0f-5f66-48f0-f36c-2107c726db44","execution":{"iopub.status.busy":"2022-07-28T07:55:53.728800Z","iopub.execute_input":"2022-07-28T07:55:53.729795Z","iopub.status.idle":"2022-07-28T07:56:17.905117Z","shell.execute_reply.started":"2022-07-28T07:55:53.729469Z","shell.execute_reply":"2022-07-28T07:56:17.903963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Exploratory Data Analysis:","metadata":{"id":"DJQrZBqcaVt3"}},{"cell_type":"code","source":"df=pd.read_csv('../input/nlp-getting-started/train.csv')\ndf_test=pd.read_csv('../input/nlp-getting-started/test.csv')\nsample_submission=pd.read_csv('../input/nlp-getting-started/sample_submission.csv')","metadata":{"id":"dkqzDqP1ZHHe","execution":{"iopub.status.busy":"2022-07-28T07:56:17.906845Z","iopub.execute_input":"2022-07-28T07:56:17.907239Z","iopub.status.idle":"2022-07-28T07:56:17.987168Z","shell.execute_reply.started":"2022-07-28T07:56:17.907193Z","shell.execute_reply":"2022-07-28T07:56:17.985964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape, df_test.shape","metadata":{"id":"lsk0k5j8ITlq","outputId":"754d94dd-47b6-4fbe-b5c9-3af39d369461","execution":{"iopub.status.busy":"2022-07-28T07:56:17.991542Z","iopub.execute_input":"2022-07-28T07:56:17.991909Z","iopub.status.idle":"2022-07-28T07:56:19.662852Z","shell.execute_reply.started":"2022-07-28T07:56:17.991877Z","shell.execute_reply":"2022-07-28T07:56:19.661683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"id":"6QC6JG9kP8Ea","outputId":"dc7f2af0-5808-4469-c332-beca7b277e1b","execution":{"iopub.status.busy":"2022-07-28T07:56:19.665376Z","iopub.execute_input":"2022-07-28T07:56:19.665809Z","iopub.status.idle":"2022-07-28T07:56:19.705630Z","shell.execute_reply.started":"2022-07-28T07:56:19.665672Z","shell.execute_reply":"2022-07-28T07:56:19.704717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.loc[:,['text','target']]","metadata":{"id":"AWdKml9vZNbv","outputId":"8d4a518b-1255-4dce-93be-b333c30193a1","execution":{"iopub.status.busy":"2022-07-28T07:56:19.707023Z","iopub.execute_input":"2022-07-28T07:56:19.707324Z","iopub.status.idle":"2022-07-28T07:56:19.728177Z","shell.execute_reply.started":"2022-07-28T07:56:19.707284Z","shell.execute_reply":"2022-07-28T07:56:19.727060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"id":"Ri873plxINjO","outputId":"68ab35c1-a74e-48f6-b4d0-0496292263ec","execution":{"iopub.status.busy":"2022-07-28T07:56:19.730894Z","iopub.execute_input":"2022-07-28T07:56:19.731320Z","iopub.status.idle":"2022-07-28T07:56:19.757760Z","shell.execute_reply.started":"2022-07-28T07:56:19.731262Z","shell.execute_reply":"2022-07-28T07:56:19.756759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"id":"cUad5rZuaNip","outputId":"36eceb68-a675-4711-e1c9-ac209cfa4654","execution":{"iopub.status.busy":"2022-07-28T07:56:19.759765Z","iopub.execute_input":"2022-07-28T07:56:19.760162Z","iopub.status.idle":"2022-07-28T07:56:19.775013Z","shell.execute_reply.started":"2022-07-28T07:56:19.760034Z","shell.execute_reply":"2022-07-28T07:56:19.773782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.target.value_counts()","metadata":{"id":"qk1t7FatZgnM","outputId":"f0ed7419-678c-4363-c1e1-c5c1ac0e207b","execution":{"iopub.status.busy":"2022-07-28T07:56:19.776409Z","iopub.execute_input":"2022-07-28T07:56:19.776812Z","iopub.status.idle":"2022-07-28T07:56:19.787001Z","shell.execute_reply.started":"2022-07-28T07:56:19.776768Z","shell.execute_reply":"2022-07-28T07:56:19.785920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see above the distribution of the classes is slightly unbalanced, this is why we should expect to have sidetrack in the prediction towards class 0.","metadata":{"id":"d4MiMBrCeOFS"}},{"cell_type":"code","source":"df2=df.copy(deep=True)\npie1=pd.DataFrame(df2['target'].replace(1,'disaster').replace(0,'non-disaster').value_counts())\npie1.reset_index(inplace=True)\npie1.plot(kind='pie', title='Pie chart of Disaster/Non-disaster tweets',y = 'target', \n          autopct='%1.1f%%', shadow=False, labels=pie1['index'], legend = False, fontsize=14, figsize=(12,12))","metadata":{"id":"_YyDWkNtZggo","outputId":"71bef4e9-b3c4-4fc4-8c98-0f43d5513994","execution":{"iopub.status.busy":"2022-07-28T07:56:19.791909Z","iopub.execute_input":"2022-07-28T07:56:19.793247Z","iopub.status.idle":"2022-07-28T07:56:20.171443Z","shell.execute_reply.started":"2022-07-28T07:56:19.793187Z","shell.execute_reply":"2022-07-28T07:56:20.170455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It would be interesting to see the number of characters contained in the text of each class, even better showing them as histograms so as to see if there is a specific pattern for any class:","metadata":{"id":"JIBT3L63aTtH"}},{"cell_type":"code","source":"sns.set(style='whitegrid')\nf, (ax1, ax2) = plt.subplots(1,2,figsize=(25,8))\n\nax1.hist(df[df['target'] == 0]['text'].str.len(), bins=30, color='b')\nax1.set_title('Non-disaster tweets')\n\nax2.hist(df[df['target'] == 1]['text'].str.len(), bins=30, color='r')\nax2.set_title('Disaster tweets')\n\nf.suptitle('Histogram number of characters in tweets')","metadata":{"id":"2rU87Q7FcEyx","outputId":"85ed9304-cb88-494d-b27c-c10de262b42c","execution":{"iopub.status.busy":"2022-07-28T07:56:20.173076Z","iopub.execute_input":"2022-07-28T07:56:20.173990Z","iopub.status.idle":"2022-07-28T07:56:20.943643Z","shell.execute_reply.started":"2022-07-28T07:56:20.173941Z","shell.execute_reply":"2022-07-28T07:56:20.942608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see above they have similar characteristics and we can not say too much about a special feature of any of them, would be better to apply this method but for the words.\n\nThis is why, we are goind to compute and display a histogram of the number of words in the headlines of each topic:","metadata":{"id":"L6X_98lWbz-R"}},{"cell_type":"code","source":"f, (ax1, ax2,) = plt.subplots(1,2,figsize=(25,8))\n\nax1.hist(df[df['target'] == 0]['text'].str.split().map(lambda x: len(x)), bins=29, color='b')\nax1.set_title('Non-disaster tweets')\n\nax2.hist(df[df['target'] == 1]['text'].str.split().map(lambda x: len(x)), bins=29, color='r')\nax2.set_title('Disaster tweets')\n\nf.suptitle('Histogram number of words in tweets')","metadata":{"id":"p6m01h4pbkDc","outputId":"c223e97e-ad2e-4d43-c441-8d06e49aa8d4","execution":{"iopub.status.busy":"2022-07-28T07:56:20.945958Z","iopub.execute_input":"2022-07-28T07:56:20.946574Z","iopub.status.idle":"2022-07-28T07:56:21.689220Z","shell.execute_reply.started":"2022-07-28T07:56:20.946526Z","shell.execute_reply":"2022-07-28T07:56:21.688311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To the instances that correspond to disaster I will add the what is contained in the keyword feature to the end of the tweet so as to get more impactful words in the sentence and help the model to find the common words in disaster tweets. In order to do this we have to obtain the indexes of those instances with target = 1 'Disaster':","metadata":{"id":"SRrqPNSkazWl"}},{"cell_type":"code","source":"disaster_index=df[df['target']==1].index.values","metadata":{"id":"yc3j_jUfqBdU","execution":{"iopub.status.busy":"2022-07-28T07:56:21.690854Z","iopub.execute_input":"2022-07-28T07:56:21.691341Z","iopub.status.idle":"2022-07-28T07:56:21.698975Z","shell.execute_reply.started":"2022-07-28T07:56:21.691282Z","shell.execute_reply":"2022-07-28T07:56:21.698102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(disaster_index)","metadata":{"id":"so9xjQFVuWjP","outputId":"9574f40a-4050-406e-d144-f2bdfca7fee2","execution":{"iopub.status.busy":"2022-07-28T07:56:21.700497Z","iopub.execute_input":"2022-07-28T07:56:21.701067Z","iopub.status.idle":"2022-07-28T07:56:21.711310Z","shell.execute_reply.started":"2022-07-28T07:56:21.700913Z","shell.execute_reply":"2022-07-28T07:56:21.710108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As an example let us print the keyword followed by the tweet of the instance 63 which does correspond to disaster:","metadata":{"id":"e5lqDXaFdl2a"}},{"cell_type":"code","source":"print(df.iloc[63,1])\nprint(df.iloc[63,3])","metadata":{"id":"G4O4x9vRdLrH","outputId":"d03e4e01-ecb2-4052-dcb1-8f2df73f96fa","execution":{"iopub.status.busy":"2022-07-28T07:56:21.713533Z","iopub.execute_input":"2022-07-28T07:56:21.714067Z","iopub.status.idle":"2022-07-28T07:56:21.725342Z","shell.execute_reply.started":"2022-07-28T07:56:21.714023Z","shell.execute_reply":"2022-07-28T07:56:21.724365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And after applying the function the text for such instance should be as follows:","metadata":{"id":"oymnfikod1GZ"}},{"cell_type":"code","source":"df.iloc[63,3] + ' ' + df.iloc[63,1]","metadata":{"id":"THUf6KtNvDD2","outputId":"19c0ffe7-32a8-4e7b-c01b-4a5380bbec59","execution":{"iopub.status.busy":"2022-07-28T07:56:21.727521Z","iopub.execute_input":"2022-07-28T07:56:21.727983Z","iopub.status.idle":"2022-07-28T07:56:21.738455Z","shell.execute_reply.started":"2022-07-28T07:56:21.727926Z","shell.execute_reply":"2022-07-28T07:56:21.737480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's apply this function to all instances in the list 'disaster_index':","metadata":{"id":"JSzpp3WveCUQ"}},{"cell_type":"code","source":"for i in disaster_index:\n  df.iloc[i,3] = str(df.iloc[i,3]) + ' ' + str(df.iloc[i,1])","metadata":{"id":"D1Ipa63Ck4BE","execution":{"iopub.status.busy":"2022-07-28T07:56:21.740195Z","iopub.execute_input":"2022-07-28T07:56:21.740660Z","iopub.status.idle":"2022-07-28T07:56:24.335096Z","shell.execute_reply.started":"2022-07-28T07:56:21.740622Z","shell.execute_reply":"2022-07-28T07:56:24.333515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"id":"3w4hL5n6sFzN","outputId":"d07a257a-6c2d-4d94-f7a2-5cbcf6fb9ae1","execution":{"iopub.status.busy":"2022-07-28T07:56:24.336352Z","iopub.execute_input":"2022-07-28T07:56:24.336920Z","iopub.status.idle":"2022-07-28T07:56:24.359708Z","shell.execute_reply.started":"2022-07-28T07:56:24.336879Z","shell.execute_reply":"2022-07-28T07:56:24.356180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Cleaning:\n\nThe tweets contained in the dataset are almost raw, this means we have to get rid of all 'impurities' such as tags, symbols, punctuations, emojis, etc. These does not add significant information to the prediction moreover makes our sentences more subjective. This process comprehend 7 key steps which will make our sentences partially-suit to be used in training of the model.","metadata":{"id":"ONu0rf7QeiMi"}},{"cell_type":"markdown","source":"### Removing URLs: \nSome tweets either disaster or non-disaster include links 'URLs' which correspond to videos or other webpages containing key information about the subject they are trying to communicate, as we want to clean the sentences we must get rid of them. The function which applies such step will be caled remove_URL:","metadata":{"id":"2mGy_NNpeu5n"}},{"cell_type":"code","source":"example=\"New competition launched :https://www.kaggle.com/c/nlp-getting-started\"","metadata":{"id":"nCOiWapIV6zE","execution":{"iopub.status.busy":"2022-07-28T07:56:24.361419Z","iopub.execute_input":"2022-07-28T07:56:24.362110Z","iopub.status.idle":"2022-07-28T07:56:24.898598Z","shell.execute_reply.started":"2022-07-28T07:56:24.362055Z","shell.execute_reply":"2022-07-28T07:56:24.897176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_URL(text):\n    url = re.compile(r'https?://\\S+|www\\.\\S+')\n    return url.sub(r'',text)\n\nremove_URL(example)","metadata":{"id":"YE5BJObeWn-Q","outputId":"786c1649-4004-4c6d-b905-b8f069468bee","execution":{"iopub.status.busy":"2022-07-28T07:56:24.900407Z","iopub.execute_input":"2022-07-28T07:56:24.900661Z","iopub.status.idle":"2022-07-28T07:56:24.950927Z","shell.execute_reply.started":"2022-07-28T07:56:24.900631Z","shell.execute_reply":"2022-07-28T07:56:24.949772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['text']=df['text'].apply(lambda x : remove_URL(x))","metadata":{"id":"gy7tu5aNWn2J","execution":{"iopub.status.busy":"2022-07-28T07:56:24.952312Z","iopub.execute_input":"2022-07-28T07:56:24.952869Z","iopub.status.idle":"2022-07-28T07:56:25.035770Z","shell.execute_reply.started":"2022-07-28T07:56:24.952821Z","shell.execute_reply":"2022-07-28T07:56:25.034787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['text']=df_test['text'].apply(lambda x : remove_URL(x))","metadata":{"id":"BFbPHF0GQ-2c","execution":{"iopub.status.busy":"2022-07-28T07:56:25.037316Z","iopub.execute_input":"2022-07-28T07:56:25.040263Z","iopub.status.idle":"2022-07-28T07:56:25.065022Z","shell.execute_reply.started":"2022-07-28T07:56:25.040222Z","shell.execute_reply":"2022-07-28T07:56:25.064220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Removing HTML tags:\n\nWe have to consider that some tweets were obtained using web scrapping, using this method the components of a publication are companied by special tags identifying them. As such tags are unuseful we must get rid of them to gather only the text. The function which applies such step will be called remove_html:","metadata":{"id":"k2mC_ZsWXLFN"}},{"cell_type":"code","source":"example = \"\"\"<div>\n<h1>Real or Fake</h1>\n<p>Kaggle </p>\n<a href=\"https://www.kaggle.com/c/nlp-getting-started\">getting started</a>\n</div>\"\"\"","metadata":{"id":"zBmhbF-UXG8h","execution":{"iopub.status.busy":"2022-07-28T07:56:25.070221Z","iopub.execute_input":"2022-07-28T07:56:25.072768Z","iopub.status.idle":"2022-07-28T07:56:25.085547Z","shell.execute_reply.started":"2022-07-28T07:56:25.072727Z","shell.execute_reply":"2022-07-28T07:56:25.083742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_html(text):\n    html=re.compile(r'<.*?>')\n    return html.sub(r'',text)\n    \nprint(remove_html(example))","metadata":{"id":"UrHXZVtTWnsg","outputId":"483747f6-243b-422f-c506-d46cfd51d562","execution":{"iopub.status.busy":"2022-07-28T07:56:25.086824Z","iopub.execute_input":"2022-07-28T07:56:25.087129Z","iopub.status.idle":"2022-07-28T07:56:25.120359Z","shell.execute_reply.started":"2022-07-28T07:56:25.087095Z","shell.execute_reply":"2022-07-28T07:56:25.118663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['text']=df['text'].apply(lambda x : remove_html(x))","metadata":{"id":"RH8zNMOkXR1i","execution":{"iopub.status.busy":"2022-07-28T07:56:25.122042Z","iopub.execute_input":"2022-07-28T07:56:25.122683Z","iopub.status.idle":"2022-07-28T07:56:25.178989Z","shell.execute_reply.started":"2022-07-28T07:56:25.122637Z","shell.execute_reply":"2022-07-28T07:56:25.178043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['text']=df_test['text'].apply(lambda x : remove_html(x))","metadata":{"id":"CGIsAvb3RHIO","execution":{"iopub.status.busy":"2022-07-28T07:56:25.180529Z","iopub.execute_input":"2022-07-28T07:56:25.181088Z","iopub.status.idle":"2022-07-28T07:56:25.215866Z","shell.execute_reply.started":"2022-07-28T07:56:25.181046Z","shell.execute_reply":"2022-07-28T07:56:25.215025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Removing Emojis:\n\nEmojis are an efficient way to show the feeling of the publishers in the message, we could translate the meaning of them to words and help to improve the scope of the message. These could be useful or confuse the algorithm when finding the same feeling for disaster and non-disaster tweets, because of this we prefer to get rid of them, the function which applies such step will be called remove_emoji:","metadata":{"id":"NRTrav5YXl4C"}},{"cell_type":"code","source":"# Reference : https://gist.github.com/slowkow/7a7f61f495e3dbb7e3d767f97bd7304b\ndef remove_emoji(text):\n    emoji_pattern = re.compile(\"[\"\n                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                           u\"\\U00002702-\\U000027B0\"\n                           u\"\\U000024C2-\\U0001F251\"\n                           \"]+\", flags=re.UNICODE)\n    return emoji_pattern.sub(r'', text)\n\nremove_emoji(\"Omg another Earthquake 😔😔\")","metadata":{"id":"qAANjLFmXRsD","outputId":"4c5a66bd-a6b8-4c1b-996b-060b6f7b6e6e","execution":{"iopub.status.busy":"2022-07-28T07:56:25.220998Z","iopub.execute_input":"2022-07-28T07:56:25.224016Z","iopub.status.idle":"2022-07-28T07:56:25.250984Z","shell.execute_reply.started":"2022-07-28T07:56:25.223974Z","shell.execute_reply":"2022-07-28T07:56:25.250004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['text']=df['text'].apply(lambda x: remove_emoji(x))","metadata":{"id":"hN_8t-cdXot2","execution":{"iopub.status.busy":"2022-07-28T07:56:25.264732Z","iopub.execute_input":"2022-07-28T07:56:25.267383Z","iopub.status.idle":"2022-07-28T07:56:25.379725Z","shell.execute_reply.started":"2022-07-28T07:56:25.267340Z","shell.execute_reply":"2022-07-28T07:56:25.378918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['text']=df_test['text'].apply(lambda x: remove_emoji(x))","metadata":{"id":"E_WF2ZSfROUx","execution":{"iopub.status.busy":"2022-07-28T07:56:25.384408Z","iopub.execute_input":"2022-07-28T07:56:25.387685Z","iopub.status.idle":"2022-07-28T07:56:25.427142Z","shell.execute_reply.started":"2022-07-28T07:56:25.387625Z","shell.execute_reply":"2022-07-28T07:56:25.426204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Removing numbers:\n\nIn the current project we will only focus on words in order to classify the tweets, obviously the number could be useful because they can mean coordinates of a disaster, code of an accident, hour of accident, number of people killed or also can mean the release year of a videogame, number of followers of an influencer, etc. The use of these numbers in the model can be pending for a next project.","metadata":{"id":"6xqPRh28jQ8x"}},{"cell_type":"code","source":"df['text']=df['text'].str.replace('\\d+', '')","metadata":{"id":"kJ6Gbh6ajQ80","execution":{"iopub.status.busy":"2022-07-28T07:56:25.428342Z","iopub.execute_input":"2022-07-28T07:56:25.428810Z","iopub.status.idle":"2022-07-28T07:56:25.493039Z","shell.execute_reply.started":"2022-07-28T07:56:25.428770Z","shell.execute_reply":"2022-07-28T07:56:25.492205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['text']=df_test['text'].str.replace('\\d+', '')","metadata":{"id":"Sv2NWlVdjQ84","execution":{"iopub.status.busy":"2022-07-28T07:56:25.494674Z","iopub.execute_input":"2022-07-28T07:56:25.495627Z","iopub.status.idle":"2022-07-28T07:56:25.523751Z","shell.execute_reply.started":"2022-07-28T07:56:25.495458Z","shell.execute_reply":"2022-07-28T07:56:25.522650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Contractions and acronyms:\nPeople world-wide make use of acronyms to speed-up the publishing of a tweet, some of them can be miswritten and others can be decomposed creating words that make sense, this process is exhaustive and requires investing a long time searching the meaning of each one, the function which replaces the contractions and acronyms by the words they stand for will be called cleaner: \n","metadata":{"id":"c7qg__RV73AQ"}},{"cell_type":"code","source":"def cleaner(tweet):\n  # Acronyms and miswritten words\n  tweet = re.sub(r\"Typhoon-Devastated\", \"typhoon devastated\", tweet)\n  tweet = re.sub(r\"TyphoonDevastated\", \"typhoon devastated\", tweet)\n  tweet = re.sub(r\"typhoondevastated\", \"typhoon devastated\", tweet)\n  tweet = re.sub(r\"MH370\", \"Malaysia Airlines Flight\", tweet)\n  tweet = re.sub(r\"MH\", \"Malaysia Airlines Flight\", tweet)\n  tweet = re.sub(r\"mh370\", \"Malaysia Airlines Flight\", tweet)\n  tweet = re.sub(r\"year-old\", \"years old\", tweet)\n  tweet = re.sub(r\"yearold\", \"years old\", tweet)\n  tweet = re.sub(r\"yr old\", \"years old\", tweet)\n  tweet = re.sub(r\"PKK\", \"Kurdistan Workers Party\", tweet)\n  tweet = re.sub(r\"MP\", \"madhya pradesh\", tweet)\n  tweet = re.sub(r\"rly\", \"railway\", tweet)\n  tweet = re.sub(r\"CDT\", \"Central Daylight Time\", tweet)\n  tweet = re.sub(r\"sensorsenso\", \"sensor senso\", tweet)\n  tweet = re.sub(r\"pm\", \"\", tweet)\n  tweet = re.sub(r\"PM\", \"\", tweet)\n  tweet = re.sub(r\"nan\", \" \", tweet)\n  tweet = re.sub(r\"terrorismturn\", \"terrorism turn\", tweet)\n  tweet = re.sub(r\"epicente\", \"epicenter\", tweet)\n  tweet = re.sub(r\"epicenterr\", \"epicenter\", tweet)\n  tweet = re.sub(r\"WAwildfire\", \"Washington Wildfire\", tweet)\n  tweet = re.sub(r\"prebreak\", \"pre break\", tweet)\n  tweet = re.sub(r\"nowplaying\", \"now playing\", tweet)\n  tweet = re.sub(r\"RT\", \"retweet\", tweet)\n  tweet = re.sub(r\"EbolaOutbreak\", \"Ebola Outbreak\", tweet)\n  tweet = re.sub(r\"LondonFire\", \"London Fire\", tweet)\n  tweet = re.sub(r\"IDFire\", \"Idaho Fire\", tweet)\n  tweet = re.sub(r\"withBioterrorism&use\", \"with Bioterrorism & use\", tweet)\n  tweet = re.sub(r\"NASAHurricane\", \"NASA Hurricane\", tweet)\n  tweet = re.sub(r\"withweapons\", \"with weapons\", tweet)\n  tweet = re.sub(r\"NuclearPower\", \"Nuclear Power\", tweet)\n  tweet = re.sub(r\"WhiteTerrorism\", \"White Terrorism\", tweet)\n  tweet = re.sub(r\"MyanmarFlood\", \"Myanmar Flood\", tweet)\n  tweet = re.sub(r\"ExtremeWeather\", \"Extreme Weather\", tweet)\n\n  # Special characters\n  tweet = re.sub(r\"%20\", \" \", tweet)\n  tweet = re.sub(r\"%\", \" \", tweet)\n  tweet = re.sub(r\"@\", \" \", tweet)\n  tweet = re.sub(r\"#\", \" \", tweet)\n  tweet = re.sub(r\"'\", \" \", tweet)\n  tweet = re.sub(r\"\\x89û_\", \" \", tweet)\n  tweet = re.sub(r\"\\x89ûò\", \" \", tweet)\n  tweet = re.sub(r\"16yr\", \"16 year\", tweet)\n  tweet = re.sub(r\"re\\x89û_\", \" \", tweet)\n  tweet = re.sub(r\"\\x89û\", \" \", tweet)\n  tweet = re.sub(r\"\\x89Û\", \" \", tweet)\n  tweet = re.sub(r\"re\\x89Û\", \"re \", tweet)\n  tweet = re.sub(r\"re\\x89û\", \"re \", tweet)\n  tweet = re.sub(r\"\\x89ûª\", \"'\", tweet)\n  tweet = re.sub(r\"\\x89û\", \" \", tweet)\n  tweet = re.sub(r\"\\x89ûò\", \" \", tweet)\n  tweet = re.sub(r\"\\x89Û_\", \"\", tweet)\n  tweet = re.sub(r\"\\x89ÛÒ\", \"\", tweet)\n  tweet = re.sub(r\"\\x89ÛÓ\", \"\", tweet)\n  tweet = re.sub(r\"\\x89ÛÏWhen\", \"When\", tweet)\n  tweet = re.sub(r\"\\x89ÛÏ\", \"\", tweet)\n  tweet = re.sub(r\"China\\x89Ûªs\", \"China's\", tweet)\n  tweet = re.sub(r\"let\\x89Ûªs\", \"let's\", tweet)\n  tweet = re.sub(r\"\\x89Û÷\", \"\", tweet)\n  tweet = re.sub(r\"\\x89Ûª\", \"\", tweet)\n  tweet = re.sub(r\"\\x89Û\\x9d\", \"\", tweet)\n  tweet = re.sub(r\"å_\", \"\", tweet)\n  tweet = re.sub(r\"\\x89Û¢\", \"\", tweet)\n  tweet = re.sub(r\"\\x89Û¢åÊ\", \"\", tweet)\n  tweet = re.sub(r\"fromåÊwounds\", \"from wounds\", tweet)\n  tweet = re.sub(r\"åÊ\", \"\", tweet)\n  tweet = re.sub(r\"åÈ\", \"\", tweet)\n  tweet = re.sub(r\"JapÌ_n\", \"Japan\", tweet)    \n  tweet = re.sub(r\"Ì©\", \"e\", tweet)\n  tweet = re.sub(r\"å¨\", \"\", tweet)\n  tweet = re.sub(r\"Surṳ\", \"Suruc\", tweet)\n  tweet = re.sub(r\"åÇ\", \"\", tweet)\n  tweet = re.sub(r\"å£3million\", \"3 million\", tweet)\n  tweet = re.sub(r\"åÀ\", \"\", tweet)\n\n  # Contractions\n  tweet = re.sub(r\"he's\", \"he is\", tweet)\n  tweet = re.sub(r\"there's\", \"there is\", tweet)\n  tweet = re.sub(r\"We're\", \"We are\", tweet)\n  tweet = re.sub(r\"That's\", \"That is\", tweet)\n  tweet = re.sub(r\"won't\", \"will not\", tweet)\n  tweet = re.sub(r\"they're\", \"they are\", tweet)\n  tweet = re.sub(r\"Can't\", \"Cannot\", tweet)\n  tweet = re.sub(r\"wasn't\", \"was not\", tweet)\n  tweet = re.sub(r\"don\\x89Ûªt\", \"do not\", tweet)\n  tweet = re.sub(r\"aren't\", \"are not\", tweet)\n  tweet = re.sub(r\"isn't\", \"is not\", tweet)\n  tweet = re.sub(r\"What's\", \"What is\", tweet)\n  tweet = re.sub(r\"haven't\", \"have not\", tweet)\n  tweet = re.sub(r\"hasn't\", \"has not\", tweet)\n  tweet = re.sub(r\"There's\", \"There is\", tweet)\n  tweet = re.sub(r\"He's\", \"He is\", tweet)\n  tweet = re.sub(r\"It's\", \"It is\", tweet)\n  tweet = re.sub(r\"You're\", \"You are\", tweet)\n  tweet = re.sub(r\"I'M\", \"I am\", tweet)\n  tweet = re.sub(r\"Im\", \"I am\", tweet)\n  tweet = re.sub(r\"shouldn't\", \"should not\", tweet)\n  tweet = re.sub(r\"wouldn't\", \"would not\", tweet)\n  tweet = re.sub(r\"i'm\", \"I am\", tweet)\n  tweet = re.sub(r\"I\\x89Ûªm\", \"I am\", tweet)\n  tweet = re.sub(r\"I'm\", \"I am\", tweet)\n  tweet = re.sub(r\"Isn't\", \"is not\", tweet)\n  tweet = re.sub(r\"Here's\", \"Here is\", tweet)\n  tweet = re.sub(r\"you've\", \"you have\", tweet)\n  tweet = re.sub(r\"you\\x89Ûªve\", \"you have\", tweet)\n  tweet = re.sub(r\"we're\", \"we are\", tweet)\n  tweet = re.sub(r\"what's\", \"what is\", tweet)\n  tweet = re.sub(r\"couldn't\", \"could not\", tweet)\n  tweet = re.sub(r\"we've\", \"we have\", tweet)\n  tweet = re.sub(r\"it\\x89Ûªs\", \"it is\", tweet)\n  tweet = re.sub(r\"doesn\\x89Ûªt\", \"does not\", tweet)\n  tweet = re.sub(r\"It\\x89Ûªs\", \"It is\", tweet)\n  tweet = re.sub(r\"Here\\x89Ûªs\", \"Here is\", tweet)\n  tweet = re.sub(r\"who's\", \"who is\", tweet)\n  tweet = re.sub(r\"I\\x89Ûªve\", \"I have\", tweet)\n  tweet = re.sub(r\"y'all\", \"you all\", tweet)\n  tweet = re.sub(r\"can\\x89Ûªt\", \"cannot\", tweet)\n  tweet = re.sub(r\"would've\", \"would have\", tweet)\n  tweet = re.sub(r\"it'll\", \"it will\", tweet)\n  tweet = re.sub(r\"we'll\", \"we will\", tweet)\n  tweet = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", tweet)\n  tweet = re.sub(r\"We've\", \"We have\", tweet)\n  tweet = re.sub(r\"he'll\", \"he will\", tweet)\n  tweet = re.sub(r\"Y'all\", \"You all\", tweet)\n  tweet = re.sub(r\"Weren't\", \"Were not\", tweet)\n  tweet = re.sub(r\"Didn't\", \"Did not\", tweet)\n  tweet = re.sub(r\"they'll\", \"they will\", tweet)\n  tweet = re.sub(r\"they'd\", \"they would\", tweet)\n  tweet = re.sub(r\"DON'T\", \"DO NOT\", tweet)\n  tweet = re.sub(r\"That\\x89Ûªs\", \"That is\", tweet)\n  tweet = re.sub(r\"they've\", \"they have\", tweet)\n  tweet = re.sub(r\"i'd\", \"I would\", tweet)\n  tweet = re.sub(r\"should've\", \"should have\", tweet)\n  tweet = re.sub(r\"You\\x89Ûªre\", \"You are\", tweet)\n  tweet = re.sub(r\"where's\", \"where is\", tweet)\n  tweet = re.sub(r\"Don\\x89Ûªt\", \"Do not\", tweet)\n  tweet = re.sub(r\"we'd\", \"we would\", tweet)\n  tweet = re.sub(r\"i'll\", \"I will\", tweet)\n  tweet = re.sub(r\"weren't\", \"were not\", tweet)\n  tweet = re.sub(r\"They're\", \"They are\", tweet)\n  tweet = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", tweet)\n  tweet = re.sub(r\"you\\x89Ûªll\", \"you will\", tweet)\n  tweet = re.sub(r\"I\\x89Ûªd\", \"I would\", tweet)\n  tweet = re.sub(r\"let's\", \"let us\", tweet)\n  tweet = re.sub(r\"it's\", \"it is\", tweet)\n  tweet = re.sub(r\"can't\", \"can not\", tweet)\n  tweet = re.sub(r\"cant\", \"can not\", tweet)\n  tweet = re.sub(r\"don't\", \"do not\", tweet)\n  tweet = re.sub(r\"dont\", \"do not\", tweet)\n  tweet = re.sub(r\"you're\", \"you are\", tweet)\n  tweet = re.sub(r\"i've\", \"I have\", tweet)\n  tweet = re.sub(r\"that's\", \"that is\", tweet)\n  tweet = re.sub(r\"i'll\", \"I will\", tweet)\n  tweet = re.sub(r\"doesn't\", \"does not\", tweet)\n  tweet = re.sub(r\"i'd\", \"I would\", tweet)\n  tweet = re.sub(r\"didn't\", \"did not\", tweet)\n  tweet = re.sub(r\"ain't\", \"am not\", tweet)\n  tweet = re.sub(r\"you'll\", \"you will\", tweet)\n  tweet = re.sub(r\"I've\", \"I have\", tweet)\n  tweet = re.sub(r\"Don't\", \"do not\", tweet)\n  tweet = re.sub(r\"I'll\", \"I will\", tweet)\n  tweet = re.sub(r\"I'd\", \"I would\", tweet)\n  tweet = re.sub(r\"Let's\", \"Let us\", tweet)\n  tweet = re.sub(r\"you'd\", \"You would\", tweet)\n  tweet = re.sub(r\"It's\", \"It is\", tweet)\n  tweet = re.sub(r\"Ain't\", \"am not\", tweet)\n  tweet = re.sub(r\"Haven't\", \"Have not\", tweet)\n  tweet = re.sub(r\"Could've\", \"Could have\", tweet)\n  tweet = re.sub(r\"youve\", \"you have\", tweet)  \n  tweet = re.sub(r\"donå«t\", \"do not\", tweet)\n\n  return tweet","metadata":{"id":"HRNcaawI8Og8","execution":{"iopub.status.busy":"2022-07-28T07:56:25.526395Z","iopub.execute_input":"2022-07-28T07:56:25.526985Z","iopub.status.idle":"2022-07-28T07:56:25.607943Z","shell.execute_reply.started":"2022-07-28T07:56:25.526928Z","shell.execute_reply":"2022-07-28T07:56:25.607038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['text'] = df['text'].apply(lambda s : cleaner(s))","metadata":{"id":"bs2qxpTd8OhF","execution":{"iopub.status.busy":"2022-07-28T07:56:25.609484Z","iopub.execute_input":"2022-07-28T07:56:25.611426Z","iopub.status.idle":"2022-07-28T07:56:28.025322Z","shell.execute_reply.started":"2022-07-28T07:56:25.610345Z","shell.execute_reply":"2022-07-28T07:56:28.024355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['text'] = df_test['text'].apply(lambda s : cleaner(s))","metadata":{"id":"KFvdBDlMRnYC","execution":{"iopub.status.busy":"2022-07-28T07:56:28.026765Z","iopub.execute_input":"2022-07-28T07:56:28.027146Z","iopub.status.idle":"2022-07-28T07:56:28.763846Z","shell.execute_reply.started":"2022-07-28T07:56:28.027072Z","shell.execute_reply":"2022-07-28T07:56:28.762898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Removing punctuations:\n\nIn this step the there are only a few tweets cleaned that still contain symbols and punctuations, as they don't add key information to the message we will get rid of them, the function which applies such step will be called remove_punct:\n","metadata":{"id":"Xj7pOUPCX1Yj"}},{"cell_type":"code","source":"def remove_punct(text):\n    table=str.maketrans('','',string.punctuation)\n    return text.translate(table)\n\nexample=\"I am a #king\"\nprint(remove_punct(example))","metadata":{"id":"CUdCO-IsXokm","outputId":"c82b7aa0-ba62-4df1-d76b-a1382515c3f6","execution":{"iopub.status.busy":"2022-07-28T07:56:28.765251Z","iopub.execute_input":"2022-07-28T07:56:28.765571Z","iopub.status.idle":"2022-07-28T07:56:28.773950Z","shell.execute_reply.started":"2022-07-28T07:56:28.765529Z","shell.execute_reply":"2022-07-28T07:56:28.772996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['text']=df['text'].apply(lambda x : remove_punct(x))","metadata":{"id":"BSA_wMvRXob3","execution":{"iopub.status.busy":"2022-07-28T07:56:28.775751Z","iopub.execute_input":"2022-07-28T07:56:28.776505Z","iopub.status.idle":"2022-07-28T07:56:28.832943Z","shell.execute_reply.started":"2022-07-28T07:56:28.776460Z","shell.execute_reply":"2022-07-28T07:56:28.832019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['text']=df_test['text'].apply(lambda x : remove_punct(x))","metadata":{"id":"rZZN0h-4RT0R","execution":{"iopub.status.busy":"2022-07-28T07:56:28.834225Z","iopub.execute_input":"2022-07-28T07:56:28.834525Z","iopub.status.idle":"2022-07-28T07:56:28.860165Z","shell.execute_reply.started":"2022-07-28T07:56:28.834484Z","shell.execute_reply":"2022-07-28T07:56:28.859186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Removing multiple spaces:\n\nNow, some sentences cleaned have different types of extra whitespaces, obviusly they don't add anything to the corpus and we will get rid of them with the following lines:","metadata":{"id":"fQ2LvmsDyKop"}},{"cell_type":"code","source":"df['text']=df['text'].str.replace('   ', ' ')\ndf['text']=df['text'].str.replace('     ', ' ')\ndf['text']=df['text'].str.replace('\\xa0 \\xa0 \\xa0', ' ')\ndf['text']=df['text'].str.replace('  ', ' ')\ndf['text']=df['text'].str.replace('—', ' ')\ndf['text']=df['text'].str.replace('–', ' ')","metadata":{"id":"AcR_SG4ux60h","execution":{"iopub.status.busy":"2022-07-28T07:56:28.863894Z","iopub.execute_input":"2022-07-28T07:56:28.864108Z","iopub.status.idle":"2022-07-28T07:56:28.920120Z","shell.execute_reply.started":"2022-07-28T07:56:28.864080Z","shell.execute_reply":"2022-07-28T07:56:28.919061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['text']=df_test['text'].str.replace('   ', ' ')\ndf_test['text']=df_test['text'].str.replace('     ', ' ')\ndf_test['text']=df_test['text'].str.replace('\\xa0 \\xa0 \\xa0', ' ')\ndf_test['text']=df_test['text'].str.replace('  ', ' ')\ndf_test['text']=df_test['text'].str.replace('—', ' ')\ndf_test['text']=df_test['text'].str.replace('–', ' ')","metadata":{"id":"0OT2HBHtRtq7","execution":{"iopub.status.busy":"2022-07-28T07:56:28.922035Z","iopub.execute_input":"2022-07-28T07:56:28.922356Z","iopub.status.idle":"2022-07-28T07:56:28.956126Z","shell.execute_reply.started":"2022-07-28T07:56:28.922316Z","shell.execute_reply":"2022-07-28T07:56:28.955182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysis of common words per class:","metadata":{"id":"KydS9AGaoCLI"}},{"cell_type":"markdown","source":"As we will select the most common words from each topic we have to make sure to avoid selecting the denominated 'stop words' because these will not be relevant in this case, this is why we will import them by downloading from nltk tool and english dictionary:","metadata":{"id":"ywahqwq_0f0e"}},{"cell_type":"code","source":"nltk.download(\"stopwords\")","metadata":{"id":"lYScQqzUb1BA","outputId":"f0b5ee8a-429c-4462-da48-ee1e3ef137e5","execution":{"iopub.status.busy":"2022-07-28T07:56:28.957751Z","iopub.execute_input":"2022-07-28T07:56:28.958070Z","iopub.status.idle":"2022-07-28T07:56:28.968239Z","shell.execute_reply.started":"2022-07-28T07:56:28.958030Z","shell.execute_reply":"2022-07-28T07:56:28.967304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords","metadata":{"id":"k-2VUfI8hEQP","execution":{"iopub.status.busy":"2022-07-28T07:56:28.969950Z","iopub.execute_input":"2022-07-28T07:56:28.970955Z","iopub.status.idle":"2022-07-28T07:56:28.977165Z","shell.execute_reply.started":"2022-07-28T07:56:28.970900Z","shell.execute_reply":"2022-07-28T07:56:28.976188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = set(stopwords.words(\"english\"))","metadata":{"id":"NT7GWkNpZPc3","execution":{"iopub.status.busy":"2022-07-28T07:56:28.978845Z","iopub.execute_input":"2022-07-28T07:56:28.979411Z","iopub.status.idle":"2022-07-28T07:56:28.990817Z","shell.execute_reply.started":"2022-07-28T07:56:28.979316Z","shell.execute_reply":"2022-07-28T07:56:28.989653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the following step we will plot the top most common words in each set we have, disaster tweets, non-disaster tweets and testing tweets:","metadata":{"id":"jxTnamwC9qol"}},{"cell_type":"code","source":"from collections import defaultdict,Counter","metadata":{"id":"mz5F2hsvdkyt","execution":{"iopub.status.busy":"2022-07-28T07:56:28.992721Z","iopub.execute_input":"2022-07-28T07:56:28.993386Z","iopub.status.idle":"2022-07-28T07:56:29.008100Z","shell.execute_reply.started":"2022-07-28T07:56:28.993345Z","shell.execute_reply":"2022-07-28T07:56:29.007108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_count = Counter(\" \".join(df[df['target']==1]['text']).split()).most_common(100)\nx=[]\ny=[]\nfor word,count in word_count:\n    if (word.casefold() not in stop_words) :\n        x.append(word)\n        y.append(count)\n\nsns.barplot(x=y[:20],y=x[:20])\nplt.title('20 most common words in Disaster tweets')","metadata":{"id":"tAGYeJVtwXtn","outputId":"4b71184d-f111-4411-fcb7-42508abb90f1","execution":{"iopub.status.busy":"2022-07-28T07:56:29.009909Z","iopub.execute_input":"2022-07-28T07:56:29.010226Z","iopub.status.idle":"2022-07-28T07:56:29.498421Z","shell.execute_reply.started":"2022-07-28T07:56:29.010187Z","shell.execute_reply":"2022-07-28T07:56:29.497560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_count = Counter(\" \".join(df[df['target']==0]['text']).split()).most_common(100)\nx=[]\ny=[]\nfor word,count in word_count:\n    if (word.casefold() not in stop_words) :\n        x.append(word)\n        y.append(count)\n\nsns.barplot(x=y[:20],y=x[:20])\nplt.title('20 most common words in Non-disaster tweets')","metadata":{"id":"oTduLOdhww5G","outputId":"0b8118d2-c0c4-4220-a77f-8bbba2a6c2de","execution":{"iopub.status.busy":"2022-07-28T07:56:29.499875Z","iopub.execute_input":"2022-07-28T07:56:29.500858Z","iopub.status.idle":"2022-07-28T07:56:29.930690Z","shell.execute_reply.started":"2022-07-28T07:56:29.500804Z","shell.execute_reply":"2022-07-28T07:56:29.929657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word_count = Counter(\" \".join(df_test['text']).split()).most_common(100)\nx=[]\ny=[]\nfor word,count in word_count:\n    if (word.casefold() not in stop_words) :\n        x.append(word)\n        y.append(count)\n\nsns.barplot(x=y[:30],y=x[:30])\nplt.title('21 most common words in Test tweets')","metadata":{"id":"FItxb0HJXmb4","outputId":"503ac327-54ff-480b-b1c3-82a177848a36","execution":{"iopub.status.busy":"2022-07-28T07:56:29.932388Z","iopub.execute_input":"2022-07-28T07:56:29.932704Z","iopub.status.idle":"2022-07-28T07:56:30.406082Z","shell.execute_reply.started":"2022-07-28T07:56:29.932665Z","shell.execute_reply":"2022-07-28T07:56:30.405141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now let's compute the N-grams in each set already mentioned, the folowing function generate_ngrams will help us with the process: ","metadata":{"id":"2ODr9Xcg-HTg"}},{"cell_type":"code","source":"# Define ngram generator function\ndef generate_ngrams(text, n_gram):\n    token = [token for token in text.lower().split(' ') if token != '' if token not in stop_words]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [' '.join(ngram) for ngram in ngrams]","metadata":{"id":"Lp2kkWAjkKXH","execution":{"iopub.status.busy":"2022-07-28T07:56:30.408588Z","iopub.execute_input":"2022-07-28T07:56:30.409471Z","iopub.status.idle":"2022-07-28T07:56:30.416610Z","shell.execute_reply.started":"2022-07-28T07:56:30.409426Z","shell.execute_reply":"2022-07-28T07:56:30.415572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The number of N-grams to compute:","metadata":{"id":"Qe2WbB5w-pHm"}},{"cell_type":"code","source":"N=50","metadata":{"id":"22gOl7ndmmDw","execution":{"iopub.status.busy":"2022-07-28T07:56:30.417906Z","iopub.execute_input":"2022-07-28T07:56:30.418823Z","iopub.status.idle":"2022-07-28T07:56:30.434871Z","shell.execute_reply.started":"2022-07-28T07:56:30.418766Z","shell.execute_reply":"2022-07-28T07:56:30.433767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Bi-grams:","metadata":{"id":"bDGhhf5ryiQI"}},{"cell_type":"markdown","source":"Bigrams analysis of test tweets:","metadata":{"id":"9BKf5Ul1yB-Z"}},{"cell_type":"code","source":"# Bigrams\ntesting_bigrams = defaultdict(int)\n\nfor instance in df_test['text']:\n    for word in generate_ngrams(instance, n_gram=2):\n        testing_bigrams[word] += 1\n   \ndf_testing_bigrams = pd.DataFrame(sorted(testing_bigrams.items(), key=lambda x: x[1])[::-1])","metadata":{"id":"9nXsd_tBYLul","execution":{"iopub.status.busy":"2022-07-28T07:56:30.436788Z","iopub.execute_input":"2022-07-28T07:56:30.437186Z","iopub.status.idle":"2022-07-28T07:56:30.504681Z","shell.execute_reply.started":"2022-07-28T07:56:30.437111Z","shell.execute_reply":"2022-07-28T07:56:30.503835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig_dims = (25, 30)\nfig, ax3 = plt.subplots(figsize=fig_dims)\n\nsns.barplot(y=df_testing_bigrams[0].values[:50], x=df_testing_bigrams[1].values[:50], color='y', ax=ax3)\nplt.tick_params(axis='x', labelsize=13)\nplt.tick_params(axis='y', labelsize=13)\n\nplt.title(f'Top {N} most common bigrams in testing tweets', fontsize=15)\n\nplt.show()","metadata":{"id":"d4We-mJDYMx6","outputId":"9367fd56-0390-4ff5-ac70-24684389ab0f","execution":{"iopub.status.busy":"2022-07-28T07:56:30.506225Z","iopub.execute_input":"2022-07-28T07:56:30.506570Z","iopub.status.idle":"2022-07-28T07:56:31.705342Z","shell.execute_reply.started":"2022-07-28T07:56:30.506528Z","shell.execute_reply":"2022-07-28T07:56:31.704415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Bigrams analysis of disaster and non-disaster tweets:","metadata":{"id":"RI-zFOLjx1M5"}},{"cell_type":"code","source":"# Bigrams\ndisaster_bigrams = defaultdict(int)\nnondisaster_bigrams = defaultdict(int)\n\nfor instance in df[df['target']==1]['text']:\n    for word in generate_ngrams(instance, n_gram=2):\n        disaster_bigrams[word] += 1\n\nfor instance in df[df['target']==0]['text']:\n    for word in generate_ngrams(instance, n_gram=2):\n        nondisaster_bigrams[word] += 1 \n   \ndf_disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items(), key=lambda x: x[1])[::-1])\ndf_nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items(), key=lambda x: x[1])[::-1])","metadata":{"id":"zQfg_tZdjCvy","execution":{"iopub.status.busy":"2022-07-28T07:56:31.706957Z","iopub.execute_input":"2022-07-28T07:56:31.707420Z","iopub.status.idle":"2022-07-28T07:56:31.845738Z","shell.execute_reply.started":"2022-07-28T07:56:31.707377Z","shell.execute_reply":"2022-07-28T07:56:31.844863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(25,30), dpi=80)\nplt.tight_layout()\n\nsns.barplot(y=df_disaster_bigrams[0].values[:N], x=df_disaster_bigrams[1].values[:N], ax=ax1, color='r')\nax1.spines['right'].set_visible(False)\nax1.tick_params(axis='x', labelsize=13)\nax1.tick_params(axis='y', labelsize=13)\n\nsns.barplot(y=df_nondisaster_bigrams[0].values[:N], x=df_nondisaster_bigrams[1].values[:N], ax=ax2, color='b')\nax2.spines['right'].set_visible(False)\nax2.tick_params(axis='x', labelsize=13)\nax2.tick_params(axis='y', labelsize=13)\n\nax1.set_title(f'Top {N} most common bigrams in Disaster tweets', fontsize=15)\nax2.set_title(f'Top {N} most common bigrams in Non-disaster tweets', fontsize=15)\n\nplt.show()\nplt.tight_layout()","metadata":{"id":"1DLrHdvEjCY1","outputId":"224cb394-a84b-4c8e-f5a3-4c49005d3089","execution":{"iopub.status.busy":"2022-07-28T07:56:31.847529Z","iopub.execute_input":"2022-07-28T07:56:31.847845Z","iopub.status.idle":"2022-07-28T07:56:35.890526Z","shell.execute_reply.started":"2022-07-28T07:56:31.847808Z","shell.execute_reply":"2022-07-28T07:56:35.889673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Tri-grams:","metadata":{"id":"r2SEiuUfr3Bt"}},{"cell_type":"markdown","source":"Trigrams analysis of test tweets:","metadata":{"id":"1Csu2qXQyVch"}},{"cell_type":"code","source":"# Trigrams\ntesting_trigrams = defaultdict(int)\n\nfor instance in df_test['text']:\n    for word in generate_ngrams(instance, n_gram=3):\n        testing_trigrams[word] += 1\n   \ndf_testing_trigrams = pd.DataFrame(sorted(testing_trigrams.items(), key=lambda x: x[1])[::-1])","metadata":{"id":"1vGO8xYrbMGq","execution":{"iopub.status.busy":"2022-07-28T07:56:35.892540Z","iopub.execute_input":"2022-07-28T07:56:35.897378Z","iopub.status.idle":"2022-07-28T07:56:35.974151Z","shell.execute_reply.started":"2022-07-28T07:56:35.897313Z","shell.execute_reply":"2022-07-28T07:56:35.973180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig_dims = (25, 30)\nfig, ax3 = plt.subplots(figsize=fig_dims)\n\nsns.barplot(y=df_testing_trigrams[0].values[:50], x=df_testing_trigrams[1].values[:50], color='y', ax=ax3)\nplt.tick_params(axis='x', labelsize=13)\nplt.tick_params(axis='y', labelsize=13)\n\nplt.title(f'Top {N} most common trigrams in testing tweets', fontsize=15)\n\nplt.show()","metadata":{"outputId":"9c1eebc2-55e3-4c7d-f15d-c81b934c434b","id":"Q2l6j0h_bMGu","execution":{"iopub.status.busy":"2022-07-28T07:56:35.975735Z","iopub.execute_input":"2022-07-28T07:56:35.976239Z","iopub.status.idle":"2022-07-28T07:56:39.191199Z","shell.execute_reply.started":"2022-07-28T07:56:35.976181Z","shell.execute_reply":"2022-07-28T07:56:39.190210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Trigrams analysis of disaster and non-disaster tweets:","metadata":{"id":"ulz7dRYAyfbT"}},{"cell_type":"code","source":"# Trigrams\ndisaster_bigrams = defaultdict(int)\nnondisaster_bigrams = defaultdict(int)\n\nfor instance in df[df['target']==1]['text']:\n    for word in generate_ngrams(instance, n_gram=3):\n        disaster_bigrams[word] += 1\n\nfor instance in df[df['target']==0]['text']:\n    for word in generate_ngrams(instance, n_gram=3):\n        nondisaster_bigrams[word] += 1 \n   \ndf_disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items(), key=lambda x: x[1])[::-1])\ndf_nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items(), key=lambda x: x[1])[::-1])","metadata":{"id":"jK7EQDdur7Dn","execution":{"iopub.status.busy":"2022-07-28T07:56:39.199480Z","iopub.execute_input":"2022-07-28T07:56:39.208604Z","iopub.status.idle":"2022-07-28T07:56:39.460748Z","shell.execute_reply.started":"2022-07-28T07:56:39.208561Z","shell.execute_reply":"2022-07-28T07:56:39.459785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(25,20), dpi=80)\nplt.tight_layout()\n\nsns.barplot(y=df_disaster_bigrams[0].values[:N], x=df_disaster_bigrams[1].values[:N], ax=ax1, color='r')\nax1.spines['right'].set_visible(False)\nax1.tick_params(axis='x', labelsize=13)\nax1.tick_params(axis='y', labelsize=13)\n\nsns.barplot(y=df_nondisaster_bigrams[0].values[:N], x=df_nondisaster_bigrams[1].values[:N], ax=ax2, color='b')\nax2.spines['right'].set_visible(False)\nax2.tick_params(axis='x', labelsize=13)\nax2.tick_params(axis='y', labelsize=13)\n\nax1.set_title(f'Top {N} most common trigrams in Disaster tweets', fontsize=15)\nax2.set_title(f'Top {N} most common trigrams in Non-disaster tweets', fontsize=15)\n\nplt.show()\nplt.tight_layout()","metadata":{"outputId":"1ac5f90d-9661-4c4b-eaea-ed16f76e6e0e","id":"hX87eykKr7Dt","execution":{"iopub.status.busy":"2022-07-28T07:56:39.466091Z","iopub.execute_input":"2022-07-28T07:56:39.468614Z","iopub.status.idle":"2022-07-28T07:56:43.230617Z","shell.execute_reply.started":"2022-07-28T07:56:39.468567Z","shell.execute_reply":"2022-07-28T07:56:43.229685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tokenizing","metadata":{"id":"So1JnWsjD0-F"}},{"cell_type":"code","source":"df[['text','target']]","metadata":{"id":"g64WyfFQDdxN","outputId":"a050f707-9708-4e9f-c7da-a187abbb1f5e","execution":{"iopub.status.busy":"2022-07-28T07:56:43.232380Z","iopub.execute_input":"2022-07-28T07:56:43.232983Z","iopub.status.idle":"2022-07-28T07:56:43.250885Z","shell.execute_reply.started":"2022-07-28T07:56:43.232940Z","shell.execute_reply":"2022-07-28T07:56:43.249887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","metadata":{"id":"OdcLvIumDdlC","execution":{"iopub.status.busy":"2022-07-28T07:56:43.252743Z","iopub.execute_input":"2022-07-28T07:56:43.253406Z","iopub.status.idle":"2022-07-28T07:56:44.156265Z","shell.execute_reply.started":"2022-07-28T07:56:43.253359Z","shell.execute_reply":"2022-07-28T07:56:44.155212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a new column in train file containing instances without stopwords:","metadata":{"id":"qxUBFBQAEKFT"}},{"cell_type":"code","source":"df['text_without_stopwords'] = df['text'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop_words)]))","metadata":{"id":"lpDbnuNFDdbS","execution":{"iopub.status.busy":"2022-07-28T07:56:44.158196Z","iopub.execute_input":"2022-07-28T07:56:44.158505Z","iopub.status.idle":"2022-07-28T07:56:44.197418Z","shell.execute_reply.started":"2022-07-28T07:56:44.158464Z","shell.execute_reply":"2022-07-28T07:56:44.196550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['text','text_without_stopwords']]","metadata":{"id":"KVlt-wkwDdNQ","outputId":"424ca2b0-cedd-4a63-ce62-a2e6b67f06a6","execution":{"iopub.status.busy":"2022-07-28T07:56:44.202810Z","iopub.execute_input":"2022-07-28T07:56:44.203061Z","iopub.status.idle":"2022-07-28T07:56:44.228194Z","shell.execute_reply.started":"2022-07-28T07:56:44.203027Z","shell.execute_reply":"2022-07-28T07:56:44.226888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a new column in test file containing instances without stopwords:","metadata":{"id":"cdCDVh2lTrIJ"}},{"cell_type":"code","source":"df_test['text_without_stopwords'] = df_test['text'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop_words)]))","metadata":{"id":"fUdrXLnYTTm0","execution":{"iopub.status.busy":"2022-07-28T07:56:44.230392Z","iopub.execute_input":"2022-07-28T07:56:44.230744Z","iopub.status.idle":"2022-07-28T07:56:44.253198Z","shell.execute_reply.started":"2022-07-28T07:56:44.230678Z","shell.execute_reply":"2022-07-28T07:56:44.252339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test[['text','text_without_stopwords']]","metadata":{"id":"K9TXYaMkTTNG","outputId":"12778f58-5378-402c-d604-2f764a1de9c9","execution":{"iopub.status.busy":"2022-07-28T07:56:44.254528Z","iopub.execute_input":"2022-07-28T07:56:44.254904Z","iopub.status.idle":"2022-07-28T07:56:44.275797Z","shell.execute_reply.started":"2022-07-28T07:56:44.254853Z","shell.execute_reply":"2022-07-28T07:56:44.274733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train-test split:\nAs we have only 7613 instances in our train dataframe we have to keep a reasonable proportion for validation set because it will not contain a huge  number of instances, 80%-20% would be nice.","metadata":{"id":"0truUH1xEgVS"}},{"cell_type":"code","source":"training_portion=0.80","metadata":{"id":"4qrJkxEwEj_E","execution":{"iopub.status.busy":"2022-07-28T07:56:44.277648Z","iopub.execute_input":"2022-07-28T07:56:44.278424Z","iopub.status.idle":"2022-07-28T07:56:44.285910Z","shell.execute_reply.started":"2022-07-28T07:56:44.278379Z","shell.execute_reply":"2022-07-28T07:56:44.284902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = int(df.shape[0]*training_portion)\n\ntrain_sentences = df['text_without_stopwords'][:train_size]\ntrain_labels = df['target'][:train_size]\n\nvalidation_sentences = df['text_without_stopwords'][train_size:]\nvalidation_labels = df['target'][train_size:]\n\nprint(train_size)\nprint(len(train_sentences))\nprint(len(train_labels))\nprint(len(validation_sentences))\nprint(len(validation_labels))","metadata":{"id":"P2Ik7tPDEYhx","outputId":"73fe5a7f-7c1a-40f6-d7b0-dd4ed0f7dfe8","execution":{"iopub.status.busy":"2022-07-28T07:56:44.287595Z","iopub.execute_input":"2022-07-28T07:56:44.287971Z","iopub.status.idle":"2022-07-28T07:56:44.302423Z","shell.execute_reply.started":"2022-07-28T07:56:44.287928Z","shell.execute_reply":"2022-07-28T07:56:44.301429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us tokenize all words in our dataset so as to analyze and get an appropriate vocab_size and max_length:","metadata":{"id":"km9jdDLjEsD7"}},{"cell_type":"code","source":"tokenizer0 = Tokenizer()\ntokenizer0.fit_on_texts(df['text'])\nword_index = tokenizer0.word_index\nlen(word_index)","metadata":{"id":"ARpSJtlnz9RY","outputId":"26213043-6d78-416b-f8f5-ff6b98c29a4c","execution":{"iopub.status.busy":"2022-07-28T07:56:44.303690Z","iopub.execute_input":"2022-07-28T07:56:44.304261Z","iopub.status.idle":"2022-07-28T07:56:44.545413Z","shell.execute_reply.started":"2022-07-28T07:56:44.304200Z","shell.execute_reply":"2022-07-28T07:56:44.543118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 16871 unique words in the training sentences, we must remember that this number would be higher if we would not have removed the stopwords.","metadata":{"id":"UZi7Xc06Ewlx"}},{"cell_type":"markdown","source":"Let's set the vocab size to 16000 so as not to slow down too much the model training later:","metadata":{"id":"Qoa9fQrWE4Jg"}},{"cell_type":"code","source":"vocab_size = 16000\noov_tok = '<OOV>'","metadata":{"id":"sfcIMjTkEzS1","execution":{"iopub.status.busy":"2022-07-28T07:56:44.550591Z","iopub.execute_input":"2022-07-28T07:56:44.553460Z","iopub.status.idle":"2022-07-28T07:56:44.560095Z","shell.execute_reply.started":"2022-07-28T07:56:44.553411Z","shell.execute_reply":"2022-07-28T07:56:44.558895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok)\ntokenizer.fit_on_texts(df['text'])\nword_index = tokenizer.word_index","metadata":{"id":"X5pg9K3yEzJq","execution":{"iopub.status.busy":"2022-07-28T07:56:44.563882Z","iopub.execute_input":"2022-07-28T07:56:44.564488Z","iopub.status.idle":"2022-07-28T07:56:44.910917Z","shell.execute_reply.started":"2022-07-28T07:56:44.564443Z","shell.execute_reply":"2022-07-28T07:56:44.909870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we can compute the length of each instance tokenized of the training set and see its distribution in order to choose maximum length which does not drop significant information at the end of the sequence:","metadata":{"id":"wXYmqOlWE_Qf"}},{"cell_type":"code","source":"train_sequences = tokenizer.texts_to_sequences(train_sentences)","metadata":{"id":"iJ1sNQXUE9PJ","execution":{"iopub.status.busy":"2022-07-28T07:56:44.916190Z","iopub.execute_input":"2022-07-28T07:56:44.918646Z","iopub.status.idle":"2022-07-28T07:56:45.083037Z","shell.execute_reply.started":"2022-07-28T07:56:44.918598Z","shell.execute_reply":"2022-07-28T07:56:45.081898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lengths=[]\nfor k in range(len(train_sequences)):\n  lengths.append(len(train_sequences[k]))","metadata":{"id":"1uAGvublE9DI","execution":{"iopub.status.busy":"2022-07-28T07:56:45.088285Z","iopub.execute_input":"2022-07-28T07:56:45.088585Z","iopub.status.idle":"2022-07-28T07:56:45.103738Z","shell.execute_reply.started":"2022-07-28T07:56:45.088547Z","shell.execute_reply":"2022-07-28T07:56:45.102701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(lengths, columns=['Lenghts']).describe()","metadata":{"id":"2XTHbRSlFEfz","outputId":"e51d04b0-f1d2-4de2-adb0-4b615bce76b7","execution":{"iopub.status.busy":"2022-07-28T07:56:45.105461Z","iopub.execute_input":"2022-07-28T07:56:45.105809Z","iopub.status.idle":"2022-07-28T07:56:45.128110Z","shell.execute_reply.started":"2022-07-28T07:56:45.105761Z","shell.execute_reply":"2022-07-28T07:56:45.127007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.hist(lengths, bins=27, alpha=0.5)\nplt.show()","metadata":{"id":"1HtZhdXjFETQ","outputId":"f149d864-9da6-4ecd-ab20-4d921798b4cb","execution":{"iopub.status.busy":"2022-07-28T07:56:45.129897Z","iopub.execute_input":"2022-07-28T07:56:45.130458Z","iopub.status.idle":"2022-07-28T07:56:45.489760Z","shell.execute_reply.started":"2022-07-28T07:56:45.130415Z","shell.execute_reply":"2022-07-28T07:56:45.488670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us set such argument to 20, truncation type 'post' and padding type 'post' as follows:","metadata":{"id":"14pk_cLYFLYK"}},{"cell_type":"code","source":"max_length = 20\ntrunc_type = 'post'\npadding_type = 'post'","metadata":{"id":"o0XuF6jpFKGg","execution":{"iopub.status.busy":"2022-07-28T07:56:45.491347Z","iopub.execute_input":"2022-07-28T07:56:45.492527Z","iopub.status.idle":"2022-07-28T07:56:45.498215Z","shell.execute_reply.started":"2022-07-28T07:56:45.492464Z","shell.execute_reply":"2022-07-28T07:56:45.497049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_padded = pad_sequences(train_sequences,maxlen=max_length,padding=padding_type,truncating=trunc_type)","metadata":{"id":"LHIMFhLNEoxY","execution":{"iopub.status.busy":"2022-07-28T07:56:45.500265Z","iopub.execute_input":"2022-07-28T07:56:45.500646Z","iopub.status.idle":"2022-07-28T07:56:45.585782Z","shell.execute_reply.started":"2022-07-28T07:56:45.500602Z","shell.execute_reply":"2022-07-28T07:56:45.584856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can print the length of the sequences before and after padding, let's see their corresponding for 3 instances of training set:","metadata":{"id":"XF5-aTePFRqv"}},{"cell_type":"code","source":"print(len(train_sequences[0]))\nprint(len(train_padded[0]))\n\nprint(len(train_sequences[1]))\nprint(len(train_padded[1]))\n\nprint(len(train_sequences[10]))\nprint(len(train_padded[10]))","metadata":{"id":"2cKWo2CCFPvc","outputId":"db7d768b-ce8a-44e4-ab7b-652a4aba0833","execution":{"iopub.status.busy":"2022-07-28T07:56:45.587306Z","iopub.execute_input":"2022-07-28T07:56:45.587610Z","iopub.status.idle":"2022-07-28T07:56:45.599685Z","shell.execute_reply.started":"2022-07-28T07:56:45.587570Z","shell.execute_reply":"2022-07-28T07:56:45.598486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The same process we just did to training set we have to do in validation set, but this will be done using the tokenizer function fitted in training:","metadata":{"id":"66d2ukXYXVSD"}},{"cell_type":"code","source":"validation_sequences = tokenizer.texts_to_sequences(validation_sentences)\nvalidation_padded = pad_sequences(validation_sequences,padding=padding_type,maxlen=max_length,truncating=trunc_type)\n\nprint(len(validation_sequences))\nprint(validation_padded.shape)","metadata":{"id":"GVAGW74FXYv7","outputId":"23f60f4d-3158-4743-b1ed-9b0a69fd1683","execution":{"iopub.status.busy":"2022-07-28T07:56:45.601303Z","iopub.execute_input":"2022-07-28T07:56:45.601900Z","iopub.status.idle":"2022-07-28T07:56:45.648472Z","shell.execute_reply.started":"2022-07-28T07:56:45.601805Z","shell.execute_reply":"2022-07-28T07:56:45.647388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As training and validation sets are ready to be used in the training of the model, it's the right moment to do exactly the same for the instances in test set in order to have them properly processed to predict their classes.","metadata":{"id":"ymijd7iSYoBC"}},{"cell_type":"code","source":"test_sequences = tokenizer.texts_to_sequences(df_test['text_without_stopwords'])\ntest_padded = pad_sequences(test_sequences,padding=padding_type,maxlen=max_length,truncating=trunc_type)\n\nprint(len(test_sequences))\nprint(test_padded.shape)","metadata":{"id":"M2FijP1aXYob","outputId":"51f6e4df-4d66-4fc2-bd8d-0d1707e0d203","execution":{"iopub.status.busy":"2022-07-28T07:56:45.649992Z","iopub.execute_input":"2022-07-28T07:56:45.650266Z","iopub.status.idle":"2022-07-28T07:56:45.731683Z","shell.execute_reply.started":"2022-07-28T07:56:45.650213Z","shell.execute_reply":"2022-07-28T07:56:45.730568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(train_labels)","metadata":{"id":"_YC6oT0oFPlZ","outputId":"75651c66-fa28-4dfd-badc-51f1019eb6f9","execution":{"iopub.status.busy":"2022-07-28T07:56:45.733010Z","iopub.execute_input":"2022-07-28T07:56:45.733311Z","iopub.status.idle":"2022-07-28T07:56:45.740976Z","shell.execute_reply.started":"2022-07-28T07:56:45.733265Z","shell.execute_reply":"2022-07-28T07:56:45.739893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling:\n\nIn this step we are going to consider building models by scratch and pre-trained as can be seen below:\n\n- Recurrent Neural Network by scratch.\n- Pre-trained 100 dimensional Glove.\n\nBefore building the models we have to create two contraints or 'callbacks', one to reduce the learning rate whenever the validation accuracy does not increase so as to lead the steps towards the global minimum loss and the second is early stopping which will stop the training if the validation accuracy does not increase after 2 consecutive epochs.","metadata":{"id":"fuqNNidLa2hI"}},{"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\nlr_reduction = ReduceLROnPlateau(monitor='val_accuracy',\n                                 patience=1, \n                                 verbose=1, \n                                 factor=0.5, \n                                 min_lr=0.000001)\n\nfrom keras.callbacks import EarlyStopping\nearly_stopping = EarlyStopping(monitor='val_accuracy', \n                               min_delta=0.005,\n                               patience=3, \n                               verbose=1, \n                               mode='auto')","metadata":{"id":"9YFXzHQ5am6S","execution":{"iopub.status.busy":"2022-07-28T07:56:45.742995Z","iopub.execute_input":"2022-07-28T07:56:45.743644Z","iopub.status.idle":"2022-07-28T07:56:45.753484Z","shell.execute_reply.started":"2022-07-28T07:56:45.743600Z","shell.execute_reply":"2022-07-28T07:56:45.752544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Recurrent Neural Network by scratch","metadata":{"id":"RsdJ_SUZcRp9"}},{"cell_type":"code","source":"embedding_dim = 32\nmodel = tf.keras.Sequential([\n              tf.keras.layers.Embedding(vocab_size,embedding_dim,input_length=max_length),\n              tf.keras.layers.GlobalAveragePooling1D(),\n              tf.keras.layers.Dense(24,activation='relu'),\n              tf.keras.layers.Dropout(0.1),\n              tf.keras.layers.Dense(1,activation='sigmoid')\n])\nmodel.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])\nmodel.summary()","metadata":{"id":"t63FusukcMfo","outputId":"ae013340-f92f-4be2-edf9-bb3209aa7ba3","execution":{"iopub.status.busy":"2022-07-28T07:56:45.755536Z","iopub.execute_input":"2022-07-28T07:56:45.755994Z","iopub.status.idle":"2022-07-28T07:56:49.281734Z","shell.execute_reply.started":"2022-07-28T07:56:45.755954Z","shell.execute_reply":"2022-07-28T07:56:49.280006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Observation: The callback which reduces the learning rate was useful as the training accuracy increases whenever the lr is reduced, however the validation accuracy stays the same in the last epochs, this is why the early stopping will not be used so as not to limit the performance of the model:","metadata":{"id":"65OIlgD4BidA"}},{"cell_type":"code","source":"num_epochs = 15\nhistory = model.fit(train_padded,train_labels,epochs=num_epochs,\n                    validation_data=(validation_padded,validation_labels),verbose=1,\n                    callbacks=[lr_reduction])","metadata":{"id":"jHZO7qTqcMT3","outputId":"6477ea51-1061-48e1-e2d7-50d51d35e660","execution":{"iopub.status.busy":"2022-07-28T07:56:49.283684Z","iopub.execute_input":"2022-07-28T07:56:49.284195Z","iopub.status.idle":"2022-07-28T07:57:10.534166Z","shell.execute_reply.started":"2022-07-28T07:56:49.284151Z","shell.execute_reply":"2022-07-28T07:57:10.533216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The first model took 15 seconds to train reaching accuracies of: 98.3% train/83.1% val.","metadata":{"id":"lLLpL7uAdNgX"}},{"cell_type":"code","source":"def plot_metrics(history):\n  acc=history.history['accuracy']\n  val_acc=history.history['val_accuracy']\n  loss=history.history['loss']\n  val_loss=history.history['val_loss']\n\n  epochs=range(1,len(history.history['accuracy'])+1) # Get number of epochs\n\n  #------------------------------------------------\n  # Plot training and validation accuracy per epoch\n  #------------------------------------------------\n  plt.plot(epochs, acc, 'r')\n  plt.plot(epochs, val_acc, 'b')\n  plt.title('Training and validation accuracy')\n  plt.xlabel(\"Epochs\")\n  plt.ylabel(\"Accuracy\")\n  plt.legend([\"Accuracy\", \"Validation Accuracy\"])\n\n  plt.figure()\n\n  #------------------------------------------------\n  # Plot training and validation loss per epoch\n  #------------------------------------------------\n  plt.plot(epochs, loss, 'r')\n  plt.plot(epochs, val_loss, 'b')\n  plt.title('Training and validation loss')\n  plt.xlabel(\"Epochs\")\n  plt.ylabel(\"Loss\")\n  plt.legend([\"Loss\", \"Validation Loss\"])\n\n  plt.figure()","metadata":{"id":"OPZXEFekamjy","execution":{"iopub.status.busy":"2022-07-28T07:57:10.536176Z","iopub.execute_input":"2022-07-28T07:57:10.536477Z","iopub.status.idle":"2022-07-28T07:57:10.547588Z","shell.execute_reply.started":"2022-07-28T07:57:10.536435Z","shell.execute_reply":"2022-07-28T07:57:10.546395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_metrics(history)","metadata":{"id":"cS3qOlVedKek","outputId":"50b428ce-2bba-4d54-fd09-4be18de984b8","execution":{"iopub.status.busy":"2022-07-28T07:57:10.559808Z","iopub.execute_input":"2022-07-28T07:57:10.560031Z","iopub.status.idle":"2022-07-28T07:57:11.289168Z","shell.execute_reply.started":"2022-07-28T07:57:10.560003Z","shell.execute_reply":"2022-07-28T07:57:11.287947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_val = model.predict(validation_padded, batch_size=32)","metadata":{"id":"7rxYBPwd1vWP","execution":{"iopub.status.busy":"2022-07-28T07:57:11.290650Z","iopub.execute_input":"2022-07-28T07:57:11.291088Z","iopub.status.idle":"2022-07-28T07:57:11.471465Z","shell.execute_reply.started":"2022-07-28T07:57:11.291044Z","shell.execute_reply":"2022-07-28T07:57:11.470431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_val[:5]","metadata":{"id":"YsWpOjI41vWQ","execution":{"iopub.status.busy":"2022-07-28T07:57:11.473609Z","iopub.execute_input":"2022-07-28T07:57:11.473932Z","iopub.status.idle":"2022-07-28T07:57:11.481156Z","shell.execute_reply.started":"2022-07-28T07:57:11.473893Z","shell.execute_reply":"2022-07-28T07:57:11.480185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Once we have this probabilities we have to convert it to discrete values, either 1 or 0, such task can be achieved by using the np.round function:","metadata":{"id":"2qVb-tJl2PqB"}},{"cell_type":"code","source":"class_pred_val= np.round(predicted_val)\nclass_pred_val[:5]","metadata":{"id":"sZuLmXo22T7w","execution":{"iopub.status.busy":"2022-07-28T07:57:11.482711Z","iopub.execute_input":"2022-07-28T07:57:11.483415Z","iopub.status.idle":"2022-07-28T07:57:11.497821Z","shell.execute_reply.started":"2022-07-28T07:57:11.483270Z","shell.execute_reply":"2022-07-28T07:57:11.496565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_labels[:5]","metadata":{"id":"bjcJZhy71vWR","execution":{"iopub.status.busy":"2022-07-28T07:57:11.499803Z","iopub.execute_input":"2022-07-28T07:57:11.500271Z","iopub.status.idle":"2022-07-28T07:57:11.513768Z","shell.execute_reply.started":"2022-07-28T07:57:11.500228Z","shell.execute_reply":"2022-07-28T07:57:11.512572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Classification report and confusion matrix:","metadata":{"id":"cO4qyzvZXn4P"}},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nprint(classification_report(class_pred_val,validation_labels))","metadata":{"id":"0nX4ZdQzH7fK","execution":{"iopub.status.busy":"2022-07-28T07:57:11.515830Z","iopub.execute_input":"2022-07-28T07:57:11.516412Z","iopub.status.idle":"2022-07-28T07:57:11.534665Z","shell.execute_reply.started":"2022-07-28T07:57:11.516370Z","shell.execute_reply":"2022-07-28T07:57:11.533428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=confusion_matrix(class_pred_val,validation_labels), display_labels=np.unique(train_labels))\ndisp.plot(cmap='Blues') \nplt.grid(False)","metadata":{"id":"uff1kA5xLICI","execution":{"iopub.status.busy":"2022-07-28T07:57:11.536552Z","iopub.execute_input":"2022-07-28T07:57:11.536894Z","iopub.status.idle":"2022-07-28T07:57:11.829638Z","shell.execute_reply.started":"2022-07-28T07:57:11.536852Z","shell.execute_reply":"2022-07-28T07:57:11.828640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pre-trained model:\n\nWe will download the Glove pre-trained model for text classification with 100 embedding dimensions, after predicting the classes for the validation set we will compare with those obtained from the previously built scratch model.\n\nAs a first step we have to tokenize again the sentences without limiting the maximum length of them as follows:","metadata":{"id":"8WK3_GsW4u1N"}},{"cell_type":"code","source":"tokenizer_glove = Tokenizer()\ntokenizer_glove.fit_on_texts(df['text'])\nword_index = tokenizer_glove.word_index\nlen(word_index)","metadata":{"id":"_AN2tUurMibd","outputId":"0ec7a88c-b1a5-4e69-f59a-bf914d8f167b","execution":{"iopub.status.busy":"2022-07-28T07:57:11.831321Z","iopub.execute_input":"2022-07-28T07:57:11.831705Z","iopub.status.idle":"2022-07-28T07:57:12.021541Z","shell.execute_reply.started":"2022-07-28T07:57:11.831639Z","shell.execute_reply":"2022-07-28T07:57:12.020586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_glove_sequences=tokenizer_glove.texts_to_sequences(train_sentences)\ntrain_glove_padded = pad_sequences(train_glove_sequences,maxlen=max_length,\n                                   padding=padding_type,truncating=trunc_type)\n\ntrain_glove_padded.shape","metadata":{"id":"gRj5HjGlicuY","outputId":"a219fa19-7de5-46ad-c26f-239169f4178d","execution":{"iopub.status.busy":"2022-07-28T07:57:12.023050Z","iopub.execute_input":"2022-07-28T07:57:12.023438Z","iopub.status.idle":"2022-07-28T07:57:12.162647Z","shell.execute_reply.started":"2022-07-28T07:57:12.023382Z","shell.execute_reply":"2022-07-28T07:57:12.161568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_glove_sequences=tokenizer_glove.texts_to_sequences(validation_sentences)\nval_glove_padded = pad_sequences(val_glove_sequences,maxlen=max_length,\n                                 padding=padding_type,truncating=trunc_type)\n\nval_glove_padded.shape","metadata":{"id":"Tw28rLiwj7i7","outputId":"0833c7ec-461b-448c-dc59-c31578e8ab93","execution":{"iopub.status.busy":"2022-07-28T07:57:12.164517Z","iopub.execute_input":"2022-07-28T07:57:12.164915Z","iopub.status.idle":"2022-07-28T07:57:12.210665Z","shell.execute_reply.started":"2022-07-28T07:57:12.164871Z","shell.execute_reply":"2022-07-28T07:57:12.209756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_glove_sequences=tokenizer_glove.texts_to_sequences(df_test['text_without_stopwords'])\ntest_glove_padded = pad_sequences(test_glove_sequences,maxlen=max_length,\n                                  padding=padding_type,truncating=trunc_type)\n\ntest_glove_padded.shape","metadata":{"id":"vkM9nPS7LaDb","outputId":"7822458b-fa24-4fd9-c11d-f46974878c6f","execution":{"iopub.status.busy":"2022-07-28T07:57:12.212107Z","iopub.execute_input":"2022-07-28T07:57:12.212474Z","iopub.status.idle":"2022-07-28T07:57:12.292952Z","shell.execute_reply.started":"2022-07-28T07:57:12.212431Z","shell.execute_reply":"2022-07-28T07:57:12.291970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_size_glove=len(word_index)","metadata":{"id":"p2iTxjit1Sur","execution":{"iopub.status.busy":"2022-07-28T07:57:12.294772Z","iopub.execute_input":"2022-07-28T07:57:12.295131Z","iopub.status.idle":"2022-07-28T07:57:12.300114Z","shell.execute_reply.started":"2022-07-28T07:57:12.295081Z","shell.execute_reply":"2022-07-28T07:57:12.298899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, as we said we will download the weights of Glove 100 dimensional version  from Stanford:","metadata":{"id":"G8PKfxJo6ZUP"}},{"cell_type":"code","source":"embeddings_index = {};\nwith open('../input/glove6b100dtxt/glove.6B.100d.txt') as f:\n    for line in f:\n        values = line.split();\n        word = values[0];\n        coefs = np.asarray(values[1:], dtype='float32');\n        embeddings_index[word] = coefs;","metadata":{"id":"3dtpURxUynaY","execution":{"iopub.status.busy":"2022-07-28T07:57:12.301980Z","iopub.execute_input":"2022-07-28T07:57:12.302593Z","iopub.status.idle":"2022-07-28T07:57:31.112210Z","shell.execute_reply.started":"2022-07-28T07:57:12.302548Z","shell.execute_reply":"2022-07-28T07:57:31.111268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following lines creates a matrix containing the weights and its dimension is the new vocabulary size by 100 embedding dimension, this will be loaded as the weights of the first embedding layer by setting the argument trainable=False.","metadata":{"id":"Qrb2pvKO62eR"}},{"cell_type":"code","source":"embeddings_matrix = np.zeros((vocab_size_glove+1, 100));\nfor word, i in word_index.items():\n    embedding_vector = embeddings_index.get(word);\n    if embedding_vector is not None:\n        embeddings_matrix[i] = embedding_vector;","metadata":{"id":"8RSyOdFXzf2H","execution":{"iopub.status.busy":"2022-07-28T07:57:31.113918Z","iopub.execute_input":"2022-07-28T07:57:31.114211Z","iopub.status.idle":"2022-07-28T07:57:31.154633Z","shell.execute_reply.started":"2022-07-28T07:57:31.114170Z","shell.execute_reply":"2022-07-28T07:57:31.153769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(embeddings_matrix))","metadata":{"id":"Y7zkWaOZyL4f","outputId":"13b6d166-7eb5-4fdf-e9c2-232deb70fdc4","execution":{"iopub.status.busy":"2022-07-28T07:57:31.156278Z","iopub.execute_input":"2022-07-28T07:57:31.156647Z","iopub.status.idle":"2022-07-28T07:57:31.162657Z","shell.execute_reply.started":"2022-07-28T07:57:31.156606Z","shell.execute_reply":"2022-07-28T07:57:31.161622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Time now to build the model using the Glove-weights, take into account that we will consider the same layers included in the previous model:","metadata":{"id":"OWkJ2nOj7fqQ"}},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n\nmodel_glove2=Sequential()\nmodel_glove2.add(Embedding(vocab_size_glove+1,100,weights=[embeddings_matrix],input_length=max_length,trainable=False))\nmodel_glove2.add(SpatialDropout1D(0.2))\nmodel_glove2.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\nmodel_glove2.add(Dense(1, activation='sigmoid'))\n\noptimizer=Adam(learning_rate=5e-3)\nmodel_glove2.compile(loss='binary_crossentropy',optimizer=optimizer,metrics=['accuracy'])\nmodel_glove2.summary()","metadata":{"id":"AJmM3Px8e81A","outputId":"0db0b726-f0ae-4443-e1dd-0403e937b057","execution":{"iopub.status.busy":"2022-07-28T07:57:31.164500Z","iopub.execute_input":"2022-07-28T07:57:31.165182Z","iopub.status.idle":"2022-07-28T07:57:31.373394Z","shell.execute_reply.started":"2022-07-28T07:57:31.165141Z","shell.execute_reply":"2022-07-28T07:57:31.372247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Above we can see this new model contains around 1.7 million parameters to compute, this would take a long time, but as we have loaded the weights only 80.501 are trainable. In contrast to the scratch model which contains over 576 thousand parameters and obviously have to be computed all of them.\n\nObservation: In this model the callback which reduces the learning rate was not useful as the accuracies didn't improve whenever the lr was reduced, this is why I'm not using them here, also the change in batch size impacted the performance:","metadata":{"id":"kXPxujHDM_mB"}},{"cell_type":"code","source":"num_epochs = 15\nhistory_glove2 = model_glove2.fit(train_glove_padded,train_labels,epochs=num_epochs,\n                    validation_data=(val_glove_padded,validation_labels),verbose=1,\n                    batch_size=32)","metadata":{"id":"IB94aNe4fcQC","outputId":"e0e06469-6c01-466a-c0fa-60eb9cefe16b","execution":{"iopub.status.busy":"2022-07-28T07:57:31.375181Z","iopub.execute_input":"2022-07-28T07:57:31.375480Z","iopub.status.idle":"2022-07-28T08:03:56.058546Z","shell.execute_reply.started":"2022-07-28T07:57:31.375437Z","shell.execute_reply":"2022-07-28T08:03:56.057481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model took 4 minutes and 33 seconds to train reaching accuracies of: 97.0% train/89.9% val.","metadata":{"id":"wZ7GIf5fCMyd"}},{"cell_type":"code","source":"plot_metrics(history_glove2)","metadata":{"id":"VvheY1dJU3nN","outputId":"33adb76c-abbf-49fa-f8a3-923dfe4e6b4f","execution":{"iopub.status.busy":"2022-07-28T08:03:56.060478Z","iopub.execute_input":"2022-07-28T08:03:56.060915Z","iopub.status.idle":"2022-07-28T08:03:56.757234Z","shell.execute_reply.started":"2022-07-28T08:03:56.060827Z","shell.execute_reply":"2022-07-28T08:03:56.756256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_val = model_glove2.predict(val_glove_padded, batch_size=32)","metadata":{"id":"gAiT2YqlqIIj","execution":{"iopub.status.busy":"2022-07-28T08:03:56.758860Z","iopub.execute_input":"2022-07-28T08:03:56.760009Z","iopub.status.idle":"2022-07-28T08:03:57.357813Z","shell.execute_reply.started":"2022-07-28T08:03:56.759966Z","shell.execute_reply":"2022-07-28T08:03:57.356780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_val[:5]","metadata":{"id":"l6AP3UDsqIIl","execution":{"iopub.status.busy":"2022-07-28T08:03:57.359323Z","iopub.execute_input":"2022-07-28T08:03:57.361429Z","iopub.status.idle":"2022-07-28T08:03:57.369410Z","shell.execute_reply.started":"2022-07-28T08:03:57.361382Z","shell.execute_reply":"2022-07-28T08:03:57.368181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Once we have this probabilities we have to convert it to discrete values, either 1 or 0, such task can be achieved by using the np.round function:","metadata":{"id":"Fq-G_kKQqIIm"}},{"cell_type":"code","source":"class_pred_val_glove= np.round(predicted_val)\nclass_pred_val_glove[:5]","metadata":{"id":"X0TjvU-oqIIn","execution":{"iopub.status.busy":"2022-07-28T08:03:57.371533Z","iopub.execute_input":"2022-07-28T08:03:57.371986Z","iopub.status.idle":"2022-07-28T08:03:57.385242Z","shell.execute_reply.started":"2022-07-28T08:03:57.371943Z","shell.execute_reply":"2022-07-28T08:03:57.384248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_labels[:5]","metadata":{"id":"Kd92UcfTqIIp","execution":{"iopub.status.busy":"2022-07-28T08:03:57.386825Z","iopub.execute_input":"2022-07-28T08:03:57.387162Z","iopub.status.idle":"2022-07-28T08:03:57.399890Z","shell.execute_reply.started":"2022-07-28T08:03:57.387120Z","shell.execute_reply":"2022-07-28T08:03:57.398662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Classification report and confusion matrix:","metadata":{"id":"x3wyC6MlqIIq"}},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\nprint(classification_report(class_pred_val_glove,validation_labels))","metadata":{"id":"V1tS_sVfqIIr","execution":{"iopub.status.busy":"2022-07-28T08:03:57.401492Z","iopub.execute_input":"2022-07-28T08:03:57.402255Z","iopub.status.idle":"2022-07-28T08:03:57.425580Z","shell.execute_reply.started":"2022-07-28T08:03:57.402180Z","shell.execute_reply":"2022-07-28T08:03:57.424439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n\ndisp = ConfusionMatrixDisplay(confusion_matrix=confusion_matrix(class_pred_val_glove,validation_labels), display_labels=np.unique(train_labels))\ndisp.plot(cmap='Blues') \nplt.grid(False)","metadata":{"id":"LxwJ6_HMqIIs","execution":{"iopub.status.busy":"2022-07-28T08:03:57.427126Z","iopub.execute_input":"2022-07-28T08:03:57.427530Z","iopub.status.idle":"2022-07-28T08:03:57.735845Z","shell.execute_reply.started":"2022-07-28T08:03:57.427488Z","shell.execute_reply":"2022-07-28T08:03:57.734785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predicting classes for test instances:","metadata":{"id":"qei-I5bRqlLW"}},{"cell_type":"code","source":"test_prediction = model_glove2.predict(test_glove_padded)\ntest_prediction = test_prediction.round().astype('int')","metadata":{"id":"gIYXZuFyfoe5","execution":{"iopub.status.busy":"2022-07-28T08:03:57.737669Z","iopub.execute_input":"2022-07-28T08:03:57.738056Z","iopub.status.idle":"2022-07-28T08:03:58.455779Z","shell.execute_reply.started":"2022-07-28T08:03:57.738012Z","shell.execute_reply":"2022-07-28T08:03:58.454769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I would like to know any feedback in order to increase the performance of the models or tell me if you found a different one even better!\n\nIf you liked this notebook I would appreciate so much your upvote if you want to see more projects/tutorials like this one. I encourage you to see my projects portfolio, am sure you will love it.\n\nThank you!","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}