{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n\nimport warnings\nwarnings.filterwarnings(action=\"ignore\")\n\nprint(\"Packages Imported!\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-12T17:12:25.642186Z","iopub.execute_input":"2022-07-12T17:12:25.642588Z","iopub.status.idle":"2022-07-12T17:12:26.446362Z","shell.execute_reply.started":"2022-07-12T17:12:25.642558Z","shell.execute_reply":"2022-07-12T17:12:26.444759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Overview of Dataset","metadata":{}},{"cell_type":"code","source":"disaster_tweets = pd.read_csv(\"../input/nlp-getting-started/train.csv\")\ndisaster_tweets.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T17:12:28.208486Z","iopub.execute_input":"2022-07-12T17:12:28.208869Z","iopub.status.idle":"2022-07-12T17:12:28.282536Z","shell.execute_reply.started":"2022-07-12T17:12:28.208839Z","shell.execute_reply":"2022-07-12T17:12:28.281525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:39.998921Z","iopub.execute_input":"2022-07-12T15:50:39.999424Z","iopub.status.idle":"2022-07-12T15:50:40.025723Z","shell.execute_reply.started":"2022-07-12T15:50:39.999383Z","shell.execute_reply":"2022-07-12T15:50:40.024596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets[\"tweets_words_count\"] = disaster_tweets.text.map(lambda x: len(x.split()))\ndisaster_tweets[\"tweets_len\"] = disaster_tweets.text.str.len()\ndisaster_tweets.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:40.577174Z","iopub.execute_input":"2022-07-12T15:50:40.577905Z","iopub.status.idle":"2022-07-12T15:50:40.612473Z","shell.execute_reply.started":"2022-07-12T15:50:40.577867Z","shell.execute_reply":"2022-07-12T15:50:40.611635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(disaster_tweets.text[0])\nprint(len(disaster_tweets.text[0]))\nprint(len(disaster_tweets.text[0].split()))","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:40.896720Z","iopub.execute_input":"2022-07-12T15:50:40.897388Z","iopub.status.idle":"2022-07-12T15:50:40.903353Z","shell.execute_reply.started":"2022-07-12T15:50:40.897322Z","shell.execute_reply":"2022-07-12T15:50:40.902396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets.columns","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:41.307936Z","iopub.execute_input":"2022-07-12T15:50:41.308871Z","iopub.status.idle":"2022-07-12T15:50:41.320445Z","shell.execute_reply.started":"2022-07-12T15:50:41.308824Z","shell.execute_reply":"2022-07-12T15:50:41.319252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion:\n- Dataset is having (7613, 5) shape including the features and target label.\n- 'keyword', 'location', 'text' are object dtype\n- 'target', 'tweets_words_count', 'tweets_len' are int dtypes\n---","metadata":{}},{"cell_type":"markdown","source":"## Summary Statistics & Missing values for Column ","metadata":{}},{"cell_type":"code","source":"# summary statistics for numerical features\ndisaster_tweets.describe().T","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:50.832277Z","iopub.execute_input":"2022-07-12T15:50:50.832685Z","iopub.status.idle":"2022-07-12T15:50:50.874186Z","shell.execute_reply.started":"2022-07-12T15:50:50.832655Z","shell.execute_reply":"2022-07-12T15:50:50.872603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summary statistics  non-numerical features\ndisaster_tweets.describe(include=\"O\").T","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:51.833580Z","iopub.execute_input":"2022-07-12T15:50:51.834035Z","iopub.status.idle":"2022-07-12T15:50:51.873964Z","shell.execute_reply.started":"2022-07-12T15:50:51.833996Z","shell.execute_reply":"2022-07-12T15:50:51.872463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets[disaster_tweets.text.str.contains(\"Boy Charged\")][[\"text\", \"target\", \"location\"]]","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:52.332880Z","iopub.execute_input":"2022-07-12T15:50:52.333556Z","iopub.status.idle":"2022-07-12T15:50:52.359176Z","shell.execute_reply.started":"2022-07-12T15:50:52.333504Z","shell.execute_reply":"2022-07-12T15:50:52.357459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(disaster_tweets.loc[3614, \"text\"].split())","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:53.129149Z","iopub.execute_input":"2022-07-12T15:50:53.129552Z","iopub.status.idle":"2022-07-12T15:50:53.136963Z","shell.execute_reply.started":"2022-07-12T15:50:53.129519Z","shell.execute_reply":"2022-07-12T15:50:53.135411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(disaster_tweets.loc[3578, \"text\"].split())","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:54.023481Z","iopub.execute_input":"2022-07-12T15:50:54.025056Z","iopub.status.idle":"2022-07-12T15:50:54.032301Z","shell.execute_reply.started":"2022-07-12T15:50:54.024983Z","shell.execute_reply":"2022-07-12T15:50:54.031423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:54.226661Z","iopub.execute_input":"2022-07-12T15:50:54.227236Z","iopub.status.idle":"2022-07-12T15:50:54.239158Z","shell.execute_reply.started":"2022-07-12T15:50:54.227203Z","shell.execute_reply":"2022-07-12T15:50:54.237894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion:\n- 75% of tweets have words less than or equal to 19. While having upto 133 text length.\n- 3341 unique location in the dataset\n- USA has most occurence with 104 frequency.\n- location and keyword have ~30% and ~2.5% missing values.\n- some tweets are duplicates too. We can drop those rows which has common \"text\" & \"location\".\n----","metadata":{}},{"cell_type":"markdown","source":"## Plotting Value Distributions\n","metadata":{}},{"cell_type":"code","source":"# disaster_tweets.tweets_len.plot(kind=\"hist\", bins=30)\n# disaster_tweets.tweets_len.plot(kind=\"box\")\n\nplt.figure(figsize=(12,4))\nplt.subplot(1, 2, 1)\nsns.boxplot(data=disaster_tweets, x='tweets_len')\nplt.tight_layout()\n\nplt.subplot(1, 2, 2)\nsns.histplot(data=disaster_tweets, x='tweets_len')\nplt.tight_layout()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T15:50:55.156691Z","iopub.execute_input":"2022-07-12T15:50:55.157058Z","iopub.status.idle":"2022-07-12T15:50:55.587938Z","shell.execute_reply.started":"2022-07-12T15:50:55.157029Z","shell.execute_reply":"2022-07-12T15:50:55.586639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion:\n- 50% percent of the texts (the box in the middle) have a length between roughly 100 and 120 characters.\n- It seems to be right skewed.\n---","metadata":{}},{"cell_type":"markdown","source":"## Target distribution:","metadata":{}},{"cell_type":"code","source":"disaster_tweets.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:24:19.857515Z","iopub.execute_input":"2022-07-12T14:24:19.858027Z","iopub.status.idle":"2022-07-12T14:24:19.868619Z","shell.execute_reply.started":"2022-07-12T14:24:19.857981Z","shell.execute_reply":"2022-07-12T14:24:19.867408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 5))\nplt.subplot(1, 2, 1)\ndisaster_tweets.target.value_counts().plot(kind=\"pie\",\n                                           labels=[\"Disaster(43%)\", \"Not a Disaster(57%)\"],\n                                           colors=['lightcoral','lightskyblue'],\n                                           fontsize=14,\n                                           ylabel=\"\");\n\nplt.subplot(1, 2, 2)\nsns.countplot(x=\"target\",data=disaster_tweets, palette=\"RdBu\")\nplt.xticks(ticks=[0, 1], fontsize=14, labels=['Not Disaster (4342)', 'Disaster (3271)']);","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:24:19.870363Z","iopub.execute_input":"2022-07-12T14:24:19.871771Z","iopub.status.idle":"2022-07-12T14:24:20.162960Z","shell.execute_reply.started":"2022-07-12T14:24:19.871722Z","shell.execute_reply":"2022-07-12T14:24:20.161344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution Plot for location","metadata":{}},{"cell_type":"code","source":"print(disaster_tweets.location.value_counts().sort_values(ascending=False)[:10])","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:24:20.164973Z","iopub.execute_input":"2022-07-12T14:24:20.166181Z","iopub.status.idle":"2022-07-12T14:24:20.185851Z","shell.execute_reply.started":"2022-07-12T14:24:20.166129Z","shell.execute_reply":"2022-07-12T14:24:20.184472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 5))\ndisaster_tweets.location.value_counts().sort_values(ascending=False)[:30].plot(kind=\"bar\",\n                                                                               color='lightcoral',\n                                                                               linewidth=2,\n                                                                               fontsize=14);\nplt.title('Location Count', fontsize = 20)\nplt.axhline(y=20, color=\"green\")\nplt.axhline(y=10, color=\"blue\")\nplt.xlabel('Regions', fontsize = 12)\nplt.ylabel('Count', fontsize = 12)\nplt.xticks(fontsize=12, rotation=70);","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:24:20.187030Z","iopub.execute_input":"2022-07-12T14:24:20.187780Z","iopub.status.idle":"2022-07-12T14:24:20.628822Z","shell.execute_reply.started":"2022-07-12T14:24:20.187733Z","shell.execute_reply":"2022-07-12T14:24:20.627258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets.groupby(\"location\")[\"tweets_len\"].agg([\"mean\"]).sort_values(by=\"mean\", ascending=False).head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:24:20.630367Z","iopub.execute_input":"2022-07-12T14:24:20.630709Z","iopub.status.idle":"2022-07-12T14:24:20.654383Z","shell.execute_reply.started":"2022-07-12T14:24:20.630656Z","shell.execute_reply":"2022-07-12T14:24:20.653504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets.groupby(\"location\")[\"tweets_words_count\"].agg([\"mean\"]).sort_values(by=\"mean\", ascending=False)[:5]","metadata":{"execution":{"iopub.status.busy":"2022-07-12T14:24:20.656108Z","iopub.execute_input":"2022-07-12T14:24:20.656844Z","iopub.status.idle":"2022-07-12T14:24:20.679784Z","shell.execute_reply.started":"2022-07-12T14:24:20.656772Z","shell.execute_reply":"2022-07-12T14:24:20.678084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- USA, New York, United States, London, Canada are locations from where most of the tweets are done.\n- london, ARBAILO, West Coast, USA has most lengthiest tweets.\n- Melbourne Australia, Miami, Elmwood Park NJ has most words in a tweet.","metadata":{"execution":{"iopub.status.busy":"2022-07-10T11:39:00.839283Z","iopub.execute_input":"2022-07-10T11:39:00.839608Z","iopub.status.idle":"2022-07-10T11:39:00.846931Z","shell.execute_reply.started":"2022-07-10T11:39:00.839584Z","shell.execute_reply":"2022-07-10T11:39:00.845605Z"}}},{"cell_type":"markdown","source":"# Text Preprocessing","metadata":{}},{"cell_type":"code","source":"from tqdm.notebook import tqdm_notebook\ntqdm_notebook.pandas()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T17:25:14.026898Z","iopub.execute_input":"2022-07-12T17:25:14.027333Z","iopub.status.idle":"2022-07-12T17:25:14.033931Z","shell.execute_reply.started":"2022-07-12T17:25:14.027295Z","shell.execute_reply":"2022-07-12T17:25:14.032433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Word Clouds","metadata":{}},{"cell_type":"code","source":"from nltk.corpus import stopwords\nfrom wordcloud import STOPWORDS, WordCloud\nfrom termcolor import colored\n\nstop_words= set(stopwords.words(\"english\"))\n\n# updating the stopwords considering the \nstop_words.update(['https', 'http', 'amp', 'CO', 't', 'u', 'new', \"I'm\", \"would\"])\n\nwc = WordCloud(width=800,\n               height=400,\n               max_words=200,\n               stopwords=stop_words,\n               background_color= \"black\", \n               colormap=\"Paired\",\n               max_font_size=150)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T17:29:47.648943Z","iopub.execute_input":"2022-07-12T17:29:47.649425Z","iopub.status.idle":"2022-07-12T17:29:47.658919Z","shell.execute_reply.started":"2022-07-12T17:29:47.649361Z","shell.execute_reply":"2022-07-12T17:29:47.657426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_tweets_text = disaster_tweets.query(\"target==1\").text\nconcat_disaster_tweets_text = disaster_tweets_text.str.cat(sep=\" \")  # str.cat -- string concatenation\n\nnon_disaster_tweets_text = disaster_tweets.query(\"target==0\").text\nconcat_non_disaster_tweets_text = non_disaster_tweets_text.str.cat(sep=\" \")\n\nprint('\\033[1m'\"\\nWord Cloud for Disaster Tweets\"'\\033[0m')\nwc.generate(concat_disaster_tweets_text)\nplt.figure(figsize=(12, 5))\nplt.imshow(wc, interpolation='bilinear')\nplt.axis('off')\nplt.show()\n\nprint('\\033[1m'\"\\nWord Cloud for Non-Disaster Tweets\"'\\033[0m')\nwc.generate(concat_non_disaster_tweets_text)\nplt.figure(figsize=(12, 5))\nplt.imshow(wc, interpolation='bilinear')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T17:29:48.540331Z","iopub.execute_input":"2022-07-12T17:29:48.540752Z","iopub.status.idle":"2022-07-12T17:29:51.192262Z","shell.execute_reply.started":"2022-07-12T17:29:48.540719Z","shell.execute_reply":"2022-07-12T17:29:51.191398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}