{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport re\nimport string\nimport operator\nfrom collections import defaultdict ,Counter\n\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nfrom wordcloud import STOPWORDS\nfrom nltk.util import ngrams\nfrom sklearn.model_selection import StratifiedKFold, StratifiedShuffleSplit\nfrom sklearn.metrics import precision_score, recall_score, f1_score\n\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom tensorflow import keras\nfrom tensorflow.keras.optimizers import SGD, Adam\nfrom tensorflow.keras.layers import Dense, Input, Dropout, GlobalAveragePooling1D\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, Callback\n\nSEED = 1337","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:12:52.591761Z","iopub.execute_input":"2022-07-16T16:12:52.592241Z","iopub.status.idle":"2022-07-16T16:12:52.604380Z","shell.execute_reply.started":"2022-07-16T16:12:52.592165Z","shell.execute_reply":"2022-07-16T16:12:52.603352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.set_theme(style=\"whitegrid\")\n!pip install tensorflow== 2.7.1\n!pip install tensorflow_hub\n!pip install bert-for-tf2\n!pip install sentencepiece\n\nimport tensorflow_hub as hub\nimport tensorflow as tf\n# from bert import tokenization\nfrom tensorflow.keras.models import Model \n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:12:52.659665Z","iopub.execute_input":"2022-07-16T16:12:52.659972Z","iopub.status.idle":"2022-07-16T16:13:13.775404Z","shell.execute_reply.started":"2022-07-16T16:12:52.659917Z","shell.execute_reply":"2022-07-16T16:13:13.774050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.insert(1, '../input/disaster-nlp-keras-bert-using-tfhub')\n\nimport tokenization\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:13.780201Z","iopub.execute_input":"2022-07-16T16:13:13.780635Z","iopub.status.idle":"2022-07-16T16:13:13.793276Z","shell.execute_reply.started":"2022-07-16T16:13:13.780548Z","shell.execute_reply":"2022-07-16T16:13:13.791866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/nlp-getting-started/train.csv')\ndf_test = pd.read_csv('../input/nlp-getting-started/test.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:13.796054Z","iopub.execute_input":"2022-07-16T16:13:13.796846Z","iopub.status.idle":"2022-07-16T16:13:13.848042Z","shell.execute_reply.started":"2022-07-16T16:13:13.796784Z","shell.execute_reply":"2022-07-16T16:13:13.847042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('df_train shape:{}'.format(df_train.shape))\nprint('df_train memory usage:{:.2} MB'.format(df_train.memory_usage().sum()/ 1024**2))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:13.849737Z","iopub.execute_input":"2022-07-16T16:13:13.850463Z","iopub.status.idle":"2022-07-16T16:13:13.863265Z","shell.execute_reply.started":"2022-07-16T16:13:13.850383Z","shell.execute_reply":"2022-07-16T16:13:13.862035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('df_train shape:{}'.format(df_test.shape))\nprint('df_train memory usage:{:.2} MB'.format(df_test.memory_usage().sum()/ 1024**2))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:13.868515Z","iopub.execute_input":"2022-07-16T16:13:13.869238Z","iopub.status.idle":"2022-07-16T16:13:13.881760Z","shell.execute_reply.started":"2022-07-16T16:13:13.869147Z","shell.execute_reply":"2022-07-16T16:13:13.879468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.isna().sum().values\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:13.887337Z","iopub.execute_input":"2022-07-16T16:13:13.888208Z","iopub.status.idle":"2022-07-16T16:13:13.900064Z","shell.execute_reply.started":"2022-07-16T16:13:13.888145Z","shell.execute_reply":"2022-07-16T16:13:13.898440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig , axes=plt.subplots(ncols=2 , figsize=(15,5))\nsns.barplot( x=df_test.isna().sum().index , y=df_test.isna().sum().values , ax=axes[0])\nsns.barplot( x=df_train.isna().sum().index , y=df_train.isna().sum().values,ax=axes[1])\naxes[0].set_title('Training Set')\naxes[1].set_title('Test Set')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:13.902364Z","iopub.execute_input":"2022-07-16T16:13:13.903040Z","iopub.status.idle":"2022-07-16T16:13:14.566868Z","shell.execute_reply.started":"2022-07-16T16:13:13.902980Z","shell.execute_reply":"2022-07-16T16:13:14.565609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in ['keyword', 'location']:\n    df_train[col].fillna(f'no_{col}', inplace=True)\n    df_test[col].fillna(f'no_{col}', inplace=True)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:14.569141Z","iopub.execute_input":"2022-07-16T16:13:14.576277Z","iopub.status.idle":"2022-07-16T16:13:14.594114Z","shell.execute_reply.started":"2022-07-16T16:13:14.576203Z","shell.execute_reply":"2022-07-16T16:13:14.592838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[df_train.keyword!='no_keyword']","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:14.600210Z","iopub.execute_input":"2022-07-16T16:13:14.603321Z","iopub.status.idle":"2022-07-16T16:13:14.634158Z","shell.execute_reply.started":"2022-07-16T16:13:14.603252Z","shell.execute_reply":"2022-07-16T16:13:14.633320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.array_equal(df_train.keyword.unique(), df_test.keyword.unique()))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:14.636551Z","iopub.execute_input":"2022-07-16T16:13:14.637172Z","iopub.status.idle":"2022-07-16T16:13:14.646625Z","shell.execute_reply.started":"2022-07-16T16:13:14.637100Z","shell.execute_reply":"2022-07-16T16:13:14.645521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train.groupby('keyword').size()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:14.647939Z","iopub.execute_input":"2022-07-16T16:13:14.648210Z","iopub.status.idle":"2022-07-16T16:13:14.665272Z","shell.execute_reply.started":"2022-07-16T16:13:14.648152Z","shell.execute_reply":"2022-07-16T16:13:14.664329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['target_mean'] = df_train.groupby('keyword')['target'].transform('mean')","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:14.667138Z","iopub.execute_input":"2022-07-16T16:13:14.667919Z","iopub.status.idle":"2022-07-16T16:13:14.683658Z","shell.execute_reply.started":"2022-07-16T16:13:14.667554Z","shell.execute_reply":"2022-07-16T16:13:14.682421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig=plt.figure(figsize=(8,75) ,dpi=100)\nsns.countplot(y=df_train.sort_values(by='target_mean' , ascending=False).keyword , hue=df_train.target )","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:14.685380Z","iopub.execute_input":"2022-07-16T16:13:14.686021Z","iopub.status.idle":"2022-07-16T16:13:21.002250Z","shell.execute_reply.started":"2022-07-16T16:13:14.685946Z","shell.execute_reply":"2022-07-16T16:13:21.001107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Meta Features\n\n* `word_count` number of words in text\n* `unique_word_count` number of unique words in text\n* `stop_word_count` number of stop words in text\n* `url_count` number of urls in text\n* `mean_word_length` average character count in words\n* `char_count` number of characters in text\n* `punctuation_count` number of punctuations in text\n* `hashtag_count` number of hashtags (**#**) in text\n* `mention_count` number of mentions (**@**) in text","metadata":{}},{"cell_type":"code","source":"df_train['word_count']=df_train['text'].apply(lambda x:len(x.split(' ')))\ndf_test['word_count']=df_test['text'].apply(lambda x:len(x.split(' ')))\n\n####################################\n\ndf_train['unique_word_count']=df_train['text'].apply(lambda x:len(set(str(x).split(' '))))\ndf_test['unique_word_count']=df_test['text'].apply(lambda x:len(set(str(x).split(' '))))\n\n\n#######################################\n\ndf_train['word_count']=df_train['text'].apply(lambda x:len(x.split(' ')))\ndf_test['word_count']=df_test['text'].apply(lambda x:len(x.split(' ')))\n\n####################################\n\ndf_train['unique_word_count']=df_train['text'].apply(lambda x:len(set(str(x).split(' '))))\ndf_test['unique_word_count']=df_test['text'].apply(lambda x:len(set(str(x).split(' '))))\n\n\n#######################################\n\n\ndf_train['stop_word_count']=df_train['text'].apply(lambda x:len(set(x.lower().split())&STOPWORDS))\ndf_test['stop_word_count']=df_test['text'].apply(lambda x:len(set(x.lower().split())&STOPWORDS))\n\n\n###########################################\n\ndf_train['url_count']=df_train['text'].apply(lambda x : len(re.findall(r'(https?://[^\\s]+)', x)))\ndf_test['url_count']=df_test['text'].apply(lambda x : len(re.findall(r'(https?://[^\\s]+)', x)))\n\n###########################################\n\ndf_train['mean_word_length']=df_train['text'].apply(lambda x : np.mean([len(w) for w in x.split()]))\ndf_test['mean_word_length']=df_test['text'].apply(lambda x : np.mean([len(w) for w in x.split()]))\n\n############################################\n\ndf_train['char_count'] = df_train['text'].apply(lambda x: len(str(x)))\ndf_test['char_count'] = df_test['text'].apply(lambda x: len(str(x)))\n############################################\n\ndf_train['punctuation_count'] = df_train['text'].apply(lambda x: len([char for char in x if char in string.punctuation]))\ndf_test['punctuation_count'] = df_test['text'].apply(lambda x: len([char for char in x if char in string.punctuation]))\n\n############################################\n\ndf_train['hashtag_count'] = df_train['text'].apply(lambda x: Counter(x)['#'])\ndf_test['hashtag_count'] = df_test['text'].apply(lambda x: Counter(x)['#'])\n\n############################################\n\ndf_train['mention_count'] = df_train['text'].apply(lambda x: Counter(x)['@'])\ndf_test['mention_count'] = df_test['text'].apply(lambda x: Counter(x)['@'])\n\n\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-16T16:13:21.003978Z","iopub.execute_input":"2022-07-16T16:13:21.004546Z","iopub.status.idle":"2022-07-16T16:13:21.891724Z","shell.execute_reply.started":"2022-07-16T16:13:21.004492Z","shell.execute_reply":"2022-07-16T16:13:21.890805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"METAFEATURES = ['word_count', 'unique_word_count', 'stop_word_count', 'url_count', 'mean_word_length',\n                'char_count', 'punctuation_count', 'hashtag_count', 'mention_count']","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:21.893762Z","iopub.execute_input":"2022-07-16T16:13:21.894689Z","iopub.status.idle":"2022-07-16T16:13:21.901260Z","shell.execute_reply.started":"2022-07-16T16:13:21.894397Z","shell.execute_reply":"2022-07-16T16:13:21.900187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sns.displot(df_train, x=\"word_count\", hue=\"target\" , kind='kde')","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:13:21.902846Z","iopub.execute_input":"2022-07-16T16:13:21.903454Z","iopub.status.idle":"2022-07-16T16:13:21.912045Z","shell.execute_reply.started":"2022-07-16T16:13:21.903379Z","shell.execute_reply":"2022-07-16T16:13:21.910958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fig, axes = plt.subplots(ncols=2, nrows=len(METAFEATURES),figsize=(20, 50), dpi=100)\n\n# for  i,feature in enumerate(METAFEATURES):\n#     sns.kdeplot(data=df_train,x=feature, hue='target',  ax=axes[i][0])\n    \n#     sns.kdeplot(data=df_train,x=feature,  ax=axes[i][1])\n#     sns.kdeplot(data=df_test,x=feature, ax=axes[i][1])\n    \n#     for j in range(2):\n#         axes[i][j].set_xlabel('')\n        \n#     axes[i][0].set_title(f'{feature} Target Distribution in Training Set', fontsize=13)\n#     axes[i][1].set_title(f'{feature} Training & Test Set Distribution', fontsize=13)\n    \n\n# plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:16:55.698280Z","iopub.execute_input":"2022-07-16T16:16:55.698606Z","iopub.status.idle":"2022-07-16T16:16:55.703433Z","shell.execute_reply.started":"2022-07-16T16:16:55.698551Z","shell.execute_reply":"2022-07-16T16:16:55.702004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that train and test data set are from same sample ","metadata":{}},{"cell_type":"code","source":"df_train.groupby('target').count()['id'].plot(kind='pie', labels=['Not Disaster (57%)', 'Disaster (43%)'])","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:16:55.780381Z","iopub.execute_input":"2022-07-16T16:16:55.780643Z","iopub.status.idle":"2022-07-16T16:16:55.912285Z","shell.execute_reply.started":"2022-07-16T16:16:55.780587Z","shell.execute_reply":"2022-07-16T16:16:55.911243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DISASTER_TWEETS = df_train['target'] == 1","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:16:55.918461Z","iopub.execute_input":"2022-07-16T16:16:55.919006Z","iopub.status.idle":"2022-07-16T16:16:55.930161Z","shell.execute_reply.started":"2022-07-16T16:16:55.918767Z","shell.execute_reply":"2022-07-16T16:16:55.929081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_ngrams(text, n_gram=1):\n    token = [token for token in text.lower().split(' ') if token != '' if token not in STOPWORDS]\n#     ngrams = zip(*[token[i:] for i in range(n_gram)])\n    ngram_sentence = ngrams(token, n_gram)\n    return [' '.join(ngram) for ngram in ngram_sentence]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:16:55.936052Z","iopub.execute_input":"2022-07-16T16:16:55.936631Z","iopub.status.idle":"2022-07-16T16:16:55.951862Z","shell.execute_reply.started":"2022-07-16T16:16:55.936398Z","shell.execute_reply":"2022-07-16T16:16:55.950631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = 100\n\n# Unigrams\ndisaster_unigrams = defaultdict(int)\nnondisaster_unigrams = defaultdict(int)\n\nfor tweet in df_train[DISASTER_TWEETS]['text']:\n    for word in generate_ngrams(tweet):\n        disaster_unigrams[word] += 1\n        \n        \nfor tweet in df_train[~DISASTER_TWEETS]['text']:\n    for word in generate_ngrams(tweet):\n        nondisaster_unigrams[word] += 1\n        \ndf_disaster_unigrams = pd.DataFrame(sorted(disaster_unigrams.items(), key=lambda x: x[1])[::-1])\ndf_nondisaster_unigrams = pd.DataFrame(sorted(nondisaster_unigrams.items(), key=lambda x: x[1])[::-1])\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:16:55.957270Z","iopub.execute_input":"2022-07-16T16:16:55.957925Z","iopub.status.idle":"2022-07-16T16:16:56.144379Z","shell.execute_reply.started":"2022-07-16T16:16:55.957850Z","shell.execute_reply":"2022-07-16T16:16:56.143502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(ncols=2, figsize=(18, 50), dpi=100)\nplt.tight_layout()\n\nsns.barplot(y=df_disaster_unigrams[0].values[:N], x=df_disaster_unigrams[1].values[:N], ax=axes[0], color='red')\nsns.barplot(y=df_nondisaster_unigrams[0].values[:N], x=df_nondisaster_unigrams[1].values[:N], ax=axes[1], color='green')\n\nfor i in range(2):\n    axes[i].spines['right'].set_visible(False)\n    axes[i].set_xlabel('')\n    axes[i].set_ylabel('')\n    axes[i].tick_params(axis='x', labelsize=13)\n    axes[i].tick_params(axis='y', labelsize=13)\n\naxes[0].set_title(f'Top {N} most common unigrams in Disaster Tweets', fontsize=15)\naxes[1].set_title(f'Top {N} most common unigrams in Non-disaster Tweets', fontsize=15)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:16:56.147885Z","iopub.execute_input":"2022-07-16T16:16:56.148254Z","iopub.status.idle":"2022-07-16T16:17:00.930404Z","shell.execute_reply.started":"2022-07-16T16:16:56.148180Z","shell.execute_reply":"2022-07-16T16:17:00.928945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disaster_bigrams = defaultdict(int)\nnondisaster_bigrams = defaultdict(int)\n\nfor tweet in df_train[DISASTER_TWEETS]['text']:\n    for word in generate_ngrams(tweet, n_gram=2):\n        disaster_bigrams[word] += 1\n        \nfor tweet in df_train[~DISASTER_TWEETS]['text']:\n    for word in generate_ngrams(tweet, n_gram=2):\n        nondisaster_bigrams[word] += 1\n        \ndf_disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items(), key=lambda x: x[1])[::-1])\ndf_nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items(), key=lambda x: x[1])[::-1])\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:17:00.932709Z","iopub.execute_input":"2022-07-16T16:17:00.933251Z","iopub.status.idle":"2022-07-16T16:17:01.130016Z","shell.execute_reply.started":"2022-07-16T16:17:00.933193Z","shell.execute_reply":"2022-07-16T16:17:01.129198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(ncols=2, figsize=(18, 50), dpi=100)\nplt.tight_layout()\n\nsns.barplot(y=df_disaster_bigrams[0].values[:N], x=df_disaster_bigrams[1].values[:N], ax=axes[0], color='red')\nsns.barplot(y=df_nondisaster_bigrams[0].values[:N], x=df_nondisaster_bigrams[1].values[:N], ax=axes[1], color='green')\n\nfor i in range(2):\n    axes[i].spines['right'].set_visible(False)\n    axes[i].set_xlabel('')\n    axes[i].set_ylabel('')\n    axes[i].tick_params(axis='x', labelsize=13)\n    axes[i].tick_params(axis='y', labelsize=13)\n\naxes[0].set_title(f'Top {N} most common bigrams in Disaster Tweets', fontsize=15)\naxes[1].set_title(f'Top {N} most common bigrams in Non-disaster Tweets', fontsize=15)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:17:01.131743Z","iopub.execute_input":"2022-07-16T16:17:01.132115Z","iopub.status.idle":"2022-07-16T16:17:06.845167Z","shell.execute_reply.started":"2022-07-16T16:17:01.132028Z","shell.execute_reply":"2022-07-16T16:17:06.844258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nglove_embeddings = np.load('../input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', allow_pickle=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:17:06.846699Z","iopub.execute_input":"2022-07-16T16:17:06.847361Z","iopub.status.idle":"2022-07-16T16:17:38.743360Z","shell.execute_reply.started":"2022-07-16T16:17:06.847302Z","shell.execute_reply":"2022-07-16T16:17:38.742276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fasttext_embeddings = np.load('../input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl', allow_pickle=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:17:38.744947Z","iopub.execute_input":"2022-07-16T16:17:38.745347Z","iopub.status.idle":"2022-07-16T16:18:06.791022Z","shell.execute_reply.started":"2022-07-16T16:17:38.745287Z","shell.execute_reply":"2022-07-16T16:18:06.789965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(glove_embeddings.keys())","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-07-16T16:18:06.793404Z","iopub.execute_input":"2022-07-16T16:18:06.793728Z","iopub.status.idle":"2022-07-16T16:18:06.809999Z","shell.execute_reply.started":"2022-07-16T16:18:06.793675Z","shell.execute_reply":"2022-07-16T16:18:06.808828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.text[0:2]","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:06.811692Z","iopub.execute_input":"2022-07-16T16:18:06.812478Z","iopub.status.idle":"2022-07-16T16:18:06.831890Z","shell.execute_reply.started":"2022-07-16T16:18:06.812391Z","shell.execute_reply":"2022-07-16T16:18:06.830744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_vocab(X):\n    \n    tweets = X.apply(lambda s: s.split()).values   \n    vocab = {}\n    \n    for tweet in tweets:\n        for word in tweet:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1                \n    return vocab\n\n\ndef check_embeddings_coverage(X, embeddings):\n    \n    vocab = build_vocab(X)    \n    covered = {}\n    oov = {}    \n    n_covered = 0\n    n_oov = 0\n    \n    for word in vocab:\n        try:\n            covered[word] = embeddings[word]\n            n_covered += vocab[word]\n        except:\n            oov[word] = vocab[word]\n            n_oov += vocab[word]\n            \n    vocab_coverage = len(covered) / len(vocab)\n    text_coverage = (n_covered / (n_covered + n_oov))\n    \n    sorted_oov = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n    return sorted_oov, vocab_coverage, text_coverage\n\ntrain_glove_oov, train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(df_train['text'], glove_embeddings)\ntest_glove_oov, test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(df_test['text'], glove_embeddings)\nprint('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))\nprint('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage))\n\ntrain_fasttext_oov, train_fasttext_vocab_coverage, train_fasttext_text_coverage = check_embeddings_coverage(df_train['text'], fasttext_embeddings)\ntest_fasttext_oov, test_fasttext_vocab_coverage, test_fasttext_text_coverage = check_embeddings_coverage(df_test['text'], fasttext_embeddings)\nprint('FastText Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_fasttext_vocab_coverage, train_fasttext_text_coverage))\nprint('FastText Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_fasttext_vocab_coverage, test_fasttext_text_coverage))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:06.833748Z","iopub.execute_input":"2022-07-16T16:18:06.834584Z","iopub.status.idle":"2022-07-16T16:18:07.087850Z","shell.execute_reply.started":"2022-07-16T16:18:06.834524Z","shell.execute_reply":"2022-07-16T16:18:07.086853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_glove_oov","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:07.089532Z","iopub.execute_input":"2022-07-16T16:18:07.089881Z","iopub.status.idle":"2022-07-16T16:18:07.150079Z","shell.execute_reply.started":"2022-07-16T16:18:07.089819Z","shell.execute_reply":"2022-07-16T16:18:07.149197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean(tweet):\n    tweet=re.sub(r'x89Û_','',tweet)\n    tweet=re.sub(r'MH370','Malaysia Airlines Flight 370',tweet)\n    tweet=re.sub(r'\\x89ÛÒ','',tweet) \n    tweet=re.sub(r're\\x89Û_','',tweet) \n    tweet=re.sub(r'\\x89Û_','',tweet) \n    tweet=re.sub(r'\\x89ÛÓ','',tweet) \n    tweet=re.sub(r'w/','',tweet) \n    tweet=re.sub(r'#GBBO','The Great British Bake Off',tweet) \n    tweet=re.sub(r'don\\x89Ûªt','do not',tweet) \n    tweet=re.sub(r\"crematoria\",\"crematoria\",tweet) \n    tweet=re.sub(r'UTC2015-08-05','UTC 2015-08-05',tweet) \n    tweet=re.sub(r'\\x89ÛÏWhen','When',tweet) \n    tweet=re.sub(r'#ISIS','Islamic State',tweet) \n    tweet=re.sub(r'&gt;&gt;','greater than',tweet) \n    tweet=re.sub(r'Time 2015-08-06','',tweet) \n    tweet=re.sub(r\"he's\",'he is',tweet) \n    tweet=re.sub(r\"we're\",'we are',tweet) \n    tweet=re.sub(r\"Typhoon-Devastated\",'Typhoon Devastated',tweet) \n    tweet=re.sub(r\"Reddit's\",'Reddit is',tweet) \n    tweet=re.sub(r\"That's\",'that is',tweet) \n    tweet=re.sub(r\"there's\",'there is',tweet) \n    tweet=re.sub(r\"won't\",'will not',tweet)              \n    tweet=re.sub(r\"they're\",'they are',tweet)              \n    tweet=re.sub(r\"she's\",'she is',tweet) \n    tweet=re.sub(r\"ain't\",'am not',tweet)  \n    tweet=re.sub(r\"Sensor-Senso\",'Sensor Senso',tweet)  \n    tweet=re.sub(r\"haven't\",'have not',tweet) \n    tweet=re.sub(r\"hasn't\",'has not',tweet)  \n    tweet=re.sub(r\"let's\",'let is',tweet) \n    \n        # Urls\n    tweet = re.sub(r\"https?:\\/\\/t.co\\/[A-Za-z0-9]+\", \"\", tweet)\n    \n    # Words with punctuations and special characters\n    punctuations = '@#!?+&*[]-%.:/();$=><|{}^' + \"'`\"\n    for p in punctuations:\n        tweet = tweet.replace(p, f' {p} ')\n    \n    return tweet","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:07.151711Z","iopub.execute_input":"2022-07-16T16:18:07.152126Z","iopub.status.idle":"2022-07-16T16:18:07.171500Z","shell.execute_reply.started":"2022-07-16T16:18:07.152035Z","shell.execute_reply":"2022-07-16T16:18:07.170212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['text_cleaned'] = df_train['text'].apply(lambda s : clean(s))\ndf_test['text_cleaned'] = df_test['text'].apply(lambda s : clean(s))\n\ntrain_glove_oov, train_glove_vocab_coverage, train_glove_text_coverage = check_embeddings_coverage(df_train['text_cleaned'], glove_embeddings)\ntest_glove_oov, test_glove_vocab_coverage, test_glove_text_coverage = check_embeddings_coverage(df_test['text_cleaned'], glove_embeddings)\nprint('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_glove_vocab_coverage, train_glove_text_coverage))\nprint('GloVe Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_glove_vocab_coverage, test_glove_text_coverage))\n\ntrain_fasttext_oov, train_fasttext_vocab_coverage, train_fasttext_text_coverage = check_embeddings_coverage(df_train['text_cleaned'], fasttext_embeddings)\ntest_fasttext_oov, test_fasttext_vocab_coverage, test_fasttext_text_coverage = check_embeddings_coverage(df_test['text_cleaned'], fasttext_embeddings)\nprint('FastText Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Training Set'.format(train_fasttext_vocab_coverage, train_fasttext_text_coverage))\nprint('FastText Embeddings cover {:.2%} of vocabulary and {:.2%} of text in Test Set'.format(test_fasttext_vocab_coverage, test_fasttext_text_coverage))","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:07.173350Z","iopub.execute_input":"2022-07-16T16:18:07.174092Z","iopub.status.idle":"2022-07-16T16:18:08.233969Z","shell.execute_reply.started":"2022-07-16T16:18:07.174011Z","shell.execute_reply":"2022-07-16T16:18:08.231692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del glove_embeddings, fasttext_embeddings, train_glove_oov, test_glove_oov, train_fasttext_oov, test_fasttext_oov\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:08.236062Z","iopub.execute_input":"2022-07-16T16:18:08.236524Z","iopub.status.idle":"2022-07-16T16:18:10.745463Z","shell.execute_reply.started":"2022-07-16T16:18:08.236435Z","shell.execute_reply":"2022-07-16T16:18:10.744322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross-validation","metadata":{}},{"cell_type":"code","source":"fig , axes=plt.subplots(figsize=(20,50))\nplt.barh(y=df_train.keyword.value_counts().index , width=df_train.keyword.value_counts().values)\nplt.barh(y=df_test.keyword.value_counts().index , width=df_test.keyword.value_counts().values)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:10.754979Z","iopub.execute_input":"2022-07-16T16:18:10.757321Z","iopub.status.idle":"2022-07-16T16:18:15.759241Z","shell.execute_reply.started":"2022-07-16T16:18:10.757254Z","shell.execute_reply":"2022-07-16T16:18:15.758219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn import feature_extraction, linear_model, model_selection, preprocessing\n# countvec=feature_extraction.text.CountVectorizer()\n# train_vectors=countvec.fit_transform(df_train.text_cleaned)\n# test_vectors = countvec.transform(df_test.text_cleaned)\n# from sklearn.naive_bayes import MultinomialNB\n# clf = MultinomialNB()\n# scores = model_selection.cross_val_score(clf, train_vectors, df_train[\"target\"], cv=3, scoring=\"f1\"  )\n# scores\n# clf.fit(train_vectors, df_train[\"target\"])\n# sample_submission = pd.read_csv(\"E:/Science/DATA/NLP-disaster-tweets/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:15.760997Z","iopub.execute_input":"2022-07-16T16:18:15.761492Z","iopub.status.idle":"2022-07-16T16:18:15.766267Z","shell.execute_reply.started":"2022-07-16T16:18:15.761439Z","shell.execute_reply":"2022-07-16T16:18:15.765214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_mislabeled = df_train.groupby(['text']).nunique().sort_values(by='target', ascending=False)\ndf_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']\ndf_mislabeled.index.tolist()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:15.767752Z","iopub.execute_input":"2022-07-16T16:18:15.768332Z","iopub.status.idle":"2022-07-16T16:18:15.861704Z","shell.execute_reply.started":"2022-07-16T16:18:15.768261Z","shell.execute_reply":"2022-07-16T16:18:15.860571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['target_relabeled'] = df_train['target'].copy() \n\ndf_train.loc[df_train['text'] == 'like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit', 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == 'Hellfire is surrounded by desires so be careful and don‰Ûªt let your desires control you! #Afterlife', 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == 'To fight bioterrorism sir.', 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == '.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4', 'target_relabeled'] = 1\ndf_train.loc[df_train['text'] == 'CLEARED:incident with injury:I-495  inner loop Exit 31 - MD 97/Georgia Ave Silver Spring', 'target_relabeled'] = 1\ndf_train.loc[df_train['text'] == '#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption', 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == 'In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!', 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == 'Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her\\n \\n#FARRAKHAN #QUOTE', 'target_relabeled'] = 1\ndf_train.loc[df_train['text'] == 'RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG', 'target_relabeled'] = 1\ndf_train.loc[df_train['text'] == \"Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...\", 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == \"wowo--=== 12000 Nigerian refugees repatriated from Cameroon\", 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == \"He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam\", 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == \"Hellfire! We don‰Ûªt even want to think about it or mention it so let‰Ûªs not do anything that leads to it #islam!\", 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == \"The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.'\", 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == \"Caution: breathing may be hazardous to your health.\", 'target_relabeled'] = 1\ndf_train.loc[df_train['text'] == \"I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????\", 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == \"#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect\", 'target_relabeled'] = 0\ndf_train.loc[df_train['text'] == \"that horrible sinking feeling when you‰Ûªve been at home on your phone for a while and you realise its been on 3G this whole time\", 'target_relabeled'] = 0","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:15.863411Z","iopub.execute_input":"2022-07-16T16:18:15.863717Z","iopub.status.idle":"2022-07-16T16:18:15.944443Z","shell.execute_reply.started":"2022-07-16T16:18:15.863663Z","shell.execute_reply":"2022-07-16T16:18:15.943618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"K = 2\nskf = StratifiedKFold(n_splits=K, random_state=SEED, shuffle=True)\n\nDISASTER = df_train['target'] == 1\nprint('Whole Training Set Shape = {}'.format(df_train.shape))\nprint('Whole Training Set Unique keyword Count = {}'.format(df_train['keyword'].nunique()))\nprint('Whole Training Set Target Rate (Disaster) {}/{} (Not Disaster)'.format(df_train[DISASTER]['target_relabeled'].count(), df_train[~DISASTER]['target_relabeled'].count()))\n\nfor fold, (trn_idx, val_idx) in enumerate(skf.split(df_train['text_cleaned'], df_train['target']), 1):\n    print('\\nFold {} Training Set Shape = {} - Validation Set Shape = {}'.format(fold, df_train.loc[trn_idx, 'text_cleaned'].shape, df_train.loc[val_idx, 'text_cleaned'].shape))\n    print('Fold {} Training Set Unique keyword Count = {} - Validation Set Unique keyword Count = {}'.format(fold, df_train.loc[trn_idx, 'keyword'].nunique(), df_train.loc[val_idx, 'keyword'].nunique()))    ","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:15.946504Z","iopub.execute_input":"2022-07-16T16:18:15.947084Z","iopub.status.idle":"2022-07-16T16:18:15.982865Z","shell.execute_reply.started":"2022-07-16T16:18:15.946816Z","shell.execute_reply":"2022-07-16T16:18:15.981877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ClassificationReport(Callback):\n    \n    def __init__(self, train_data=(), validation_data=()):\n        super(Callback, self).__init__()\n        \n        self.X_train, self.y_train = train_data\n        self.train_precision_scores = []\n        self.train_recall_scores = []\n        self.train_f1_scores = []\n        \n        self.X_val, self.y_val = validation_data\n        self.val_precision_scores = []\n        self.val_recall_scores = []\n        self.val_f1_scores = [] \n               \n    def on_epoch_end(self, epoch, logs={}):\n        train_predictions = np.round(self.model.predict(self.X_train, verbose=0))        \n        train_precision = precision_score(self.y_train, train_predictions, average='macro')\n        train_recall = recall_score(self.y_train, train_predictions, average='macro')\n        train_f1 = f1_score(self.y_train, train_predictions, average='macro')\n        self.train_precision_scores.append(train_precision)        \n        self.train_recall_scores.append(train_recall)\n        self.train_f1_scores.append(train_f1)\n        \n        val_predictions = np.round(self.model.predict(self.X_val, verbose=0))\n        val_precision = precision_score(self.y_val, val_predictions, average='macro')\n        val_recall = recall_score(self.y_val, val_predictions, average='macro')\n        val_f1 = f1_score(self.y_val, val_predictions, average='macro')\n        self.val_precision_scores.append(val_precision)        \n        self.val_recall_scores.append(val_recall)        \n        self.val_f1_scores.append(val_f1)\n        \n        print('\\nEpoch: {} - Training Precision: {:.6} - Training Recall: {:.6} - Training F1: {:.6}'.format(epoch + 1, train_precision, train_recall, train_f1))\n        print('Epoch: {} - Validation Precision: {:.6} - Validation Recall: {:.6} - Validation F1: {:.6}'.format(epoch + 1, val_precision, val_recall, val_f1))  ","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:15.984276Z","iopub.execute_input":"2022-07-16T16:18:15.984591Z","iopub.status.idle":"2022-07-16T16:18:16.000729Z","shell.execute_reply.started":"2022-07-16T16:18:15.984535Z","shell.execute_reply":"2022-07-16T16:18:15.998887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nbert_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1', trainable=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:16.003819Z","iopub.execute_input":"2022-07-16T16:18:16.004517Z","iopub.status.idle":"2022-07-16T16:18:53.759761Z","shell.execute_reply.started":"2022-07-16T16:18:16.004457Z","shell.execute_reply":"2022-07-16T16:18:53.758862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DisasterDetector:\n    \n    def __init__(self, bert_layer, max_seq_length=128, lr=0.0001, epochs=15, batch_size=32):\n        \n        # BERT and Tokenization params\n        self.bert_layer = bert_layer\n        \n        self.max_seq_length = max_seq_length        \n        vocab_file = self.bert_layer.resolved_object.vocab_file.asset_path.numpy()\n        do_lower_case = self.bert_layer.resolved_object.do_lower_case.numpy()\n        self.tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n        \n        # Learning control params\n        self.lr = lr\n        self.epochs = epochs\n        self.batch_size = batch_size\n        \n        self.models = []\n        self.scores = {}\n        \n        \n    def encode(self, texts):\n                \n        all_tokens = []\n        all_masks = []\n        all_segments = []\n\n        for text in texts:\n            text = self.tokenizer.tokenize(text)\n            text = text[:self.max_seq_length - 2]\n            input_sequence = ['[CLS]'] + text + ['[SEP]']\n            pad_len = self.max_seq_length - len(input_sequence)\n\n            tokens = self.tokenizer.convert_tokens_to_ids(input_sequence)\n            tokens += [0] * pad_len\n            pad_masks = [1] * len(input_sequence) + [0] * pad_len\n            segment_ids = [0] * self.max_seq_length\n\n            all_tokens.append(tokens)\n            all_masks.append(pad_masks)\n            all_segments.append(segment_ids)\n\n        return np.array(all_tokens), np.array(all_masks), np.array(all_segments)\n    \n    \n    def build_model(self):\n        \n        input_word_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_word_ids')\n        input_mask = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='input_mask')\n        segment_ids = Input(shape=(self.max_seq_length,), dtype=tf.int32, name='segment_ids')    \n        \n        pooled_output, sequence_output = self.bert_layer([input_word_ids, input_mask, segment_ids])   \n        clf_output = sequence_output[:, 0, :]\n        out = Dense(1, activation='sigmoid')(clf_output)\n        \n        model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)\n        optimizer = SGD(learning_rate=self.lr, momentum=0.8)\n        model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n        \n        return model\n    \n    \n    def train(self, X):\n        \n        for fold, (trn_idx, val_idx) in enumerate(skf.split(X['text_cleaned'], X['keyword'])):\n            \n            print('\\nFold {}\\n'.format(fold))\n        \n            X_trn_encoded = self.encode(X.loc[trn_idx, 'text_cleaned'].str.lower())\n            y_trn = X.loc[trn_idx, 'target_relabeled']\n            X_val_encoded = self.encode(X.loc[val_idx, 'text_cleaned'].str.lower())\n            y_val = X.loc[val_idx, 'target_relabeled']\n        \n            # Callbacks\n            metrics = ClassificationReport(train_data=(X_trn_encoded, y_trn), validation_data=(X_val_encoded, y_val))\n            \n            # Model\n            model = self.build_model()        \n            model.fit(X_trn_encoded, y_trn, validation_data=(X_val_encoded, y_val), callbacks=[metrics], epochs=self.epochs, batch_size=self.batch_size)\n            \n            self.models.append(model)\n            self.scores[fold] = {\n                'train': {\n                    'precision': metrics.train_precision_scores,\n                    'recall': metrics.train_recall_scores,\n                    'f1': metrics.train_f1_scores                    \n                },\n                'validation': {\n                    'precision': metrics.val_precision_scores,\n                    'recall': metrics.val_recall_scores,\n                    'f1': metrics.val_f1_scores                    \n                }\n            }\n                    \n                \n    def plot_learning_curve(self):\n        \n        fig, axes = plt.subplots(nrows=K, ncols=2, figsize=(20, K * 6), dpi=100)\n    \n        for i in range(K):\n            \n            # Classification Report curve\n            sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[i].history.history['val_accuracy'], ax=axes[i][0], label='val_accuracy')\n            sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['precision'], ax=axes[i][0], label='val_precision')\n            sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['recall'], ax=axes[i][0], label='val_recall')\n            sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.scores[i]['validation']['f1'], ax=axes[i][0], label='val_f1')        \n\n            axes[i][0].legend() \n            axes[i][0].set_title('Fold {} Validation Classification Report'.format(i), fontsize=14)\n\n            # Loss curve\n            sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['loss'], ax=axes[i][1], label='train_loss')\n            sns.lineplot(x=np.arange(1, self.epochs + 1), y=clf.models[0].history.history['val_loss'], ax=axes[i][1], label='val_loss')\n\n            axes[i][1].legend() \n            axes[i][1].set_title('Fold {} Train / Validation Loss'.format(i), fontsize=14)\n\n            for j in range(2):\n                axes[i][j].set_xlabel('Epoch', size=12)\n                axes[i][j].tick_params(axis='x', labelsize=12)\n                axes[i][j].tick_params(axis='y', labelsize=12)\n\n        plt.show()\n        \n        \n    def predict(self, X):\n        \n        X_test_encoded = self.encode(X['text_cleaned'].str.lower())\n        y_pred = np.zeros((X_test_encoded[0].shape[0], 1))\n\n        for model in self.models:\n            y_pred += model.predict(X_test_encoded) / len(self.models)\n\n        return y_pred","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:53.761513Z","iopub.execute_input":"2022-07-16T16:18:53.761910Z","iopub.status.idle":"2022-07-16T16:18:53.807014Z","shell.execute_reply.started":"2022-07-16T16:18:53.761839Z","shell.execute_reply":"2022-07-16T16:18:53.805313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from bert import tokenization\nimport os\nos.getcwd()\n","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:53.808993Z","iopub.execute_input":"2022-07-16T16:18:53.809592Z","iopub.status.idle":"2022-07-16T16:18:53.831792Z","shell.execute_reply.started":"2022-07-16T16:18:53.809444Z","shell.execute_reply":"2022-07-16T16:18:53.830844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = DisasterDetector(bert_layer, max_seq_length=128, lr=0.0001, epochs=10, batch_size=32)\n\nclf.train(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:18:53.835806Z","iopub.execute_input":"2022-07-16T16:18:53.836208Z","iopub.status.idle":"2022-07-16T16:54:34.783221Z","shell.execute_reply.started":"2022-07-16T16:18:53.836136Z","shell.execute_reply":"2022-07-16T16:54:34.782322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclf.plot_learning_curve()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:54:34.785074Z","iopub.execute_input":"2022-07-16T16:54:34.785516Z","iopub.status.idle":"2022-07-16T16:54:36.268275Z","shell.execute_reply.started":"2022-07-16T16:54:34.785457Z","shell.execute_reply":"2022-07-16T16:54:36.267180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = clf.predict(df_test)\n\nmodel_submission = pd.read_csv(\"/kaggle/input/nlp-getting-started/sample_submission.csv\")\nmodel_submission['target'] = np.round(y_pred).astype('int')\nmodel_submission.to_csv('model_submission.csv', index=False)\nmodel_submission.describe()","metadata":{"execution":{"iopub.status.busy":"2022-07-16T16:54:36.269848Z","iopub.execute_input":"2022-07-16T16:54:36.270450Z","iopub.status.idle":"2022-07-16T16:55:09.068253Z","shell.execute_reply.started":"2022-07-16T16:54:36.270392Z","shell.execute_reply":"2022-07-16T16:55:09.067235Z"},"trusted":true},"execution_count":null,"outputs":[]}]}