{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport warnings\nimport torch\nimport numpy as np \nimport pandas as pd \nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nseed = 42\nsns.set_theme()\npd.set_option('display.max_colwidth', None)\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nmodel_name = \"distilbert-base-uncased\"\nTRAIN_FILE = \"../input/nlp-getting-started/train.csv\"\nTEST_FILE = \"../input/nlp-getting-started/test.csv\"\nSUBMISSION_FILE = \"../input/nlp-getting-started/sample_submission.csv\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-03T12:11:41.398596Z","iopub.execute_input":"2022-08-03T12:11:41.399945Z","iopub.status.idle":"2022-08-03T12:11:44.828827Z","shell.execute_reply.started":"2022-08-03T12:11:41.399795Z","shell.execute_reply":"2022-08-03T12:11:44.827489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Natural Language Processing with Disaster Tweets\n\n**Task:**\n<br/>\n\nIn this competition we have collection of ~10,000 tweets that were hand classified. For each tweet we have a label determining whether the tweet is about disaster or not.\n\n<br/>\n\n**Solution:**\n\nThis competition is a text binary classifications task. For tokenization, model training and finall predictions we will use Transformers library created by the Hugging Face🤗 company.","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(TRAIN_FILE)\ntest_df = pd.read_csv(TEST_FILE)\n\n\ntrain_df.rename(columns={\"target\": \"labels\"}, inplace=True)\nnum_labels = len(train_df.labels.value_counts())\ntrain_df.sample(5, random_state=seed)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:44.831385Z","iopub.execute_input":"2022-08-03T12:11:44.832448Z","iopub.status.idle":"2022-08-03T12:11:44.939336Z","shell.execute_reply.started":"2022-08-03T12:11:44.832403Z","shell.execute_reply":"2022-08-03T12:11:44.938116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of trainining samples: {train_df.shape[0]}\")\nprint(f\"Number of test samples: {test_df.shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:44.941151Z","iopub.execute_input":"2022-08-03T12:11:44.941615Z","iopub.status.idle":"2022-08-03T12:11:44.949273Z","shell.execute_reply.started":"2022-08-03T12:11:44.941572Z","shell.execute_reply":"2022-08-03T12:11:44.947808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:44.953786Z","iopub.execute_input":"2022-08-03T12:11:44.954848Z","iopub.status.idle":"2022-08-03T12:11:44.970165Z","shell.execute_reply.started":"2022-08-03T12:11:44.954809Z","shell.execute_reply":"2022-08-03T12:11:44.968455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preprocessing","metadata":{}},{"cell_type":"markdown","source":"We drop keyword and location columns since they contain a lot of null values but keyword could be handy for final predictions. This texts were collected from social network so they contain lot of links, hashtags and html tags. We will remove them before tokenization process.","metadata":{}},{"cell_type":"code","source":"for df in (train_df, test_df):\n    df.drop([\"id\", \"keyword\", \"location\"], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:44.972666Z","iopub.execute_input":"2022-08-03T12:11:44.973922Z","iopub.status.idle":"2022-08-03T12:11:44.984752Z","shell.execute_reply.started":"2022-08-03T12:11:44.973884Z","shell.execute_reply":"2022-08-03T12:11:44.983000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nfrom bs4 import BeautifulSoup, MarkupResemblesLocatorWarning\nwarnings.filterwarnings(\"ignore\", category=MarkupResemblesLocatorWarning, module='bs4')\n\ndef clean_text(text):\n    text = BeautifulSoup(text, 'html.parser').get_text() # Remove html tags\n    text = re.sub(\"http[s]?\\:\\/\\/\\S+\", \" \", text) # Remove links\n    text = re.sub(\"[ \\t\\n]+\", \" \", text) # Remove tabs, newlines and multiple spaces\n    text = re.sub(\"[^a-zA-Z]\", \" \", text) \n    \n    return text.strip().lower()\n\n\nfor df in (train_df, test_df):\n    df.text = df.text.apply(lambda x: clean_text(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:44.987055Z","iopub.execute_input":"2022-08-03T12:11:44.987526Z","iopub.status.idle":"2022-08-03T12:11:46.611464Z","shell.execute_reply.started":"2022-08-03T12:11:44.987488Z","shell.execute_reply":"2022-08-03T12:11:46.610000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.sample(5, random_state=seed)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:46.614381Z","iopub.execute_input":"2022-08-03T12:11:46.615392Z","iopub.status.idle":"2022-08-03T12:11:46.633083Z","shell.execute_reply.started":"2022-08-03T12:11:46.615347Z","shell.execute_reply":"2022-08-03T12:11:46.631574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(\"Text length\")\nplt.xlabel(\"n words\")\nplt.ylabel(\"Count\")\nplt.hist(train_df.text.apply(lambda x: x.split(\" \")).str.len())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:46.636467Z","iopub.execute_input":"2022-08-03T12:11:46.638064Z","iopub.status.idle":"2022-08-03T12:11:46.987132Z","shell.execute_reply.started":"2022-08-03T12:11:46.638004Z","shell.execute_reply":"2022-08-03T12:11:46.985788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that most tweets have less than 30 words.","metadata":{}},{"cell_type":"code","source":"train_df = train_df.sample(frac=1, random_state=seed).reset_index(drop=True) # Shuffle training data","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:46.988718Z","iopub.execute_input":"2022-08-03T12:11:46.989588Z","iopub.status.idle":"2022-08-03T12:11:47.000267Z","shell.execute_reply.started":"2022-08-03T12:11:46.989556Z","shell.execute_reply":"2022-08-03T12:11:46.998567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import Dataset\n\ntrain_ds = Dataset.from_pandas(train_df)\ntest_ds = Dataset.from_pandas(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:47.008136Z","iopub.execute_input":"2022-08-03T12:11:47.009179Z","iopub.status.idle":"2022-08-03T12:11:47.780740Z","shell.execute_reply.started":"2022-08-03T12:11:47.009136Z","shell.execute_reply":"2022-08-03T12:11:47.779409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will split our training data into training and validation sets so we can evaluate model performance later. ","metadata":{}},{"cell_type":"code","source":"train_ds = train_ds.train_test_split(test_size=0.1, seed=seed)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:47.782263Z","iopub.execute_input":"2022-08-03T12:11:47.783107Z","iopub.status.idle":"2022-08-03T12:11:47.807577Z","shell.execute_reply.started":"2022-08-03T12:11:47.783074Z","shell.execute_reply":"2022-08-03T12:11:47.806318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We utilize GPU if possible since model fine-tuning on CPU is slow.","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:47.809657Z","iopub.execute_input":"2022-08-03T12:11:47.810132Z","iopub.status.idle":"2022-08-03T12:11:47.909378Z","shell.execute_reply.started":"2022-08-03T12:11:47.810090Z","shell.execute_reply":"2022-08-03T12:11:47.907725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModel, AutoTokenizer\n\n\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel = AutoModel.from_pretrained(model_name).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:11:47.911208Z","iopub.execute_input":"2022-08-03T12:11:47.912835Z","iopub.status.idle":"2022-08-03T12:12:20.427246Z","shell.execute_reply.started":"2022-08-03T12:11:47.912752Z","shell.execute_reply":"2022-08-03T12:12:20.425848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(batch):\n    return tokenizer(batch[\"text\"], padding=True, truncation=True)\n\n\ntrain_encoded = train_ds.map(tokenize, batched=True, batch_size=None)\ntest_encoded = test_ds.map(tokenize, batched=True, batch_size=None)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:12:20.430431Z","iopub.execute_input":"2022-08-03T12:12:20.431024Z","iopub.status.idle":"2022-08-03T12:12:21.908176Z","shell.execute_reply.started":"2022-08-03T12:12:20.430947Z","shell.execute_reply":"2022-08-03T12:12:21.906823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_encoded.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"labels\"])\ntest_encoded.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:12:21.910391Z","iopub.execute_input":"2022-08-03T12:12:21.911348Z","iopub.status.idle":"2022-08-03T12:12:21.920847Z","shell.execute_reply.started":"2022-08-03T12:12:21.911303Z","shell.execute_reply":"2022-08-03T12:12:21.919350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we extract last hidden state from pre-trained model. We will use these vectors later for further analysis.","metadata":{}},{"cell_type":"code","source":"def extract_embedding(batch):\n    inputs = {k:v.to(device) for k,v in batch.items() \n              if k in tokenizer.model_input_names}\n    \n    with torch.no_grad():\n        pred = model(**inputs).last_hidden_state\n        \n    return {\"embedding\": pred[:, 0].cpu().numpy()}\n\n  \ntrain_embedding = train_encoded.map(extract_embedding, batched=True, batch_size=None)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:12:21.923459Z","iopub.execute_input":"2022-08-03T12:12:21.924871Z","iopub.status.idle":"2022-08-03T12:12:28.189427Z","shell.execute_reply.started":"2022-08-03T12:12:21.924816Z","shell.execute_reply":"2022-08-03T12:12:28.187911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize classes in 2D\n\nWe can utilize UMAP reduction technique and apply it to reduce dimensionality of extracted hidden states. Then we can plot this reduced hidden states and check if we found any pattern in classes distribution.","metadata":{}},{"cell_type":"code","source":"import umap\nfrom sklearn.preprocessing import MinMaxScaler\n\n\ntrain_scaled = MinMaxScaler().fit_transform(train_embedding['train']['embedding'].numpy())\nreductor = umap.UMAP(n_neighbors=5, n_components=2, min_dist=0.3).fit(train_scaled)\ntrain_reduced = pd.DataFrame({\"X\": reductor.embedding_[:,0], \n                              \"Y\": reductor.embedding_[:,1], \n                              \"labels\": train_embedding['train']['labels']})","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:12:28.191847Z","iopub.execute_input":"2022-08-03T12:12:28.192706Z","iopub.status.idle":"2022-08-03T12:13:19.367116Z","shell.execute_reply.started":"2022-08-03T12:12:28.192644Z","shell.execute_reply":"2022-08-03T12:13:19.365371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.scatterplot(data=train_reduced, x=\"X\", y=\"Y\", hue=\"labels\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:19.370395Z","iopub.execute_input":"2022-08-03T12:13:19.371510Z","iopub.status.idle":"2022-08-03T12:13:19.925935Z","shell.execute_reply.started":"2022-08-03T12:13:19.371445Z","shell.execute_reply":"2022-08-03T12:13:19.924526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that points overlaps but we can recognize places with higher density for each class. The train_scaled and train_reduced variables can be deleted since we no longer need them for training and evaluation. ","metadata":{}},{"cell_type":"code","source":"del train_scaled, train_reduced","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:19.927223Z","iopub.execute_input":"2022-08-03T12:13:19.927682Z","iopub.status.idle":"2022-08-03T12:13:19.935287Z","shell.execute_reply.started":"2022-08-03T12:13:19.927583Z","shell.execute_reply":"2022-08-03T12:13:19.933835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Baseline model","metadata":{}},{"cell_type":"markdown","source":"Let´s start with Logistic regression model which we apply to extracted hidden states. First, we fit the model on train data split and then evaluate on validation split.","metadata":{}},{"cell_type":"code","source":"X_train = train_embedding['train']['embedding']\ny_train = train_embedding['train']['labels']\n\nX_valid = train_embedding['test']['embedding']\ny_valid = train_embedding['test']['labels']","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:19.943285Z","iopub.execute_input":"2022-08-03T12:13:19.944889Z","iopub.status.idle":"2022-08-03T12:13:20.027412Z","shell.execute_reply.started":"2022-08-03T12:13:19.944832Z","shell.execute_reply":"2022-08-03T12:13:20.026037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlogistic_regression = LogisticRegression(max_iter=2000, random_state=seed)\nlogistic_regression.fit(X_train, y_train)\n\nprint(f\"Logistic regression score: {logistic_regression.score(X_valid, y_valid):.4f}\")","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:20.029699Z","iopub.execute_input":"2022-08-03T12:13:20.030184Z","iopub.status.idle":"2022-08-03T12:13:24.683720Z","shell.execute_reply.started":"2022-08-03T12:13:20.030140Z","shell.execute_reply":"2022-08-03T12:13:24.682201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Logistic regression model performance is not bad but let´s check whether we can achieve better performance with model fine-tuning. Since later we will use Dataset object we can now delete train/valid variables.","metadata":{}},{"cell_type":"code","source":"del X_train, X_valid, y_train, y_valid","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:24.686000Z","iopub.execute_input":"2022-08-03T12:13:24.686971Z","iopub.status.idle":"2022-08-03T12:13:24.693922Z","shell.execute_reply.started":"2022-08-03T12:13:24.686920Z","shell.execute_reply":"2022-08-03T12:13:24.692270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fine-Tuning DistilBERT model","metadata":{}},{"cell_type":"markdown","source":" We will utilize DistilBERT for sequence classification task. This model is already pre-trained so we only fine-tune the model for our task. Fine-tuning process will be handled using Trainer class.","metadata":{}},{"cell_type":"code","source":"from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer\n\n\nbatch_size = 32\nlogging_steps = len(train_encoded[\"train\"]) // batch_size\ntraining_args = TrainingArguments(output_dir=\"distilbert_disaster\", \n                                  report_to=\"tensorboard\",\n                                  evaluation_strategy=\"epoch\",\n                                  num_train_epochs=2,\n                                  learning_rate=2e-5,\n                                  logging_steps=logging_steps,\n                                  per_device_train_batch_size=batch_size,\n                                  per_device_eval_batch_size=batch_size,                                \n                                  weight_decay=0.01,\n                                  save_strategy=\"no\",\n                                  disable_tqdm=False,\n                                  push_to_hub=False,\n                                  log_level=\"error\")\n\nmodel = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=num_labels).to(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:24.697293Z","iopub.execute_input":"2022-08-03T12:13:24.698754Z","iopub.status.idle":"2022-08-03T12:13:29.618581Z","shell.execute_reply.started":"2022-08-03T12:13:24.698682Z","shell.execute_reply":"2022-08-03T12:13:29.617150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import load_metric\nfrom sklearn.metrics import f1_score\n\nmetric = load_metric(\"accuracy\")\n\ndef compute_metrics(pred):\n    logits, labels = pred\n    preds = np.argmax(logits, axis=-1)  \n    acc = metric.compute(predictions=preds, references=labels)\n    return {\"accuracy\": acc[\"accuracy\"], \"f1\": f1_score(labels, preds)}","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:29.620261Z","iopub.execute_input":"2022-08-03T12:13:29.621143Z","iopub.status.idle":"2022-08-03T12:13:30.699479Z","shell.execute_reply.started":"2022-08-03T12:13:29.621110Z","shell.execute_reply":"2022-08-03T12:13:30.698000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import Trainer\n\ntrainer = Trainer(model=model, args=training_args, \n                  compute_metrics=compute_metrics,\n                  train_dataset=train_encoded[\"train\"],\n                  eval_dataset=train_encoded[\"test\"],\n                  tokenizer=tokenizer)\n\ntrainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:13:30.704924Z","iopub.execute_input":"2022-08-03T12:13:30.707816Z","iopub.status.idle":"2022-08-03T12:14:18.541268Z","shell.execute_reply.started":"2022-08-03T12:13:30.707735Z","shell.execute_reply":"2022-08-03T12:14:18.539777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluation","metadata":{}},{"cell_type":"markdown","source":"We can now evaluate fine-tuned DistilBERT model performance on validation split.","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\nvalid_preds = trainer.predict(train_encoded[\"test\"])\nvalid_preds = np.argmax(valid_preds.predictions, axis=-1)\n\ncnf_matrix = confusion_matrix(train_encoded[\"test\"][\"labels\"], valid_preds)\n\nsns.heatmap(cnf_matrix, annot=True, fmt=\"d\")\nplt.xlabel(\"True class\")\nplt.ylabel(\"Pred class\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:14:18.543146Z","iopub.execute_input":"2022-08-03T12:14:18.543736Z","iopub.status.idle":"2022-08-03T12:14:19.792756Z","shell.execute_reply.started":"2022-08-03T12:14:18.543692Z","shell.execute_reply":"2022-08-03T12:14:19.791171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that quite a lot of real disaster texts were classified as not disaster. Let's take a look at some misclassified samples.","metadata":{}},{"cell_type":"code","source":"misclasified = np.nonzero(valid_preds != train_encoded[\"test\"][\"labels\"].numpy())[0][:10]\ntrue_labels = train_encoded[\"test\"].select(misclasified)\npd.DataFrame({\"text\": true_labels['text'], \n              \"labels\": true_labels['labels'].numpy(), \n              \"preds\":valid_preds[misclasified]})","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:14:19.794508Z","iopub.execute_input":"2022-08-03T12:14:19.795093Z","iopub.status.idle":"2022-08-03T12:14:19.819482Z","shell.execute_reply.started":"2022-08-03T12:14:19.795047Z","shell.execute_reply":"2022-08-03T12:14:19.818176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = trainer.predict(test_encoded)\npreds = np.argmax(preds.predictions, axis=-1)\nsubmission = pd.read_csv(SUBMISSION_FILE)\nsubmission.target = preds\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T12:14:19.821230Z","iopub.execute_input":"2022-08-03T12:14:19.823105Z","iopub.status.idle":"2022-08-03T12:14:24.084082Z","shell.execute_reply.started":"2022-08-03T12:14:19.823062Z","shell.execute_reply":"2022-08-03T12:14:24.082608Z"},"trusted":true},"execution_count":null,"outputs":[]}]}