{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Most basic stuff for EDA.\n\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_columns', 50)\npd.set_option('display.max_rows', 150)\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Core packages for text processing.\n\nimport string\nimport re\n\n# Core packages for Training.\n\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\n# Core packages for Transformers from Huggingface.\n\n# !pip install transformers\n# !pip install datasets\nfrom datasets import load_dataset, load_metric\nfrom transformers import AutoTokenizer\nfrom transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer \n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-07T09:20:18.700288Z","iopub.execute_input":"2022-07-07T09:20:18.700967Z","iopub.status.idle":"2022-07-07T09:20:28.768331Z","shell.execute_reply.started":"2022-07-07T09:20:18.700869Z","shell.execute_reply":"2022-07-07T09:20:28.767165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tweets = pd.read_csv(\"../input/nlp-getting-started/train.csv\")\ntest_tweets = pd.read_csv(\"../input/nlp-getting-started/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:28.774117Z","iopub.execute_input":"2022-07-07T09:20:28.776853Z","iopub.status.idle":"2022-07-07T09:20:28.892536Z","shell.execute_reply.started":"2022-07-07T09:20:28.776811Z","shell.execute_reply":"2022-07-07T09:20:28.891530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_tweets.info())","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:28.898164Z","iopub.execute_input":"2022-07-07T09:20:28.900831Z","iopub.status.idle":"2022-07-07T09:20:28.939501Z","shell.execute_reply.started":"2022-07-07T09:20:28.900780Z","shell.execute_reply":"2022-07-07T09:20:28.938535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_tweets.shape)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:28.945088Z","iopub.execute_input":"2022-07-07T09:20:28.947529Z","iopub.status.idle":"2022-07-07T09:20:28.955182Z","shell.execute_reply.started":"2022-07-07T09:20:28.947463Z","shell.execute_reply":"2022-07-07T09:20:28.954032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tweets.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:28.956598Z","iopub.execute_input":"2022-07-07T09:20:28.957700Z","iopub.status.idle":"2022-07-07T09:20:28.982161Z","shell.execute_reply.started":"2022-07-07T09:20:28.957655Z","shell.execute_reply":"2022-07-07T09:20:28.981109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's do some EDA on the Train data from tweets","metadata":{}},{"cell_type":"code","source":"sns.set_style('whitegrid')\nsns.countplot(y=train_tweets['target'])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:28.983434Z","iopub.execute_input":"2022-07-07T09:20:28.984021Z","iopub.status.idle":"2022-07-07T09:20:29.272950Z","shell.execute_reply.started":"2022-07-07T09:20:28.983984Z","shell.execute_reply":"2022-07-07T09:20:29.271878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))\ntweet_len_disaster=train_tweets[train_tweets['target']==1]['text'].str.len()\nax1.hist(tweet_len_disaster,color='red')\nax1.set_title('disaster tweets')\ntweet_len_not_disaster=train_tweets[train_tweets['target']==0]['text'].str.len()\nax2.hist(tweet_len_disaster,color='green')\nax2.set_title('Not disaster tweets')\nfig.suptitle('Characters in tweets')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:29.277608Z","iopub.execute_input":"2022-07-07T09:20:29.280545Z","iopub.status.idle":"2022-07-07T09:20:29.844799Z","shell.execute_reply.started":"2022-07-07T09:20:29.280499Z","shell.execute_reply":"2022-07-07T09:20:29.843316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tweets","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:29.848684Z","iopub.execute_input":"2022-07-07T09:20:29.849073Z","iopub.status.idle":"2022-07-07T09:20:29.877982Z","shell.execute_reply.started":"2022-07-07T09:20:29.849033Z","shell.execute_reply":"2022-07-07T09:20:29.876948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tweets['location'].value_counts().head(n=20)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:29.882022Z","iopub.execute_input":"2022-07-07T09:20:29.882867Z","iopub.status.idle":"2022-07-07T09:20:29.896133Z","shell.execute_reply.started":"2022-07-07T09:20:29.882829Z","shell.execute_reply":"2022-07-07T09:20:29.895230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (9, 6))\nax = plt.axes()\nax.set_facecolor('white')\nax = ((train_tweets.location.value_counts())[:10]).plot(kind = 'bar', color = 'lightcoral', linewidth = 2, edgecolor = 'white')\nplt.title('Location Count', fontsize = 14)\nplt.xlabel('Location', fontsize = 12)\nplt.ylabel('Count', fontsize = 12)\nax.xaxis.set_tick_params(labelsize = 12, rotation = 30)\nax.yaxis.set_tick_params(labelsize = 12)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:29.901000Z","iopub.execute_input":"2022-07-07T09:20:29.901559Z","iopub.status.idle":"2022-07-07T09:20:30.271855Z","shell.execute_reply.started":"2022-07-07T09:20:29.901525Z","shell.execute_reply":"2022-07-07T09:20:30.270287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Let's Clean the text ","metadata":{}},{"cell_type":"markdown","source":"As we know,twitter tweets always have to be cleaned before we go onto modelling.So we will do some basic cleaning such as spelling correction,removing punctuations,removing html tags and emojis etc.So let's start.","metadata":{}},{"cell_type":"code","source":"# Some basic helper functions to clean text by removing urls, emojis, html tags and punctuations.\n\ndef remove_URL(text):\n    url = re.compile(r'https?://\\S+|www\\.\\S+')\n    return url.sub(r'', text)\n\n\ndef remove_emoji(text):\n    emoji_pattern = re.compile(\n        '['\n        u'\\U0001F600-\\U0001F64F'  # emoticons\n        u'\\U0001F300-\\U0001F5FF'  # symbols & pictographs\n        u'\\U0001F680-\\U0001F6FF'  # transport & map symbols\n        u'\\U0001F1E0-\\U0001F1FF'  # flags (iOS)\n        u'\\U00002702-\\U000027B0'\n        u'\\U000024C2-\\U0001F251'\n        ']+',\n        flags=re.UNICODE)\n    return emoji_pattern.sub(r'', text)\n\n\ndef remove_html(text):\n    html = re.compile(r'<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});')\n    return re.sub(html, '', text)\n\n\ndef remove_punct(text):\n    table = str.maketrans('', '', string.punctuation)\n    return text.translate(table)\n\n# Applying helper functions on Train Dataset\n\ntrain_tweets['text_clean'] = train_tweets['text'].apply(lambda x: remove_URL(x))\ntrain_tweets['text_clean'] = train_tweets['text_clean'].apply(lambda x: remove_emoji(x))\ntrain_tweets['text_clean'] = train_tweets['text_clean'].apply(lambda x: remove_html(x))\ntrain_tweets['text_clean'] = train_tweets['text_clean'].apply(lambda x: remove_punct(x))\n\n# Applying helper functions on Test Dataset\n\ntest_tweets['text_clean'] = test_tweets['text'].apply(lambda x: remove_URL(x))\ntest_tweets['text_clean'] = test_tweets['text_clean'].apply(lambda x: remove_emoji(x))\ntest_tweets['text_clean'] = test_tweets['text_clean'].apply(lambda x: remove_html(x))\ntest_tweets['text_clean'] = test_tweets['text_clean'].apply(lambda x: remove_punct(x))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:30.273494Z","iopub.execute_input":"2022-07-07T09:20:30.274099Z","iopub.status.idle":"2022-07-07T09:20:30.894830Z","shell.execute_reply.started":"2022-07-07T09:20:30.274058Z","shell.execute_reply":"2022-07-07T09:20:30.893811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training LSTM Model at first","metadata":{}},{"cell_type":"code","source":"train_tweets","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:30.896252Z","iopub.execute_input":"2022-07-07T09:20:30.896789Z","iopub.status.idle":"2022-07-07T09:20:30.917179Z","shell.execute_reply.started":"2022-07-07T09:20:30.896752Z","shell.execute_reply":"2022-07-07T09:20:30.916250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tweets","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:30.918726Z","iopub.execute_input":"2022-07-07T09:20:30.919227Z","iopub.status.idle":"2022-07-07T09:20:30.937856Z","shell.execute_reply.started":"2022-07-07T09:20:30.919192Z","shell.execute_reply":"2022-07-07T09:20:30.936643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = train_tweets.drop(['id','keyword','location','text'],axis=1)\ndf_test = test_tweets.drop(['keyword','location','text'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:30.939240Z","iopub.execute_input":"2022-07-07T09:20:30.939762Z","iopub.status.idle":"2022-07-07T09:20:30.948361Z","shell.execute_reply.started":"2022-07-07T09:20:30.939727Z","shell.execute_reply":"2022-07-07T09:20:30.947346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Move texts and targets from train & test dataset to the list:\ntraining_message = []\ntesting_message = []\ntraining_labels = []\nfor i in range(len(df_train)):\n    training_message.append(df_train.loc[i,'text_clean'])\n    training_labels.append(df_train.loc[i,'target'])\n    \nfor i in range(len(df_test)):\n    testing_message.append(df_test.loc[i,'text_clean'])","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:30.950045Z","iopub.execute_input":"2022-07-07T09:20:30.950637Z","iopub.status.idle":"2022-07-07T09:20:31.350493Z","shell.execute_reply.started":"2022-07-07T09:20:30.950593Z","shell.execute_reply":"2022-07-07T09:20:31.349371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_size = 23000\nembedding_dim = 32\nmax_length = 100\ntrunc_type='post'\npadding_type='post'\noov_tok = \"<OOV>\"","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:31.352001Z","iopub.execute_input":"2022-07-07T09:20:31.352596Z","iopub.status.idle":"2022-07-07T09:20:31.358526Z","shell.execute_reply.started":"2022-07-07T09:20:31.352555Z","shell.execute_reply":"2022-07-07T09:20:31.357151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tokenization Process:\ntokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\ntokenizer.fit_on_texts(training_message)\nword_index = tokenizer.word_index","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:31.360223Z","iopub.execute_input":"2022-07-07T09:20:31.360886Z","iopub.status.idle":"2022-07-07T09:20:31.777384Z","shell.execute_reply.started":"2022-07-07T09:20:31.360849Z","shell.execute_reply":"2022-07-07T09:20:31.776114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# texts_to_sequences Process:\ntraining_message = tokenizer.texts_to_sequences(training_message)\ntesting_message = tokenizer.texts_to_sequences(testing_message)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:31.782594Z","iopub.execute_input":"2022-07-07T09:20:31.783035Z","iopub.status.idle":"2022-07-07T09:20:32.486846Z","shell.execute_reply.started":"2022-07-07T09:20:31.782998Z","shell.execute_reply":"2022-07-07T09:20:32.478518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pad_sequences Process:\ntraining_padded = pad_sequences(training_message,maxlen=max_length, truncating=trunc_type, padding=padding_type)\ntesting_padded = pad_sequences(testing_message,maxlen=max_length, truncating=trunc_type, padding=padding_type)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:32.488148Z","iopub.execute_input":"2022-07-07T09:20:32.488546Z","iopub.status.idle":"2022-07-07T09:20:32.659290Z","shell.execute_reply.started":"2022-07-07T09:20:32.488508Z","shell.execute_reply":"2022-07-07T09:20:32.653293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_padded = np.array(training_padded)\ntesting_padded = np.array(testing_padded)\ntraining_labels = np.array(training_labels)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:32.660443Z","iopub.execute_input":"2022-07-07T09:20:32.660808Z","iopub.status.idle":"2022-07-07T09:20:32.676330Z","shell.execute_reply.started":"2022-07-07T09:20:32.660772Z","shell.execute_reply":"2022-07-07T09:20:32.675503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(12)),\n    tf.keras.layers.Dense(1, activation = 'sigmoid')\n])\n\nmodel.compile(\n    loss='binary_crossentropy',\n    optimizer='Adamax',\n    metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall()]\n)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:32.677777Z","iopub.execute_input":"2022-07-07T09:20:32.678429Z","iopub.status.idle":"2022-07-07T09:20:39.367727Z","shell.execute_reply.started":"2022-07-07T09:20:32.678385Z","shell.execute_reply":"2022-07-07T09:20:39.366696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    training_padded, \n    training_labels, \n    epochs = 50, \n    batch_size = 64,  \n    validation_split=0.2\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:20:39.369106Z","iopub.execute_input":"2022-07-07T09:20:39.370109Z","iopub.status.idle":"2022-07-07T09:21:49.374380Z","shell.execute_reply.started":"2022-07-07T09:20:39.370052Z","shell.execute_reply":"2022-07-07T09:21:49.373316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history[\"accuracy\"], color=\"r\")\nplt.plot(history.history[\"val_accuracy\"], color=\"g\")\nplt.legend([\"Training\", \"Validation\"])\nplt.xlabel(\"epochs\")\nplt.ylabel(\"accuracy\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:21:49.376437Z","iopub.execute_input":"2022-07-07T09:21:49.376812Z","iopub.status.idle":"2022-07-07T09:21:49.615490Z","shell.execute_reply.started":"2022-07-07T09:21:49.376771Z","shell.execute_reply":"2022-07-07T09:21:49.614445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predict = model.predict(testing_padded).round().astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:21:49.618762Z","iopub.execute_input":"2022-07-07T09:21:49.619040Z","iopub.status.idle":"2022-07-07T09:21:50.532976Z","shell.execute_reply.started":"2022-07-07T09:21:49.619013Z","shell.execute_reply":"2022-07-07T09:21:50.531899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LSTM Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({'id':df_test['id'],'target':test_predict.ravel()})\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:21:50.535732Z","iopub.execute_input":"2022-07-07T09:21:50.536018Z","iopub.status.idle":"2022-07-07T09:21:50.552649Z","shell.execute_reply.started":"2022-07-07T09:21:50.535991Z","shell.execute_reply":"2022-07-07T09:21:50.551426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transformers from HuggingFace","metadata":{}},{"cell_type":"code","source":"train_dataset = load_dataset('csv', data_files='../input/nlp-getting-started/train.csv',split=\"train\")\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:33:51.937527Z","iopub.execute_input":"2022-07-07T09:33:51.937943Z","iopub.status.idle":"2022-07-07T09:33:52.073886Z","shell.execute_reply.started":"2022-07-07T09:33:51.937907Z","shell.execute_reply":"2022-07-07T09:33:52.072913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric = load_metric('glue', 'sst2')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:33:52.327619Z","iopub.execute_input":"2022-07-07T09:33:52.328664Z","iopub.status.idle":"2022-07-07T09:33:52.577420Z","shell.execute_reply.started":"2022-07-07T09:33:52.328613Z","shell.execute_reply":"2022-07-07T09:33:52.576402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:33:52.786371Z","iopub.execute_input":"2022-07-07T09:33:52.787421Z","iopub.status.idle":"2022-07-07T09:33:52.795360Z","shell.execute_reply.started":"2022-07-07T09:33:52.787376Z","shell.execute_reply":"2022-07-07T09:33:52.794186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = train_dataset.rename_columns({\"target\" : \"label\",\"text\" : \"sentence\"})\ntrain_dataset = train_dataset.train_test_split(test_size=0.1)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:33:54.803446Z","iopub.execute_input":"2022-07-07T09:33:54.803930Z","iopub.status.idle":"2022-07-07T09:33:54.825097Z","shell.execute_reply.started":"2022-07-07T09:33:54.803889Z","shell.execute_reply":"2022-07-07T09:33:54.823881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset['train'][0]","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:34:09.854883Z","iopub.execute_input":"2022-07-07T09:34:09.855932Z","iopub.status.idle":"2022-07-07T09:34:09.863466Z","shell.execute_reply.started":"2022-07-07T09:34:09.855882Z","shell.execute_reply":"2022-07-07T09:34:09.862321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_checkpoint = \"bert-base-uncased\"\nbatch_size = 16\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:40:43.559244Z","iopub.execute_input":"2022-07-07T09:40:43.559690Z","iopub.status.idle":"2022-07-07T09:40:43.564847Z","shell.execute_reply.started":"2022-07-07T09:40:43.559647Z","shell.execute_reply":"2022-07-07T09:40:43.563693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:40:44.468709Z","iopub.execute_input":"2022-07-07T09:40:44.469116Z","iopub.status.idle":"2022-07-07T09:40:45.418391Z","shell.execute_reply.started":"2022-07-07T09:40:44.469082Z","shell.execute_reply":"2022-07-07T09:40:45.417367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_function(examples):\n  return tokenizer(examples['sentence'], truncation=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:40:46.108886Z","iopub.execute_input":"2022-07-07T09:40:46.109888Z","iopub.status.idle":"2022-07-07T09:40:46.115682Z","shell.execute_reply.started":"2022-07-07T09:40:46.109839Z","shell.execute_reply":"2022-07-07T09:40:46.114531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoded_dataset = train_dataset.map(preprocess_function, batched=True)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:40:46.968434Z","iopub.execute_input":"2022-07-07T09:40:46.968829Z","iopub.status.idle":"2022-07-07T09:40:46.985366Z","shell.execute_reply.started":"2022-07-07T09:40:46.968797Z","shell.execute_reply":"2022-07-07T09:40:46.984128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_labels = 2\nmodel = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:40:47.619440Z","iopub.execute_input":"2022-07-07T09:40:47.620200Z","iopub.status.idle":"2022-07-07T09:40:49.290757Z","shell.execute_reply.started":"2022-07-07T09:40:47.620161Z","shell.execute_reply":"2022-07-07T09:40:49.289502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name = model_checkpoint.split(\"/\")[-1]\nmetric_name = 'accuracy'\nargs = TrainingArguments(\n    f\"{model_name}-finetuned-dst_clf\",\n    per_device_train_batch_size=batch_size,\n    evaluation_strategy = \"epoch\",\n    save_strategy = \"epoch\",\n    per_device_eval_batch_size=batch_size,\n    num_train_epochs=3,\n    weight_decay=0.01,\n    metric_for_best_model =metric_name,\n    learning_rate=2e-5\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:40:49.339117Z","iopub.execute_input":"2022-07-07T09:40:49.339467Z","iopub.status.idle":"2022-07-07T09:40:49.350945Z","shell.execute_reply.started":"2022-07-07T09:40:49.339420Z","shell.execute_reply":"2022-07-07T09:40:49.349572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_metrics(eval_pred):\n    predictions, labels = eval_pred\n    predictions = np.argmax(predictions, axis=1)\n    return metric.compute(predictions=predictions, references=labels)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:40:50.944043Z","iopub.execute_input":"2022-07-07T09:40:50.945235Z","iopub.status.idle":"2022-07-07T09:40:50.955110Z","shell.execute_reply.started":"2022-07-07T09:40:50.945186Z","shell.execute_reply":"2022-07-07T09:40:50.954163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:34:31.007759Z","iopub.execute_input":"2022-07-07T09:34:31.008105Z","iopub.status.idle":"2022-07-07T09:34:31.013056Z","shell.execute_reply.started":"2022-07-07T09:34:31.008074Z","shell.execute_reply":"2022-07-07T09:34:31.011977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.memory_summary(device=None, abbreviated=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:34:33.232951Z","iopub.execute_input":"2022-07-07T09:34:33.233298Z","iopub.status.idle":"2022-07-07T09:34:33.241958Z","shell.execute_reply.started":"2022-07-07T09:34:33.233267Z","shell.execute_reply":"2022-07-07T09:34:33.240741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = Trainer(\n    model,\n    args,\n    train_dataset=encoded_dataset[\"train\"],\n    eval_dataset=encoded_dataset['test'],\n    tokenizer=tokenizer,\n    compute_metrics=compute_metrics\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:41:02.683984Z","iopub.execute_input":"2022-07-07T09:41:02.684374Z","iopub.status.idle":"2022-07-07T09:41:04.567748Z","shell.execute_reply.started":"2022-07-07T09:41:02.684338Z","shell.execute_reply":"2022-07-07T09:41:04.566312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:30:18.644788Z","iopub.status.idle":"2022-07-07T09:30:18.645245Z","shell.execute_reply.started":"2022-07-07T09:30:18.645013Z","shell.execute_reply":"2022-07-07T09:30:18.645037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:30:18.647098Z","iopub.status.idle":"2022-07-07T09:30:18.648087Z","shell.execute_reply.started":"2022-07-07T09:30:18.647821Z","shell.execute_reply":"2022-07-07T09:30:18.647848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.evaluate()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:30:18.649329Z","iopub.status.idle":"2022-07-07T09:30:18.650325Z","shell.execute_reply.started":"2022-07-07T09:30:18.650059Z","shell.execute_reply":"2022-07-07T09:30:18.650084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_output = model.cpu()","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:30:18.651700Z","iopub.status.idle":"2022-07-07T09:30:18.652673Z","shell.execute_reply.started":"2022-07-07T09:30:18.652358Z","shell.execute_reply":"2022-07-07T09:30:18.652388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(text):\n  token_output = tokenizer(text, truncation=True,return_tensors='pt')\n  output = model.forward(input_ids=token_output['input_ids'],attention_mask=token_output['attention_mask'])\n  return np.argmax(output['logits'].detach().numpy(), axis=1)[0]\n  \n\neval_df = pd.read_csv(\"../input/nlp-getting-started/test.csv\")\neval_df['target'] = eval_df['text'].apply(predict)\neval_df[['id','target']].to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:30:18.713721Z","iopub.execute_input":"2022-07-07T09:30:18.714499Z","iopub.status.idle":"2022-07-07T09:30:18.833358Z","shell.execute_reply.started":"2022-07-07T09:30:18.714437Z","shell.execute_reply":"2022-07-07T09:30:18.831754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install GPUtil\n\nimport torch\nfrom GPUtil import showUtilization as gpu_usage\nfrom numba import cuda\n\ndef free_gpu_cache():\n    print(\"Initial GPU Usage\")\n    gpu_usage()                             \n\n    torch.cuda.empty_cache()\n\n    cuda.select_device(0)\n    cuda.close()\n    cuda.select_device(0)\n\n    print(\"GPU Usage after emptying the cache\")\n    gpu_usage()\n\nfree_gpu_cache()                           \n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T09:39:23.781688Z","iopub.execute_input":"2022-07-07T09:39:23.782700Z","iopub.status.idle":"2022-07-07T09:39:34.785113Z","shell.execute_reply.started":"2022-07-07T09:39:23.782658Z","shell.execute_reply":"2022-07-07T09:39:34.783543Z"},"trusted":true},"execution_count":null,"outputs":[]}]}