{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Text classification: We are going to developp a model able to classify text in 6 categories.","metadata":{}},{"cell_type":"markdown","source":"## Import","metadata":{}},{"cell_type":"code","source":"import os\nimport torch\nfrom torchtext import data\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport random","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:32:22.456931Z","iopub.execute_input":"2021-09-30T08:32:22.457218Z","iopub.status.idle":"2021-09-30T08:32:24.292152Z","shell.execute_reply.started":"2021-09-30T08:32:22.457160Z","shell.execute_reply":"2021-09-30T08:32:24.290973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reproducibility","metadata":{}},{"cell_type":"code","source":"SEED = 2021\n\ndef reproducibility(seed=2021):\n    \n    # seed for random, os, numpy and torch librayr\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    \n    # seed fo cuda\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nreproducibility(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:32:24.294274Z","iopub.execute_input":"2021-09-30T08:32:24.294916Z","iopub.status.idle":"2021-09-30T08:32:24.308894Z","shell.execute_reply.started":"2021-09-30T08:32:24.294852Z","shell.execute_reply":"2021-09-30T08:32:24.307736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"data_path = \"../input/genderbased-violence-tweet-classification/Gender-Based Violence Tweet Classification Challenge/Train.csv\"\nmodels_folder = \"./\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"donnees = pd.read_csv(data_path, index_col=\"Tweet_ID\")\n\nsns.set_style(\"dark\")\nsns.set_context(\"notebook\")\nplt.figure(figsize=(8, 5))\nplt.subplot(111)\nsns.countplot(data=donnees, x=\"type\")\nplt.xticks(rotation=90)\nplt.legend(loc=\"upper right\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:32:24.311035Z","iopub.execute_input":"2021-09-30T08:32:24.312095Z","iopub.status.idle":"2021-09-30T08:32:25.130286Z","shell.execute_reply.started":"2021-09-30T08:32:24.311752Z","shell.execute_reply":"2021-09-30T08:32:25.129192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_data_per_class = donnees[\"type\"].value_counts()\nnum_data_per_class","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:32:25.135584Z","iopub.execute_input":"2021-09-30T08:32:25.138266Z","iopub.status.idle":"2021-09-30T08:32:25.171012Z","shell.execute_reply.started":"2021-09-30T08:32:25.138205Z","shell.execute_reply":"2021-09-30T08:32:25.170043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"percentage_data_per_class = 100*donnees[\"type\"].value_counts()/donnees.shape[0]\npercentage_data_per_class","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:32:25.176295Z","iopub.execute_input":"2021-09-30T08:32:25.179224Z","iopub.status.idle":"2021-09-30T08:32:25.238504Z","shell.execute_reply.started":"2021-09-30T08:32:25.179162Z","shell.execute_reply":"2021-09-30T08:32:25.237555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"1 / (100*donnees[\"type\"].value_counts()/donnees.shape[0])","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:32:25.243328Z","iopub.execute_input":"2021-09-30T08:32:25.245647Z","iopub.status.idle":"2021-09-30T08:32:25.276166Z","shell.execute_reply.started":"2021-09-30T08:32:25.245588Z","shell.execute_reply":"2021-09-30T08:32:25.275191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Resampling","metadata":{}},{"cell_type":"code","source":"labels = [\"sexual_violence\", \"Physical_violence\", \"emotional_violence\",\n         \"economic_violence\", \"Harmful_Traditional_practice\"]","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:32:25.281314Z","iopub.execute_input":"2021-09-30T08:32:25.284226Z","iopub.status.idle":"2021-09-30T08:32:25.291783Z","shell.execute_reply.started":"2021-09-30T08:32:25.284158Z","shell.execute_reply":"2021-09-30T08:32:25.290476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import math\npercent = 0.7\n\nnew_train_data = pd.concat([donnees.loc[donnees[\"type\"]==t, :].sample(math.ceil(percent*num_data_per_class[t])) for t in labels]\n                , axis=0)\n\nnew_valid_data = pd.concat([donnees.loc[donnees[\"type\"]==t, :].sample(math.floor((1-percent)*num_data_per_class[t])) for t in labels]\n                , axis=0)\n\nnew_train_data.to_csv(\"./new_train.csv\")\nnew_valid_data.to_csv(\"./new_valid.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:34:17.577225Z","iopub.execute_input":"2021-09-30T08:34:17.577734Z","iopub.status.idle":"2021-09-30T08:34:18.364653Z","shell.execute_reply.started":"2021-09-30T08:34:17.577604Z","shell.execute_reply":"2021-09-30T08:34:18.363805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"# Now, let us see how to preprocess the text using field objects.\n# There are 2 different types of field objects – Field and LabelField.\n# Field concern the preprocessing tehcnics to apply on text ans LabelField concern ones toapply for Label.\n\nTEXT = data.Field(tokenize='spacy', batch_first=True, include_lengths=True)\nLABEL = data.LabelField()\n\n# This variable is used to read file data.\nfields = [('Tweet_ID',None), ('tweet',TEXT), ('type', LABEL)]\n\n# Loading custom dataset    \ntrain_data=data.TabularDataset(path = './new_train.csv'\n                                  ,format = 'csv', fields = fields, skip_header = True)\nvalid_data=data.TabularDataset(path = './new_valid.csv'\n                                  ,format = 'csv', fields = fields, skip_header = True)\n\n# Print preprocessed text\nprint(vars(train_data.examples[0]))","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:34:26.746293Z","iopub.execute_input":"2021-09-30T08:34:26.746636Z","iopub.status.idle":"2021-09-30T08:34:44.624122Z","shell.execute_reply.started":"2021-09-30T08:34:26.746535Z","shell.execute_reply":"2021-09-30T08:34:44.622908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split data to train and validation set ","metadata":{}},{"cell_type":"code","source":"# Check whether cuda is available\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  \n\n# Set batch size\nBATCH_SIZE = 64\n\n# Load iterators\ntrain_iterator, valid_iterator = data.BucketIterator.splits(\n    (train_data, valid_data),\n    batch_size = BATCH_SIZE,\n    sort_key = lambda x: len(x.tweet),\n    sort_within_batch=True,\n    device = device)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:34:44.736321Z","iopub.execute_input":"2021-09-30T08:34:44.736668Z","iopub.status.idle":"2021-09-30T08:34:44.748154Z","shell.execute_reply.started":"2021-09-30T08:34:44.736611Z","shell.execute_reply":"2021-09-30T08:34:44.746623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing input and output sequences","metadata":{}},{"cell_type":"code","source":"# Initialize glove embeddings\nTEXT.build_vocab(train_data, min_freq=3, vectors = \"glove.6B.100d\")  \nLABEL.build_vocab(train_data)\n\n# No. of unique tokens in text\nprint(\"Size of TEXT vocabulary:\",len(TEXT.vocab))\n\n# No. of unique tokens in label\nprint(\"Size of LABEL vocabulary:\",len(LABEL.vocab))\n\n# Commonly used words\nprint(\"Commonly used words\", TEXT.vocab.freqs.most_common(10))  \n\n# Word dictionary\nprint(\"Word dictionary\", TEXT.vocab.stoi.items())   ","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build the model","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\n\nclass classifier(nn.Module):\n    \n    #define all the layers used in model\n    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, \n                 bidirectional, dropout):\n        \n        #Constructor\n        super().__init__()          \n        \n        #embedding layer\n        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n        \n        #lstm layer\n        self.lstm = nn.LSTM(embedding_dim, \n                           hidden_dim, \n                           num_layers=n_layers, \n                           bidirectional=bidirectional, \n                           dropout=dropout,\n                           batch_first=True)\n        \n        #dense layer\n        self.fc = nn.Linear(hidden_dim * 2, output_dim)\n        \n        #activation function\n        self.act = nn.Sigmoid()\n        \n    def forward(self, text, text_lengths):\n        \n        #text = [batch size,sent_length]\n        embedded = self.embedding(text)\n        #embedded = [batch size, sent_len, emb dim]\n      \n        #packed sequence\n        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths, batch_first=True)\n        \n        packed_output, (hidden, cell) = self.lstm(packed_embedded)\n        #hidden = [batch size, num layers * num directions,hid dim]\n        #cell = [batch size, num layers * num directions,hid dim]\n        \n        #concat the final forward and backward hidden state\n        hidden = torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1)\n                \n        #hidden = [batch size, hid dim * num directions]\n        dense_outputs=self.fc(hidden)\n\n        #Final activation function\n        outputs=self.act(dense_outputs)\n        \n        return outputs","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:38:19.947903Z","iopub.execute_input":"2021-09-30T08:38:19.948286Z","iopub.status.idle":"2021-09-30T08:38:19.960107Z","shell.execute_reply.started":"2021-09-30T08:38:19.948217Z","shell.execute_reply":"2021-09-30T08:38:19.958946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define hyperparameters\nsize_of_vocab = len(TEXT.vocab)\nembedding_dim = 100\nnum_hidden_nodes = 32\nnum_output_nodes = 5\nnum_layers = 2\nbidirection = True\ndropout = 0.2\n\n#instantiate the model\nmodel = classifier(size_of_vocab, embedding_dim, num_hidden_nodes,num_output_nodes, num_layers, \n                   bidirectional = True, dropout = dropout)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:38:31.797962Z","iopub.execute_input":"2021-09-30T08:38:31.798263Z","iopub.status.idle":"2021-09-30T08:38:31.823856Z","shell.execute_reply.started":"2021-09-30T08:38:31.798208Z","shell.execute_reply":"2021-09-30T08:38:31.822953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#architecture\nprint(model)\n\n#No. of trianable parameters\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n    \nprint(f'The model has {count_parameters(model):,} trainable parameters')\n\n#Initialize the pretrained embedding\npretrained_embeddings = TEXT.vocab.vectors\nmodel.embedding.weight.data.copy_(pretrained_embeddings)\n\nprint(pretrained_embeddings.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:38:46.455996Z","iopub.execute_input":"2021-09-30T08:38:46.456295Z","iopub.status.idle":"2021-09-30T08:38:46.472074Z","shell.execute_reply.started":"2021-09-30T08:38:46.456239Z","shell.execute_reply":"2021-09-30T08:38:46.471193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\n\n#define optimizer and loss\noptimizer = optim.Adam(model.parameters())\ncriterion = nn.BCELoss(weight=torch.tensor([0.012145, 0.066683, 0.609063, 1.827189, 2.109043])) \n\n#define metric\ndef binary_accuracy(preds, y):\n    #round predictions to the closest integer\n    rounded_preds = torch.round(preds)\n    \n    correct = (rounded_preds == y).float() \n    acc = correct.sum().sum() / len(correct)\n    return acc\n    \n#push to cuda if available\nmodel = model.to(device)\ncriterion = criterion.to(device)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T09:00:35.477883Z","iopub.execute_input":"2021-09-30T09:00:35.478175Z","iopub.status.idle":"2021-09-30T09:00:35.488841Z","shell.execute_reply.started":"2021-09-30T09:00:35.478120Z","shell.execute_reply":"2021-09-30T09:00:35.487010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, iterator, optimizer, criterion):\n    \n    #initialize every epoch \n    epoch_loss = 0\n    epoch_acc = 0\n    \n    #set the model in training phase\n    model.train()  \n    \n    for batch in iterator:\n        \n        #resets the gradients after every batch\n        optimizer.zero_grad()   \n        \n        #retrieve text and no. of words\n        text, text_lengths = batch.tweet   \n        #convert to 1D tensor\n        predictions = model(text, text_lengths)\n        \n        #find label\n        batch_size = batch.type.shape[0]\n        target = torch.zeros((batch_size, num_output_nodes))\n        target[[i for i in range(batch_size)], batch.type.to(\"cpu\").numpy()] = 1.0\n        target = target.to(device)\n        \n        #compute the loss\n        loss = criterion(predictions, target)        \n        \n        #compute the binary accuracy\n        acc = binary_accuracy(predictions, target)   \n        \n        #backpropage the loss and compute the gradients\n        loss.backward()       \n        \n        #update the weights\n        optimizer.step()      \n        \n        #loss and accuracy\n        epoch_loss += loss.item()  \n        epoch_acc += acc.item()    \n        \n    return epoch_loss / len(iterator), epoch_acc / len(iterator)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T09:00:40.183631Z","iopub.execute_input":"2021-09-30T09:00:40.183959Z","iopub.status.idle":"2021-09-30T09:00:40.193010Z","shell.execute_reply.started":"2021-09-30T09:00:40.183898Z","shell.execute_reply":"2021-09-30T09:00:40.190963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, iterator, criterion):\n    \n    #initialize every epoch\n    epoch_loss = 0\n    epoch_acc = 0\n\n    #deactivating dropout layers\n    model.eval()\n    \n    #deactivates autograd\n    with torch.no_grad():\n    \n        for batch in iterator:\n        \n            #retrieve text and no. of words\n            text, text_lengths = batch.tweet\n            \n            #convert to 1d tensor\n            predictions = model(text, text_lengths)\n            \n            #compute the label\n            batch_size = batch.type.shape[0]\n            target = torch.zeros((batch_size, num_output_nodes))\n            target[[i for i in range(batch_size)], batch.type.to(\"cpu\").numpy()] = 1.0\n            target = target.to(device)\n            \n            #compute loss and accuracy\n            loss = criterion(predictions, target)        \n            acc = binary_accuracy(predictions, target)\n            \n            #keep track of loss and accuracy\n            epoch_loss += loss.item()\n            epoch_acc += acc.item()\n        \n    return epoch_loss / len(iterator), epoch_acc / len(iterator)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T09:01:23.813074Z","iopub.execute_input":"2021-09-30T09:01:23.813454Z","iopub.status.idle":"2021-09-30T09:01:23.821996Z","shell.execute_reply.started":"2021-09-30T09:01:23.813384Z","shell.execute_reply":"2021-09-30T09:01:23.820707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_EPOCHS = 5\nbest_valid_loss = float('inf')\n\ntrain_loss = np.zeros((1, N_EPOCHS))\nvalid_loss = np.zeros((1, N_EPOCHS))\ntrain_acc = np.zeros((1, N_EPOCHS))\nvalid_acc = np.zeros((1, N_EPOCHS))\n\nfor epoch in range(N_EPOCHS):\n     \n    #train the model\n    train_loss[0, epoch], train_acc[0, epoch] = train(model, train_iterator, optimizer, criterion)\n    \n    #evaluate the model\n    valid_loss[0, epoch], valid_acc[0, epoch] = evaluate(model, valid_iterator, criterion)\n    \n    #save the best model\n    if valid_loss[0, epoch] < best_valid_loss:\n        best_valid_loss = valid_loss[0, epoch]\n        torch.save(model.state_dict(), models_folder+'saved_weights('+str(epoch)+').pt')\n    \n    print(f\"EPOCH {epoch}%\")\n    print(f'\\tTrain Loss: {train_loss[0, epoch]:.3f} | Train Acc: {train_acc[0, epoch]*100:.2f}%')\n    print(f'\\t Val. Loss: {valid_loss[0, epoch]:.3f} |  Val. Acc: {valid_acc[0, epoch]*100:.2f}%')\n","metadata":{"execution":{"iopub.status.busy":"2021-09-30T09:02:21.156708Z","iopub.execute_input":"2021-09-30T09:02:21.157042Z","iopub.status.idle":"2021-09-30T09:03:24.654848Z","shell.execute_reply.started":"2021-09-30T09:02:21.156988Z","shell.execute_reply":"2021-09-30T09:03:24.653841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the complexity graph\nplt.plot(train_loss[0,:], label = \"train\")\nplt.plot(valid_loss[0,:], label = \"validation\")\nplt.xlabel('epoch')\n# Set the y axis label of the current axis.\nplt.ylabel('error')\n# Set a title of the current axes.\nplt.title('Complexity graph')\n# show a legend on the plot\nplt.legend()\n# Display a figure.\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:42:52.205129Z","iopub.execute_input":"2021-09-30T08:42:52.205462Z","iopub.status.idle":"2021-09-30T08:42:52.461290Z","shell.execute_reply.started":"2021-09-30T08:42:52.205407Z","shell.execute_reply":"2021-09-30T08:42:52.460021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load weights\npath='./saved_weights(4).pt'\nmodel.load_state_dict(torch.load(path));\nmodel.eval();\n\n#inference \nimport spacy\nnlp = spacy.load('en') # I replace \"en\" by \"fr\"\n\nCLASS = [\"Sexual violence\", \"Physical violence\", \"Emotional Violence\",\n         \"Economic Violence\", \"Harmful traditional practice\"]\n\ndef predict(model, sentence):\n    tokenized = [tok.text for tok in nlp.tokenizer(sentence)]  #tokenize the sentence \n    indexed = [TEXT.vocab.stoi[t] for t in tokenized]          #convert to integer sequence\n    \n    length = len(indexed)\n    length_tensor = torch.LongTensor([length]).to('cpu')\n    \n    my_tensor = torch.LongTensor(indexed).to(device)           #convert to tensor\n    my_tensor = torch.reshape(my_tensor, (1, length))          #reshape in form of batch,no. of words\n    \n    prediction = model(my_tensor, length_tensor)               #prediction \n    i = prediction.argmax().to('cpu').numpy()\n    \n    return CLASS[i]","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:43:26.788966Z","iopub.execute_input":"2021-09-30T08:43:26.789317Z","iopub.status.idle":"2021-09-30T08:43:27.400677Z","shell.execute_reply.started":"2021-09-30T08:43:26.789258Z","shell.execute_reply":"2021-09-30T08:43:27.399828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make predictions\npredict(model, \"he humiliate me in front of my children\")","metadata":{"execution":{"iopub.status.busy":"2021-09-30T08:45:49.883408Z","iopub.execute_input":"2021-09-30T08:45:49.883755Z","iopub.status.idle":"2021-09-30T08:45:49.896280Z","shell.execute_reply.started":"2021-09-30T08:45:49.883700Z","shell.execute_reply":"2021-09-30T08:45:49.894850Z"},"trusted":true},"execution_count":null,"outputs":[]}]}