{"cells":[{"metadata":{"_uuid":"13d7c8ae7ec76419a20756533f0b66b0a727b14e"},"cell_type":"markdown","source":"###  importing necessary things"},{"metadata":{"trusted":true,"_uuid":"4f6ad4da5267f048d9d97ca19a19a215c1dbc757"},"cell_type":"code","source":"import torch\nfrom torchtext import data\n\nSEED = 1234\nimport pandas as pd\nimport numpy as np\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport torchtext\n\nimport nltk\n\nimport random\nfrom sklearn.metrics import classification_report\n\nimport pyprind\n%matplotlib inline  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5b28ff172982ef0a7d96ff71540ee252958a677"},"cell_type":"markdown","source":"## preparing sheets for train, validation and test set from entire data\n\ntorchtext can load a custom dataset in csv and json. I chose csv for my convinience."},{"metadata":{"trusted":true,"_uuid":"8fdabf7c6bdd2aff8a7bdb1eeea976be561eb2fa"},"cell_type":"code","source":"main_df = pd.read_csv(\"../input/train.csv\")\nprint(main_df.shape)\nmain_df = main_df.sample(n=main_df.shape[0])\nmain_df = main_df[[\"question_text\", \"target\"]]\nmain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2058ed746742ebc045a524f802d2ba134097f18b"},"cell_type":"code","source":"main_df.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3284e9a82e44977227a2f94cbca8d882db10d7b"},"cell_type":"code","source":"o_class = main_df.loc[main_df.target == 0, :]\nl_class = main_df.loc[main_df.target == 1, :]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9ec844069657964e32a3e815ad2fc85aba605de"},"cell_type":"markdown","source":"preparing balanced test and validation set"},{"metadata":{"trusted":true,"_uuid":"2f2f6caa8ab778e965c8ca8d8ae10c7a2cb8faa9"},"cell_type":"code","source":"# splitting test and train \ntest_o = o_class.iloc[:10000, :]\ntest_l = l_class.iloc[:10000, :]\n\nvalid_o = o_class.iloc[10000:20000, :]\nvalid_l = l_class.iloc[10000:20000, :]\n\ntrain_o = o_class.iloc[20000:, :]\ntrain_l = l_class.iloc[20000:, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9ea9a9bdad2bfd9d6175efc14ab66ceb31d55c3"},"cell_type":"code","source":"train = pd.concat([train_o, train_l], axis=0)\nprint(train.shape)\n\nvalid = pd.concat([valid_o, valid_l], axis=0)\nprint(valid.shape)\n\ntest = pd.concat([test_o, test_l], axis=0)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47c8476d41ef118eea4b024771337f536cb27c02"},"cell_type":"code","source":"train.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd064751fc60acadb207578a2b0136b7dbb23ec9"},"cell_type":"code","source":"test.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f211e308476f96252a74069239d31c6212136b25"},"cell_type":"code","source":"valid.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee697ef941286df14c4451e22c294bd1fbdce9e4"},"cell_type":"code","source":"!mkdir torchtext_data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8724d0cee26d9be5fad83ed3618aeb23135a20f9"},"cell_type":"markdown","source":"Saving files to disk"},{"metadata":{"trusted":true,"_uuid":"6a461b30557c96d478e45f9bae4da9f2dc6ac53b"},"cell_type":"code","source":"train.to_csv(\"torchtext_data/train.csv\", index=False)\ntest.to_csv(\"torchtext_data/test.csv\", index=False)\nvalid.to_csv(\"torchtext_data/valid.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad01428d498ebd50d4dc87aeb2368891b1646b36"},"cell_type":"code","source":"# freeing up some memory\ndel main_df, train, test, valid, train_l, train_o, test_l, test_o, valid_l,valid_o, o_class, l_class","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e25298e8db4aed80109ea1446ef6ad63a55fd55e"},"cell_type":"markdown","source":"I am going to use spacy tokenizer. "},{"metadata":{"trusted":true,"_uuid":"f8294f0289ba0f7eac11f5cee4995e2ed208c592"},"cell_type":"code","source":"import spacy\nspacy_en = spacy.load('en')\n# nltk.download('punkt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7306cd2279b7310ae054ad08f9137c49f8737553"},"cell_type":"code","source":"is_cuda = torch.cuda.is_available()\nprint(\"Cuda Status on system is {}\".format(is_cuda))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"61c0ccf6d212d020b889c85d12a7ab3994254819"},"cell_type":"markdown","source":"### using torchtext to load text data "},{"metadata":{"trusted":true,"_uuid":"1560985656a01056d03dbaadbb282f846a66c027"},"cell_type":"code","source":"# sample tokenizer which you can use\ndef tokenizer(text):\n    return [tok for tok in nltk.word_tokenize(text)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75c3dcd28e083e800cb4337c962747c377f40024"},"cell_type":"code","source":"# tokenizer = \"spacy\" uses spacy's tokenizer\nTEXT = data.Field(sequential=True, tokenize=\"spacy\")\nLABEL = data.LabelField(dtype=torch.long, sequential=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64d9c121e8af2d9c7d21408e4ac8358bd77f3829"},"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89d295d09bac577243d4b92808f027d5586c4c53"},"cell_type":"code","source":"# loading train, test and validation data \ntrain_data, valid_data, test_data = data.TabularDataset.splits(\n    path=\"torchtext_data/\", train=\"train.csv\", \n    validation=\"valid.csv\", test=\"test.csv\",format=\"csv\", skip_header=True, \n    fields=[('Text', TEXT), ('Label', LABEL)]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac9b65ee4356af7bb899384998edcb4ad6179149"},"cell_type":"code","source":"print(f'Number of training examples: {len(train_data)}')\nprint(f'Number of valid examples: {len(valid_data)}')\nprint(f'Number of testing examples: {len(test_data)}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8abf2d6f2d9833c7e7864f84b821952f66b5e7c9"},"cell_type":"code","source":"TEXT.build_vocab(train_data, vectors=torchtext.vocab.Vectors(\"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\"), \n                 max_size=20000, min_freq=10)\nLABEL.build_vocab(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a64effa7ba23288eadc014310313de64f8d4a3fe"},"cell_type":"code","source":"# if you dont wanna load any word vectors\n# TEXT.build_vocab(train_data, max_size=50000)\n# LABEL.build_vocab(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"742ff47bf79abb7e77a5233ed812d38ea20e775b"},"cell_type":"code","source":"print(f\"Unique tokens in TEXT vocabulary: {len(TEXT.vocab)}\")\nprint(f\"Unique tokens in LABEL vocabulary: {len(LABEL.vocab)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebdd4d50b73bb5439d66738022d7d8b8a45ba8aa"},"cell_type":"code","source":"BATCH_SIZE = 20\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# keep in mind the sort_key option \ntrain_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n    (train_data, valid_data, test_data), sort_key=lambda x: len(x.Text),\n    batch_size=BATCH_SIZE,\n    device=device)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aeaf0d81fc294e79f8a58c171e88e05cc6bec7c"},"cell_type":"code","source":"LABEL.vocab.freqs\n# torch.cuda.empty_cache()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1625024046771ec2ade425e5fbba9a3730c2284b"},"cell_type":"code","source":"class RNN(nn.Module):\n    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):\n        super().__init__()\n        \n        self.embedding = nn.Embedding(input_dim, embedding_dim)\n        self.rnn = nn.RNN(embedding_dim, hidden_dim)\n        self.fc = nn.Linear(hidden_dim, output_dim)\n        \n    def forward(self, x):\n\n        #x = [sent len, batch size]\n        \n        embedded = self.embedding(x)\n        \n        #embedded = [sent len, batch size, emb dim]\n        \n        output, hidden = self.rnn(embedded)\n        \n        #output = [sent len, batch size, hid dim]\n        #hidden = [1, batch size, hid dim]\n        \n        assert torch.equal(output[-1,:,:], hidden.squeeze(0))\n        \n        out = self.fc(hidden)\n        return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2923d4b92199582891d27b86edd9b2802b8bd00"},"cell_type":"code","source":"INPUT_DIM = len(TEXT.vocab)\nEMBEDDING_DIM = 300\nHIDDEN_DIM = 374\nOUTPUT_DIM = 2\n\nmodel = RNN(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"810402dfc6f9e1726f999bc5dbecf27e06ce1ce9"},"cell_type":"code","source":"pretrained_embeddings = TEXT.vocab.vectors\n\nprint(pretrained_embeddings.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b2b1ffda745b767a6a311a165aac33906354fd0"},"cell_type":"code","source":"model.embedding.weight.data = pretrained_embeddings.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"363f747a56e6c8fa034bc90fd49b27226096163d"},"cell_type":"code","source":"class_weights = torch.tensor([1.0, 15.0]).cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45cf2e2ec10dd321c2ef146a63d54d6c34b47474"},"cell_type":"code","source":"optimizer = optim.SGD(model.parameters(), lr=1e-3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a1221bcd7373b37e6877432174b7d21079d8d7c"},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss(weight=class_weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"263c73d30dafa9216824bc8189ccb4b5276ef0d2"},"cell_type":"code","source":"model = model.to(device)\ncriterion = criterion.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03ba833073a278af4b6f45e9fae8f5c9e7225ebb"},"cell_type":"code","source":"def binary_accuracy(preds, y):\n    \"\"\"\n    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n    \"\"\"\n\n    preds, ind= torch.max(F.softmax(preds, dim=-1), 1)\n    correct = (ind == y).float()\n    acc = correct.sum()/float(len(correct))\n    return acc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c38f546938c0b2dd7416f80bc5c59c022d70584"},"cell_type":"code","source":"def train(model, iterator, optimizer, criterion):\n    \n    epoch_loss = 0\n    epoch_acc = 0\n    \n    model.train()\n    bar = pyprind.ProgBar(len(iterator), bar_char='█')\n    for batch in iterator:\n        \n        optimizer.zero_grad()\n                \n        predictions = model(batch.Text).squeeze(0)\n#         print(predictions.shape, batch.Label.shape, model(batch.Text).shape)\n        loss = criterion(predictions, batch.Label)\n#         print(loss.shape)\n        acc = binary_accuracy(predictions, batch.Label)\n        \n        loss.backward()\n        \n        optimizer.step()\n        \n        epoch_loss += loss.item()\n        epoch_acc += acc.item()\n        bar.update()\n    return epoch_loss / len(iterator), epoch_acc / len(iterator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca252870207019254e0de7a95b522f46a60ae33b"},"cell_type":"code","source":"def evaluate(model, iterator, criterion):\n    \n    epoch_loss = 0\n    epoch_acc = 0\n    \n    model.eval()\n    \n    with torch.no_grad():\n        bar = pyprind.ProgBar(len(iterator), bar_char='█')\n        for batch in iterator:\n\n            predictions = model(batch.Text).squeeze(0)\n            \n            loss = criterion(predictions, batch.Label)\n            \n            acc = binary_accuracy(predictions, batch.Label)\n\n            epoch_loss += loss.item()\n            epoch_acc += acc.item()\n            bar.update()\n    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d71751b08c218c42b75e786428df5f0696147de2"},"cell_type":"code","source":"N_EPOCHS = 2\n\nfor epoch in range(N_EPOCHS):\n\n    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)\n    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)\n    \n    print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}% | Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:.2f}% |')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0504ec0e7b273e5db118a36f9e5aafdc86160c4"},"cell_type":"code","source":"test_loss, test_acc = evaluate(model, test_iterator, criterion)\n\nprint(f'| Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}% |')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba9f4f2472d368922dbee3a5e16d821aa5dda59a"},"cell_type":"code","source":"def predict_sentiment(sentence):\n    tokenized = [tok for tok in sentence.split()]\n    indexed = [TEXT.vocab.stoi[t] for t in tokenized]\n    tensor = torch.LongTensor(indexed).to(device)\n    \n    tensor = tensor.unsqueeze(1)\n#     print(tensor.shape)\n    prediction = model(tensor)\n#     print(prediction)\n    preds, ind= torch.max(F.softmax(prediction.squeeze(0), dim=-1), 1)\n#     print(preds)\n    return preds, ind","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bd7138d2bcba357baa1e3ed7e85f27b40e5ff51"},"cell_type":"code","source":"text = \"My voice range is A2-C5. My chest voice goes up to F4. Included sample in my higher chest range. What is my voice type?\"\npredict_sentiment(text)[1].item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30227e1bb05ddf1525b17fbdea511f2bc01c6c01"},"cell_type":"code","source":"# calculating classification report\ntest = pd.read_csv(\"torchtext_data/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64238bc08f711af066b2a4817e5f742eb45bdaae"},"cell_type":"code","source":"pre = [predict_sentiment(k)[1].item() for k in test.question_text]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eec0b5c6b7e55fb70e403cfbc55adbfdc602f591"},"cell_type":"code","source":"print(classification_report(test.target, pre))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"944ddabc6ca3dc7c1ba9cbeed5e9fdc3fa36f735"},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/test.csv\")\nprint(test_df.shape)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e58fa74dabe768cd4f704edd4909569c353c12b"},"cell_type":"code","source":"test_predictions = [int(predict_sentiment(k)[1].item()) for k in test_df.question_text]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3792698f40b1b57456d65a7465aff393af6d14f8"},"cell_type":"code","source":"out_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = test_predictions\nprint(out_df.shape)\nout_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16e60e20f9a1f1a41775f27cfec470b3607b310b"},"cell_type":"code","source":"out_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71e8bc382b814ed3081abebc9a1b3dc0ee1badb1"},"cell_type":"code","source":"out_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4aedb28f909cb89bbe2118bd95efa8d0d52540ef"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}