{"cells":[{"metadata":{},"cell_type":"markdown","source":"********# EDA <a id=\"1\"></a>\n"},{"metadata":{},"cell_type":"markdown","source":"> ## Preparing the ground <a id=\"1.1\"></a>"},{"metadata":{},"cell_type":"markdown","source":"### Install and import necessary packages"},{"metadata":{"trusted":true},"cell_type":"code","source":"!/opt/conda/bin/python3.7 -m pip install --upgrade pip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"!pip install -q pyicu\n!pip install -q pycld2\n!pip install -q polyglot\n!pip install -q textstat\n!pip install -q googletrans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wandb login 718aa8083f5b57ce1fc331cf71855becebf6c109","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport re\nimport folium\nimport textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport palettable\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\n\nfrom collections import Counter\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.corpus import stopwords\n\nstopword=set(STOPWORDS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1.Load the datasets & Brief description\nWe first have a brief view of the information of these datasets, including the percentage of their own data\n\nThen we'll show top 5 & bottum 5 in each of them to see the format of the four datasets to roughly see the format of these data.\n\nAlso, we plotted a histogram of the length distribution of the data.\n\n\\*Note: The test_with_label dataset is almost the same with test dataset, except that it has a label \"True/False\" for every tweet. So we'll leave out the same part between those two datasets."},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = \"../input/mydata/\"\nos.listdir(DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_PATH = DATA_PATH + \"Constraint_English_Test.xlsx\"\nVAL_PATH = DATA_PATH + \"Constraint_English_Val.xlsx\"\nTRAIN_PATH = DATA_PATH + \"Constraint_English_Train.xlsx\"\nTEST_WITH_LABEL_PATH = DATA_PATH + \"english_test_with_labels.xlsx\"\n\nval_data = pd.read_excel(VAL_PATH)\ntest_data = pd.read_excel(TEST_PATH)\ntrain_data = pd.read_excel(TRAIN_PATH)\ntest_label_data= pd.read_excel(TEST_WITH_LABEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_dpi=96\nplt.figure(figsize=(480/my_dpi,480/my_dpi),dpi=my_dpi)\nplt.pie(x=[len(train_data),len(val_data),len(test_data)],labels=['train','val','test'],\n       colors=plt.cm.get_cmap('Set3')(range(5)),autopct='%.2f%%')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data['tweet_length']=train_data['tweet'].apply(lambda x: len(x.split(\" \")))\ntrain_data['tweet_length'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head().append(train_data.tail())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.xlabel(\"lengths of sentences in train\")\nplt.ylabel(\"frequency\")\nplt.hist(train_data['tweet_length'],bins=100,rwidth=0.8)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def new_len(x):\n    if type(x) is str:\n        return len(x.split())\n    else:\n        return 0\n\ntrain_data[\"words\"] = train_data[\"tweet\"].apply(new_len)\nnums = train_data.query(\"words != 0 and words < 200\").sample(frac=0.1)[\"words\"]\nfig = ff.create_distplot(hist_data=[nums],\n                         group_labels=[\"All comments\"],\n                         colors=[\"coral\"])\n\nfig.update_layout(title_text=\"Comment words\", xaxis_title=\"Comment words\", template=\"simple_white\", showlegend=False)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_data['tweet_length']=val_data['tweet'].apply(lambda x: len(x.split(\" \")))\nval_data['tweet_length'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_data.head().append(val_data.tail())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.xlabel(\"lengths of sentences in val\")\nplt.ylabel(\"frequency\")\nplt.hist(val_data['tweet_length'],bins=100,rwidth=0.8)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def new_len(x):\n    if type(x) is str:\n        return len(x.split())\n    else:\n        return 0\n\nval_data[\"words\"] = val_data[\"tweet\"].apply(new_len)\nnums = val_data.query(\"words != 0 and words < 200\").sample(frac=0.1)[\"words\"]\nfig = ff.create_distplot(hist_data=[nums],\n                         group_labels=[\"All comments\"],\n                         colors=[\"coral\"])\n\nfig.update_layout(title_text=\"Comment words\", xaxis_title=\"Comment words\", template=\"simple_white\", showlegend=False)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data['tweet_length']=test_data['tweet'].apply(lambda x: len(x.split(\" \")))\ntest_data['tweet_length'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head().append(test_data.tail())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.xlabel(\"lengths of sentences in test\")\nplt.ylabel(\"frequency\")\nplt.hist(test_data['tweet_length'],bins=100,rwidth=0.8)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def new_len(x):\n    if type(x) is str:\n        return len(x.split())\n    else:\n        return 0\n\ntest_data[\"words\"] = test_data[\"tweet\"].apply(new_len)\nnums = test_data.query(\"words != 0 and words < 200\").sample(frac=0.1)[\"words\"]\nfig = ff.create_distplot(hist_data=[nums],\n                         group_labels=[\"All comments\"],\n                         colors=[\"coral\"])\n\nfig.update_layout(title_text=\"Comment words\", xaxis_title=\"Comment words\", template=\"simple_white\", showlegend=False)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that that, there's totally 10700 tweets in these four datasets. \n\n* In the train dataset, there's totally 6420 tweets, and the average length is about 26.93, the shortest sentence length is 3, and the longest sentence length is 1409. \n\n* In the validation dataset, there's totally 2140 tweets, and the average length is about 26.73; the shortest sentence length is 3, and the longest sentence length is 297. \n\n* In the test dataset, there's totally 2140 tweets, and the average length is about 27.38; the shortest sentence length is 4, and the longest sentence length is 1433.  \n\nWe notice that in all these datasets, 75% of sentences are within 37 words, which means most part of our data is in normal length, but there still some sentences with extra long length. And this point can also be seen in each histogram."},{"metadata":{},"cell_type":"markdown","source":"### 2.Labels\nIn part 2, we'll show some features of the labels.\n\nTo be specific, the 'Real/Fake' features will be shown in heatmaps, bar charts and funnel charts."},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = train_data.groupby('label').count()['tweet'].reset_index().sort_values(by='tweet',ascending=False)\ncmap = sns.cubehelix_palette(start = 1.5, rot = 3, gamma=0.8, as_cmap = True)\ntemp.style.background_gradient(cmap=cmap)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8,6))\nsns.countplot(x='label',data=train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure(go.Funnelarea(\n    text =temp.label,\n    values = temp.tweet,\n    title = {\"position\": \"top center\", \"text\": \"Funnel-Chart of Real/Fake Distribution in Train\"}\n    ))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp = val_data.groupby('label').count()['tweet'].reset_index().sort_values(by='tweet',ascending=False)\ncmap = sns.cubehelix_palette(start = 1.5, rot = 3, gamma=0.8, as_cmap = True)\ntemp.style.background_gradient(cmap=cmap)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8,6))\nsns.countplot(x='label',data=val_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure(go.Funnelarea(\n    text =temp.label,\n    values = temp.tweet,\n    title = {\"position\": \"top center\", \"text\": \"Funnel-Chart of Real/Fake Distribution in Validation\"}\n    ))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that in each dataset, the real new is 52.3% and the fake news is 47.7%, which means their proportions are not largely different, and the data is not very much in one category or the other.\nSo we infer that in the test data, the proportion of true news and fake news should also be similar."},{"metadata":{},"cell_type":"markdown","source":"### 3.Words\nIn part 3, we'll analyze some characteristics of words in sentences from deffirent type. "},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\nmask_pic=plt.imread('../input/mypictures/mask2.jpg')\ntext = ' '.join([nonan(abstract) for abstract in train_data[\"tweet\"]])\nwordcloud = WordCloud(max_font_size=None, background_color='white', collocations=False,\n                      width=1200, height=1000,mask=mask_pic,stopwords=stopword).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in tweets from train')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"And from all these sentences, the WordCloud of the real news and the fake news are representatively shown below:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\nmask_pic=plt.imread('../input/mypictures/mask2.jpg')\nfor index in range(len(val_data)):\n    if(train_data['label'][index]=='real'):\n        text = ' '.join([nonan(train_data['tweet'][index])])\nwordcloud = WordCloud(max_font_size=None, background_color='white', collocations=False,\n                      width=1200, height=1000,mask=mask_pic,stopwords=stopword).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in real news from train dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\nmask_pic=plt.imread('../input/mypictures/mask2.jpg')\nfor index in range(len(val_data)):\n    if(train_data['label'][index]=='fake'):\n        text = ' '.join([nonan(train_data['tweet'][index])])\nwordcloud = WordCloud(max_font_size=None, background_color='white', collocations=False,\n                      width=1200, height=1000,mask=mask_pic,stopwords=stopword).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in fake news from train dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_stopword(x):\n    return [y for y in x if y not in stopwords.words('english')]\n\ntrain_data['temp_list'] = train_data['tweet'].apply(lambda x:str(x).split())\ntrain_data['temp_list'] = train_data['temp_list'].apply(lambda x:remove_stopword(x))\n    \ntop = Counter([item for sublist in train_data['temp_list'] for item in sublist])\ntemp = pd.DataFrame(top.most_common(20))\ntemp.columns = ['Common_words','count']\ntemp=temp.T\ntemp.style.background_gradient()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\nmask_pic=plt.imread('../input/mypictures/covid.jpg')\ntext = ' '.join([nonan(abstract) for abstract in val_data[\"tweet\"]])\nwordcloud = WordCloud(max_font_size=None, background_color='white', collocations=False,\n                      width=1200, height=1000,mask=mask_pic,stopwords=stopword).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in tweets from validation dataset')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also, we plotted the WordCloud of the real news and the fake news:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\ncovid_pic=plt.imread('../input/mypictures/covid.jpg')\nfor index in range(len(val_data)):\n    if(val_data['label'][index]=='real'):\n        text = ' '.join([nonan(val_data['tweet'][index])])\nwordcloud = WordCloud(max_font_size=None, background_color='white',mask=covid_pic, collocations=False,\n                      width=1200, height=1000,stopwords=stopword).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in real news from validation dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\ncovid_pic=plt.imread('../input/mypictures/covid.jpg')\nfor index in range(len(val_data)):\n    if(val_data['label'][index]=='fake'):\n        text = ' '.join([nonan(val_data['tweet'][index])])\nwordcloud = WordCloud(max_font_size=None, background_color='white',mask=covid_pic, collocations=False,\n                      width=1200, height=1000,stopwords=stopword).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in fake news from validation dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_stopword(x):\n    return [y for y in x if y not in stopwords.words('english')]\n\nval_data['temp_list'] = val_data['tweet'].apply(lambda x:str(x).split())\nval_data['temp_list'] = val_data['temp_list'].apply(lambda x:remove_stopword(x))\n\ntop = Counter([item for sublist in val_data['temp_list'] for item in sublist])\ntemp = pd.DataFrame(top.most_common(20))\ntemp.columns = ['Common_words','count']\ntemp=temp.T\ntemp.style.background_gradient()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\nlung_pic=plt.imread('../input/mypictures/lung.jpg')\ntext = ' '.join([nonan(abstract) for abstract in train_data[\"tweet\"]])\nwordcloud = WordCloud(max_font_size=None, background_color='white',mask=lung_pic, collocations=False,\n                      width=1200, height=1000,stopwords=stopword).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in tweets')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_stopword(x):\n    return [y for y in x if y not in stopwords.words('english')]\n\ntest_data['temp_list'] = test_data['tweet'].apply(lambda x:str(x).split())\ntest_data['temp_list'] = test_data['temp_list'].apply(lambda x:remove_stopword(x))\n\ntop = Counter([item for sublist in test_data['temp_list'] for item in sublist])\ntemp = pd.DataFrame(top.most_common(20))\ntemp.columns = ['Common_words','count']\ntemp=temp.T\ntemp.style.background_gradient()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the wordcloud above, we can see the most common words in the tweets. These words include \"COVID\", \"case\", \"people\", \"coronavirus\", \"tests\", \"number\", \"deaths\" among other words. "},{"metadata":{},"cell_type":"markdown","source":"# RNN model\nWe first tried an RNN model, "},{"metadata":{"trusted":true},"cell_type":"code","source":"# -*- coding: utf-8 -*-\n\nimport numpy as np\nimport pandas as pd\nimport torch\nimport math\nfrom torch import nn\nfrom torchtext import data\nfrom torchtext.vocab import Vectors\nimport spacy\nimport sys\nimport torch.optim as optim\n\n\n\nfrom sklearn.metrics import accuracy_score\nclass Config(object):\n    embed_size = 200\n    hidden_layers = 2\n    hidden_size = 32\n    bidirectional = True\n    output_size = 2\n    max_epochs = 100\n    lr = 0.25\n    batch_size = 64\n    max_sen_len = 20 # Sequence length for RNN\n    dropout_keep = 0.8\n    \n\nclass TextRNN(nn.Module):\n\n    def __init__(self, config, vocab_size, word_embeddings):\n        super(TextRNN, self).__init__()\n        self.config = config\n        # Embedding Layer\n        self.embeddings = nn.Embedding(vocab_size, self.config.embed_size)\n        self.embeddings.weight = nn.Parameter(word_embeddings, requires_grad=False)\n        self.lstm = nn.LSTM(input_size = self.config.embed_size,\n                            hidden_size = self.config.hidden_size,\n                            num_layers = self.config.hidden_layers,\n                            dropout = self.config.dropout_keep,\n                            bidirectional = self.config.bidirectional)\n        self.dropout = nn.Dropout(self.config.dropout_keep)\n\n        # Fully-Connected Layer\n        self.fc = nn.Linear(\n            self.config.hidden_size * self.config.hidden_layers * (1+self.config.bidirectional),\n            self.config.output_size\n        )\n        # Softmax non-linearity\n        self.softmax = nn.Softmax()\n\n    def forward(self, x):\n        # x.shape = (max_sen_len, batch_size)\n        embedded_sent = self.embeddings(x)\n        # embedded_sent.shape = (max_sen_len=20, batch_size=64,embed_size=300)\n        lstm_out, (h_n,c_n) = self.lstm(embedded_sent)\n        final_feature_map = self.dropout(h_n) # shape=(num_layers * num_directions, 64, hidden_size)\n        # Convert input to (64, hidden_size * hidden_layers * num_directions) for linear layer\n        final_feature_map = torch.cat([final_feature_map[i,:,:] for i in range(final_feature_map.shape[0])], dim=1)\n        final_out = self.fc(final_feature_map)\n        return self.softmax(final_out)\n\n    \n    def add_optimizer(self, optimizer):\n        self.optimizer = optimizer\n\n    def add_loss_op(self, loss_op):\n        self.loss_op = loss_op\n\n    def reduce_lr(self):\n        print(\"Reducing LR\")\n        for g in self.optimizer.param_groups:\n            g['lr'] = g['lr'] / 2\n\n                \n    def run_epoch(self, train_iterator, val_iterator, epoch):\n        train_losses = []\n        val_accuracies = []\n        losses = []\n        # Reduce learning rate as number of epochs increase\n        if (epoch == int(self.config.max_epochs/3)) or (epoch == int(2*self.config.max_epochs/3)):\n            self.reduce_lr()\n\n        for i, batch in enumerate(train_iterator):\n            self.optimizer.zero_grad()\n            if torch.cuda.is_available():\n                x = batch.text.cuda()\n                y = (batch.label).type(torch.cuda.LongTensor)\n            else:\n                x = batch.text\n                y = (batch.label).type(torch.LongTensor)\n            y_pred = self.__call__(x)\n            loss = self.loss_op(y_pred, y)\n            loss.backward()\n            losses.append(loss.data.cpu().numpy())\n            self.optimizer.step()\n\n    \n            if i % 100 == 0:\n                print(\"Iter: {}\".format(i+1))\n                avg_train_loss = np.mean(losses)\n                train_losses.append(avg_train_loss)\n                print(\"\\tAverage training loss: {:.5f}\".format(avg_train_loss))\n                losses = []\n\n                # Evalute Accuracy on validation set\n                val_accuracy = evaluate_model(self, val_iterator)\n                print(\"\\tVal Accuracy: {:.4f}\".format(val_accuracy))\n                self.train()           \n        return train_losses, val_accuracies\n    \nclass Dataset(object):\n    def __init__(self, config):\n        self.config = config\n        self.train_iterator = None\n        self.val_iterator = None\n        self.vocab = []\n        self.word_embeddings = {}    \n\n    \n#r'C:/Users/Shelley/Desktop/train.csv'\n    def get_pandas_df(self, filename):\n        '''\n        Load the data into Pandas.DataFrame object\n        This will be used to convert data to torchtext ob ject\n        '''\n        df = pd.read_excel(filename)\n        for i in range(df.shape[0]):\n            if df.at[i,'label']=='real':\n                df.at[i,'label']=1\n            else:\n                df.at[i,'label']=0\n        full_df = pd.DataFrame({\"text\":df['tweet'], \"label\":df['label']})\n        return full_df\n    \n    def load_data(self, train_file, val_file):\n        '''\n        Loads the data from files\n        Sets up iterators for training, validation and test data\n        Also create vocabulary and word embeddings based on the data     \n\n        Inputs:\n            w2v_file (String): absolute path to file containing word embeddings (GloVe/Word2Vec)\n            train_file (String): absolute path to training file\n            test_file (String): absolute path to test file\n            val_file (String): absolute path to validation file\n        '''\n        \n        NLP =spacy.load('en')\n        tokenizer = lambda sent: [x.text for x in NLP.tokenizer(sent) if x.text != \" \"]       \n\n        # Creating Field for data\n        TEXT = data.Field(sequential=True, tokenize=tokenizer, lower=True, fix_length=self.config.max_sen_len)\n        LABEL = data.Field(sequential=False, use_vocab=False)\n        datafields = [(\"text\",TEXT),(\"label\",LABEL)]       \n\n        # Load data from pd.DataFrame into torchtext.data.Dataset\n        train_df = self.get_pandas_df(train_file)\n        train_examples = [data.Example.fromlist(i, datafields) for i in train_df.values.tolist()]\n        train_data = data.Dataset(train_examples, datafields)\n             \n\n        # If validation file exists, load it. Otherwise get validation data from training data\n       \n        val_df = self.get_pandas_df(val_file)\n        val_examples = [data.Example.fromlist(i, datafields) for i in val_df.values.tolist()]\n        val_data = data.Dataset(val_examples, datafields)\n        \n\n        TEXT.build_vocab(train_data, vectors=\"glove.6B.200d\")\n        self.word_embeddings = TEXT.vocab.vectors\n        self.vocab = TEXT.vocab\n        \n        self.train_iterator = data.BucketIterator(\n            (train_data),\n            batch_size=self.config.batch_size,\n            sort_key=lambda x: len(x.text),\n            repeat=False,\n            shuffle=True)        \n       \n        self.val_iterator = data.BucketIterator(\n            (val_data),\n            batch_size=self.config.batch_size,\n            sort_key=lambda x: len(x.text),\n            repeat=False,\n            shuffle=True)\n\n        #print (\"Loaded {} training examples\".format(len(train_data)))\n        #print (\"Loaded {} test examples\".format(len(test_data)))\n        #print (\"Loaded {} validation examples\".format(len(val_data)))\n\n\ndef evaluate_model(model, iterator):\n    all_preds = []\n    all_y = []\n    for idx,batch in enumerate(iterator):\n        if torch.cuda.is_available():\n            x = batch.text.cuda()\n        else:\n            x = batch.text\n    \n        y_pred = model(x)\n        predicted = torch.max(y_pred.cpu().data, 1)[1]\n        all_preds.extend(predicted.numpy())\n        all_y.extend(batch.label.numpy())\n    score = accuracy_score(all_y, np.array(all_preds).flatten())\n    return score\n\nif __name__=='__main__':\n\n    config = Config()\n    train_file = '../input/mydata/Constraint_English_Train.xlsx'\n    val_file = '../input/mydata/Constraint_English_Val.xlsx'\n    dataset = Dataset(config)\n    dataset.load_data(train_file, val_file)\n\n    # Create Model with specified optimizer and loss function\n\n    ##############################################################\n\n    model = TextRNN(config, len(dataset.vocab), dataset.word_embeddings)\n    if torch.cuda.is_available():\n        model.cuda()\n    model.train()\n    optimizer = optim.SGD(model.parameters(), lr=config.lr)\n    NLLLoss = nn.NLLLoss()\n    model.add_optimizer(optimizer)\n    model.add_loss_op(NLLLoss)\n    ##############################################################\n    train_losses = []\n    val_accuracies = []\n    for i in range(config.max_epochs):\n        #print (\"Epoch: {}\".format(i))\n        train_loss,val_accuracy = model.run_epoch(dataset.train_iterator, dataset.val_iterator, i)\n        train_losses.append(train_loss)\n        val_accuracies.append(val_accuracy)\n\n    train_acc = evaluate_model(model, dataset.train_iterator)\n    val_acc = evaluate_model(model, dataset.val_iterator)\n\n    print ('Final Training Accuracy: {:.4f}'.format(train_acc))\n\n    print ('Final Validation Accuracy: {:.4f}'.format(val_acc))\n\n    \n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that the accuracy on the train dataset is about 0.9607, and the accuracy on the validation is about 0.9051.\nSo the RNN model can also get a good score on this kind of issue."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}