{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Giới thiệu bài toán:\n Quora là nền tảng hỏi đáp được tạo ra với mục đích học hỏi và chia sẻ kiến thức. Tuy nhiên, bên cạnh những câu hỏi mang tính tìm tòi, học hỏi đơn thuần thì còn có những câu hỏi được đặt ra với mục đích gây tranh cãi, mang tính tiêu cực. Vì vậy, xây dựng mô hình phân loại những câu hỏi chân thành(sincere question) và những câu hỏi không chân thành(insincere question) là nhiệm vụ cần thiết.\n
\n **Input:** Câu hỏi(dữ liệu text)\n
\n **Output:** Label Sincere (0), Insincere (1)\n","metadata":{}},{"cell_type":"markdown","source":"# 2. Data","metadata":{}},{"cell_type":"code","source":"import os\nimport json\nimport string\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nfrom sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nimport lightgbm as lgb\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:30:54.612176Z","iopub.execute_input":"2022-01-06T15:30:54.612444Z","iopub.status.idle":"2022-01-06T15:30:57.219294Z","shell.execute_reply.started":"2022-01-06T15:30:54.612415Z","shell.execute_reply":"2022-01-06T15:30:57.2184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dữ liệu gồm những file sau:","metadata":{}},{"cell_type":"code","source":"!ls ../input/","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:30:57.221173Z","iopub.execute_input":"2022-01-06T15:30:57.221446Z","iopub.status.idle":"2022-01-06T15:30:58.002606Z","shell.execute_reply.started":"2022-01-06T15:30:57.221413Z","shell.execute_reply":"2022-01-06T15:30:58.001492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* train.csv - the training set\n* test.csv - the test set\n* sample_submission.csv - A sample submission in correct format\n* embeddings/ - Folder containing word embeddings ","metadata":{}},{"cell_type":"code","source":"!ls ../input/quora-insincere-questions-classification/","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:30:58.004405Z","iopub.execute_input":"2022-01-06T15:30:58.004679Z","iopub.status.idle":"2022-01-06T15:30:58.774956Z","shell.execute_reply.started":"2022-01-06T15:30:58.004644Z","shell.execute_reply":"2022-01-06T15:30:58.773855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* GoogleNews-vectors-negative300 - https://code.google.com/archive/p/word2vec/\n* glove.840B.300d - https://nlp.stanford.edu/projects/glove/\n* paragram_300_sl999 - https://cogcomp.org/page/resource_view/106\n* wiki-news-300d-1M - https://fasttext.cc/docs/en/english-vectors.html","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\nprint(\"Train shape : \", train_df.shape)\nprint(\"Test shape : \", test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:30:58.777127Z","iopub.execute_input":"2022-01-06T15:30:58.777381Z","iopub.status.idle":"2022-01-06T15:31:05.025713Z","shell.execute_reply.started":"2022-01-06T15:30:58.777348Z","shell.execute_reply":"2022-01-06T15:31:05.025022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:05.02998Z","iopub.execute_input":"2022-01-06T15:31:05.030396Z","iopub.status.idle":"2022-01-06T15:31:05.050288Z","shell.execute_reply.started":"2022-01-06T15:31:05.030366Z","shell.execute_reply":"2022-01-06T15:31:05.049374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nhìn vào phân bố targer ta thấy các giá trị không cân bằng, số lượng nhãn '0' lớn hơn nhiều lần nhãn '1'.","metadata":{}},{"cell_type":"code","source":"## target count ##\ncnt_srs = train_df['target'].value_counts()\ntrace = go.Bar(\n x=cnt_srs.index,\n y=cnt_srs.values,\n marker=dict(\n color=cnt_srs.values,\n colorscale = 'Picnic',\n reversescale = True\n ),\n)\n\nlayout = go.Layout(\n title='Target Count',\n font=dict(size=18)\n)\n\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"TargetCount\")\n\n## target distribution ##\nlabels = (np.array(cnt_srs.index))\nsizes = (np.array((cnt_srs / cnt_srs.sum())*100))\n\ntrace = go.Pie(labels=labels, values=sizes)\nlayout = go.Layout(\n title='Target distribution',\n font=dict(size=18),\n width=600,\n height=600,\n)\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"usertype\")","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:05.051822Z","iopub.execute_input":"2022-01-06T15:31:05.05226Z","iopub.status.idle":"2022-01-06T15:31:06.178282Z","shell.execute_reply.started":"2022-01-06T15:31:05.05221Z","shell.execute_reply":"2022-01-06T15:31:06.177386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Khoảng 6% training set là insincere questions (target=1) và phần còn lại là sincere.\n\n","metadata":{}},{"cell_type":"markdown","source":"**Word Cloud:**\nTần số xuất hiện của từ trong tập dữ liệu, tính bằng cách tạo một 'word cloud' trên cột 'question_text'","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n\n# Thanks : https://www.kaggle.com/aashita/word-clouds-of-various-shapes ##\ndef plot_wordcloud(text, mask=None, max_words=200, max_font_size=100, figure_size=(24.0,16.0), \n title = None, title_size=40, image_color=False):\n stopwords = set(STOPWORDS)\n more_stopwords = {'one', 'br', 'Po', 'th', 'sayi', 'fo', 'Unknown'}\n stopwords = stopwords.union(more_stopwords)\n\n wordcloud = WordCloud(background_color='black',\n stopwords = stopwords,\n max_words = max_words,\n max_font_size = max_font_size, \n random_state = 42,\n width=800, \n height=400,\n mask = mask)\n wordcloud.generate(str(text))\n \n plt.figure(figsize=figure_size)\n if image_color:\n image_colors = ImageColorGenerator(mask);\n plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation=\"bilinear\");\n plt.title(title, fontdict={'size': title_size, \n 'verticalalignment': 'bottom'})\n else:\n plt.imshow(wordcloud);\n plt.title(title, fontdict={'size': title_size, 'color': 'black', \n 'verticalalignment': 'bottom'})\n plt.axis('off');\n plt.tight_layout() \n \nplot_wordcloud(train_df[\"question_text\"], title=\"Word Cloud of Questions\")","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:06.179846Z","iopub.execute_input":"2022-01-06T15:31:06.180115Z","iopub.status.idle":"2022-01-06T15:31:07.397669Z","shell.execute_reply.started":"2022-01-06T15:31:06.180075Z","shell.execute_reply":"2022-01-06T15:31:07.396772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Word Frequency plot of sincere & insincere questions:**","metadata":{}},{"cell_type":"code","source":"from collections import defaultdict\ntrain1_df = train_df[train_df[\"target\"]==1]\ntrain0_df = train_df[train_df[\"target\"]==0]\n\n## custom function for ngram generation ##\ndef generate_ngrams(text, n_gram=1):\n token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n ngrams = zip(*[token[i:] for i in range(n_gram)])\n return [\" \".join(ngram) for ngram in ngrams]\n\n## custom function for horizontal bar chart ##\ndef horizontal_bar_chart(df, color):\n trace = go.Bar(\n y=df[\"word\"].values[::-1],\n x=df[\"wordcount\"].values[::-1],\n showlegend=False,\n orientation = 'h',\n marker=dict(\n color=color,\n ),\n )\n return trace\n\n## Get the bar chart from sincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n for word in generate_ngrams(sent):\n freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n## Get the bar chart from insincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n for word in generate_ngrams(sent):\n freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n subplot_titles=[\"Frequent words of sincere questions\", \n \"Frequent words of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Word Count Plots\")\npy.iplot(fig, filename='word-plots')\n\n#plt.figure(figsize=(10,16))\n#sns.barplot(x=\"ngram_count\", y=\"ngram\", data=fd_sorted.loc[:50,:], color=\"b\")\n#plt.title(\"Frequent words for Insincere Questions\", fontsize=16)\n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:07.399092Z","iopub.execute_input":"2022-01-06T15:31:07.399875Z","iopub.status.idle":"2022-01-06T15:31:19.903853Z","shell.execute_reply.started":"2022-01-06T15:31:07.399831Z","shell.execute_reply":"2022-01-06T15:31:19.902999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Quan sát ta thấy:\n* Một vài từ có tần suất cao xuất hiện chung ở cả hai lớp như 'people', 'will', 'think',...\n* Những top words khác trong class sincere là 'best', 'good',...\n* Những top words khác trong class insincere là 'trump', 'women', 'white',...\n","metadata":{}},{"cell_type":"markdown","source":"**Phân bố bigram:**","metadata":{}},{"cell_type":"code","source":"freq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n for word in generate_ngrams(sent,2):\n freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'orange')\n\n\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n for word in generate_ngrams(sent,2):\n freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'orange')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,horizontal_spacing=0.15,\n subplot_titles=[\"Frequent bigrams of sincere questions\", \n \"Frequent bigrams of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Bigram Count Plots\")\npy.iplot(fig, filename='word-plots')","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:19.905299Z","iopub.execute_input":"2022-01-06T15:31:19.905552Z","iopub.status.idle":"2022-01-06T15:31:39.099725Z","shell.execute_reply.started":"2022-01-06T15:31:19.905521Z","shell.execute_reply":"2022-01-06T15:31:39.098521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Data preprocessing","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:39.100988Z","iopub.execute_input":"2022-01-06T15:31:39.101214Z","iopub.status.idle":"2022-01-06T15:31:42.844416Z","shell.execute_reply.started":"2022-01-06T15:31:39.101188Z","shell.execute_reply":"2022-01-06T15:31:42.843447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xử lý data :**\n
\nBỏ các từ không mang nghĩa và dấu (stopword, punctual), đổi chữ hoa thành chữ thường, lemmatize (đưa chữ về một dạng thống nhất).","metadata":{}},{"cell_type":"code","source":"from gensim.utils import simple_preprocess \nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\n\nstop_words = set(stopwords.words('english')) \nwordnet_lemmatizer = WordNetLemmatizer()\ndef preprocessing(corpus):\n res = []\n for doc in corpus:\n words = []\n for word in simple_preprocess(doc):\n if word not in stop_words:\n word1 = wordnet_lemmatizer.lemmatize(word, pos = \"n\")\n word2 = wordnet_lemmatizer.lemmatize(word1,pos = \"v\")\n word3 = wordnet_lemmatizer.lemmatize(word2, pos = (\"a\"))\n words.append(word3)\n pass\n pass\n res.append(' '.join(words)) \n pass\n return res","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:42.845851Z","iopub.execute_input":"2022-01-06T15:31:42.846099Z","iopub.status.idle":"2022-01-06T15:31:43.691688Z","shell.execute_reply.started":"2022-01-06T15:31:42.846071Z","shell.execute_reply":"2022-01-06T15:31:43.69078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ví dụ:","metadata":{}},{"cell_type":"code","source":"preprocessing(['Is a PhD really worth pursuing? How many PhD students can really get ‘productive’ or real time use output from their PhD research?'])","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:43.693151Z","iopub.execute_input":"2022-01-06T15:31:43.693381Z","iopub.status.idle":"2022-01-06T15:31:45.943768Z","shell.execute_reply.started":"2022-01-06T15:31:43.693355Z","shell.execute_reply":"2022-01-06T15:31:45.942788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Áp dụng lên tập train và test:","metadata":{}},{"cell_type":"code","source":"%%time\ntrain['question_text'] = preprocessing(train['question_text'])\ntest['question_text'] = preprocessing(test['question_text'])","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:31:45.945007Z","iopub.execute_input":"2022-01-06T15:31:45.945222Z","iopub.status.idle":"2022-01-06T15:35:23.066626Z","shell.execute_reply.started":"2022-01-06T15:31:45.945196Z","shell.execute_reply":"2022-01-06T15:35:23.065658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Trích xuất đặc trưng","metadata":{}},{"cell_type":"markdown","source":"Chọn TF-IDF làm đặc trưng, công thức là:\n
\n\n
\ntrong đó term frequency đại diện cho số lần một term xuất hiện trong docs và inverse doc frequency đại diện cho số doc xuất hiện term đấy. Một term đặc trưng của một văn bản thì sẽ là term xuất hiện nhiều nhất trong văn bản đấy. Nhưng khi so sánh giữa nhiều văn bản với nhau, những từ xuất hiện ở càng ít văn bản thì lại càng trở nên đặc trưng và có ý nghĩa phân loại hơn những từ xuất hiện ở đa số văn bản, vì thế nên công thức chia cho IDF để giảm tần số của những từ đại trà đấy. Dùng scikit để trích xuất những term gồm một hoặc 2 từ liền nhau (bi-gram)","metadata":{},"attachments":{"88cd67c1-421b-4113-aa81-09ffc1859656.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYUAAABXCAYAAADrolnYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAACuASURBVHhe7Z0NbFNXmvf/u6O6jBy1a5bWTFUvHXsrnFZrqPAgxY1IqJp0tyGzTUYljFgjkWSYBI0SqZjdJZkuMGxgOkn7TrIVoV2SapOi4lQQOsHdElfFYYuZpY6GeERjXmS/BUcQ0y5WUSyIu8x9n3Puvc6180HShpKk5yfBvT735tzz8Zznec7XvX8mERAIBAKBgPhz5SgQCAQCgTAKAoFAIBhDGAWBQCAQpBBGQSAQCAQphFEQCAQCQQphFAQCgUCQQhgFgUAgEKQQRkEgEAgEKYRREAgEAkEKYRQEAoFAkEIYBYFAIBCkEEZBIBAIBCmEURAIBAJBCmEUBAKBQJBCGAWBQCAQpBBGQSAQCAQphFEQCAQCQQphFAQCgUCQQhgFgUAgEKQQRkEgEAgEKYRREAjmEIlP++C/lFB+Ce45iSj8p0L4LtWIMAqC8dyKou+t3dj6i63Y7mqBJ5JULgjuJrGPW1DbHsNio14JmYw4/K8Vo7i4GJVbKul4AAFhR+4OeiMWX2lDbbMfMSVooSOMgiCDBAJv1aPpqBFFxWbELnhx4FiA1JDgbpK82IWmV+Io27Ye1kVK4KQY4HipBz2vrEfsKqmqlRaY7mRHBF8THawbXCiL70PT4RC+C+6RMAp34lYCicR3yFO+6ofnfTIBuYuRONfHjYFhSRaEzrmLJCM49kYnkpudKPiBEjYN4tdl39XwpAlGfjZPuJ2kNpWYRwrWiIJN5UgeasWxCwtfFwijMBXJIDq3bMCG6mOIKEELnXh4AAF2ssyGF150oXzLDjSst5G/JLhbxH/vRucFB0rXmJWQ6RG71MePDrOJH+cHSQQPVWLDhmocu6gEzQfM+SjNjaDznb4FP4wkjMJUXArBy1zlHPP88sS+ASlF84gROnMeSoodMN1xOEPw9YnAd8QPPJsH+xIlaFpEMHiWHR3Ifmw+9eOGEPqQNSoHzI/IIfMDA+xrCoD+4/AvcA9RGIUpiEUG+PCJw2r6jgyfxMgoyGcWo0E+EdxdSMZ8pGQcNsvMZCw+jDBTTmYrTA/LQfOCq2EMcEcre97Ng+gtK8iUkRH/48K2Cn8mEcq5II0EAq0bsPt9M8qbm1EyYc8+geiZXnhO+RH+XAfciiNrVRnKf5one9fX/Gj5VTuCSR1MOeWo+sli6jp3ojc8QhdNyN9ahSKzDskL3dj37x5Eb5jgqKxC+Sodgu91wt1PHdUvo4gtsqOE3bssgeD7bnT/PorkKClw2FBUXoGSJ2ardYXQWbwdXST6rrd2IC/Nc43B37wb7eeT0JkcKP95GRYHO9H5QRg8N3nVqCo2jw0z3Y4jeLIbvpODiFJo4sskrM9UoKLUmq78ElH0dbVTXkfIF0tAt6oc5c8k4H3zOAwVjShZEkL3G218mWbCaEd5dbnGo04i8iH97flslNfmZfTm7lA3lJ/AwQNoPxtG0lKGutoiGGN+eI55MHCNdO7ncRhztPdrSSIe9ML9nz4eN9UgdJZCODcXwEz3Jj7tRmu7H5EbLI4q1Gy2U94UbkXgecONWC6rZwNiJ+pR+XoUztc6sP5x5Z4JiAc9cL/nQfBLKr1bRhTk6tB+yAuUNuDI5ozhPSr70Gkven/nQ+h+enI0xvPi3ETpyxCV5BdBeN/1wBe+Dt399HuRBYUby1FAcpniNpXVu264Pw7RjxHEVXnU3jNNEmcPYMMeD8yVzWj++4ka1VxuUxF0bapF57IaHNxTsHBHD5hREIwx8t+t0rp16yb81/rfI8pdROwTqa3OKa3b1iqdvjIqh90clDpq1kmu312mH8NS706n1PaHUWnwHTUOl9QRuC6NhI5Ku35Gv189LV0fHZA6nLuk3ivDku/X8j2ubS6p1T8sx0nX29jfOhul5lcrpMYPLktyKi5Lx3/J7t8r+T7nAV+b4Y/2Kukb/8/ZFZbv8e6SnO0D0mjInbrm+o9PpOtfDkpHd1bQ70bp9HV+qzQaPi7tpfw5f3VUGvxSDpNiPmnvOqfUGtCWoU9qdNJ9r/qk4f9lAWoZsH9t0sAoK8MaqWOA/kZ5bs0xOT2cz1mcdG/NUUkTOo26GZ+fmro6qfpXbmlAzUNAlgPnO4NygMrNy1Lvbym/VB+9n8l5GSaZcbF7WVld6ZV21XRILMlyvddIRzWJU8t670esfkekT/aze6aqQ5KfrjrJSXLhPi8/b/R8h1TD8k3/Gv9LSbAK5b11G13b1iF9ol5S8r5um1sKK8XB4r3sbZYqqE5SMqX+rZPu4/cwWBpZWbqlQX7TqBQ+tmt8mU/FCMWrpHfcv/2fKPJMzPk2dV2Jj8mmErQAEcNHGehXV6Gnpwc9zeXgfszzO3GY/aZ/VasV7+FWCF2/2Y3uoAU126rg+IHiMS0yYOkD5G396SvqJgfhi5cgfyX9vilftlZWw7kKGPC0I3CVev6PLYX+0wC6VhVQHHQfc3bIVzH+2IWqHMUP0emQxY7xPkQtdXA9pw5lZSGLX/Aj9j884GtjXLuD5+/wy0VyAP3uUPLc8SIrhRiCfddRkmcDbvJEAsvLUb3JDgSPo515X2YLljJ3+FofWnYdgP9WCVzbS2Cl8uD8hYE8qzh5ncpvXoZN6IsXoKqSvPzvsUAjcp5V02DF0lAvOg1lKLXpEftsgAcv1o95cInIIOWeWGnBozyEmE7dTJCfyOgK1GxfD5vi0uuyFvNj/Mp1zXLcGPr216PlQz3W/3MNCpaxtETQ97qH+liUevJkg95OGDaUwqaPIXqe/c1iZH2fHRnkBYdYig2wPMrqdwQJHvmjME4ynxA7sQ/bO4IwbK7GesV71T2Rj8In2JkZlkc0w3xK3j3X8uD6Ryfs6qVFVvwoh44XyKM+J29oiJ1qQX2zF/oNdahRZCrS1wLPBTp5WI/7+F1EhOrg/TiKfvoCrOym5BAGqHeFSBjDYwUzNXo7qrg8NaOcN6oi7Dwsy1dPtV2W53nRpkiGuaDFEL/BAxYkwihMQuxigJo74Hhy/Fhv5EQbOlnjKS1BnrqE8HaCwjvReSkPZU+bkSChsW+0U7MdQugku8GGwtWsRRioO7sfDU0HsbfUjHjSgKq12dDHwxg4R5fNJShdo+mYxq/jMj9Zj4q07jZ1dbnScWC2hv9jl7iKJf2+dGy4g8E011OkZOjxQxd9PMhW4OBG05Bbjf0NjTi4t4R+JxA40kaKnq5vLIJNHXZJxhE43Inu5U4UrpRLUy1Dw8ZCOFTDQQwNDfKjjcpdupUF59+uoPKPwP9+kELtVB9jZRO96OHHvMdNqeGT6dRNWn4uycam4CdF0O4PiMdI8RGOpyypskic7UbbSfrbZ8lQPaE+0YyiPQ28PncV65HMojw+SXmM+HGc1ecqB2xqOpJhBN9nJ/mwLmPHOGJn2PF+9t94bvjhfp2tBctYmZQgg/MpO7EjWxOs5t38QinyJpln8ESiFG8A3QfZcuMClBVbU2Vnfo7ywepydxHU9UyRP/p4O0je1OyOe8CEgi2lY0ZnulwNI8Aiy82GJaNRzZs2xZ2Xb+6IzWWEUZiQOELnmBIyw/popnREMPAR8wtJFPvd2P1yPepd27F9Vwt6k/lobHXBQV6ffnkBSlZT01In1sx2WFRhp0ZlW27kxsa4ugRF5J4mL4XA1/2szubKViUV/nx2qqFyyEs+zeOdrYlGUjQXWYTAimUpv1tGb0VBqZ2eH0M4yB8K++NqI9PDZLOCb8JNDCLA9jgQMS95okrZ1O/pROSxCnS8sh7yMHSIjIdchvlWbW7j5EkzrWHACrORl03BcqZg5clY5ObDrpYh1YO8+saGFRa1jqZXN2n54VrKQXFotVQC4fPMQFL9L1PzGSPD5OG9BsdTpHDkQI5+mU2pTyPspQW8dxQ5JytTxzP2sbHnIZIFdsy1wKRq4imIn+uDl52stMOq7UkMkZFkR61MJIM4zcvUTJ60tkwZpOxYYgjz93WInfXAwzKSuwLZGoPM5VKtSwW9Xu4xeV/ZhOIt27GvbQDGjTtRo50/miZxcrR4q7Ka0p2OBdum5ifCKExEMorwx3Q0PJ3miXHUVR+kSCr+hTwr7iU2onHPDlQVkwLQagsiESXvkI6GnHTBzET1WIvI69WihmeugFI9OPMzK6aMd/rEcZl7rXmkCCdp7gkqF+Z5TVQujCuKsiIPzNU0VjYNe2qwfo0VBu5lEVejcqPOfFYijEFW7ilPWkbNqyPHNqZMvohCth8rYFIVwwzrJpWfTCVwY4AMAB3N+WSc5CB+bz87sZFBnMyNVCEl18dTDMcTY/fGSBnxOhunFCdCNUyURZslbVIzEjrNjZPNrJEJMjgBXqYrYMmw6Sztg7xuDbD/0EDGn+9EgW0l1Qk/mxzjs1XYWeqAlZXx1RCVSzv2Ve+D96p8ffokEQ2z/BjwdJojQCzYNjU/EUZhIv5fCHyQZK1VEY4keTlBRNk44u2vwEalAWqo0/Am1CGOqTcYsfFnJo42WNLW6ames9ZjZTDPSg7P/xtK4Y0QvEcDFMs3gBQ199XUuYGJuDQInpscc7qHpfInlmIi96/SlNg4bsR5o8ZKUmDaZ6kecJonrSkDTa8tNZ/A64ji6+lGIDazuknlRzsnQSTOB7iHLiuHBEInKO7PEnLeQAYk1VuZBNVgMWOT8vDH5hPGlCL1hnLZcZT/SifJh0sYjrS6T+9NYSiA7g9DSHz1FVdoyKG8Z9j0ZCgg53N5CZ62UUqUeK2mKWsJiUt+9J1KIHvzDjS+2YOed/fDtZbVQQABpVc5fSLKkA8Z/B/yACS/iCA4lJhfbYrPiTlg/Ev+a0EijMIEqJ5Yaqz6ah9a9gwgzsacl5ggt+mJGjKJ4tl2tPTw5kloNhhlrgXUonpy2u4wI+U5p48dp4ZTVq6jhsGe6UbniO6OXt9UqN5XpoLUElE8zInmWTiPmsGnianhjH8ZQBKRnha0n6WSNRipRIgstpxTJYlQQPGAJ/Rgl2KxZqgjGpG9aIeF6uiLIDzvjkBHSmL6dTOWH+2cBFMm/hPMJNiwjo1XX/XDfYjipnvSDFgGsZNNqD86FjfnkcWafFAd8/kEB8ypAtbDwJXgEGJf8AAN6qQmsPhBTWlfC8Kv6U1F+jvhS+igV8t+HDH09TAlaoXz50Wk8gwwmafIyLU+NNV3IzLkRdMv9qGpsRm9arYWUfk+lU0n6QaaLf/cvWUTtr4+xUvj1KEZMuKywad0vb4LA9fvm0dtKo7YEDsaoE8tHlh4CKMwBX/1EIlEIoSuV3uRXV8GGxdm8iRezCOx8MLfr5l8uxFB38F61L93H/LzFGm7pniMmWPCmajDLpkKWQ3PHPtUYe+8SZAX2KWDs+CbvYoiTcnys0zGPK9Jh0/0NhRusgJnfAjwxiPD1sJ3v7IVeyPZKGDLex62wcE85PNhRG+xOxKIvLcPew+z+MkDfkzrwZESe5yV52Bqci9xrhNth2RP9dElBsT6fRh+/kew6mZQN9TAh/mYhXZOgpHACI+aDUvF4T/mhm5TIdW9GQVb5Lh9ZyaI+2MLKp5X4laV3Hm27p24TYbmzTZ0sfO0SVYdjI+Q687yJmcnDbPdSdJGVz9TVO01Pw78pg2DPLlZ0OmYd6uTPduJyp4Uo7e5Hi0RO8qbG7D+cblmzWurkEdxePv8lFsFNql7qh31v/TBUkHGgw/1mZD3CxeKUkUWQK/HD+vG6rEwIvIHtvKHPPAT+9DLJ8CngBtK6n0dbkKvtQ5lvFHNlzZFRoFNRFOcC/kFhGLz2kSwDUb/1oTuS6SSltixbnMZ8vjyQ5Ukoqfextu/CyB6vwEGZlqXWJC/tgR5pPRSgnTJg/qXu2GsbkRNziSKlEj0H0D1rgDydzWjfNXYc9RwB4VXacKZcAbeasGBk1HgAQtKXtr2tTYSjRGD9+VKtJyzoebNhkleyhaFp74e3Q9XobHWQQ14MqjBH21D50chJA1yWWQ9Ykfec4XkgWnyoJSx+6IeJuoBGKwWxNnmLJSg4Ui5YoAV2Ku833kbx/uj0D1Icf51IcqLTQgeakX3+TgMlhJU15Ii46uHplk3LD+urXDfX4WGhrHVNox4P3mmb/jojixYil3YlppUzYh7lPpDpmzk5xbBsUqe5FRJDvXh7bfc8MfI439Qh6z7yDBQ+sdt2op0o7a2HUtfOogda8cP5zAv+cBBH2IPUfz3ZXNZtMU9aNl/fIL0Udn3dMJ9IogRgx6JeBZszxWhhE12axNH8PQdOo5AlLzhB5OUMxOy1zpQlDN2L9uE13bIT89RenR/boT9hTIUUV611cN6L91HfAiGvdD9uAM71kwkHXJPsYl5/A9QPD8uR9ka7Zj+PGhT1IvaV9GE4Uk33i0MhFFYIMRJeTT1yMso78iyErgqNbtsyTM6sGE3POZyNDezpaX3gAtd2OTqRDzXhY5/Yl7j/IWNlQ+Rf2peoioVMrq7yOj2T7Q7PoLu2lq0m3fgMBnbDN09j5DfABDMOYLylRnKdIGQONOCDXvDU7zhYIHAjILgO8jnyu7Rn3VIA4EOie0O3eVVdnzeA4Y/qONpqPvg3qVhNhgd6JCcfKetZof3H9p4mOudQWmijbBsd/Vs7Ey/p9xku5abpdPqDvYFh7Kb+dc+OlvYiDmF7yjxT73oDsZheOh+hPq9wHInynKnXo1y94gheIZPc99xRcxc56tbbJWSAfZKeXNX4mI39r3WDd2zLrheHNsopsW4pgxOsx9HT7HB8vlIEqFjbgxuLIRdu+9hIRHx4ejHVjhLc+Z1L3Y6iOGj7yrKmL7nkh76ZQ5U/FzzSopvDWXoJFMX5rhwsC7zBXfzBTan0oqWj2IwLCJ1+ZAF+X9XxjdTTUXyQhfqXYMofHPnjD60MycY8qD+twk4/3U6X42bj8jDf71PNqPhxZlv2ptvCKMgEMwR2Dea648thWvBKtf5CPWCDtejKVaChlrHPHVUZoYwCgLBHCLxaR8G9HY40la7Ce4ZiShfJrtiTcYr3xcwwigIBAKBIIWYaBYIBAJBCmEUBAKBQJBCGAWBQCAQpBBGQSAQCAQp5sREc3FxsXImEAgEgq8D+7zpbCBWHwkEAoEghRg+EggEAkEKYRQEAoFAkEIYBYFAIBCkEEZBIBCkwV614b+k+QKa4N7CXrVxKoRvq0aEUfhOEYf/tWK+2qtySyUdDyAwV9s++9raW7ux9Rdbsd3VAk9k/FefBbMPeylfbXsMizM/1TaOeSRL8x29EYuvtKG2eYpvYM8ic94oJBMJCHUwWxjgeKkHPa+sR+wqidec/dZsAoG36tF01IiiYjNiF7w4cCxAakhwN0le7ELTK3GUbZvOW1rniywtBHSwbnChLL4PTYdDd10fzmGjQIrhzUr8ZMMG1PdElTDBbBC/LvsbBvaRcn42x7jqh+d9MgG5i5E418eNgWFJ1nfmLZX3hGQEx97oRHKzc0bfc5jzsjQZt5NIzCuH04iCTeVIHmrFsQt3N9Vz2CiMIPEFZf4HDuQ/+V14i/m3R+xSHz86zNrP1c8d4uEBBNjJMhteeNGF8i070LDetuA/bnIvif/ejc4LDpSumdnHh+e6LE1MEsFDldiwoRrHLipB8wFzPkpzI+h8p++uDiPNYaNgRF5dB3re3IEis1AHs0cEg2fZ0YHsx+am751SNI8YoTPnoaTYAZP46MxdJALfET/wbB7sS5SgaTH3ZWlihhD6kPU/HTA/IofMDwywrykA+o/Dfxe/3Commr9rxIcRZgJltsL0sBw0t4iRUZDPLMaF/jXcOUJkAD6SCYfNMrMhujkvS5NwNYwBZhNysufdPIjesoJMGRnxP949qzD3XnNxOwZ/xwF4+qOILXGg/Of/AMcP1J5CEpGeV9HaE0bsIQdqasthvy8Iz+Fu+IeSSH5Onaoni1CxOf17w4mLHrQ0dyN8AzCtrULVZjsf/0xe7MbuPe2IrtyB5pccmg9yJxA90wvPKT/Cn9Ozb8WRtaoM5T/Nkz3Wa360/KodwaQOppxyVP1kMXVHO9EbHqGLJuRvrRrXu4n1d8H9jg8hOh+5oYe9uBpVxRnfe70dR/BkN3wnBxGlK4kvk7A+U4GK0q//1ad40AP3ex4Ev6QYbhlRkKtD+yEvUNqAI5szhmTo+aHTXvT+jtJ5P5VGNAZjThmcmwpgzkhA8osgvO964Atfh+5++r3IgsKN5Sj4xr26EDqLt6OLRN/11g7kpXmuJBvNu9F+PgmdiclGGRYHO9H5QRi85PMyynS65ZmIoq+rHe7+EZKBBHSrylH+TALeN4/DUNGIkiUhdL/RxpdpJox2lFeT3KXSRTL5If3t+WyU12Z+V/oOckT5CRw8gPazYSQtZairLYIx5ofnmAcD16juPo/z8h+7X0uS6tYL93/6eNw6+q2zFMK5meqK7k182o3Wdj8iN1gcVaghmU/JN/s+9xtuxHKrUL7KgNiJelS+HoXztQ6sf1y5ZwLuqSyRXgi864b7Y96CEF9kR8kE7Ww6JM4ewIY9Hpgrm9H89xMNl81O+09e6Ma+f/cgesMERyUrax2C73WSnJGe+pL0m5qHZQkE33ej+/dRJEfJKYINReUVKHkio6A4EXRtqkXnshoc3FNwd+ZxmFGYO4xKg2/XSK53BqXRz33S3nXrpHX7P5FG1KsDHZJzZ680HFOubauT6upaJd9nyh3R41IdC/+1T7ouh0gSu9fZKPlikjT4Dl1b55Tc/1e+NOzdRb9Z2F7J97kcJsU+kdrqnBR3q3T6yqgcdnNQ6qhZJ7l+d5l+DEu9O51S2x8orTw+9s8ldQSuSyOho9Kun9HvV0+PPZ8Y+e9WyUn3uM/L6RwN033raqSjYf6TMxo+Lu2lv3X+6qg0+KUSyPPplFoDagnMBEpfV136c893SDU8veukxv/SppCgfLduo2vbOqRP1EtKvtdtc0thpShYvJe9zVIFpavxg8ty3ah/66T7+D0zZ/ijvUpZjv/n7JJjZfXlbB+QRkPu1DXXf3wiXf9yUDq6s4J+N0qnlbRPuzwprNFJ973qk4b/lwUMS75fq89ukwZGWX3XSB0D9DfKc2uOaXKpymnN0fS831GOxuenpq5Oqv6VWxpQ8xBo5eFOag9p3Lws9f6W8kty3avI/jDJmIvdy8rqSq+0q6ZDYkmWZTRd1tSy3vvRMP0akT7Zz+7RtIFx3GtZYmlkZemWBvlNo1L4GLXdzDKfihGKV0nvuH8aHXPneptm+x8dkDqcu6TeK6o8uSTXNpfU6mdlTtD1Nva3VIfNr1aM5V+6LB3/Jbt/svq4rsTHZFMJmmXm1vDR1T64zz7NPTkMheGnIINBXXWSRIi8bftzDhhvJ7lniGsGFNZWIU/9nu2iLGSx4/mYsnwxieAHbUhuciLv4QgGz7CwbKijEsZnq+DK5WcwsJ7FrRC6frMb3UELarZVjfVQFhmwlK4n//QVpTEIX7wE+Svp9035srWyGs5VwICnHYGr1Jt+bKmm1xFB7yEP4sVleIFb/iSGzgUQpvDwFTmVuNaHll0H4L9VAtd2TS/nLwyUMrrntvJ7BsRO7MP2jiAMm6uxXvE4dE/ko/AJdmaG5ZGxFKr59lzLg+sfnbCrlxZZ8aMcOl4gL+icvAg9dqoF9c1e6DfUoeY5E6+bSF8LPBfo5GE97uN3zRzj2h38LY+HXy6SA+h3B/1mYR0vMm8uhmDfdZTk2YCbvPaB5eWo3mQHgsfRzrwvswVLWdqnW548303oixegqpK8/O+xQCNynlXTYMXSUC86DWUotekR+2yABy/WK/JGJCKDXE7ZksxHeQgxHTmaID+R0RWo2b4eNqX8dVmL+TF+5bpmOW4Mffvr0fKhHuv/uQYFXPYj6Hvdw3uhRvJkg95OGDaUwqaPIXqe/c1iZH2fHRnkBYd4y4LlUeZnjiDBI38UxknmE+65LEWoDt6Po+inL8DKbkoOYYB6V4iEMTxWMFOjt6OKy1MzynnnoAg7D8vy1VNtl3XMLLZ//acBdK0qoDjoPl691FP6sQtVOYpvr9PJuireh6ilDi4l/6DQLH7Bj9j/8IAMSIa5oJGOu8EDZp05ZRQSlEnbxhxYddSdOtNNIQase4oMBCeO5IM1KHxST739MIIUYn6hlJS9fJXzPzEMsuNzNhJVBv1NlhNlq6kilHFT5DpgSwm/EbYcB5BjgZHqP3KiDZ1MIEtLkKcuy7udoPBOdF7KQ9nTZp5G+0Y7xT+E0El2gw2Fq9nTDNRF3I+GpoPYWyo/naM+92aSmqPC97JgIoNUyltMAoEjbaSYWN6LYFOHCZJxBA53onu5E4UrZXGZNjf8cL/O1u9krCZJkJL4lJ3Yka1NopLvceWpwROJUrwBdB9kS0QLUFZsTQ0XmJ9rQENDIw7uLqLO8zcjdomrWNLvWsNKMM31FCkZSvfQRR8PshU4eD0bcquxnz1/bwn9nn55qvk2bCyEQzUcxNAQlyLYnrRAupUF59+uoAYbgf99JnV2ODSr4aIXPfyY97gpVR7TkaO0/FySjU3BT4qg3R8Qj5HiIxxPWVJlkTjbjbaT9LfPkqF6IlUDKNpDdUCyt6tYz2WetRNE/Dh+ji6vIplX05GktvM+O8mHdRk7xhHjztL97L/xzAFZivzRR6XPlLBmd9wDJhRsKR0zOtPlahgBrgeyYcloVrPZ/uNJA6rWZkMfD2OA1YG5hMpPM9gTv47L/GQ9KtKGsGKIcUNOzu9keePOy2RG45szp4yCfnkBSlaTKNwYoAZIAVSQ9uXyNabA7aUF5PUlET4vN0S7VVuYVKmh0yRk5ANQA5VR/4aunZMFq2CNZmyViMeHYeZrrCMY+Ij5WlS9/W7sfrke9a7t2L6rBb3JfDS2uuAgY5JKozpZZbbDogoQCaptuVGx+ArfzyI/jfhwHzYVV2L7K+0YePgfsLO2iI/9IjGIAFuTT8S85Dkpz63f04nIYxXoeGU9ZjpsGj/XBy87WWmHVev9DZFiY8fns8eUdzKI00dYvs3k/aSXJxdQVmiE+fs6xM564GFJzV2BbI0S5fm2WXHHTbB3hBTNRbksVixL+d0yeisKSu2U7hjCQV7wsD+uNjI9TOrzp12eITIecn3np8lRnDxplmkDVpiNMK4uQcFypmBVpyIfdrW+SWbk1Tc2rLCoUjU9OUrLD9dSDopDW4AJknNWW2ZS3mo+Y9QuqNdJZ46nSOHIgRz9Mpsie+Nl3vGMPIfGoR44N0G5FpimIVdzQZb0ernH5H1lE4q3bMe+tgEYN+5ETeac3DSIXwzIDqXVlO50zHL7Z3JTRF2+5KUQ+Fq61dlUKmOkwrXlx6Ce52ke772bvJ+Tq49UQTTnrUgrSJkhhJnlpe5fNvd0VFilMqkrgp15SWnQtT6l4T2uvcYaZBxPM6WgrqSgeyr+hbwV7nk1onHPDlQVU6PKiFLtrRhy0it7HD8oQNXLJXCQsLDnhT72oH3vVuz7UFlpfEVpXOQxuJrGntuwpwbr11hh4F7BTFCVCaXNRj0gfiajGk2bWe2qEqQkqAiIFbBk6GE2ASsPuRlg/6GBFDbfPQDbSkoXP5tt4rjMn5dHinCS5k5p4vVveDrNQ00x3fK8GpUbdeazEmEMfsxOVE9aRvVWHTm2sbx/EYVsP1bApCqGGcpRKj+ZSiDlGOWTcZKD+L397MRGBvFONTAm844nxu6NkTJioeOV4kTMDVliw7w7Sx2wsjK+GqJyace+6n3wXpWvT58komGWH4Pc5rXcpfav9gLHHFUZNdxB9aCNWpUz8zMT6b5vhzloFGIIfMQqTvE2rlI3k7rtXNYY15SGmLmtXvXknqfuLIVHqZvtvaB0N9UuY6a3w7rXN5TeyO2vwEZ6ARL+aVhoddhgyk07N9iLrPqQsJZjB3Ure44dwf6X8ngjCPSH5Dz9ialCIvev0hrd1yfJu7gMR8rDZKR7wBiicv0whMRXX3EhVIfQtCRDAfBcLi/B0zZqCEq8VtPspHQcpKi5r6bODUzEpUE5TTnmdA9LZbrleSPOGzVWkgLTPkv1gNM8abXsyGt/dOzm1HzCWitdofh6uhGIzUyOUvnRzkkQifMB2THiyiGB0AmK+7OEnDeQAUn1ViZBNVjM2KRkfmw+YUwpUm+Iz6uN8l/p3HtZSlzyo+9UAtmbd6DxzR70vLsfrrW8BSGg9CqnT0QZ8iGD/0MegOQXEQSHSE/cjfZPuix6npWIDZY0ZaWRp7RyVR1b0n1/Q/VzIwTv0QDFkgGfE3PA+Jf816wz94wCm8hh3tDKdVTgVExnOuG7lcUVKSPxmdwQ0z2XJIIfd3OBLCFPTk/dWN/r1OQfUCpihCqBHbPY0j2VBAInPMjeWEhVQCwxQW4nEzUOqt6z7Wjp4SJPaDbtZK6vSxGD9zXqETQ2ofmk8nff08H05I+QTacpT+1RM/VtCKro8ZvX2RLcFrSfnYnwqxNRwOIHNWm7FoRf4wFH+qlcEzro1eePI4a+Hib4Vjh/XkRlZIDJrNWeGVzrQ1M91cHXmBRXUb2vTAWpJaJ4mI4nJ1lTP93yNBip9og0mUgiFFA84Ak92KVYrBnqiJJTwXBYTNB9EYTn3RHoSElMX47G8qOdk2Bl7z/BTIIN69h49VU/3IcobronzYBlEDvZhPqjY3FzHlmsyQd563w+wQFzqoD1MHAlOITYFzxAwz2WpSEvmn6xD02NzehVs7WIyvcp3oLSDDRb/rl7yyZsfX2Kl8apQzNkxGWDT+l6fRcGrt93F9o/ofaOtENMjFRvNH0+JuXYKrovdtaNzhFdhhzGERtiRwP0qcUDs8vcMwo3R+SVRWycn4Tj6AfZcD47VnLqRGT+41q18RWSfAmFg7qtOkSOd6LvWc1k0TIrCljJng8jeosFUDfyRCsO3HSiZLVaqWSdX2RevBf+fs2E1o0I+g7Wo/69+5Cfp6Qj1VvJ6HlMgGltDVzPKX93O47AB8fhX+5EtRqmt6FwkxU440OAV7YMW7vd/cpW7I1ko0BdjjJNzHYnN3SDnynN45ofB37ThkEeTRZ0OuaR6GRvZKLnkzB7m+vRErGjvLkB6x+X1ZV5bRXyKA5vn59MqgKbiDvVjvpf+mCpoAY/4+GuMdKULD/LZMzzmnT4ZLrl+bANDuYhp2SC8vHePuw9zOInD/gxrQdHSuxxXqKpyb3EuU60HZKN9aNLDIj1+zD8/I9g1c1AjqiBD/MxC+2cBCPB/BiCDUvF4T/mhm5TIWwUd8EWOW7fmQni/tiCiueVuFUld56teyfY/p8329DFztMmWXUwPkKuO8ubnJ007r0smZD3CxeKUkUWQK/HD+vG6rEwIvIHtvKHnL8T+9DLJ8CngBtK6n0dbkKvtQ5lNpamu9D+1aHMTCdHDc+cT1Bhui9BPasuHZwFma93IaPAJqIzR0pmkTn4jeYYAm8dwIGTUfK6HCir/AfkkaKXYS/Jq8buQD52/p9y2DWFkox40PKaG6FkFoyry1FTqZlcI5JDfXj7ECnk8wkYTQYY/qYEzhfpnjQlRsbi1Nt4+3cBRO+ne5jJXGJB/loyMKRIUpVzyYP6l7thrG5ETc4kyolB3b/utzrhp26Kjln1P1FnfVUJyp6n52pWmbB8hY62ofOjEJIG+TlZj9iR91wheQxfr+aZZ3PgoA+xh4zQ35eNdZvLYItTGe0/Tv5iFizFLmxLTdTR83s64T4RxIhBj0Q8C7bnilDCJigzHq+WYyBKHsyDSSoxE7LXOlCUM/7emUE9q5cr0XLOhpo3GyZ5KVsUnvp6dD9chcZa7WbDTKZZnmwT1781wX1RDxP1AAxWC+JscxZK0HCknJSwch+Dvcr7nbdxvJ/k8kGK868LUV5sQvBQK7rPx2GwlKBaXTwwXTli+XFthfv+KjQ0pK/civeTZ/qGb4K6yoh7lPpDpmzk5xbBsSp9kQOvq7fc8MfI439Qh6z7yDBQ+sdt2op0o7a2HUtfOogda8cP59xLWWKb8NoO+ek5So/uz42wv1CGIsqrtnpY76X7iA/BsBe6H3dgx5qJpEPuKTYxj/8BiufH5Shbox3Tn932n+g/gOpdAeTvakb5qrGnqOEOCq/ShDOFH3irhes+PGBByUvbxm/OI0d5X0UThifdePfNER/unw+Qd9T+WjfkBYp3ggkTGcwpbNXdIk7Ko6lneqnEshK4yHCnkkme0YENu+Exl6O5mS0tvQdc6MImVyfiuS50/JM89zNfYWPlQ+SfmpeoSoWM7i4yuv1m8tibUZJWwBF019ai3bwDh8nYZujueQQ5ja0bEMw5gvKVGcp0gZA404INe8MT1OEswoyCQHBP+FzZPfqzDmkg0CGx3aG7vMqOz3vA8Ad1PA11H9y7NMwGfOc/32mr2eH9hzYext8WIAelIe/un2pX8zzgJtu13CydVnewLziU3czaNzbcBebenILgO0P8Uy+6g3EYHrofoX4vsNyJstzxwxffDjEEz/Bp7ru3uupb4qtbbJWSAfZKeXNX4mI39lFPU/esC64XxzaKaTGuKYPT7MfRU2ywfD6SROiYG4MbC2HX7ntYSER8OPqxFc7SnLvaixXDR4J7hzKm77mkh36ZAxU/T3+R4beDMnSSqQtzXDhYl/mCu/kCm1NpRctHMRgWkbp8yIL8vyvjm6mmInmhC/WuQRS+uXNGH9qZEwx5UP/bBJz/Op2vxs1H5OG/3ieb0fDizDftzQRhFAQCQQr2jeb6Y0vhWrDKdT5CvaDD9WiKlaCh1nHXHRVhFAQCQRqJT/swoLfDob5oUnBvSUT5MtkVa77+K/RngjAKAoFAIEghJpoFAoFAkEIYBYFAIBCkEEZBIBAIBCmEURAIBAJBCmEUBAKBQJBCGAWBQCAQpBBGQSAQCAQphFEQCAQCQQphFAQCgUCgAPx/wlmNVuFKCKEAAAAASUVORK5CYII="}}},{"cell_type":"code","source":"vector = TfidfVectorizer( ngram_range = (1,2))\ntrain_feature_matrics = vector.fit_transform(train['question_text'].values.astype('U'))\ntest_feature_matrics = vector.transform(test['question_text'].values.astype('U'))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:35:23.069274Z","iopub.execute_input":"2022-01-06T15:35:23.069521Z","iopub.status.idle":"2022-01-06T15:36:32.126758Z","shell.execute_reply.started":"2022-01-06T15:35:23.06949Z","shell.execute_reply":"2022-01-06T15:36:32.126021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feature_matrics","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:36:32.127891Z","iopub.execute_input":"2022-01-06T15:36:32.128164Z","iopub.status.idle":"2022-01-06T15:36:32.133805Z","shell.execute_reply.started":"2022-01-06T15:36:32.128132Z","shell.execute_reply":"2022-01-06T15:36:32.133155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Model","metadata":{}},{"cell_type":"markdown","source":"Chọn F1 là metric chính để đánh giá model. Vì data bị lệch nên accuracy có vẻ không tốt bằng F1","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, confusion_matrix, f1_score","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:36:32.135279Z","iopub.execute_input":"2022-01-06T15:36:32.135495Z","iopub.status.idle":"2022-01-06T15:36:32.146663Z","shell.execute_reply.started":"2022-01-06T15:36:32.135466Z","shell.execute_reply":"2022-01-06T15:36:32.145695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Học tf-idf đã extract ở trên bằng mô hình logistic regression. Thử tunning tham số regularize C.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\ntrain_x, valid_x, train_y, valid_y = train_test_split(train_feature_matrics, train['target'], test_size=0.25, shuffle=False)\nC = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000]","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:36:32.147906Z","iopub.execute_input":"2022-01-06T15:36:32.148182Z","iopub.status.idle":"2022-01-06T15:36:32.246818Z","shell.execute_reply.started":"2022-01-06T15:36:32.148149Z","shell.execute_reply":"2022-01-06T15:36:32.24588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for c in C:\n model = LogisticRegression(solver='liblinear', penalty='l2', C=c)\n model.fit(train_x, train_y)\n prediction = model.predict(valid_x)\n f1 = f1_score(valid_y, prediction)\n acc = accuracy_score(valid_y, prediction)\n print(\"Regularization: \", c)\n print(\"F1 score: \",f1)\n print(\"Acc score: \",acc)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:36:32.247843Z","iopub.execute_input":"2022-01-06T15:36:32.248093Z","iopub.status.idle":"2022-01-06T15:50:07.700626Z","shell.execute_reply.started":"2022-01-06T15:36:32.248057Z","shell.execute_reply":"2022-01-06T15:50:07.699438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Có thể thấy giá trị C tốt nhất là 30. Vì hàm predict sẽ trả về một sample là sincere hay không bằng cách so sánh xác suất đầu ra với một giá trị mặc định threshold = 0.5 (decision bound = 0), thử thay đổi mức threshold này.","metadata":{}},{"cell_type":"code","source":"thresholds = [0.5, 0.1, 0.14, 0.15, 0.16, 0.17, 0.25, 0.3, 0.4, 0.55, 0.7]\nc = 30\nmodel = LogisticRegression(solver='liblinear', penalty='l2', C=c)\nmodel.fit(train_x, train_y)\npredict = model.predict_proba(valid_x)[:,1]\nfor t in thresholds:\n predict_t = np.where(predict > t, 1, 0)\n f1 = f1_score(valid_y, predict_t)\n print(\"Threshold: \", t)\n print(\"F1 score: \", f1)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:50:07.702989Z","iopub.execute_input":"2022-01-06T15:50:07.703864Z","iopub.status.idle":"2022-01-06T15:51:42.998158Z","shell.execute_reply.started":"2022-01-06T15:50:07.703815Z","shell.execute_reply":"2022-01-06T15:51:42.997084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Có vẻ chọn t = 0.16 là tốt nhất cho tập validate.","metadata":{}},{"cell_type":"code","source":"t = 0.16\npredict = model.predict_proba(valid_x)[:,1]\npredict = np.where(predict > t, 1, 0)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:54:09.82202Z","iopub.execute_input":"2022-01-06T15:54:09.822334Z","iopub.status.idle":"2022-01-06T15:54:09.867797Z","shell.execute_reply.started":"2022-01-06T15:54:09.822301Z","shell.execute_reply":"2022-01-06T15:54:09.866946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix(valid_y, predict)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:54:15.566111Z","iopub.execute_input":"2022-01-06T15:54:15.566381Z","iopub.status.idle":"2022-01-06T15:54:16.038685Z","shell.execute_reply.started":"2022-01-06T15:54:15.566353Z","shell.execute_reply":"2022-01-06T15:54:16.037811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Vì số sample có nhãn 1 chỉ chiếm 6% tổng số sample, thử sử dụng class weight để cải tiến model. Coi 1 sample có label 1 như 14 samples có label 0.","metadata":{}},{"cell_type":"code","source":"class_weight = {0: 1., 1: 14.}\nthresholds = [0.1, 0.25, 0.3, 0.35, 0.4, 0.55, 0.6, 0.7, 0.8, 0.85, 0.9]\nc = 30\nmodel = LogisticRegression(solver='liblinear', penalty='l2', C=c, class_weight = class_weight)\nmodel.fit(train_x, train_y)\npredict = model.predict_proba(valid_x)[:,1]\nfor t in thresholds:\n predict_t = np.where(predict > t, 1, 0)\n f1 = f1_score(valid_y, predict_t)\n print(\"Threshold: \", t)\n print(\"F1 score: \", f1)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:54:43.599826Z","iopub.execute_input":"2022-01-06T15:54:43.600123Z","iopub.status.idle":"2022-01-06T15:58:15.410378Z","shell.execute_reply.started":"2022-01-06T15:54:43.600089Z","shell.execute_reply":"2022-01-06T15:58:15.409331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Có vẻ F1 của model này kém hơn model không có class weight, chọn model trước làm model cuối cùng để test.","metadata":{}},{"cell_type":"code","source":"model = LogisticRegression(solver='liblinear', penalty='l2', C=30)\nmodel.fit(train_feature_matrics, train['target'])\npredict = model.predict_proba(test_feature_matrics)[:,1]\npredict = np.where(predict > 0.17, 1, 0)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T15:59:19.016237Z","iopub.execute_input":"2022-01-06T15:59:19.016507Z","iopub.status.idle":"2022-01-06T16:02:22.040428Z","shell.execute_reply.started":"2022-01-06T15:59:19.01648Z","shell.execute_reply":"2022-01-06T16:02:22.039306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['prediction'] = predict\nresults = test[['qid', 'prediction']]\nresults.to_csv('submission.csv', index=False)\nresults.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-06T16:02:45.699705Z","iopub.execute_input":"2022-01-06T16:02:45.699978Z","iopub.status.idle":"2022-01-06T16:02:46.652881Z","shell.execute_reply.started":"2022-01-06T16:02:45.699949Z","shell.execute_reply":"2022-01-06T16:02:46.651954Z"},"trusted":true},"execution_count":null,"outputs":[]}]}