{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction 👋\n\n__`Quora Insincere Questions Classification`__ là một cuộc thi Machine Learning trên nền tảng Kaggle. Em xin phép lấy bài tập này để làm bài tổng kết cuối kỳ môn Học máy - INT3405E 20 (GV: Trần Quốc Long).\n\n🏆 __Goal__ \n> Phân loại những câu hỏi có nội dung độc hại không phù hợp\n* `input`: dữ liệu dạng `text` là câu hỏi trên `Quora` bằng tiếng anh\n* `output`: `0` hoặc `1` tương đương với câu hỏi `non-toxic` hoặc `toxic` \n\n💡 __Idea__\n> Áp dụng mô hình CNN để giải bài toán:\n* CNN sử dụng các bộ lọc để trích xuất ra mối quan hệ địa phương trong những bức ảnh. Trong bài toán NLP này, ta sẽ tận dụng ưu điểm đó để xác định ngữ cảnh của câu giữa các từ.\n* Để làm được điều đó, ta sẽ chuyển các câu hỏi thành các ma trận `m x n` với `n` là số chiều của mỗi từ sau khi thực hiện `word embedding`, `m` là số từ trong câu hỏi.\n\n","metadata":{"id":"0gCssH2YzSQ9","papermill":{"duration":0.055591,"end_time":"2022-01-11T06:06:37.717884","exception":false,"start_time":"2022-01-11T06:06:37.662293","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Libraries\nimport string\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os\nimport zipfile","metadata":{"id":"qcuPAfitRKZa","papermill":{"duration":1.014908,"end_time":"2022-01-11T06:06:38.777363","exception":false,"start_time":"2022-01-11T06:06:37.762455","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-14T14:22:27.281993Z","iopub.execute_input":"2022-01-14T14:22:27.282558Z","iopub.status.idle":"2022-01-14T14:22:28.291977Z","shell.execute_reply.started":"2022-01-14T14:22:27.282422Z","shell.execute_reply":"2022-01-14T14:22:28.290773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make DataFrame show full screen\npd.set_option('display.max_colwidth', None)\n\n# Root path\nROOT_PATH = \"/kaggle/input/quora-insincere-questions-classification\"\n\n!ls $ROOT_PATH","metadata":{"id":"lRCfKF6xubmU","outputId":"ae9e600e-e2eb-4dc9-d5cc-a93e5b959e16","papermill":{"duration":0.815261,"end_time":"2022-01-11T06:06:39.634997","exception":false,"start_time":"2022-01-11T06:06:38.819736","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-14T14:22:28.293874Z","iopub.execute_input":"2022-01-14T14:22:28.294546Z","iopub.status.idle":"2022-01-14T14:22:29.073927Z","shell.execute_reply.started":"2022-01-14T14:22:28.294500Z","shell.execute_reply":"2022-01-14T14:22:29.072880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data 📔\n* `train.csv`: train_set, gồm 3 cột `qid` - id, `question_text` - nội dung câu hỏi, `target` - nhãn (0 là non-toxic, 1 là toxic)\n* `test.csv`: test_set, gồm 2 cột `qid`, `question_text`, không chứa cột nhãn `target`, mình sẽ dùng để làm file `submission`\n* `embeddings`: folder chứa các tập file `embeddings_word` lớn có thể dùng trong quá trình làm bài\n>Ở bài tập này ta sử dụng GloVe - một dự án nguồn mở của Stanford nhằm tạo ra các vector để biểu diễn từ.","metadata":{"id":"mmYmkuA7_Okw","papermill":{"duration":0.042026,"end_time":"2022-01-11T06:06:39.719786","exception":false,"start_time":"2022-01-11T06:06:39.677760","status":"completed"},"tags":[]}},{"cell_type":"code","source":"TRAIN_DF = ROOT_PATH + \"/train.csv\"\n\ntrain_df = pd.read_csv(TRAIN_DF)\ntrain_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:06:39.810004Z","iopub.status.busy":"2022-01-11T06:06:39.809323Z","iopub.status.idle":"2022-01-11T06:06:44.758529Z","shell.execute_reply":"2022-01-11T06:06:44.759035Z","shell.execute_reply.started":"2022-01-11T04:16:45.094265Z"},"id":"Gwe1FD1fRTt4","outputId":"9c7c5e6e-741f-405f-9687-9a8294ab4d54","papermill":{"duration":4.996911,"end_time":"2022-01-11T06:06:44.759240","exception":false,"start_time":"2022-01-11T06:06:39.762329","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_DF = ROOT_PATH + \"/test.csv\"\n\ntest_df = pd.read_csv(TEST_DF)\ntest_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:06:44.850703Z","iopub.status.busy":"2022-01-11T06:06:44.849999Z","iopub.status.idle":"2022-01-11T06:06:46.097476Z","shell.execute_reply":"2022-01-11T06:06:46.096949Z","shell.execute_reply.started":"2022-01-11T04:16:50.448867Z"},"id":"Z7wuucXtbrab","outputId":"4144e439-5437-41a1-dc56-3298ab0b05cf","papermill":{"duration":1.294714,"end_time":"2022-01-11T06:06:46.097618","exception":false,"start_time":"2022-01-11T06:06:44.802904","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"🎨 Ta sẽ khảo sát tính cân bằng của dữ liệu Ta sẽ khảo sát tính cân bằng của dữ liệu","metadata":{"id":"zmqUX2eJ7cot","papermill":{"duration":0.043795,"end_time":"2022-01-11T06:06:46.185767","exception":false,"start_time":"2022-01-11T06:06:46.141972","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_non_toxic = train_df[train_df.target == 0]\ndata_toxic = train_df[train_df.target == 1]\n\nprint(f\"Non toxic length: {len(data_non_toxic)}\")\nprint(f\"Toxic length: {len(data_toxic)}\")\n\n# Plot\nfig, ax = plt.subplots(figsize=(10, 5))\nplt.bar(x=[\"Non toxic\", \"Toxic\"], height=[len(data_non_toxic), len(data_toxic)])\nplt.title(\"Imbalanced Data\", fontsize=20)\nplt.xlabel(\"Data\")\nplt.ylabel(\"Length\");","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:06:46.276889Z","iopub.status.busy":"2022-01-11T06:06:46.276251Z","iopub.status.idle":"2022-01-11T06:06:46.585985Z","shell.execute_reply":"2022-01-11T06:06:46.585498Z","shell.execute_reply.started":"2022-01-11T04:17:00.620869Z"},"id":"tQLHz-hT8GIv","outputId":"eab6798a-fdb3-4ec5-9b92-d3bdd62051d3","papermill":{"duration":0.356238,"end_time":"2022-01-11T06:06:46.586153","exception":false,"start_time":"2022-01-11T06:06:46.229915","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Dữ liệu đang bị mất cân bằng__ 😰\n\n👉 Với kỹ thuật _Undersampling_, ta sẽ tiến hành giảm số lượng dữ liệu `data_non_toxic` để tập dữ liệu được cân bằng hơn, giúp năng cao hiệu năng của mô hình\n* 240,000 câu hỏi non-toxic\n* 80,000 câu hỏi toxic\n\n\n","metadata":{"id":"-4ZXp47k-3Xj","papermill":{"duration":0.044392,"end_time":"2022-01-11T06:06:46.676929","exception":false,"start_time":"2022-01-11T06:06:46.632537","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# We get 240k non-toxic question and 80k toxic question \ntrain_df = pd.concat([data_non_toxic[:240000], data_toxic[:80000]])\ntrain_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:06:46.773091Z","iopub.status.busy":"2022-01-11T06:06:46.771792Z","iopub.status.idle":"2022-01-11T06:06:46.811344Z","shell.execute_reply":"2022-01-11T06:06:46.810817Z","shell.execute_reply.started":"2022-01-11T04:18:43.457936Z"},"id":"izbxW6y4N5wB","outputId":"cf6097ce-03cf-4b4a-cc0e-7fbc061cc01e","papermill":{"duration":0.089947,"end_time":"2022-01-11T06:06:46.811489","exception":false,"start_time":"2022-01-11T06:06:46.721542","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text Preprocessing 🈲\n👉 Như mọi khi, ta sẽ tiến hành _tiền xử lý dữ liệu_ trước khi _vector hóa dữ liệu_\n* `Unicode`: chuyển văn bản về dạng unicode\n* `Expand contractions`: mở rộng các từ: `i'll`, `she's`,... thành `i will`, `she is`,...\n* `Lowercase`: chuyển hóa các từ thành chữ viết thường\n* `Remove punctuation`: Loại bỏ dấu câu\n* `Tokenize`\n* `Remove stopword`: Loại bỏ các từ mà khi không có chúng thì ý nghĩa của câu vẫn không thay đổi\n* `Lemmatization`: Chuyển hóa văn bản về dạng gốc nhưng vẫn giữ được ngữ cảnh của văn bản\n\n🚫 Lưu ý\n\nTa dùng `Lemmatization` thay vì `Stemming` để có thể giữ được ngữ cảnh của từng từ.\n>`goes` được chuyển thành `goe` khi dùng `Stemming` nhưng khi dùng `Lemmatization` thì chúng được chuyển thành `go`\n\nTa không loại bỏ các chữ số vì có thể chúng mang ý nghĩa lớn trong câu.\n>`How did Quebec nationalists see their province as a nation in the 1960s?`  (`target`: 0)\n\n> `Does 1 plus 1 actually equal 7? Can this claim be refuted?` (`target`: 1)","metadata":{"id":"Gk479OpAhhTh","papermill":{"duration":0.045006,"end_time":"2022-01-11T06:06:46.902325","exception":false,"start_time":"2022-01-11T06:06:46.857319","status":"completed"},"tags":[]}},{"cell_type":"code","source":"contractions = { \n\"ain't\": \"am not / are not / is not / has not / have not\",\n\"aren't\": \"are not / am not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he had / he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he shall / he will\",\n\"he'll've\": \"he shall have / he will have\",\n\"he's\": \"he has / he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how has / how is / how does\",\n\"I'd\": \"I had / I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I shall / I will\",\n\"I'll've\": \"I shall have / I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it had / it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it shall / it will\",\n\"it'll've\": \"it shall have / it will have\",\n\"it's\": \"it has / it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she had / she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she shall / she will\",\n\"she'll've\": \"she shall have / she will have\",\n\"she's\": \"she has / she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as / so is\",\n\"that'd\": \"that would / that had\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that has / that is\",\n\"there'd\": \"there had / there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there has / there is\",\n\"they'd\": \"they had / they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they shall / they will\",\n\"they'll've\": \"they shall have / they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we had / we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what shall / what will\",\n\"what'll've\": \"what shall have / what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what has / what is\",\n\"what've\": \"what have\",\n\"when's\": \"when has / when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where has / where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who shall / who will\",\n\"who'll've\": \"who shall have / who will have\",\n\"who's\": \"who has / who is\",\n\"who've\": \"who have\",\n\"why's\": \"why has / why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you had / you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you shall / you will\",\n\"you'll've\": \"you shall have / you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:06:47.007570Z","iopub.status.busy":"2022-01-11T06:06:47.006746Z","iopub.status.idle":"2022-01-11T06:06:47.012418Z","shell.execute_reply":"2022-01-11T06:06:47.012876Z","shell.execute_reply.started":"2022-01-11T04:18:46.509587Z"},"papermill":{"duration":0.065483,"end_time":"2022-01-11T06:06:47.013072","exception":false,"start_time":"2022-01-11T06:06:46.947589","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from unidecode import unidecode\nimport string\nimport re\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import WordNetLemmatizer\n\ndef clean(text: str): \n  uni_text = str(unidecode(text).encode(\"ascii\"), \"ascii\")\n\n  expand_con = uni_text\n  for word in contractions:\n    expand_con = re.sub(rf\"{word}\", contractions[word], expand_con) \n\n  lower = expand_con.lower();\n\n  remove_punc = \"\".join([i for i in lower if i not in string.punctuation])\n\n  tokens = remove_punc.strip().split()\n\n  # Keep `not` in question cause it is a word with multi meaning, that can change a non-toxic to toxic and vice versa.\n  sw_list = stopwords.words()\n  sw_list.remove('not')\n  remove_sw = [word for word in tokens if not word in sw_list]\n\n  # If remove stopword will remove all of `tokens`, keep at most 5 words\n  if (len(remove_sw) != 0):\n    lemma = [WordNetLemmatizer().lemmatize(word) for word in remove_sw]\n  else:\n    lemma = [WordNetLemmatizer().lemmatize(word) for word in tokens[:5]]\n  return lemma","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:06:47.114869Z","iopub.status.busy":"2022-01-11T06:06:47.113987Z","iopub.status.idle":"2022-01-11T06:06:47.800291Z","shell.execute_reply":"2022-01-11T06:06:47.799687Z","shell.execute_reply.started":"2022-01-11T04:18:50.047595Z"},"id":"QK0TqtkVV-V-","papermill":{"duration":0.741276,"end_time":"2022-01-11T06:06:47.800427","exception":false,"start_time":"2022-01-11T06:06:47.059151","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean data\ninput = []\nfor idx, ques in enumerate(train_df['question_text']):\n  input.append(clean(ques))\n  if idx % 50000 == 0:\n    print(f\"Completed: {idx}/320000\")\nprint(\"Done\")","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:06:47.903909Z","iopub.status.busy":"2022-01-11T06:06:47.903279Z","iopub.status.idle":"2022-01-11T06:36:53.818789Z","shell.execute_reply":"2022-01-11T06:36:53.819467Z","shell.execute_reply.started":"2022-01-11T04:18:52.450833Z"},"papermill":{"duration":1805.973709,"end_time":"2022-01-11T06:36:53.819889","exception":false,"start_time":"2022-01-11T06:06:47.846180","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"🧶 Thêm cột dữ liệu đã xử lý vào tập dữ liệu hiện tại","metadata":{"id":"LS5EHOmrMAZQ","papermill":{"duration":0.04842,"end_time":"2022-01-11T06:36:53.917394","exception":false,"start_time":"2022-01-11T06:36:53.868974","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Add new column after clean data into data\ntrain_df['clean_ques_text'] = input\ntrain_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:36:54.020541Z","iopub.status.busy":"2022-01-11T06:36:54.019462Z","iopub.status.idle":"2022-01-11T06:36:54.479367Z","shell.execute_reply":"2022-01-11T06:36:54.478789Z","shell.execute_reply.started":"2022-01-11T04:48:52.964140Z"},"id":"kb8D_EnKL-O4","outputId":"ef7943f2-96ec-4a7f-a8dd-d36c07132647","papermill":{"duration":0.513425,"end_time":"2022-01-11T06:36:54.479533","exception":false,"start_time":"2022-01-11T06:36:53.966108","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data analysis 👓","metadata":{"id":"2pwd23pfG42e","papermill":{"duration":0.049111,"end_time":"2022-01-11T06:36:54.579323","exception":false,"start_time":"2022-01-11T06:36:54.530212","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"👉 Mục đích của ta khi dùng mạng CNN là biến đổi mỗi câu hỏi thành một ma trận. \n\n> Vì thế, ta sẽ khảo sát độ dài của các `question` để phục vụ cho việc xác định kích thước của ma trận.","metadata":{"id":"D_hA2UtcODe3","papermill":{"duration":0.049383,"end_time":"2022-01-11T06:36:54.678301","exception":false,"start_time":"2022-01-11T06:36:54.628918","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Init array for questions length\ntmp = [[str(idx), len(ques)] for idx, ques in enumerate(input)]\nlength_df = pd.DataFrame(tmp, columns=[ 'id', 'length'])\nlength_df.sort_values(by='length',ascending=False, inplace=True)\n\nlength_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:36:54.796353Z","iopub.status.busy":"2022-01-11T06:36:54.791377Z","iopub.status.idle":"2022-01-11T06:36:56.239520Z","shell.execute_reply":"2022-01-11T06:36:56.238728Z","shell.execute_reply.started":"2022-01-11T04:49:15.203845Z"},"papermill":{"duration":1.51207,"end_time":"2022-01-11T06:36:56.239692","exception":false,"start_time":"2022-01-11T06:36:54.727622","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot\nfig, ax = plt.subplots(figsize=(16, 7))\nplt.bar(x=length_df[\"id\"][::100], height=length_df[\"length\"][::100])\nplt.title(\"Questions Length\", fontsize=20)\nplt.xticks([])\nplt.xlabel(\"ID\")\nplt.ylabel(\"Length\");","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:36:56.365114Z","iopub.status.busy":"2022-01-11T06:36:56.362173Z","iopub.status.idle":"2022-01-11T06:37:03.210446Z","shell.execute_reply":"2022-01-11T06:37:03.209615Z","shell.execute_reply.started":"2022-01-11T04:49:18.849491Z"},"id":"fl1LwaNN7BkS","outputId":"c643e12b-b521-4a42-9847-b4265f3a9aea","papermill":{"duration":6.920427,"end_time":"2022-01-11T06:37:03.210613","exception":false,"start_time":"2022-01-11T06:36:56.290186","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vocabulary count\nfrom collections import Counter\n\ndef word_frequency(list):\n  counted = Counter(list)\n  word_freq = pd.DataFrame(counted.items(),columns=['word','frequency'])\n  return word_freq.sort_values(by='frequency',ascending=False)\n\nall = list()\nfor idx, ques in enumerate(input):\n  all.extend(ques)\n\nvocab = word_frequency(all)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:03.328947Z","iopub.status.busy":"2022-01-11T06:37:03.322255Z","iopub.status.idle":"2022-01-11T06:37:04.422070Z","shell.execute_reply":"2022-01-11T06:37:04.421389Z","shell.execute_reply.started":"2022-01-11T04:49:26.462216Z"},"id":"mL1sbpQXwAhj","papermill":{"duration":1.160161,"end_time":"2022-01-11T06:37:04.422246","exception":false,"start_time":"2022-01-11T06:37:03.262085","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example\nprint(f'We have {len(all)} words')\nprint(f'Vocab has {len(vocab)} words')\nvocab.head()","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:04.539286Z","iopub.status.busy":"2022-01-11T06:37:04.533288Z","iopub.status.idle":"2022-01-11T06:37:04.543972Z","shell.execute_reply":"2022-01-11T06:37:04.543353Z","shell.execute_reply.started":"2022-01-11T04:49:27.291726Z"},"id":"WKEaFOOfxxac","outputId":"1912dca2-5f5a-4535-8515-2883f0d0d49d","papermill":{"duration":0.067375,"end_time":"2022-01-11T06:37:04.544143","exception":false,"start_time":"2022-01-11T06:37:04.476768","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vector hóa dữ liệu 🎭\n\nTrong bài tập này, ta sử dụng `GloVe` để vector hóa các từ sau khi thực hiện `clean data`.\n\n>`GloVe` là viết tắt của `Global Vectors`, một dự án mã nguồn mở của Stanford nhằm tạo ra các vector biểu diễn cho các từ.\n\nSau khi `clean data`, tập từ vựng của chúng ta khá ít (~100,000 từ). Do đó, để tiết kiệm bộ nhớ cũng như tăng tốc tính toán, ta sẽ sử dụng vector 50 chiều để biểu diễn từ.\n>Ta sử dụng “Wikipedia 2014 + Gigaword 5”, đây là file nhỏ nhất (“ glove.6B.zip”) có dung lượng 822 MB. Nó được huấn luyện trên một kho ngữ liệu chứa 6 tỷ từ trong tập từ vựng chứa 400,000 từ khác nhau.\n\n🚫 Lưu ý: Khi triển khai trên `Google Colab`, em đã triển khai với `GloVe` vector 50 chiều, nhưng trong cuộc thi trên Kaggle thì chỉ có sẵn cho GloVe vector 300 chiều. Vì thế, em sẽ dùng thư viện `Word2Vec` để tự triển khai cho phù hợp với những gì đã thực hành trên `Google Colab`","metadata":{"id":"IrLZ1at-nkff","papermill":{"duration":0.051743,"end_time":"2022-01-11T06:37:04.648166","exception":false,"start_time":"2022-01-11T06:37:04.596423","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from gensim.models import Word2Vec\n\nmodel = Word2Vec(input, vector_size=50, window=5, min_count=0, workers=4, sg=1)\nmodel.wv.save(\"word.model\")","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:04.759171Z","iopub.status.busy":"2022-01-11T06:37:04.758310Z","iopub.status.idle":"2022-01-11T06:37:40.882152Z","shell.execute_reply":"2022-01-11T06:37:40.880760Z","shell.execute_reply.started":"2022-01-11T04:49:34.593930Z"},"papermill":{"duration":36.181859,"end_time":"2022-01-11T06:37:40.882345","exception":false,"start_time":"2022-01-11T06:37:04.700486","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.models import KeyedVectors\n\nglove = KeyedVectors.load('./word.model')","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:40.999717Z","iopub.status.busy":"2022-01-11T06:37:40.990140Z","iopub.status.idle":"2022-01-11T06:37:41.091422Z","shell.execute_reply":"2022-01-11T06:37:41.090814Z","shell.execute_reply.started":"2022-01-11T04:50:09.807441Z"},"papermill":{"duration":0.156445,"end_time":"2022-01-11T06:37:41.091570","exception":false,"start_time":"2022-01-11T06:37:40.935125","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EMBEDDINGS = \"../input/glove6b/glove.6B.50d.txt\"\n\n# # Init embedding words\n# glove = {}\n# with open(EMBEDDINGS, 'rb') as f:\n#     for l in f:\n#         line = l.decode().split()\n#         glove[line[0]] = np.array(line[1:]).astype(np.float)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:41.198819Z","iopub.status.busy":"2022-01-11T06:37:41.198166Z","iopub.status.idle":"2022-01-11T06:37:41.201190Z","shell.execute_reply":"2022-01-11T06:37:41.200557Z","shell.execute_reply.started":"2022-01-10T17:23:24.509359Z"},"id":"seyHXpGJr22x","papermill":{"duration":0.058378,"end_time":"2022-01-11T06:37:41.201357","exception":false,"start_time":"2022-01-11T06:37:41.142979","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example\nprint(f\"Embedding words: {len(glove)}\")\nprint(glove['convert'])","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:41.309055Z","iopub.status.busy":"2022-01-11T06:37:41.307998Z","iopub.status.idle":"2022-01-11T06:37:41.314562Z","shell.execute_reply":"2022-01-11T06:37:41.315187Z","shell.execute_reply.started":"2022-01-11T04:50:09.907336Z"},"id":"j8YbxV0muhHO","outputId":"f3d171d8-0d7b-406f-97c3-02bfb84683c3","papermill":{"duration":0.061743,"end_time":"2022-01-11T06:37:41.315358","exception":false,"start_time":"2022-01-11T06:37:41.253615","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Question to Matrix ✨\nTa sẽ tiến hành chuyển các câu hỏi thành dạng ma trận, với kích thước là số từ trong câu hỏi đó.\n\n>Để có thể chuyển `question` thành `matrix`, ta cần xác định kích thước của `matrix` - hay chính là số từ trong `question`. \n\n>Dựa vào biểu đồ bên trên, ta thấy sau khi tiến hành `clean data`, các câu hỏi chủ yếu có độ dài từ 5-25.\n\n>Câu hỏi có độ dài lớn nhất là 40. Đây là 1 con số nhỏ, vì thế, để có thể lấy được tất cả thông tin của mọi câu hỏi trong tập dữ liệu, ta sẽ khởi tạo độ lớn của ma trận `max_seq = 40`.","metadata":{"id":"PGNgR5Nk8-Vs","papermill":{"duration":0.051326,"end_time":"2022-01-11T06:37:41.418157","exception":false,"start_time":"2022-01-11T06:37:41.366831","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"","metadata":{"papermill":{"duration":0.051424,"end_time":"2022-01-11T06:37:41.521192","exception":false,"start_time":"2022-01-11T06:37:41.469768","status":"completed"},"tags":[]}},{"cell_type":"code","source":"max_seq = 40\nembedding_size = 50\n    \ndef question_embedding(question):\n  matrix = np.zeros((max_seq, embedding_size))\n  ques_len = len(question)\n\n  if (ques_len > 0):\n    for i in range(max_seq):\n      index = i % ques_len\n      if (i >= ques_len):\n          break\n      if(question[index] in glove):\n          matrix[i] = glove[question[index]]\n\n  if (ques_len > max_seq or ques_len == 0):\n    print(question)\n    print(f\"Invalid: This question has {ques_len} words\")\n\n  matrix = np.array(matrix)\n  return matrix","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:41.629392Z","iopub.status.busy":"2022-01-11T06:37:41.628395Z","iopub.status.idle":"2022-01-11T06:37:41.641809Z","shell.execute_reply":"2022-01-11T06:37:41.642310Z","shell.execute_reply.started":"2022-01-11T04:50:35.226554Z"},"id":"DYziwVKczCyp","papermill":{"duration":0.068698,"end_time":"2022-01-11T06:37:41.642495","exception":false,"start_time":"2022-01-11T06:37:41.573797","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example\nprint(train_df['question_text'][0])\n\nclean_ques = train_df['clean_ques_text'][0]\nprint(clean_ques)\n\nprint(\"Embedding for last word: \")\nprint(question_embedding(clean_ques)[len(clean_ques)-1])\nprint(\"Embedding for last word + 1 (all is zero): \")\nprint(question_embedding(clean_ques)[len(clean_ques)])","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:41.758569Z","iopub.status.busy":"2022-01-11T06:37:41.757812Z","iopub.status.idle":"2022-01-11T06:37:41.775557Z","shell.execute_reply":"2022-01-11T06:37:41.774814Z","shell.execute_reply.started":"2022-01-11T04:50:38.812067Z"},"id":"NhK2wQJkJNAa","outputId":"55f49066-9558-488a-8206-b85a5a4e7133","papermill":{"duration":0.081338,"end_time":"2022-01-11T06:37:41.775722","exception":false,"start_time":"2022-01-11T06:37:41.694384","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mô tả mô hình 🎉\n💡 Áp dụng mô hình CNN để giải bài toán:\n* Chi tiết về [mô hình CNN](https://excessive-source-1c9.notion.site/07-10-2021-M-ng-t-ch-ch-p-CNN-aa7ebecc13a04b7bb609f2e7a9de7812#b63518c9c6d5494cbd1625c035c5ca10)\n* CNN sử dụng các bộ lọc để trích xuất ra mối quan hệ địa phương trong những bức ảnh. Trong bài toán NLP này, ta sẽ tận dụng ưu điểm đó để xác định ngữ cảnh của câu giữa các từ.\n* Để làm được điều đó, ta sẽ chuyển các câu hỏi thành các ma trận `m x n` với `n` là số chiều của mỗi từ sau khi thực hiện `word embedding`, `m` là số từ trong câu hỏi.\n\n🚫 Không giống đối với đầu vào là một ảnh, do mỗi `vector` trong `matrix` biểu diễn cho 1 từ nên số chiều của `filters` trong `Convolution2D` sẽ chính bằng số chiều của `vector`.\n\nTa sẽ chia tập dữ liệu thành 2 tập nhỏ hơn dùng để huấn luyện mô hình:\n* `train_set`: chiếm 80% - bao gồm `train_data` và `train_label`\n* `valid_set`: chiếm 20% - bao gồm `valid_data` và `valid_label`","metadata":{"id":"qMuE6_VX-xZW","papermill":{"duration":0.052462,"end_time":"2022-01-11T06:37:41.882269","exception":false,"start_time":"2022-01-11T06:37:41.829807","status":"completed"},"tags":[]}},{"cell_type":"code","source":"data_non_toxic = train_df[train_df.target == 0]\ndata_toxic = train_df[train_df.target == 1]\n\ntrain_set = pd.concat([data_non_toxic[:192000], data_toxic[:64000]])\nvalid_set = pd.concat([data_non_toxic[192000:], data_toxic[64000:]])\n\n# Suffle DataFrame\ntrain_set = train_set.sample(frac=1).reset_index(drop=True)\nvalid_set = valid_set.sample(frac=1).reset_index(drop=True)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:41.993122Z","iopub.status.busy":"2022-01-11T06:37:41.992374Z","iopub.status.idle":"2022-01-11T06:37:42.425920Z","shell.execute_reply":"2022-01-11T06:37:42.425171Z","shell.execute_reply.started":"2022-01-11T04:50:47.070827Z"},"id":"Bk-dxRh9JwS0","papermill":{"duration":0.4911,"end_time":"2022-01-11T06:37:42.426116","exception":false,"start_time":"2022-01-11T06:37:41.935016","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Handle `train_set`\ntrain_data = []\ntrain_label = []\n\nfor ques in train_set['clean_ques_text']:\n  train_data.append(question_embedding(ques))\ntrain_data = np.array(train_data)\n\nfor y in train_set['target']:\n  label = np.zeros(2)\n  label[int(y)] = int(1)\n  train_label.append(label)\ntrain_label = np.array(train_label)\n\n# Handle `valid_set`\nvalid_data = []\nvalid_label = []\n\nfor ques in valid_set['clean_ques_text']:\n  valid_data.append(question_embedding(ques))\nvalid_data = np.array(valid_data)\n\nfor y in valid_set['target']:\n  label = np.zeros(2)\n  label[int(y)] = int(1)\n  valid_label.append(label)\nvalid_label = np.array(valid_label)\n\nlen(train_data), len(train_label), len(valid_data), len(valid_label)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:37:42.542184Z","iopub.status.busy":"2022-01-11T06:37:42.541424Z","iopub.status.idle":"2022-01-11T06:38:03.406485Z","shell.execute_reply":"2022-01-11T06:38:03.407983Z","shell.execute_reply.started":"2022-01-11T04:50:52.423538Z"},"id":"H9QvzJ-QWRDn","outputId":"1d5d0738-9f85-43d3-ce57-47a9b403bdf0","papermill":{"duration":20.929138,"end_time":"2022-01-11T06:38:03.408253","exception":false,"start_time":"2022-01-11T06:37:42.479115","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Huấn luyện mô hình 🍀","metadata":{"papermill":{"duration":0.054876,"end_time":"2022-01-11T06:38:03.518728","exception":false,"start_time":"2022-01-11T06:38:03.463852","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from tensorflow.keras import layers\nfrom tensorflow import keras \nimport tensorflow as tf\nfrom keras.preprocessing import sequence\nimport datetime\n\nsequence_length = 40\nembedding_size = 50\nepochs = 20\nbatch_size = 32\nlearning_rate = 0.001\ndecay_rate = learning_rate / epochs\ndropout_rate = 0.2\nnum_classes = 2\nfilter_sizes = 3\nnum_filters = 25","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:38:03.634788Z","iopub.status.busy":"2022-01-11T06:38:03.633976Z","iopub.status.idle":"2022-01-11T06:38:10.510302Z","shell.execute_reply":"2022-01-11T06:38:10.509406Z","shell.execute_reply.started":"2022-01-11T04:51:12.008267Z"},"id":"M5doG9M1L8n1","papermill":{"duration":6.937001,"end_time":"2022-01-11T06:38:10.510523","exception":false,"start_time":"2022-01-11T06:38:03.573522","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train_data.reshape(train_data.shape[0], sequence_length, embedding_size, 1).astype('float32')\ny_train = np.array(train_label)\n\nx_valid = valid_data.reshape(valid_data.shape[0], sequence_length, embedding_size, 1).astype('float32')\ny_valid = np.array(valid_label)\n\nmodel = keras.Sequential()\nmodel.add(layers.Convolution2D(num_filters, (filter_sizes, embedding_size),\n                        padding='valid',\n                        input_shape=(sequence_length, embedding_size, 1), activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(38, 1)))\nmodel.add(layers.Dropout(dropout_rate))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(128, activation='relu'))\nmodel.add(layers.Dense(2, activation='softmax'))\n\nadam = tf.optimizers.Adam(learning_rate=learning_rate, decay=decay_rate)\nmodel.compile(loss='categorical_crossentropy',\n              optimizer=adam,\n              metrics=['accuracy'])\nprint(model.summary())","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:38:10.630237Z","iopub.status.busy":"2022-01-11T06:38:10.629474Z","iopub.status.idle":"2022-01-11T06:38:11.605984Z","shell.execute_reply":"2022-01-11T06:38:11.606541Z","shell.execute_reply.started":"2022-01-11T04:51:27.025308Z"},"id":"WKKv1YXhMMbR","outputId":"e1039947-496f-4821-a1fb-28dbc20062e2","papermill":{"duration":1.041986,"end_time":"2022-01-11T06:38:11.606736","exception":false,"start_time":"2022-01-11T06:38:10.564750","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"🛫 _Training..._ ","metadata":{"papermill":{"duration":0.053225,"end_time":"2022-01-11T06:38:11.714156","exception":false,"start_time":"2022-01-11T06:38:11.660931","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Save the best model to .h5 file\nbest_model=keras.callbacks.ModelCheckpoint(filepath=\"best_model.h5\",monitor='val_loss',save_best_only=True,verbose=1)\n\nmodel.fit(x_train, y_train, batch_size=batch_size, verbose=1, epochs=epochs, validation_data=(x_valid, y_valid), callbacks=[best_model])\n\n# Show `classification_report`\nfrom sklearn.metrics import classification_report\n\ny_pred = model.predict(x_valid)\ny_pred = (y_pred >= 0.5) \nprint(\"\\n\")\nprint(classification_report(y_valid, y_pred))","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:38:11.827852Z","iopub.status.busy":"2022-01-11T06:38:11.826799Z","iopub.status.idle":"2022-01-11T06:51:18.301776Z","shell.execute_reply":"2022-01-11T06:51:18.302408Z","shell.execute_reply.started":"2022-01-11T04:51:33.064020Z"},"papermill":{"duration":786.532951,"end_time":"2022-01-11T06:51:18.302618","exception":false,"start_time":"2022-01-11T06:38:11.769667","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"🪂 Free memory","metadata":{"papermill":{"duration":4.201628,"end_time":"2022-01-11T06:51:26.889056","exception":false,"start_time":"2022-01-11T06:51:22.687428","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gc\n\ndel data_non_toxic\ndel data_toxic\ndel train_set\ndel valid_set\ndel x_train\ndel y_train\ndel x_valid\ndel y_valid\ndel train_data\ndel train_label\ndel valid_data\ndel valid_label\ndel model\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:51:35.427811Z","iopub.status.busy":"2022-01-11T06:51:35.427142Z","iopub.status.idle":"2022-01-11T06:51:36.129293Z","shell.execute_reply":"2022-01-11T06:51:36.129808Z","shell.execute_reply.started":"2022-01-11T05:04:32.521810Z"},"papermill":{"duration":5.014763,"end_time":"2022-01-11T06:51:36.129983","exception":false,"start_time":"2022-01-11T06:51:31.115220","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test 🚀","metadata":{"id":"RxdDxPP9R153","papermill":{"duration":4.244657,"end_time":"2022-01-11T06:51:44.586723","exception":false,"start_time":"2022-01-11T06:51:40.342066","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from keras.models import load_model\nmodel_sentiment = load_model(\"best_model.h5\")","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:51:53.120756Z","iopub.status.busy":"2022-01-11T06:51:53.119745Z","iopub.status.idle":"2022-01-11T06:51:53.209825Z","shell.execute_reply":"2022-01-11T06:51:53.210338Z","shell.execute_reply.started":"2022-01-11T05:04:51.843865Z"},"papermill":{"duration":4.333151,"end_time":"2022-01-11T06:51:53.210541","exception":false,"start_time":"2022-01-11T06:51:48.877390","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text = \"Why do people hate Adolf Hitler? Do you have any specific and logical reasons?\"\ntext = clean(text)\n\nmaxtrix_embedding = np.expand_dims(question_embedding(text), axis=0)\nmaxtrix_embedding = np.expand_dims(maxtrix_embedding, axis=3)\n\nresult = model_sentiment.predict(maxtrix_embedding)\nprint(\"Result: \", result)\nresult = np.argmax(result)\nprint(\"Label predict: \", result)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:52:01.641826Z","iopub.status.busy":"2022-01-11T06:52:01.638384Z","iopub.status.idle":"2022-01-11T06:52:01.763342Z","shell.execute_reply":"2022-01-11T06:52:01.762630Z","shell.execute_reply.started":"2022-01-11T05:04:59.816287Z"},"id":"QiEwxMNnSE5V","papermill":{"duration":4.328519,"end_time":"2022-01-11T06:52:01.763509","exception":false,"start_time":"2022-01-11T06:51:57.434990","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission 🎁","metadata":{"papermill":{"duration":4.23991,"end_time":"2022-01-11T06:52:10.319693","exception":false,"start_time":"2022-01-11T06:52:06.079783","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# test.csv\nTEST_DF = ROOT_PATH + \"/test.csv\"\n\ntest_df = pd.read_csv(TEST_DF)\ntest_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:52:18.786434Z","iopub.status.busy":"2022-01-11T06:52:18.785621Z","iopub.status.idle":"2022-01-11T06:52:20.175520Z","shell.execute_reply":"2022-01-11T06:52:20.174630Z","shell.execute_reply.started":"2022-01-11T05:05:05.537073Z"},"papermill":{"duration":5.649554,"end_time":"2022-01-11T06:52:20.175669","exception":false,"start_time":"2022-01-11T06:52:14.526115","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean data\ninput = []\nfor idx, ques in enumerate(test_df['question_text']):\n  input.append(clean(ques))\n  if idx % 50000 == 0:\n    print(f\"Completed: {idx}/375806\")\nprint(\"Done\")","metadata":{"execution":{"iopub.execute_input":"2022-01-11T06:52:28.903846Z","iopub.status.busy":"2022-01-11T06:52:28.903170Z","iopub.status.idle":"2022-01-11T07:28:16.409102Z","shell.execute_reply":"2022-01-11T07:28:16.409668Z","shell.execute_reply.started":"2022-01-11T05:05:14.111944Z"},"papermill":{"duration":2151.882304,"end_time":"2022-01-11T07:28:16.409994","exception":false,"start_time":"2022-01-11T06:52:24.527690","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(input)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:28:25.042927Z","iopub.status.busy":"2022-01-11T07:28:25.041894Z","iopub.status.idle":"2022-01-11T07:28:25.056337Z","shell.execute_reply":"2022-01-11T07:28:25.056886Z","shell.execute_reply.started":"2022-01-11T05:40:39.949595Z"},"papermill":{"duration":4.268042,"end_time":"2022-01-11T07:28:25.057100","exception":false,"start_time":"2022-01-11T07:28:20.789058","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add new column after clean data into data\ntest_df['clean_ques_text'] = input\ntest_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:28:33.646997Z","iopub.status.busy":"2022-01-11T07:28:33.646337Z","iopub.status.idle":"2022-01-11T07:28:34.322456Z","shell.execute_reply":"2022-01-11T07:28:34.321843Z","shell.execute_reply.started":"2022-01-11T05:41:16.986079Z"},"papermill":{"duration":4.976522,"end_time":"2022-01-11T07:28:34.322607","exception":false,"start_time":"2022-01-11T07:28:29.346085","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del input\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:28:42.710348Z","iopub.status.busy":"2022-01-11T07:28:42.709582Z","iopub.status.idle":"2022-01-11T07:28:44.873374Z","shell.execute_reply":"2022-01-11T07:28:44.872705Z","shell.execute_reply.started":"2022-01-11T05:41:27.186732Z"},"papermill":{"duration":6.347985,"end_time":"2022-01-11T07:28:44.873526","exception":false,"start_time":"2022-01-11T07:28:38.525541","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare test data\nsequence_length = 40\nembedding_size = 50\n\ntest_data = []\nfor ques in test_df['clean_ques_text'][:200000]:\n  test_data.append(question_embedding(ques))\ntest_data = np.array(test_data)\nx_test = test_data.reshape(test_data.shape[0], sequence_length, embedding_size, 1).astype('float32')","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:28:53.418457Z","iopub.status.busy":"2022-01-11T07:28:53.417718Z","iopub.status.idle":"2022-01-11T07:29:03.779309Z","shell.execute_reply":"2022-01-11T07:29:03.778529Z"},"papermill":{"duration":14.521727,"end_time":"2022-01-11T07:29:03.779509","exception":false,"start_time":"2022-01-11T07:28:49.257782","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict\ny_test = model_sentiment.predict(x_test)\ny_test = [np.argmax(idx) for idx in y_test]\n\nTEMP_PATH = \"prediction1.csv\"\n\ndf_write = pd.DataFrame(y_test, columns=['prediction'])\ndf_write.to_csv(TEMP_PATH, index=False)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:29:12.384904Z","iopub.status.busy":"2022-01-11T07:29:12.384225Z","iopub.status.idle":"2022-01-11T07:29:28.729981Z","shell.execute_reply":"2022-01-11T07:29:28.729321Z","shell.execute_reply.started":"2022-01-11T05:48:12.470849Z"},"papermill":{"duration":20.638631,"end_time":"2022-01-11T07:29:28.730173","exception":false,"start_time":"2022-01-11T07:29:08.091542","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_test\ndel test_data\ndel y_test\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:29:38.091454Z","iopub.status.busy":"2022-01-11T07:29:37.187054Z","iopub.status.idle":"2022-01-11T07:29:38.100399Z","shell.execute_reply":"2022-01-11T07:29:38.099790Z","shell.execute_reply.started":"2022-01-11T05:50:59.548491Z"},"papermill":{"duration":5.158353,"end_time":"2022-01-11T07:29:38.100565","exception":false,"start_time":"2022-01-11T07:29:32.942212","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare test data\ntest_data = []\nfor ques in test_df['clean_ques_text'][200000:]:\n  test_data.append(question_embedding(ques))\ntest_data = np.array(test_data)\nx_test = test_data.reshape(test_data.shape[0], sequence_length, embedding_size, 1).astype('float32')","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:29:46.601487Z","iopub.status.busy":"2022-01-11T07:29:46.600412Z","iopub.status.idle":"2022-01-11T07:29:55.447967Z","shell.execute_reply":"2022-01-11T07:29:55.447179Z","shell.execute_reply.started":"2022-01-11T05:51:18.108716Z"},"papermill":{"duration":13.062035,"end_time":"2022-01-11T07:29:55.448140","exception":false,"start_time":"2022-01-11T07:29:42.386105","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict\ny_test = model_sentiment.predict(x_test)\ny_test = [np.argmax(idx) for idx in y_test]\n\nTEMP_PATH = \"prediction2.csv\"\n\ndf_write = pd.DataFrame(y_test, columns=['prediction'])\ndf_write.to_csv(TEMP_PATH, index=False)","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:30:04.146601Z","iopub.status.busy":"2022-01-11T07:30:04.145886Z","iopub.status.idle":"2022-01-11T07:30:14.396257Z","shell.execute_reply":"2022-01-11T07:30:14.396771Z","shell.execute_reply.started":"2022-01-11T05:51:45.227875Z"},"papermill":{"duration":14.623855,"end_time":"2022-01-11T07:30:14.397017","exception":false,"start_time":"2022-01-11T07:29:59.773162","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_test\ndel test_data\ndel y_test\ndel model_sentiment\ngc.collect()","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:30:22.817481Z","iopub.status.busy":"2022-01-11T07:30:22.816782Z","iopub.status.idle":"2022-01-11T07:30:23.675753Z","shell.execute_reply":"2022-01-11T07:30:23.675205Z","shell.execute_reply.started":"2022-01-11T06:05:13.562114Z"},"papermill":{"duration":5.091607,"end_time":"2022-01-11T07:30:23.675914","exception":false,"start_time":"2022-01-11T07:30:18.584307","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1 = pd.read_csv(\"prediction1.csv\")\ndf2 = pd.read_csv(\"prediction2.csv\")\n\ndf = pd.concat([df1, df2], ignore_index=True)\ndf","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:30:32.374613Z","iopub.status.busy":"2022-01-11T07:30:32.373186Z","iopub.status.idle":"2022-01-11T07:30:32.420110Z","shell.execute_reply":"2022-01-11T07:30:32.420652Z","shell.execute_reply.started":"2022-01-11T06:03:36.105565Z"},"papermill":{"duration":4.252676,"end_time":"2022-01-11T07:30:32.420845","exception":false,"start_time":"2022-01-11T07:30:28.168169","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add `prediction` column into test_dataframe\ntest_df[\"prediction\"] = df[\"prediction\"]\ntest_df","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:30:41.091683Z","iopub.status.busy":"2022-01-11T07:30:41.090676Z","iopub.status.idle":"2022-01-11T07:30:41.109409Z","shell.execute_reply":"2022-01-11T07:30:41.109918Z","shell.execute_reply.started":"2022-01-11T06:04:21.324851Z"},"papermill":{"duration":4.333733,"end_time":"2022-01-11T07:30:41.110153","exception":false,"start_time":"2022-01-11T07:30:36.776420","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Export submission.csv\nsubmission = test_df[['qid', 'prediction']]\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission","metadata":{"execution":{"iopub.execute_input":"2022-01-11T07:30:49.523526Z","iopub.status.busy":"2022-01-11T07:30:49.522820Z","iopub.status.idle":"2022-01-11T07:30:50.602768Z","shell.execute_reply":"2022-01-11T07:30:50.602220Z","shell.execute_reply.started":"2022-01-11T06:04:32.888561Z"},"papermill":{"duration":5.293173,"end_time":"2022-01-11T07:30:50.602928","exception":false,"start_time":"2022-01-11T07:30:45.309755","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}