{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# MSSV: 19020063\n# Name: Cao Đình Hoàng Minh\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # plot some graphs\nfrom wordcloud import WordCloud, STOPWORDS\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import layers\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-08T01:45:57.682703Z","iopub.execute_input":"2022-01-08T01:45:57.683483Z","iopub.status.idle":"2022-01-08T01:46:02.136264Z","shell.execute_reply.started":"2022-01-08T01:45:57.683386Z","shell.execute_reply":"2022-01-08T01:46:02.135509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_data = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:46:02.137992Z","iopub.execute_input":"2022-01-08T01:46:02.138257Z","iopub.status.idle":"2022-01-08T01:46:07.476108Z","shell.execute_reply.started":"2022-01-08T01:46:02.138222Z","shell.execute_reply":"2022-01-08T01:46:07.475395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()\nprint(\"\\n\")\ntest_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:46:07.478128Z","iopub.execute_input":"2022-01-08T01:46:07.478327Z","iopub.status.idle":"2022-01-08T01:46:07.836094Z","shell.execute_reply.started":"2022-01-08T01:46:07.478302Z","shell.execute_reply":"2022-01-08T01:46:07.834759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:46:07.838112Z","iopub.execute_input":"2022-01-08T01:46:07.83866Z","iopub.status.idle":"2022-01-08T01:46:07.853798Z","shell.execute_reply.started":"2022-01-08T01:46:07.838619Z","shell.execute_reply":"2022-01-08T01:46:07.852898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:46:07.856038Z","iopub.execute_input":"2022-01-08T01:46:07.856253Z","iopub.status.idle":"2022-01-08T01:46:07.866107Z","shell.execute_reply.started":"2022-01-08T01:46:07.856229Z","shell.execute_reply":"2022-01-08T01:46:07.865251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincereQuestions = train_data[train_data['target']==0]\ninsincereQuestions = train_data[train_data['target']==1]\nnumOfSincere = sincereQuestions.shape[0]\nnumOfInsincere = insincereQuestions.shape[0]\nprint(\"Number Of Sincere Questions:\", numOfSincere)\nprint(\"Number Of Insincere Questions:\", numOfInsincere)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:46:07.867419Z","iopub.execute_input":"2022-01-08T01:46:07.867668Z","iopub.status.idle":"2022-01-08T01:46:07.95665Z","shell.execute_reply.started":"2022-01-08T01:46:07.867635Z","shell.execute_reply":"2022-01-08T01:46:07.955797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"quantity = [numOfSincere, numOfInsincere]\nlabels = ['Sincere Questions', 'Insincere Questions']\nplt.pie(quantity, labels=labels, autopct='%1.2f%%', shadow=False)\nplt.title('Target Distribution')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:46:07.958162Z","iopub.execute_input":"2022-01-08T01:46:07.958422Z","iopub.status.idle":"2022-01-08T01:46:08.069398Z","shell.execute_reply.started":"2022-01-08T01:46:07.958385Z","shell.execute_reply":"2022-01-08T01:46:08.068703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincereWordcloud = WordCloud(width=800, height=450, background_color='white', min_font_size=10).generate(\" \".join(sincereQuestions.question_text))\nplt.figure(figsize=(16,9), facecolor=None)\nplt.imshow(sincereWordcloud)\nplt.axis(\"off\")\nplt.title(\"Common Words in Sincere Questions\", fontsize=30,color='k')\nplt.tight_layout(pad=0)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:46:08.070484Z","iopub.execute_input":"2022-01-08T01:46:08.070907Z","iopub.status.idle":"2022-01-08T01:47:01.252902Z","shell.execute_reply.started":"2022-01-08T01:46:08.070868Z","shell.execute_reply":"2022-01-08T01:47:01.252193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincereWordcloud = WordCloud(width=800, height=450, background_color='white', min_font_size=10).generate(\" \".join(insincereQuestions.question_text))\nplt.figure(figsize=(16,9), facecolor=None)\nplt.imshow(insincereWordcloud)\nplt.axis(\"off\")\nplt.title(\"Common Words in Insincere Questions\", fontsize=30,color='k')\nplt.tight_layout(pad=0)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:47:01.253884Z","iopub.execute_input":"2022-01-08T01:47:01.25411Z","iopub.status.idle":"2022-01-08T01:47:07.359829Z","shell.execute_reply.started":"2022-01-08T01:47:01.254081Z","shell.execute_reply":"2022-01-08T01:47:07.359118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ratio = 0.8\nvalid_ratio = 1 - train_ratio\nnumOfTrain = int(train_ratio * (numOfSincere + numOfInsincere))\ntrain_sen = []\nval_sen = []\ntest_sen = []\n\nfor i in range(0, len(train_data['question_text'])):\n    if i < numOfTrain:\n        train_sen.append(train_data['question_text'].loc[i])\n    else:\n        val_sen.append(train_data['question_text'].loc[i])\n        \nfor i in range(0, len(test_data['question_text'])):\n    test_sen.append(test_data['question_text'].loc[i])\n\ntrain_label = []\nval_label = []\nfor i in range(0, len(train_data['target'])):\n    if i < numOfTrain:\n        train_label.append(float(train_data['target'].loc[i]))\n    else:\n        val_label.append(float(train_data['target'].loc[i]))\n        ","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:47:07.362401Z","iopub.execute_input":"2022-01-08T01:47:07.362766Z","iopub.status.idle":"2022-01-08T01:47:57.255775Z","shell.execute_reply.started":"2022-01-08T01:47:07.36273Z","shell.execute_reply":"2022-01-08T01:47:57.255035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allStopWords = \"a abaft abafter abaftest about abouter aboutest above abover abovest accordingly aer aest afore after afterer afterest afterward afterwards again against aid ain albeit all aller allest alls allyou almost along alongside already also although always am amid amidst among amongst an and andor anear anent another any anybody anyhow anyone anything anywhere apart aparter apartest appear appeared appearing appears appropriate appropriated appropriater appropriates appropriatest appropriating are aren aren't ares around as ases aside asides aslant astraddle astraddler astraddlest astride astrider astridest at athwart atop atween aught aughts available availabler availablest awfully b be became because become becomes becoming becominger becomingest becomings been before beforehand beforehander beforehandest behind behinds being below beneath beside besides better bettered bettering betters between betwixt beyond bist both but buts by by-and-by byandby c can cannot canst cant canted cantest canting cants cer certain certainer certainest cest chez circa co come-on come-ons comeon comeons concerning concerninger concerningest consequently considering could couldn couldn't couldst cum d dday ddays describe described describes describing despite despited despites despiting did didn didn't different differenter differentest do doe does doesn doesn't doing doings don don't done doner dones donest dos dost doth down downs downward downwarder downwardest downwards during e each eg eight either else elsewhere enough ere et etc even evened evenest evens evenser evensest ever every everybody everyone everything everywhere ex except excepted excepting excepts exes f fact facts failing failings few fewer fewest figupon figuponed figuponing figupons five followthrough for forby forbye fore forer fores forever former formerer formerest formerly formers fornenst forwhy four fourscore frae from fs further furthered furtherer furtherest furthering furthermore furthers g get gets getting go gone good got gotta gotten h had hadn hadn't hadst hae hardly has hasn hasn't hast hath have haven haven't haves having he hence her here hereafter hereafters hereby herein hereupon hers herself him himself his hither hitherer hitherest hoo hoos how how-do-you-do howbeit howdoyoudo however huh humph i idem idemer idemest ie if ifs immediate immediately immediater immediatest in inasmuch inc indeed indicate indicated indicates indicating info information insofar instead into inward inwarder inwardest inwards is isn isn't it it's its itself j just k l latter latterer latterest latterly latters layabout layabouts less lest ll lot lots lotted lotting m ma main make many mauger maugre mayest me meanwhile meanwhiles midst midsts might mightn mightn't mights more moreover most mostly much mucher muchest must musth musths mustn mustn't musts my myself n natheless nathless neath neaths necessarier necessariest necessary needn needn't neither nethe nethermost never nevertheless nigh nigher nighest nine no no-one nobodies nobody noes none noone nor nos not nothing nothings notwithstanding now nowhere nowheres o of off offest offs often oftener oftenest oh on once one oneself onest only ons onto or orer orest other others otherwise otherwiser otherwisest ought oughts our ours ourself ourselves out outed outest outs outside outwith over overall overaller overallest overalls overs own owned owning owns owt p particular particularer particularest particularly particulars per perhaps plaintiff please pleased pleases plenties plenty pro probably provide provided provides providing q qua que quite r rath rathe rather rathest re really regarding relate related relatively res respecting respectively s said saider saidest same samer sames samest sans sanserif sanserifs sanses saved sayid sayyid seem seemed seeminger seemingest seemings seems send sent senza serious seriouser seriousest seven several severaler severalest shall shalled shalling shalls shan shan't she she's should should've shoulded shoulding shouldn shouldn't shoulds since sine sines sith six so sobeit soer soest some somebody somehow someone something sometime sometimer sometimes sometimest somewhat somewhere stop stopped such summat sup supped supping sups syn syne t ten than that that'll the thee their theirs them themselves then thence thener thenest there thereafter thereby therefore therein therer therest thereupon these they thine thing things this thises thorough thorougher thoroughest thoroughly those thou though thous thouses three thro through througher throughest throughout thru thruer thruest thus thy thyself till tilled tilling tills to together too toward towarder towardest towards two u umpteen under underneath unless unlike unliker unlikest until unto up upon uponed uponing upons upped upping ups us use used usedest username usually v various variouser variousest ve verier veriest versus very via vis-a-vis vis-a-viser vis-a-visest viz vs w was wasn wasn't wast we were weren weren't wert what whatever whateverer whateverest whatsoever whatsoeverer whatsoeverest wheen when whenas whence whencesoever whenever whensoever where whereafter whereas whereby wherefrom wherein whereinto whereof whereon wheresoever whereto whereupon wherever wherewith wherewithal whether which whichever whichsoever while whiles whilst whither whithersoever who whoever whom whomever whose whoso whosoever why will with withal within without won won't would woulded woulding wouldn wouldn't woulds x y ye yet yon yond yonder you you'd you'll you're you've your yours yourself yourselves z zillion\"\nstopWords = allStopWords.split(\" \")\ndef removeStop(sentence):\n    words = sentence.split(\" \")\n    ans = \"\"\n    for s in words:\n        try: \n            index = stopWords.index(s)\n        except:\n            ans += (s + \" \")\n    return ans","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:47:57.262394Z","iopub.execute_input":"2022-01-08T01:47:57.263016Z","iopub.status.idle":"2022-01-08T01:47:57.272531Z","shell.execute_reply.started":"2022-01-08T01:47:57.262914Z","shell.execute_reply":"2022-01-08T01:47:57.270871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(0, len(train_sen)):\n    if not isinstance(train_sen[i], str):\n        train_sen[i] = str(train_sen[i])\n    train_sen[i] = removeStop(train_sen[i])\n\nfor i in range(0, len(val_sen)):\n    if not isinstance(val_sen[i], str):\n        val_sen[i] = str(val_sen[i])\n    val_sen[i] = removeStop(val_sen[i])\n\nfor i in range(0, len(test_sen)):\n    if not isinstance(test_sen[i], str):\n        test_sen[i] = str(test_sen[i])\n    test_sen[i] = removeStop(test_sen[i])","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:47:57.273984Z","iopub.execute_input":"2022-01-08T01:47:57.274488Z","iopub.status.idle":"2022-01-08T01:51:04.108651Z","shell.execute_reply.started":"2022-01-08T01:47:57.274449Z","shell.execute_reply":"2022-01-08T01:51:04.107875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_size = 30000\nembedding_dim = 64\nmax_length = 64\ntrunc_type = 'post'\npad_type = 'post'\noov_tok = \"<OOV>\"","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:51:04.110066Z","iopub.execute_input":"2022-01-08T01:51:04.110337Z","iopub.status.idle":"2022-01-08T01:51:04.115524Z","shell.execute_reply.started":"2022-01-08T01:51:04.1103Z","shell.execute_reply":"2022-01-08T01:51:04.114887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)\ntokenizer.fit_on_texts(train_sen)\n\ntrain_seq = tokenizer.texts_to_sequences(train_sen)\ntrain_pad = pad_sequences(train_seq,maxlen=max_length, padding=pad_type, truncating=trunc_type)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:51:04.116697Z","iopub.execute_input":"2022-01-08T01:51:04.117099Z","iopub.status.idle":"2022-01-08T01:51:39.676251Z","shell.execute_reply.started":"2022-01-08T01:51:04.117065Z","shell.execute_reply":"2022-01-08T01:51:39.675494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_seq = tokenizer.texts_to_sequences(val_sen)\nval_pad = pad_sequences(val_seq,maxlen=max_length, padding=pad_type, truncating=trunc_type)\n\ntest_seq = tokenizer.texts_to_sequences(test_sen)\ntest_pad = pad_sequences(test_seq,maxlen=max_length, padding=pad_type, truncating=trunc_type)\n\ninput1 = keras.Input(len(train_pad[0]))\n\nembedded_vector = layers.Embedding(vocab_size, embedding_dim)(input1)\n\nhidden_class_1 = layers.LSTM(64)(embedded_vector)\nhidden_class_1 = keras.Model(inputs=input1, outputs=hidden_class_1)\n\nhidden_class = hidden_class_1.output\noutput = layers.Dense(2, activation='softmax')(hidden_class)\n\nmodel = keras.Model(inputs=[hidden_class_1.input], outputs=output)\n\nmodel.summary()\ntrain_label = np.array(train_label)\nval_label = np.array(val_label)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:51:39.677514Z","iopub.execute_input":"2022-01-08T01:51:39.67775Z","iopub.status.idle":"2022-01-08T01:51:55.353364Z","shell.execute_reply.started":"2022-01-08T01:51:39.677718Z","shell.execute_reply":"2022-01-08T01:51:55.352595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_function = keras.losses.SparseCategoricalCrossentropy()\nmodel.compile(optimizer='Adam', loss=loss_function, metrics=['accuracy'])\nhistory = model.fit(x=[train_pad], y=train_label, batch_size=32, epochs=5)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-08T01:51:55.354746Z","iopub.execute_input":"2022-01-08T01:51:55.355193Z","iopub.status.idle":"2022-01-08T02:06:59.128843Z","shell.execute_reply.started":"2022-01-08T01:51:55.355152Z","shell.execute_reply":"2022-01-08T02:06:59.128151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_recall, test_acc = model.evaluate(x=[val_pad], y=val_label, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T02:06:59.130515Z","iopub.execute_input":"2022-01-08T02:06:59.130762Z","iopub.status.idle":"2022-01-08T02:07:40.487657Z","shell.execute_reply.started":"2022-01-08T02:06:59.130727Z","shell.execute_reply":"2022-01-08T02:07:40.486756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_y = model.predict([test_pad], batch_size=1024, verbose=1)\nans = []\nfor prediction in pred_y:\n    ans.append((prediction[1]*2).astype(int))\n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-08T02:07:40.489364Z","iopub.execute_input":"2022-01-08T02:07:40.491211Z","iopub.status.idle":"2022-01-08T02:07:44.23876Z","shell.execute_reply.started":"2022-01-08T02:07:40.491166Z","shell.execute_reply":"2022-01-08T02:07:44.238012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_submit = pd.DataFrame({\"qid\":test_data[\"qid\"].values})\ndata_submit['prediction'] = ans\ndata_submit.to_csv(\"submission.csv\", index=False)\ndata_submit","metadata":{"execution":{"iopub.status.busy":"2022-01-08T02:07:44.240123Z","iopub.execute_input":"2022-01-08T02:07:44.240374Z","iopub.status.idle":"2022-01-08T02:07:45.274346Z","shell.execute_reply.started":"2022-01-08T02:07:44.240338Z","shell.execute_reply":"2022-01-08T02:07:45.273619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(ans.count(1))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T02:07:45.275655Z","iopub.execute_input":"2022-01-08T02:07:45.275896Z","iopub.status.idle":"2022-01-08T02:07:45.40197Z","shell.execute_reply.started":"2022-01-08T02:07:45.275862Z","shell.execute_reply":"2022-01-08T02:07:45.401059Z"},"trusted":true},"execution_count":null,"outputs":[]}]}