{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from collections import defaultdict\nimport nltk\nimport sklearn\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.metrics import accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18a91763ff858d98a023e9c1f86c0091bf556414"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ae13297562985e5d496c3665f98afb2321fb34a"},"cell_type":"code","source":"def clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct,\"\")\n    return x\n\ndef split_text(x):\n    x = wordninja.split(x)\n    return '-'.join(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32f98809384148d674b85fd46433ae3d3539d00a"},"cell_type":"code","source":"trainSet = pd.read_csv('../input/train.csv')\ntestSet = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1fb61dfd0826d5b876693ea2f4e1c17633c1aee"},"cell_type":"code","source":"trainSet[\"question_text\"] = trainSet[\"question_text\"].apply(lambda x: clean_text(x))\ntestSet[\"question_text\"] = testSet[\"question_text\"].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"380878376da3e48022395de150381ee32e3dff5e"},"cell_type":"code","source":"trainSet[\"question_text\"] = trainSet[\"question_text\"].str.lower()\ntestSet[\"question_text\"] = testSet[\"question_text\"].str.lower()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"574a47c5790d88e330f8e2d00d7bc2610e56da79"},"cell_type":"code","source":"train_int,val_int = train_test_split(trainSet, test_size=0.001, random_state=2018)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"285aa55ed1a8d3a99846c3e16a607f6a10830624"},"cell_type":"code","source":"train_X = train_int[\"question_text\"].fillna(\"_##_\").values\nval_X = val_int[\"question_text\"].fillna(\"_##_\").values\ntest_X = testSet[\"question_text\"].fillna(\"_##_\").values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b806a083fa401bc881c74f278e8ca760ab200a5"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=120000)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da3a541c6e52c5f51b25373fb9159f84b53bbf8f"},"cell_type":"code","source":"trainX = np.array(train_X)\nvalX = np.array(val_X)\ntestX = np.array(test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37e78e0aa53f251390ba33d348729e33bef638cb"},"cell_type":"code","source":"trainX = pad_sequences(trainX,maxlen = 70)\nvalX = pad_sequences(valX,maxlen = 70)\ntestX = pad_sequences(testX,maxlen = 70)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90410eea33925063dbc9c489eaf19f7254c82df1"},"cell_type":"code","source":"B_model = BernoulliNB(alpha=0.1,binarize = 5000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c58438111b4960563443245e3e4ffa701350b62"},"cell_type":"code","source":"B_model.fit(trainX,train_int.target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8379c1e5b006f0280311536a18b74a4b606d5be1"},"cell_type":"code","source":"predXB = B_model.predict(valX)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"602580d14168d0be31e35b274427f17fcd500e16"},"cell_type":"code","source":"accuracy_score(val_int.target,predXB,normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a94a7c484bf4fd2326de1ae6331765502445db86"},"cell_type":"code","source":"predXBtest = B_model.predict(testX)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93f254d82f6de734e41838734adfef66bb2e5dd9"},"cell_type":"code","source":"result = pd.Series(predXBtest,name = \"Target\")\nsubmission = pd.concat([pd.Series(testSet.qid,name = \"qid\"),result],axis = 1)\nsubmission.to_csv(\"submit1.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9132d507aaad2a9020b2c6f91f8ebfb9f89a9be9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}