{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d283df9fcf69cd3d64a0239e78739f2ae0a281d8"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom tqdm import tqdm\n\nfrom nltk.corpus import stopwords\n\nfrom gensim.models import Word2Vec\nfrom gensim.models import Phrases\nfrom gensim.models.phrases import Phraser\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.utils import np_utils\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, Embedding, Dropout,Bidirectional, Reshape, Flatten, CuDNNGRU, CuDNNLSTM\nfrom keras.models import Model, Sequential\nfrom keras.initializers import Constant\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef797ea8467a75d46bbe89e52fa43dcbbd58c491"},"cell_type":"code","source":"def f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcd668b5be771bbee2ba600e655f6a0f09245f5a"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct,\"\")\n        \n        \nclean_text([train_df,test_df])\ntrain_df['lower_questions'] = train_df['question_text'].apply(lambda x: x.lower())\ntest_df['lower_questions'] = test_df['question_text'].apply(lambda x: x.lower())\ntrain_X = train_df['lower_questions']\ntest_X = test_df['lower_questions']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d679dcc9c9711dbd2dd838123a7713934f8408f"},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n# TFIDF 설정\ntfv = TfidfVectorizer()\n\n# Fit TFIDF (훈련)\ntfv.fit(pd.concat([train_X, test_X])) # Learn vocabulary and idf from training set.\n\n# 변환\nX =  tfv.transform(train_X)\nX_test = tfv.transform(test_X)\n\ntrain_y = train_df['target'].values\n#train_y = np_utils.to_categorical(train_y)\nprint(X.shape)\nprint(X_test.shape)\nprint(train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa385e080a97217a54ed83b0ba292b333c343dff","scrolled":true},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nclf = MultinomialNB()\nclf.fit(X, train_y)\ny_test_predict = clf.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e76c6b7eab6973f8dd05ec1b411598d0f56055c"},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = y_test_predict\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}