{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom tqdm import tqdm\n\nfrom nltk.corpus import stopwords\n\nfrom gensim.models import Word2Vec\nfrom gensim.models import Phrases\nfrom gensim.models.phrases import Phraser\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.utils import np_utils\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, Embedding, Dropout,Bidirectional, Reshape, Flatten, CuDNNGRU, CuDNNLSTM\nfrom keras.models import Model, Sequential\nfrom keras.initializers import Constant\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fcddf986d34ccb0e78bc162a1212fd0b7e4155e"},"cell_type":"code","source":"#f1 스코어\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c241b613b7dcf62c884ebfe1bd563200c1e382a8"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab393bcc0cd61910c5ce9dd3d29b61feea08813e"},"cell_type":"code","source":"# 불용어 만들기\nstopWords = stopwords.words('english')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80fcbb86c08c0cd3a9038cbb20bcc97fd47a34ea"},"cell_type":"code","source":"# 데이터 정제\ndef cleanData(sentence):\n    processedList = \"\"\n    \n    # convert to lowercase, ignore all special characters - keep only alpha-numericals and spaces (not removing full-stop here)\n    sentence = re.sub(r'[^A-Za-z0-9\\s.]',r'',str(sentence).lower())\n    sentence = re.sub(r'\\n',r' ',sentence)\n    \n    # remove stop words\n    sentence = \" \".join([word for word in sentence.split() if word not in stopWords])\n    \n    return sentence\n\n\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x :cleanData(x))\ntest_df['question_text'] = test_df['question_text'].apply(lambda x :cleanData(x))\ntrain_X = train_df['question_text']\ntest_X = test_df['question_text']\nprint(train_X.shape)\nprint(test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22b9a0245bbb528c8305928e9ece59ec7309b16c"},"cell_type":"code","source":"# corpus 만들기\ntmp_corpus = train_X.apply(lambda x: x.split(\".\"))\n#tmp_corpus.head(3)\n#tmp_corpus[1]\n\ncorpus = []\nfor i in tqdm(range(len(tmp_corpus))):\n    for line in tmp_corpus[i]:\n        words = [x for x in line.split()]\n        corpus.append(words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c32e9f6078785677ddd5d88221bfba094ec923b2"},"cell_type":"code","source":"#keras로 전처리\nmaxlen = 70 \ntokenizer = Tokenizer()\ntokenizer.fit_on_texts(list(train_X))\n\ntrain_X = tokenizer.texts_to_sequences(train_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\ntrain_y = train_df['target'].values\ntrain_y = np_utils.to_categorical(train_y)\n\nprint(train_X.shape)\nprint(test_X.shape)\nprint(train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02bdb0e6c8dda373da7cf6f6dab237c69ce805f6"},"cell_type":"code","source":"#gensim으로 Word2Vec 만들기\nmodel = Word2Vec(corpus, sg =1, window = 3, size = 100, min_count = 5, workers = 4 , iter = 100)\nfilename = 'gensim_word2vec.txt'\nmodel.wv.save_word2vec_format(filename, binary = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a024ca61b93cace6c805a54789a2678a7f0c34f0"},"cell_type":"code","source":"#Word2Vec 사용하기\nimport os\nembedding_index = {}\nf = open(os.path.join(\"\",'gensim_word2vec.txt'), encoding = 'utf-8')\nfor line in f:\n    values = line.split()\n    word = values[0]\n    coefs = np.asarray(values[1:])\n    embedding_index[word] = coefs\nf.close()\n\nword_index = tokenizer.word_index\n\nnum_words = len(word_index) + 1\nembedding_matrix = np.zeros((num_words, 100))\n\nfor word, i in word_index.items():\n    if i > num_words:\n        continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d38fa435d4b1415c77c3e54756dd49023dbf759","scrolled":true},"cell_type":"code","source":"#model 만들기\nmodel1 = Sequential()\nmodel1.add(Embedding(num_words, 100, embeddings_initializer= Constant(embedding_matrix),trainable=False, input_length = 70))\nmodel1.add(Bidirectional(CuDNNLSTM(150, return_sequences=True)))\nmodel1.add(Bidirectional(CuDNNLSTM(125, return_sequences=True)))\nmodel1.add(Bidirectional(CuDNNLSTM(100, return_sequences=True)))\nmodel1.add(Flatten())\nmodel1.add(Dense(100, activation = 'relu'))\nmodel1.add(Dense(50, activation = 'relu'))\nmodel1.add(Dense(2, activation = 'sigmoid'))\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])\n#model1.summary()\nmodel1.fit(train_X, train_y, batch_size = 500, epochs = 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ed1fae8a0efe095d174a7ba912899f480e720b3"},"cell_type":"code","source":"pred_y = np.argmax(model1.predict(test_X), axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e391e7101d6b97f3411a74709e76e6991e42c26a"},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}