{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport re\nfrom tqdm import tqdm\n\nfrom nltk.corpus import stopwords\n\nfrom gensim.models import Word2Vec\nfrom gensim.models import Phrases\nfrom gensim.models.phrases import Phraser\n\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.utils import np_utils\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, Embedding, Dropout,Bidirectional, Reshape, Flatten, CuDNNGRU, CuDNNLSTM\nfrom keras.models import Model, Sequential\nfrom keras.initializers import Constant\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40b8277493c2b7f4901d0b12175683f0013a01a8"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ef2581505bb7e851aa9ad78d55e5d7c9b13d840"},"cell_type":"code","source":"# 불용어 만들기\nstopWords = stopwords.words('english')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"951b769e9aa289dc4036ef8ed586cb087ef3bc17"},"cell_type":"code","source":"# 데이터 정제\ndef cleanData(sentence):\n    processedList = \"\"\n    \n    # convert to lowercase, ignore all special characters - keep only alpha-numericals and spaces (not removing full-stop here)\n    sentence = re.sub(r'[^A-Za-z0-9\\s.]',r'',str(sentence).lower())\n    sentence = re.sub(r'\\n',r' ',sentence)\n    \n    # remove stop words\n    sentence = \" \".join([word for word in sentence.split() if word not in stopWords])\n    \n    return sentence\n\n\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x :cleanData(x))\ntest_df['question_text'] = test_df['question_text'].apply(lambda x :cleanData(x))\ntrain_X = train_df['question_text']\ntest_X = test_df['question_text']\nprint(train_X.shape)\nprint(test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a80a5c79f6cc18f9e407fff9f0e1afed962184fe"},"cell_type":"code","source":"# corpus 만들기\ntmp_corpus = train_X.apply(lambda x: x.split(\".\"))\ncorpus = []\nfor i in tqdm(range(len(tmp_corpus))):\n    for line in tmp_corpus[i]:\n        words = [x for x in line.split()]\n        corpus.append(words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"418fa8e04e313886f6e90273d79df4cc50ccfc4e"},"cell_type":"code","source":"#keras로 전처리\nmaxlen = 70 \ntokenizer = Tokenizer()\ntokenizer.fit_on_texts(list(train_X))\n\ntrain_X = tokenizer.texts_to_sequences(train_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\ntrain_y = train_df['target'].values\ntrain_y = np_utils.to_categorical(train_y)\n\nprint(train_X.shape)\nprint(test_X.shape)\nprint(train_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ee11dca5686f9fe17e55bdf99a73e8ab950322b"},"cell_type":"code","source":"#gensim으로 Word2Vec 만들기\nmodel = Word2Vec(corpus, sg =1, window = 3, size = 100, min_count = 5, workers = 4 , iter = 100)\nfilename = 'gensim_word2vec.txt'\nmodel.wv.save_word2vec_format(filename, binary = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f175b517daabb410a1edcbc1be2b021836625b6a"},"cell_type":"code","source":"#Word2Vec 사용하기\nimport os\nembedding_index = {}\nf = open(os.path.join(\"\",'gensim_word2vec.txt'), encoding = 'utf-8')\nfor line in f:\n    values = line.split()\n    word = values[0]\n    coefs = np.asarray(values[1:])\n    embedding_index[word] = coefs\nf.close()\n\nword_index = tokenizer.word_index\n\nnum_words = len(word_index) + 1\nembedding_matrix = np.zeros((num_words, 100))\n\nfor word, i in word_index.items():\n    if i > num_words:\n        continue\n    embedding_vector = embedding_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e2ba1f9f5c699f397bac35ab4d2003293f38e74"},"cell_type":"code","source":"#model 만들기\nmodel1 = Sequential()\nmodel1.add(Embedding(num_words, 100, embeddings_initializer= Constant(embedding_matrix),trainable=False, input_length = 70))\nmodel1.add(Bidirectional(CuDNNLSTM(70, return_sequences=True)))\nmodel1.add(Bidirectional(CuDNNLSTM(70, return_sequences=True)))\nmodel1.add(Bidirectional(CuDNNLSTM(70, return_sequences=True)))\nmodel1.add(Flatten())\nmodel1.add(Dense(100, activation = 'relu'))\nmodel1.add(Dense(100, activation = 'relu'))\nmodel1.add(Dense(2, activation = 'sigmoid'))\nmodel1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])\n#model1.summary()\nmodel1.fit(train_X, train_y, batch_size = 500, epochs = 7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d46bbdb1cfdad94fa7eab03be30864704583d5a"},"cell_type":"code","source":"pred_y = np.argmax(model1.predict(test_X), axis = 1)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}