# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.

train_df = pd.read_csv("../input/train.csv")
test_df = pd.read_csv("../input/test.csv")
df = pd.concat([train_df ,test_df],sort=True)
df['target'].fillna('unknown', inplace = True)

## 사전 만들기
def build_vocab(texts):
    sentences = texts.apply(lambda x: x.split()).values
    vocab = {}
    for sentence in sentences:
        for word in sentence:
            try:
                vocab[word] += 1
            except KeyError:
                vocab[word] = 1
    return vocab

vocab = build_vocab(df['question_text'])

## 소문자로 변환
df['lower_question'] = df['question_text'].apply(lambda x: x.lower())
vocab = build_vocab(df['lower_question'])

puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\', '•',  '~', '@', '£', 
 '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', 
 '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', 
 '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', 
 '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]

def clean_text(x):
    x = str(x)
    for punct in puncts:
        x = x.replace(punct, f' {punct} ')
        
        
clean_text([train_df,test_df])

train_df['lower_questions'] = train_df['question_text'].apply(lambda x: x.lower())
train_X = train_df['lower_questions']

import warnings
warnings.filterwarnings('ignore')
from sklearn.feature_extraction.text import TfidfVectorizer
# TFIDF 설정
tfv = TfidfVectorizer()

# Fit TFIDF (훈련)
tfv.fit(list(train_X)) # Learn vocabulary and idf from training set.

# 변환
X =  tfv.transform(train_X) # Transform documents to document-term matrix.
#X_test = tfv.transform(test_desc)

train_y = train_df['target'].values
X.shape

from keras.models import Sequential
from keras.layers import CuDNNLSTM, Dropout, Activation, Reshape, Flatten, Bidirectional, Dense, Activation
from keras.optimizers import Adam
def f1(y_true, y_pred):
    def recall(y_true, y_pred):
        """Recall metric.

        Only computes a batch-wise average of recall.

        Computes the recall, a metric for multi-label classification of
        how many relevant items are selected.
        """
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
        recall = true_positives / (possible_positives + K.epsilon())
        return recall

    def precision(y_true, y_pred):
        """Precision metric.

        Only computes a batch-wise average of precision.

        Computes the precision, a metric for multi-label classification of
        how many selected items are relevant.
        """
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
        precision = true_positives / (predicted_positives + K.epsilon())
        return precision
    precision = precision(y_true, y_pred)
    recall = recall(y_true, y_pred)
    return 2*((precision*recall)/(precision+recall+K.epsilon()))

import keras.backend.tensorflow_backend as K
with K.tf.device('/gpu:0'):
    model = Sequential()
    model.add(Dense(100, input_shape = (195000,),activation = 'relu'))
    model.add(Dense(1,activation = 'sigmoid'))
    model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics=[f1])
    model.fit(X,train_y, epochs=9,batch_size = 6000,verbose = False)
    
test_df['lower_question'] = test_df['question_text'].apply(lambda x: x.lower())
test_X = test_df['lower_question']
tfv = TfidfVectorizer()

# Fit TFIDF (훈련)
tfv.fit(list(test_X)) # Learn vocabulary and idf from training set.
# 변환
X_test =  tfv.transform(test_X) # Transform documents to document-term matrix.

pred_y = model.predict(X_test)

test_df = pd.read_csv("../input/test.csv", usecols=["qid"])
out_df = pd.DataFrame({"qid":test_df["qid"].values})
out_df['prediction'] = pred_y
out_df.to_csv("submission.csv", index=False)