{"cells":[{"metadata":{},"cell_type":"markdown","source":"Proyecto 3 Juan Montenegro, Ivan Loscher"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\n\nfrom keras.models import Model, Sequential\nfrom keras.layers import Dense, Embedding, Input, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, LSTM, concatenate\nfrom keras.preprocessing import text, sequence\n\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\n\nprint(os.listdir(\"../input\"))\n\n        \ntrain_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Procesamiento de Datos**\n\nIncluyendo el procesamiento de palabras, separando y arreglandolas\n\nTokenizandolas y tambien creando la matriz de word embedding usando GloVe\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"#procesamiento de palabras, reemplazamos aquellas que vemos necesarias asi como acomodar ciertos signos, esto se aplica a los sets\nreemplazar = {r\"i'm\": 'i am',\n                r\"'re\": ' are',\n                r\"ain't\": 'is not',\n                r\"let's\": 'let us',\n                r\"didn't\": 'did not',\n                r\"'s\":  ' is',\n                r\"'ve\": ' have',\n                r\"can't\": 'can not',\n                r\"cannot\": 'can not',\n                r\"shan’t\": 'shall not',\n                r\"n't\": ' not',\n                r\"'d\": ' would',\n                r\"'ll\": ' will',\n                r\"'scuse\": 'excuse',\n                ',': ' ,',\n                '.': ' .',\n                '!': ' !',\n                '?': ' ?',\n                '\\s+': ' '}\ndef limpiar(text):\n    text = text.lower()\n    for s in reemplazar:\n        text = text.replace(s, reemplazar[s])\n    text = ' '.join(text.split())\n    return text\n\nX_train= train_df['question_text'].apply(lambda p: limpiar(p))\nX_train = X_train.fillna(\"dieter\").values\nX_test= test_df['question_text'].apply(lambda p: limpiar(p))\nX_test = X_test.fillna(\"dieter\").values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"maxlen = 50 #palabras maximas en un documento\nmax_carac = 50000 #maximas caracteristicas\nembed_tama = 300 #tamano del embedding\nbatch_size = 256 #batch size a utilizar, no queremos que sea tan alto ni tan bajo\nepochs = 3 #tardan bastante, pero el modelo llega a ser lo suficientemente preciso para no tener que usar una gran cantidad de epochs\n\ntokenizer = text.Tokenizer(num_words=max_carac) #tokenizer permite vectorizar un cuerpo de texto, convirtiendo cada cuerpo en una sequencia de ints\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#usando el archivo embedding que nos ofrece, crear la matriz de embedding usando GloVe\ndef get_coefs(word,*arr): \n    return word, np.asarray(arr, dtype='float32')\n\ndef cargar_archivo_embedding(file):  #cargamos e indexamos el embedding a usar, en este caso glove para la representacion de palabras \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        index_e = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        index_e = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n    return index_e\n\nglove = cargar_archivo_embedding('../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#creacion de la matriz a traves de los datos anteriores\ndef hacer_matriz_embedding(embedding, tokenizer, features):\n    todos_embs = np.stack(embedding.values())\n    emb_mean,emb_std = todos_embs.mean(), todos_embs.std()\n    embed_tama = todos_embs.shape[1]\n    index_palabras = tokenizer.word_index\n    matriz_embedding = np.random.normal(emb_mean, emb_std, (features, embed_tama))\n    \n    for word, i in index_palabras.items():\n        if i >= features:\n            continue\n        vector_embedding = embedding.get(word)\n        if vector_embedding is not None: \n            matriz_embedding[i] = vector_embedding\n    \n    return matriz_embedding\n\nembed_mat = hacer_matriz_embedding(glove, tokenizer, max_carac) #recibe glove, tokenizer procesado y las caracteristicas\nprint(embed_mat)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Creacion del Modelo**\nLSTM Bidireccional aplicando el Word Embedding que usamos junto a Pooling\nFuncion de Activacion Sigmoide"},{"metadata":{"trusted":true},"cell_type":"code","source":"#creacion del modelo, usamos LSTM y word embedding, se decidio esta LSTM bidireccional que en el recorrido recuerde las palabras importantes para el contexto\ndef embed_model():\n    model = Sequential()\n    inp = Input(shape=(maxlen, )) #instanciar keras tensor\n    x = Embedding(max_carac, embed_tama, weights=[embed_mat])(inp) #embedding a traves del procesamiento que hicimos\n    x = Bidirectional(LSTM(64, return_sequences=True))(x) #LSTM y Bidirectional, dimension de 64 que retorna el output de la secuencia\n    avg_pool = GlobalAveragePooling1D()(x) #global average pooling para data temporal \n    max_pool = GlobalMaxPooling1D()(x) #max pooling para data espacial\n    conc = concatenate([avg_pool, max_pool]) #concatenando ambos poolings\n    outp = Dense(1, activation=\"sigmoid\")(conc) #funcion de activacion sigmoide \n    \n    model = Model(inputs = inp,outputs = outp)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model\n\nmodel = embed_model()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Entrenamiento del Modelo**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#entrenar\nX_t, X_val, y_t, y_val = train_test_split(x_train, y, test_size = 0.1, random_state= np.random) #dividimos entre train y Y\nhistorial = model.fit(X_t, y_t, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True) #entrenamos con nuestro modelo ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Prediccion**\nUsando el modelo, generar resultados"},{"metadata":{"trusted":true},"cell_type":"code","source":"# corremos el modelo en un test para generar nuestras predicciones respecto a ello\ny_pred = model.predict(x_test, batch_size=batch_size)\ny_pred.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Generando el archivo de output**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#output\ny_te = (y_pred[:,0] > 0.5).astype(np.int) #clasificacion\n\n\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te}) #generar archivo de output\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}