{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport plotly_express as px\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"sodata = pd.read_csv(\"/kaggle/input/stack-overflow-questions/stack-overflow-data.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sodata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tags_counts = sodata.groupby('tags').aggregate({'post':np.count_nonzero}).reset_index().rename(columns = {'post':'tag_count'})\npx.bar(tags_counts,x='tags',y='tag_count',color='tags')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Creating train-test Split\nfrom sklearn.model_selection import train_test_split\nX = sodata[['post']]\ny = sodata[['tags']]\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import make_pipeline\n\nvec = CountVectorizer()\nclf = LogisticRegression()\npipe = make_pipeline(vec, clf)\npipe.fit(X_train.post, y_train.tags)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\n\ndef print_report(pipe):\n    y_actuals = y_test['tags']\n    y_preds = pipe.predict(X_test['post'])\n    report = metrics.classification_report(y_actuals, y_preds)\n    print(report)\n    print(\"accuracy: {:0.3f}\".format(metrics.accuracy_score(y_actuals, y_preds)))\n\nprint_report(pipe)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf.classes_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, tag in enumerate(clf.classes_):\n    coefficients = clf.coef_[i]\n    weights = list(zip(vec.get_feature_names(),coefficients))\n    print('Tag:',tag)\n    print('Most Positive Coefficients:')\n    print(sorted(weights,key=lambda x: -x[1])[:10])\n    print('Most Negative Coefficients:')\n    print(sorted(weights,key=lambda x: x[1])[:10])\n    print(\"--------------------------------------\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import eli5\neli5.show_weights(clf, vec=vec, top=50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Misclassified Examples"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_preds = pipe.predict(sodata['post'])\nsodata['predicted_label'] = y_preds\nmisclassified_examples = sodata[(sodata['tags']!=sodata['predicted_label'])&(sodata['tags']=='python')&(sodata['predicted_label']=='java')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"misclassified_examples.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eli5.show_prediction(clf, misclassified_examples['post'].values[1], vec=vec)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Going Deep\nLets create a deep model for this simple task."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some imports, we are not gong to use all the imports in this workbook but in subsequent workbooks we surely will.\nimport os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.utils import to_categorical\nfrom sklearn.preprocessing import LabelEncoder\n\n\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport os\nimport time\nimport gc\nimport re\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define some Global Variables\n\nmax_features = 100000 # Maximum Number of words we want to include in our dictionary\nmaxlen = 72 # No of words in question we want to create a sequence with\nembed_size = 300# Size of word to vec embedding we are using","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some preprocesssing that will be common to all the text classification methods you will see. \n\n\n# Loading the data\ndef load_and_prec():\n\n    ## split to train and val\n    train_df, val_df = train_test_split(sodata, test_size=0.2, random_state=2019)\n\n    ## fill up the missing values\n    train_X = train_df[\"post\"].values\n    val_X = val_df[\"post\"].values\n\n\n    ## Tokenize the sentences\n    '''\n    keras.preprocessing.text.Tokenizer tokenizes(splits) the texts into tokens(words).\n    Signature:\n    Tokenizer(num_words=None, filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n', \n    lower=True, split=' ', char_level=False, oov_token=None, document_count=0, **kwargs)\n\n    The num_words parameter keeps a prespecified number of words in the text only. \n    It also filters some non wanted tokens by default and converts the text into lowercase.\n\n    It keeps an index of words(dictionary of words which we can use to assign a unique number to a word) \n    which can be accessed by tokenizer.word_index.\n    For example - For a text corpus the tokenizer word index might look like. \n    The words in the indexed dictionary are sort of ranked in order of frequencies,\n    {'the': 1,'what': 2,'is': 3, 'a': 4, 'to': 5, 'in': 6, 'of': 7, 'i': 8, 'how': 9}\n    \n    The texts_to_sequence function converts every word(token) to its respective index in the word_index\n    \n    So Lets say we started with \n    train_X as something like ['This is a sentence','This is another bigger sentence']\n    and after fitting our tokenizer we get the word_index as {'this':1,'is':2,'sentence':3,'a':4,'another':5,'bigger':6}\n    The texts_to_sequence function will tokenize the sentences and replace words with individual tokens to give us \n    train_X = [[1,2,4,3],[1,2,5,6,3]]\n    '''\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    val_X = tokenizer.texts_to_sequences(val_X)\n\n\n    ## Pad the sentences. We need to pad the sequence with 0's to achieve consistent length across examples.\n    '''\n    We had train_X = [[1,2,4,3],[1,2,5,6,3]]\n    lets say maxlen=6\n        We will then get \n        train_X = [[1,2,4,3,0,0],[1,2,5,6,3,0]]\n    '''\n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    val_X = pad_sequences(val_X, maxlen=maxlen)\n\n\n    ## Get the target values\n    train_y = train_df['tags'].values\n    val_y = val_df['tags'].values\n    \n    encoder = LabelEncoder()\n    encoder.fit(train_y)\n    train_y = encoder.transform(train_y)\n    val_y = encoder.transform(val_y)\n    train_y = to_categorical(train_y)\n    val_y = to_categorical(val_y)\n\n    \n    \n    #shuffling the data\n    np.random.seed(2018)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    \n    return train_X, val_X,  train_y, val_y, val_df, tokenizer, encoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, val_X,  train_y, val_y, val_df,tokenizer , encoder = load_and_prec()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Word 2 vec Embedding\n\ndef load_glove(word_index):\n    '''We want to create an embedding matrix in which we keep only the word2vec for words which are in our word_index\n    '''\n    EMBEDDING_FILE = '../input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = -0.005838499,0.48782197\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix = load_glove(tokenizer.word_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/yekenot/2dcnn-textclassifier\ndef model_cnn(embedding_matrix):\n    filter_sizes = [1,2,3,5]\n    num_filters = 36\n\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    x = Reshape((maxlen, embed_size, 1))(x)\n\n    maxpool_pool = []\n    for i in range(len(filter_sizes)):\n        conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),\n                                     kernel_initializer='he_normal', activation='elu')(x)\n        maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))\n\n    z = Concatenate(axis=1)(maxpool_pool)   \n    z = Flatten()(z)\n    z = Dropout(0.1)(z)\n\n    outp = Dense(20, activation=\"softmax\")(z)\n\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model_cnn(embedding_matrix)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, epochs=5):\n    filepath=\"weights_best.h5\"\n    checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.0001, verbose=2)\n    earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')\n    callbacks = [checkpoint, reduce_lr]\n    for e in range(epochs):\n        model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y),callbacks=callbacks)\n    model.load_weights(filepath)\n\ntrain(model, epochs = 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now we have a trained model and we want to use ELI5 to understand how our model is performing.\nBut before that let us see the performance on the validation data set.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import metrics\n\ndef print_report_nn(model):\n    a=np.argmax(val_y, axis = 1)\n    y_actuals = encoder.inverse_transform(a)\n    y_preds = model.predict([val_X], batch_size=1024, verbose=0)\n    prediction_ = np.argmax(y_preds, axis = 1)\n    prediction_ = encoder.inverse_transform(prediction_)\n    report = metrics.classification_report(y_actuals, prediction_)\n    print(report)\n    print(\"accuracy: {:0.3f}\".format(metrics.accuracy_score(y_actuals, prediction_)))\n\nprint_report_nn(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Surprisingly, not better than the previous model. That may be because the model is not tuned enough or we might want to use LSTM. But for our case that is to understand ELI5 this model works fine."},{"metadata":{},"cell_type":"markdown","source":"Getting the weights using ELI5. Need to create a function which takes an array of texts and  return a matrix of shape (n_samples, n_classes) with probability values - a row per document and a column per output label.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_complex(docs):\n    # preprocess the docs\n    val_X = tokenizer.texts_to_sequences(docs)\n    val_X = pad_sequences(val_X, maxlen=maxlen)\n    y_preds = model.predict([val_X], batch_size=1024, verbose=0)\n    return y_preds\n                            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import eli5\nfrom eli5.lime import TextExplainer\n\nte = TextExplainer(random_state=2019)\nte.fit(misclassified_examples['post'].values[1], predict_complex)\nte.show_prediction(target_names=list(encoder.classes_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}