{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport datetime\nimport os, gc\nimport pandas as pd\nimport numpy as np\nimport pkg_resources\nimport seaborn as sns\nimport time\nimport scipy.stats as stats\n\nfrom sklearn import metrics\nfrom sklearn import model_selection\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.utils import to_categorical\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, Embedding, Dropout, Reshape, SpatialDropout1D, Activation, LSTM, GRU, Conv1D, Conv2D, MaxPool2D, GlobalMaxPooling1D, Flatten, Concatenate, Bidirectional, GlobalAveragePooling1D, GlobalMaxPool1D, MaxPooling1D, concatenate\nfrom keras.layers import Embedding\nfrom keras.layers import Input\nfrom keras.layers import Conv1D\nfrom keras.layers import MaxPooling1D\nfrom keras.layers import Flatten\nfrom keras.layers import Dropout\nfrom keras.layers import Dense\nfrom keras.optimizers import RMSprop\nfrom keras.models import Model\nfrom keras.models import load_model\n\nimport gensim\nfrom gensim.models import KeyedVectors","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load and pre-process the data set"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')\nprint('loaded %d records' % len(train))\n\n# Make sure all comment_text values are strings\ntrain['comment_text'] = train['comment_text'].astype(str) \n\n# List all identities\nidentity_columns = [\n    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n\n# Convert taget and identity columns to booleans\ndef convert_to_bool(df, col_name):\n    df[col_name] = np.where(df[col_name] >= 0.5, True, False)\n    \ndef convert_dataframe_to_bool(df):\n    bool_df = df.copy()\n    for col in ['target'] + identity_columns:\n        convert_to_bool(bool_df, col)\n    return bool_df\n\ntrain = convert_dataframe_to_bool(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split the data into 80% train and 20% validate sets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, validate_df = model_selection.train_test_split(train, test_size=0.2)\nprint('%d train comments, %d validate comments' % (len(train_df), len(validate_df)))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create a text tokenizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_NUM_WORDS = 95000 #10000\nTOXICITY_COLUMN = 'target'\nTEXT_COLUMN = 'comment_text'\n\n# Create a text tokenizer.\ntokenizer = Tokenizer(num_words=MAX_NUM_WORDS)\ntokenizer.fit_on_texts(train_df[TEXT_COLUMN])\n\n# All comments must be truncated or padded to be the same length.\nMAX_SEQUENCE_LENGTH = 250\ndef pad_text(texts, tokenizer):\n    return pad_sequences(tokenizer.texts_to_sequences(texts), maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define and train a Convolutional Neural Net for classifying toxic comments"},{"metadata":{"trusted":true},"cell_type":"code","source":"#EMBEDDINGS_PATH = '../input/glove-global-vectors-for-word-representation/glove.6B.100d.txt'\nEMBEDDINGS_DIMENSION = 300 #100\nDROPOUT_RATE = 0.3\nLEARNING_RATE = 0.00005\nNUM_EPOCHS = 10\nBATCH_SIZE = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## some config values \n# EMBED_SIZE = 300 # how big is each word vector EMBEDDINGS_DIMENSION\n# MAX_FEATURES = 95000 # how many unique words to use (i.e num rows in embedding vector) MAX_NUM_WORDS\n# MAX_NUMWORDS = 70 # max number of words in a question to use MAX_SEQUENCE_LENGTH\n\ndef load_glove():\n    print('Loading glove embeddings...')\n    EMBEDDING_FILE = '../input/glove840b300dtxt/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore'))\n    return make_matrix(embeddings_index)\n    \ndef load_fasttext():  \n    print('Loading wiki-news embeddings...')\n    EMBEDDING_FILE = '../input/wikinews300d1mvec/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    return make_matrix(embeddings_index)\n\ndef load_para():\n    print('Loading paragram embeddings...')\n    EMBEDDING_FILE = '../input/paragram-300-sl999/paragram_300_sl999/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    return make_matrix(embeddings_index)\n\ndef load_wordvec():\n    print('Loading word2vec embeddings...')\n    EMBEDDING_FILE = '../input/googlenewsvectorsnegative300/GoogleNews-vectors-negative300.bin'\n    embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n    word_index = tokenizer.word_index\n    nb_words = min(MAX_NUM_WORDS, len(word_index))\n    embedding_matrix = np.zeros((nb_words, EMBEDDINGS_DIMENSION))\n    for word, i in word_index.items():\n        if i >= MAX_NUM_WORDS: continue\n        if word in embeddings_index.vocab:\n            embedding_matrix[i] = embeddings_index.word_vec(word)\n    return embedding_matrix\n\ndef make_matrix(embeddings_index):\n    #print('Making matrix...')\n    #embedding_matrix = np.zeros((len(tokenizer.word_index) + 1,EMBEDDINGS_DIMENSION))\n    #num_words_in_embedding = 0\n    #for word, i in tokenizer.word_index.items():\n    #    embedding_vector = embeddings_index.get(word)\n    #    if embedding_vector is not None:\n    #        num_words_in_embedding += 1\n    #        # words not found in embedding index will be all-zeros.\n    #        embedding_matrix[i] = embedding_vector\n    #return embedding_matrix\n\n    print('Making matrix...')\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    word_index = tokenizer.word_index\n    nb_words = min(MAX_NUM_WORDS, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= MAX_NUM_WORDS: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    return embedding_matrix\n\nprint('Loading Glove...')\nmatrix_glove = load_glove()\n\n#print('Loading Fasttext...')\n#matrix_fasttext = load_fasttext()\n\nprint('Loading Paragram...')\nmatrix_para = load_para()\n\nprint('Loading Wordvec...')\nmatrix_wordvec = load_wordvec()\n\nembedding_matrix = np.mean((matrix_glove\n                            #, matrix_fasttext\n                            , matrix_para\n                            , matrix_wordvec\n                            ), axis = 0)\nnp.shape(embedding_matrix)\ndel matrix_glove\n#del matrix_fasttext\ndel matrix_para\ndel matrix_wordvec\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_embeddings(path, dim):\n    # Load embeddings\n    print('loading embeddings from ', path)\n    embeddings_index = {}\n    with open(path) as f:\n        for line in f:\n            values = line.split()\n            word = values[0]\n            coefs = np.asarray(values[1:], dtype='float32')\n            embeddings_index[word] = coefs\n\n    embedding_matrix = np.zeros((len(tokenizer.word_index) + 1,dim))\n    num_words_in_embedding = 0\n    for word, i in tokenizer.word_index.items():\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            num_words_in_embedding += 1\n            # words not found in embedding index will be all-zeros.\n            embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix\n\n#test_emb = load_embeddings('../input/glove840b300dtxt/glove.840B.300d.txt', EMBEDDINGS_DIMENSION)\n#test_emb.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_data(train_df, validate_df, tokenizer):    \n    print('preparing data')\n    train_text = pad_text(train_df[TEXT_COLUMN], tokenizer)\n    train_labels = to_categorical(train_df[TOXICITY_COLUMN])\n    validate_text = pad_text(validate_df[TEXT_COLUMN], tokenizer)\n    validate_labels = to_categorical(validate_df[TOXICITY_COLUMN])\n    \n    return train_text, train_labels, validate_text, validate_labels, tokenizer\n\ntrain_text, train_labels, validate_text, validate_labels, tokenizer = prepare_data(train_df, validate_df, tokenizer)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(train_text, train_labels, validate_text, validate_labels, tokenizer):\n    # Prepare data\n\n    #embedding_matrix = load_embeddings(EMBEDDINGS_PATH, EMBEDDINGS_DIMENSION)\n    \n    # Create model layers.\n    def get_gru_neural_net_layers():\n        print('Modeling...')\n        sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n        x = Embedding(MAX_NUM_WORDS, EMBEDDINGS_DIMENSION, weights=[embedding_matrix], trainable=False)(sequence_input)\n        x = Bidirectional(GRU(128, return_sequences=True))(x)\n        x = GlobalMaxPool1D()(x)\n        x = Dense(16, activation=\"relu\")(x)\n        x = Dropout(0.1)(x)\n        preds = Dense(2, activation='softmax')(x)\n        return sequence_input, preds\n    \n    def get_convolutional_neural_net_layers():\n        \"\"\"Returns (input_layer, output_layer)\"\"\"\n        sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n        #embedding_layer = Embedding(len(tokenizer.word_index) + 1,\n        #                            EMBEDDINGS_DIMENSION,\n        #                            weights=[embedding_matrix],\n        #                            input_length=MAX_SEQUENCE_LENGTH,\n         #                           trainable=False)\n        #x = embedding_layer(sequence_input)\n        x = Embedding(MAX_NUM_WORDS, EMBEDDINGS_DIMENSION, weights=[embedding_matrix], trainable=False)(sequence_input)\n        \n        x = Conv1D(128, 2, activation='relu', padding='same')(x)\n        x = MaxPooling1D(5, padding='same')(x)\n        x = Conv1D(128, 3, activation='relu', padding='same')(x)\n        x = MaxPooling1D(5, padding='same')(x)\n        x = Conv1D(128, 4, activation='relu', padding='same')(x)\n        x = MaxPooling1D(40, padding='same')(x)\n        x = Flatten()(x)\n        x = Dropout(DROPOUT_RATE)(x)\n        x = Dense(128, activation='relu')(x)\n        preds = Dense(2, activation='softmax')(x)\n        return sequence_input, preds\n\n    # Compile model.\n    print('compiling model')\n    input_layer, output_layer = get_convolutional_neural_net_layers()\n    model = Model(input_layer, output_layer)\n    model.compile(loss='categorical_crossentropy',\n                  optimizer=RMSprop(lr=LEARNING_RATE),\n                  metrics=['acc'])\n\n    # Train model.\n    print('training model')\n    model.fit(train_text,\n              train_labels,\n              batch_size=BATCH_SIZE,\n              epochs=NUM_EPOCHS,\n              validation_data=(validate_text, validate_labels),\n              verbose=2)\n\n    return model\n\nmodel = train_model(train_text, train_labels, validate_text, validate_labels, tokenizer)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate model predictions on the validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_NAME = 'emb_model'\nvalidate_df[MODEL_NAME] = model.predict(pad_text(validate_df[TEXT_COLUMN], tokenizer))[:, 1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define bias metrics, then evaluate our new model for bias using the validation set predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"SUBGROUP_AUC = 'subgroup_auc'\nBPSN_AUC = 'bpsn_auc'  # stands for background positive, subgroup negative\nBNSP_AUC = 'bnsp_auc'  # stands for background negative, subgroup positive\n\ndef compute_auc(y_true, y_pred):\n    try:\n        return metrics.roc_auc_score(y_true, y_pred)\n    except ValueError:\n        return np.nan\n\ndef compute_subgroup_auc(df, subgroup, label, model_name):\n    subgroup_examples = df[df[subgroup]]\n    return compute_auc(subgroup_examples[label], subgroup_examples[model_name])\n\ndef compute_bpsn_auc(df, subgroup, label, model_name):\n    \"\"\"Computes the AUC of the within-subgroup negative examples and the background positive examples.\"\"\"\n    subgroup_negative_examples = df[df[subgroup] & ~df[label]]\n    non_subgroup_positive_examples = df[~df[subgroup] & df[label]]\n    examples = subgroup_negative_examples.append(non_subgroup_positive_examples)\n    return compute_auc(examples[label], examples[model_name])\n\ndef compute_bnsp_auc(df, subgroup, label, model_name):\n    \"\"\"Computes the AUC of the within-subgroup positive examples and the background negative examples.\"\"\"\n    subgroup_positive_examples = df[df[subgroup] & df[label]]\n    non_subgroup_negative_examples = df[~df[subgroup] & ~df[label]]\n    examples = subgroup_positive_examples.append(non_subgroup_negative_examples)\n    return compute_auc(examples[label], examples[model_name])\n\ndef compute_bias_metrics_for_model(dataset,\n                                   subgroups,\n                                   model,\n                                   label_col,\n                                   include_asegs=False):\n    \"\"\"Computes per-subgroup metrics for all subgroups and one model.\"\"\"\n    records = []\n    for subgroup in subgroups:\n        record = {\n            'subgroup': subgroup,\n            'subgroup_size': len(dataset[dataset[subgroup]])\n        }\n        record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)\n        record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)\n        record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)\n        records.append(record)\n    return pd.DataFrame(records).sort_values('subgroup_auc', ascending=True)\n\nbias_metrics_df = compute_bias_metrics_for_model(validate_df, identity_columns, MODEL_NAME, TOXICITY_COLUMN)\nbias_metrics_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Calculate the final score"},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_overall_auc(df, model_name):\n    true_labels = df[TOXICITY_COLUMN]\n    predicted_labels = df[model_name]\n    return metrics.roc_auc_score(true_labels, predicted_labels)\n\ndef power_mean(series, p):\n    total = sum(np.power(series, p))\n    return np.power(total / len(series), 1 / p)\n\ndef get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):\n    bias_score = np.average([\n        power_mean(bias_df[SUBGROUP_AUC], POWER),\n        power_mean(bias_df[BPSN_AUC], POWER),\n        power_mean(bias_df[BNSP_AUC], POWER)\n    ])\n    return (OVERALL_MODEL_WEIGHT * overall_auc) + ((1 - OVERALL_MODEL_WEIGHT) * bias_score)\n    \nget_final_metric(bias_metrics_df, calculate_overall_auc(validate_df, MODEL_NAME))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction on Test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')\nsubmission = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['prediction'] = model.predict(pad_text(test[TEXT_COLUMN], tokenizer))[:, 1]\nsubmission.to_csv('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}