{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# imports\nimport numpy as np\nimport pandas as pd\nimport os\nfrom tqdm import tqdm, trange\nimport string\nimport re\nimport sys\nimport math\nfrom keras.models import Sequential\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional, Dropout, BatchNormalization, Activation\nimport random\nfrom keras.preprocessing.sequence import pad_sequences\nfrom time import time\nfrom sklearn import metrics\nfrom keras.optimizers import Adam\nfrom keras.regularizers import l1_l2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31a80dab52d3774d04b17de6aec268d2d6219d3e"},"cell_type":"markdown","source":"**Section 1. Data exploration**\n=====================\n========================================================\n\n*(disclaimer: sometimes I use words like \"caustic\", \"toxic\" and such, read them all as \"insincere\")*"},{"metadata":{"trusted":true,"_uuid":"92503940d7f560316373e85104c31eecfc342a7f"},"cell_type":"code","source":"# loading training data\ndf=pd.read_csv('../input/train.csv')\ndf=df[['question_text','target']]\ndf=df.dropna() # just in case","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dceb830db41f280e17049f1a960b008d87e68aed"},"cell_type":"code","source":"# now lets look at training data more closely, we need to see how skewed the training data is\npos_pct=df['target'].value_counts()[1]/len(df)\nprint(f'Caustic questions percentage is {pos_pct*100:.2f}%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"989ce326ea77de18292a392c7b6ed25333d6f243"},"cell_type":"code","source":"trans_table={key:' ' for key in string.punctuation} # punctuation removal table\ndef get_words(s):\n    '''Function for counting words in a question'''\n    s=s.translate(str.maketrans(trans_table)).lower().strip() # remove punctuation\n    s=re.sub(' +',' ',s) # get rid of multiple spaces inside\n    s=s.split(' ')\n    return len(s)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"386919f3d087a9d6a79d2334a88a036f462bfaa7"},"cell_type":"code","source":"# what is the average number of words in a question?\ndf['n_words']=df['question_text'].apply(get_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71a3f606fb7b036d6785905c77b4f8683a7af76e"},"cell_type":"code","source":"# distribution for good and taxic questions\nmax_word_len=60\ndf[(df['target']==0) & (df['n_words']<=max_word_len)]['n_words'].value_counts().sort_index().plot(\n    label='Sincere',legend=True, color=(0,0.3,0))\nax=df[(df['target']==1) & (df['n_words']<=max_word_len)]['n_words'].value_counts().sort_index().plot(\n    figsize=(7,3.5), logy=True,xlim=(0,max_word_len+1), label='Toxic',legend=True,\n    color='r', title='Questions distribution',xticks=np.arange(0,max_word_len+1,5)).set_xlabel('Words')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e983ad3c514b7653413b799231634fc75579092"},"cell_type":"markdown","source":"**EDA conclusions:**\n1. Data is heavily imbalanced, we have slightly over 6% of positive examples.\n2. Most questions are in 1 - 60 words range\n3. There are no non-toxic questions that have 1 word in it (at least in the training set, but its hard to imagine a question with only one word, that has a legitimate answer)"},{"metadata":{"_uuid":"e6d97252678b9b80f1740cb69f79ae70cff36018"},"cell_type":"markdown","source":"**Section 2. Data preparation**\n=====================\n========================================================"},{"metadata":{"trusted":true,"_uuid":"f8de096d26610024508c0ba203a57dc5cd490ca1"},"cell_type":"code","source":"# first, we dump all questions longer than 60 words\ndf=df[df['n_words']<=60]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fe5f4b726b55c4e443897d327915d2f78caec9c"},"cell_type":"code","source":"## comment this block before committing unfinished model! Saves up to 4 minutes of time\n# load embeddings and create embedding vocabulary\n\nembedding_path='glove.840B.300d/glove.840B.300d.txt' # <embedding folder>/<embedding file name>\nvocab={}\nwith open('../input/embeddings/' + embedding_path,'r') as f:\n    for line_number,line in enumerate(tqdm(f)):\n        key,values=line.split(' ')[0],line.split(' ')[1:]\n        if not any(char in string.punctuation for char in key): \n            # we do this filter because  we will remove punctuation anyway\n            vocab[key]=np.asarray(values,dtype='float32')\nprint(f'Total of {len(vocab.keys())} words in vocabulary')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"966f8dc0a496ecf0810b642438104e731a366151"},"cell_type":"code","source":"# string preprocessing function, where we get rid of punctuation, \n# split string to a list of words\n# after that we encode words to 300-D vectors\ntrans_table={key:' ' for key in string.punctuation} # punctuation removal table\nunknown_word=np.zeros(300) # token for unknown word\n\ndef str_prep(s):\n    s=s.translate(str.maketrans(trans_table)).lower().strip() # remove punctuation\n    s=re.sub(' +',' ',s) # get rid of multiple spaces inside\n    s=s.split(' ')\n    ar=np.asarray([vocab.get(x,unknown_word) for x in s],dtype='float32')\n    return ar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d1bf7fda7351e0e3fd608cd1a5eff1512cc02b7"},"cell_type":"code","source":"# simpler batch generator, padding to 60 words\ndef batch_gen(df,min_batch_size=1024,transformation_func=str_prep, training_mode=True):\n    n_batches=math.ceil(len(df)/min_batch_size)\n    while True:\n        df=df.sample(frac=1).reset_index(drop=True)\n        for batch in range(n_batches):\n            start=batch*min_batch_size\n            end=start+min_batch_size\n            if batch==n_batches-1:\n                X=df['question_text'][start:].apply(transformation_func)\n                X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n                y=np.array(df['target'][start:])\n            else:\n                X=df['question_text'][start:end].apply(transformation_func)\n                X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n                y=np.array(df['target'][start:end])\n            yield X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5ac28b20e8fda9e8c4594938abe993653f63ebe"},"cell_type":"code","source":"# train/validation split function\n\ndef train_val_split(df,frac=0.2):\n    df=df.sample(frac=1).reset_index(drop=True) # random shuffling\n\n    df_sincere=df[df['target']==0]\n    df_taxic=df[df['target']==1]\n    \n    sincere_border=int(len(df_sincere)*(1-frac))\n    taxic_border=int(len(df_taxic)*(1-frac))\n    \n    df_train=pd.concat([df_sincere[:sincere_border],df_taxic[:taxic_border]])\n    df_val=pd.concat([df_sincere[sincere_border:],df_taxic[taxic_border:]])\n    \n    print('Training data peppered (prepared)!')\n    return df_train.sample(frac=1).reset_index(drop=True), df_val.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4272cf088074d774b4aad37446a176ce5674de03"},"cell_type":"markdown","source":"**Section 3. Modelling**\n=====================\n========================================================"},{"metadata":{"trusted":true,"_uuid":"bb06f935d56790c4458a8814e147b0df2c23d4e0"},"cell_type":"code","source":"# train model on data\ndef train_model(train_df,val_df,n_epochs=8,batch_size=1024):\n    model = Sequential()\n    model.add(Bidirectional(CuDNNLSTM(64, return_sequences=True),\n                             input_shape=(60, 300)))\n    model.add(Bidirectional(CuDNNLSTM(32)))\n    model.add(Dense(64))\n    model.add(Activation('relu'))\n    model.add(Dense(32))\n    model.add(Activation('relu'))\n    model.add(Dense(16))\n    model.add(Activation('relu'))\n    model.add(Dense(1,activation='sigmoid'))\n\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    model.fit_generator(batch_gen(train_df, min_batch_size=batch_size), epochs=n_epochs,\n                        steps_per_epoch=math.ceil(len(train_df)/batch_size),\n                        validation_data=batch_gen(val_df, min_batch_size=batch_size),\n                        validation_steps=math.ceil(len(val_df)/batch_size),\n                        verbose=True)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9faaf24178fdd8629cec8209ba562bd880cc79f6"},"cell_type":"code","source":"# get predictions with trained model\ndef get_predictions(df, model, min_batch_size=1024,transformation_func=str_prep):\n    predictions=np.ndarray(shape=(0,1))\n    print(f'Total values to predict: {len(df)}')\n    n_batches=math.ceil(len(df)/min_batch_size)\n    for batch in trange(n_batches):\n        start=batch*min_batch_size\n        end=start+min_batch_size\n        if batch==n_batches-1:\n            X=df['question_text'][start:].apply(transformation_func)\n            X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n            y_predicted=model.predict(X)\n        else:\n            X=df['question_text'][start:end].apply(transformation_func)\n            X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n            y_predicted=model.predict(X)\n        predictions=np.append(predictions,y_predicted,axis=0)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecf0499858efee7e45ae78284719aa59da4f571f"},"cell_type":"code","source":"# selecting best threshold value for current model\ndef select_best_threshold(y_true, y_predicted):\n    # code for finding threshold taken from\n    # https://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork\n    # dont forget to upvote that kernel:\n    thresholds = []\n    for thresh in np.arange(0.0, 1, 0.001):\n        thresh = np.round(thresh, 3)\n        res=metrics.f1_score(y_true, (y_predicted>thresh).astype(int))\n        thresholds.append([thresh, res])\n    thresholds.sort(key=lambda x: x[1], reverse=True)\n    best_thresh = thresholds[0][0]\n    print(\"Best threshold: \", best_thresh)\n    return best_thresh","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3640353a87d1613ee2e7933d98312bdeaa2b0a7"},"cell_type":"markdown","source":"Training stack of models"},{"metadata":{"trusted":true,"_uuid":"06e79414a22718a68ddbc8adfedd825c067cdbea"},"cell_type":"code","source":"stack_size=3\nmodel_stack={}\nfor ind in range(stack_size):\n    print(f'\\n=== Model number {ind+1} ===\\n')\n    # creating training and validation sets, these will be different for each model in stack\n    train_df, val_df=train_val_split(df, frac=0.1)\n    # training current model\n    model=train_model(train_df,val_df, n_epochs=5)\n    # predicting on validation data\n    y_predicted=get_predictions(val_df,model)\n    y_true=val_df['target'].values\n    # selecting threshold\n    best_thresh=select_best_threshold(y_true, y_predicted)\n    model_stack[ind]=(model, best_thresh)\n    print(f'Model number {ind+1} finished!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13e811d2e9e58c53d54c2464de6539b18a04ec52"},"cell_type":"code","source":"# predict on test set\ntest_set=pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95c7136467d8f56300da0aabe667dbe31b51a584"},"cell_type":"code","source":"test_set.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"183da27169957b7476608e1d730730f4e802e07c"},"cell_type":"code","source":"predictions={}\nfor key, (model, threshold) in model_stack.items():\n    predictions[key]=(get_predictions(test_set,model)>threshold).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35dd6f086e53866c5d2b3c284a9426ab4138279d"},"cell_type":"code","source":"preds=np.stack([value for value in predictions.values()],axis=1).reshape(-1,stack_size)\npreds=np.average(preds,axis=1)\npreds=np.around(preds,decimals=0).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd26e7526efd46c1df1c53c48674599db8179df8"},"cell_type":"code","source":"test_set['prediction']=preds\nto_submit=test_set[['qid','prediction']]\n\nto_submit.to_csv(\"submission.csv\", index=False)\nprint('Submissions saved to file!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"449d537fc06e6047ca7d9e0b58f018dbf89e572a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}