{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import re\nimport os\nimport sys\nimport math\nimport string\nimport random\nimport datetime\n\nimport numpy as np\nimport pandas as pd\n\nfrom time import time\nfrom tqdm import tqdm, trange\n\nfrom sklearn import metrics\n\nimport keras.layers\nfrom keras import backend as K\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.models import Sequential, Model\nfrom keras.layers import CuDNNLSTM, CuDNNGRU, Dense, Bidirectional, Dropout, BatchNormalization, Activation, Input\nfrom keras.optimizers import Adam\nfrom keras.regularizers import l1_l2\nfrom keras.engine.topology import Layer\nfrom keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e12b76691a4da17a8e507907f5c2d630603b80ea"},"cell_type":"code","source":"print(f'Starting at {datetime.datetime.now()}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92503940d7f560316373e85104c31eecfc342a7f"},"cell_type":"code","source":"# training data\ndf=pd.read_csv('../input/train.csv')\ndf=df[['question_text','target']]\ndf=df.dropna()\nmax_word_len=60","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fe5f4b726b55c4e443897d327915d2f78caec9c"},"cell_type":"code","source":"# embeddings\nembedding_path='glove.840B.300d/glove.840B.300d.txt'\nvocab={}\nwith open('../input/embeddings/' + embedding_path,'r') as f:\n    for line_number,line in enumerate(tqdm(f)):\n        key,values=line.split(' ')[0],line.split(' ')[1:]\n        if not any(char in string.punctuation for char in key): \n            vocab[key]=np.asarray(values,dtype='float32')\nprint(f'Total of {len(vocab.keys())} words in vocabulary')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e2fc87558eeee75cd8e2b8cb4dce7c1437f23cb"},"cell_type":"code","source":"trans_table={key:' ' for key in string.punctuation}\nunknown_word=np.zeros(300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"966f8dc0a496ecf0810b642438104e731a366151"},"cell_type":"code","source":"def str_prep(s):\n    s=s.translate(str.maketrans(trans_table)).lower().strip() # remove punctuation\n    s=re.sub(' +',' ',s) # get rid of multiple spaces inside\n    s=s.split(' ')\n    ar=np.asarray([vocab.get(x,unknown_word) for x in s],dtype='float32')\n    return ar","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d1bf7fda7351e0e3fd608cd1a5eff1512cc02b7"},"cell_type":"code","source":"def batch_gen(df,min_batch_size=1024,transformation_func=str_prep, training_mode=True):\n    n_batches=math.ceil(len(df)/min_batch_size)\n    while True:\n        df=df.sample(frac=1).reset_index(drop=True)\n        for batch in range(n_batches):\n            start=batch*min_batch_size\n            end=start+min_batch_size\n            if batch==n_batches-1:\n                X=df['question_text'][start:].apply(transformation_func)\n                X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n                y=np.array(df['target'][start:])\n            else:\n                X=df['question_text'][start:end].apply(transformation_func)\n                X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n                y=np.array(df['target'][start:end])\n            yield X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5ac28b20e8fda9e8c4594938abe993653f63ebe"},"cell_type":"code","source":"def train_val_split(df,frac=0.2):\n    df=df.sample(frac=1).reset_index(drop=True) # random shuffling\n    df_sincere=df[df['target']==0]\n    df_taxic=df[df['target']==1]\n    sincere_border=int(len(df_sincere)*(1-frac))\n    taxic_border=int(len(df_taxic)*(1-frac))\n    df_train=pd.concat([df_sincere[:sincere_border],df_taxic[:taxic_border]])\n    df_val=pd.concat([df_sincere[sincere_border:],df_taxic[taxic_border:]])\n    print('Training data peppered (prepared)!')\n    return df_train.sample(frac=1).reset_index(drop=True), df_val.sample(frac=1).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4272cf088074d774b4aad37446a176ce5674de03"},"cell_type":"markdown","source":"**Section 3. Modelling**\n=====================\n========================================================"},{"metadata":{"trusted":true,"_uuid":"9f3cdae01d4c851a90744f0e506016d728e88477"},"cell_type":"code","source":"# attention layer\n# idea stolen from https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n\nclass Attention(Layer):\n    def __init__(self, step_dim, W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None, bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        self.W = self.add_weight((input_shape[-1],), initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],), initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n        if self.bias:\n            eij += self.b\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b8b671c2d2ce2fd34d74e960bbd1f6c6e5fb47b"},"cell_type":"code","source":"# # experimental model with attention layer\n# def train_model(train_df,val_df,n_epochs=5,batch_size=1024):\n    \n#     #########################################\n    \n#     inp = Input(shape=(max_word_len, 300))\n#     lstm1=Bidirectional(CuDNNLSTM(32, return_sequences=True))(inp)\n#     lstm2=Bidirectional(CuDNNLSTM(32, return_sequences=True))(lstm1)\n#     attention1=Attention(max_word_len)(lstm2)\n#     dense1=Dense(64, activation='elu')(attention1)\n#     dense2=Dense(64, activation='elu')(dense1)\n#     outp = Dense(1, activation=\"sigmoid\")(dense2)\n    \n#     #########################################\n#     opt=Adam(lr=0.00321)\n#     model = Model(inputs=inp, outputs=outp)\n#     model.compile(loss='binary_crossentropy',\n#                   optimizer=opt,\n#                   metrics=['accuracy'])\n#     model.fit_generator(batch_gen(train_df, min_batch_size=batch_size), epochs=n_epochs,\n#                         steps_per_epoch=math.ceil(len(train_df)/batch_size),\n#                         validation_data=batch_gen(val_df, min_batch_size=batch_size),\n#                         validation_steps=math.ceil(len(val_df)/batch_size),\n#                         verbose=True)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08e4b57703514e3335c048e2b919775189b40b6e"},"cell_type":"code","source":"# # experimental model - GRU vs LSTM vs BOTH\n# def train_model(train_df,val_df,n_epochs=5,batch_size=1024):\n    \n#     '''\n#     Quick reminder for me\n#     Your output shape depends on how you configure the net. \n#     If your LSTM/RNN has return_sequences=False, you'll have one\n#     label per sequence; if you set return_sequences=True, \n#     you'll have one label per timestep.\n#     '''\n    \n#     input_layer = Input(shape=(max_word_len, 300))\n#     #########################################\n#     # left branch\n#     lstm_left=Bidirectional(CuDNNLSTM(32, return_sequences=True))(input_layer)\n#     gru_left=Bidirectional(CuDNNGRU(32, return_sequences=True))(lstm_left)\n#     attention_left=Attention(max_word_len)(gru_left)\n#     dense_left=Dense(256, activation='elu')(attention_left)\n    \n#     # right branch\n#     gru_right=Bidirectional(CuDNNGRU(32, return_sequences=True))(input_layer)\n#     lstm_right=Bidirectional(CuDNNLSTM(32, return_sequences=True))(gru_right)\n#     attention_right=Attention(max_word_len)(lstm_right)\n#     dense_right=Dense(256, activation='elu')(attention_right)   \n    \n#     # only lstm\n#     only_lstm1=Bidirectional(CuDNNLSTM(32, return_sequences=True))(input_layer)\n#     only_lstm2=Bidirectional(CuDNNLSTM(32, return_sequences=True))(only_lstm1)\n#     attention_only_lstm=Attention(max_word_len)(only_lstm2)\n#     dense_only_lstm=Dense(256, activation='elu')(attention_only_lstm)\n    \n#     # only gru\n#     only_gru1=Bidirectional(CuDNNGRU(32, return_sequences=True))(input_layer)\n#     only_gru2=Bidirectional(CuDNNGRU(32, return_sequences=True))(only_gru1)\n#     attention_only_gru=Attention(max_word_len)(only_gru2)\n#     dense_only_gru=Dense(256, activation='elu')(attention_only_gru) \n    \n#     # concatenation\n#     concatenate_layer=keras.layers.concatenate([dense_left,dense_right,\n#                                                dense_only_lstm,dense_only_gru],\n#                                                axis=1)\n#     out=Dense(128, activation='elu')(concatenate_layer)\n#     output_layer = Dense(1, activation='sigmoid', name='Output')(out)\n    \n#     #########################################\n#     opt=Adam(lr=0.003)\n#     model = Model(inputs=input_layer, outputs=output_layer)\n#     model.compile(loss='binary_crossentropy',\n#                   optimizer=opt,\n#                   metrics=['accuracy'])\n#     model.fit_generator(batch_gen(train_df, min_batch_size=batch_size), epochs=n_epochs,\n#                         steps_per_epoch=math.ceil(len(train_df)/batch_size),\n#                         validation_data=batch_gen(val_df, min_batch_size=batch_size),\n#                         validation_steps=math.ceil(len(val_df)/batch_size),\n#                         verbose=True)\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18d6b60253f71de491c512127af6b7ea685e651f"},"cell_type":"code","source":"# experimental model - GRU and LSTM\ndef train_model(train_df,val_df,n_epochs=5,batch_size=1024):\n    \n    '''\n    Quick reminder for me\n    Your output shape depends on how you configure the net. \n    If your LSTM/RNN has return_sequences=False, you'll have one\n    label per sequence; if you set return_sequences=True, \n    you'll have one label per timestep.\n    '''\n    \n    input_layer = Input(shape=(max_word_len, 300))\n    #########################################\n    # only lstm\n    only_lstm1=Bidirectional(CuDNNLSTM(32, return_sequences=True))(input_layer)\n    only_lstm2=Bidirectional(CuDNNLSTM(32, return_sequences=True))(only_lstm1)\n    attention_only_lstm=Attention(max_word_len)(only_lstm2)\n    dense_only_lstm=Dense(64, activation='elu')(attention_only_lstm)\n    \n    # only gru\n    only_gru1=Bidirectional(CuDNNGRU(32, return_sequences=True))(input_layer)\n    only_gru2=Bidirectional(CuDNNGRU(32, return_sequences=True))(only_gru1)\n    attention_only_gru=Attention(max_word_len)(only_gru2)\n    dense_only_gru=Dense(64, activation='elu')(attention_only_gru) \n    \n    # concatenation\n    concatenate_layer=keras.layers.concatenate([dense_only_lstm,dense_only_gru],\n                                               axis=1)\n    out=Dense(64, activation='elu')(concatenate_layer)\n    output_layer = Dense(1, activation='sigmoid', name='Output')(out)\n    \n    #########################################\n    opt=Adam(lr=0.003)\n    model = Model(inputs=input_layer, outputs=output_layer)\n    model.compile(loss='binary_crossentropy',\n                  optimizer=opt,\n                  metrics=['accuracy'])\n    model.fit_generator(batch_gen(train_df, min_batch_size=batch_size), epochs=n_epochs,\n                        steps_per_epoch=math.ceil(len(train_df)/batch_size),\n                        validation_data=batch_gen(val_df, min_batch_size=batch_size),\n                        validation_steps=math.ceil(len(val_df)/batch_size),\n                        verbose=True)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9faaf24178fdd8629cec8209ba562bd880cc79f6"},"cell_type":"code","source":"def get_predictions(df, model, min_batch_size=1024,transformation_func=str_prep):\n    predictions=np.ndarray(shape=(0,1))\n    print(f'Total values to predict: {len(df)}')\n    n_batches=math.ceil(len(df)/min_batch_size)\n    for batch in trange(n_batches):\n        start=batch*min_batch_size\n        end=start+min_batch_size\n        if batch==n_batches-1:\n            X=df['question_text'][start:].apply(transformation_func)\n            X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n            y_predicted=model.predict(X)\n        else:\n            X=df['question_text'][start:end].apply(transformation_func)\n            X=pad_sequences(X,maxlen=60,dtype='float32',padding='post')\n            y_predicted=model.predict(X)\n        predictions=np.append(predictions,y_predicted,axis=0)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecf0499858efee7e45ae78284719aa59da4f571f"},"cell_type":"code","source":"def select_best_threshold(y_true, y_predicted):\n    thresholds = []\n    for thresh in np.arange(0.0, 1, 0.001):\n        thresh = np.round(thresh, 3)\n        res=metrics.f1_score(y_true, (y_predicted>thresh).astype(int))\n        thresholds.append([thresh, res])\n    thresholds.sort(key=lambda x: x[1], reverse=True)\n    best_thresh = thresholds[0][0]\n    print(\"Best threshold: \", best_thresh)\n    return best_thresh","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3640353a87d1613ee2e7933d98312bdeaa2b0a7"},"cell_type":"markdown","source":"Training stack of models"},{"metadata":{"trusted":true,"_uuid":"06e79414a22718a68ddbc8adfedd825c067cdbea"},"cell_type":"code","source":"stack_size=3\nmodel_stack={}\nt_start=time()\nfor ind in range(stack_size):\n    print(f'\\n=== Model number {ind+1} ===\\n')\n    train_df, val_df=train_val_split(df,frac=0.2)\n    model=train_model(train_df,val_df, n_epochs=5)\n    y_predicted=get_predictions(val_df,model)\n    y_true=val_df['target'].values\n    best_thresh=select_best_threshold(y_true, y_predicted)\n    model_stack[ind]=(model, best_thresh)\n    print(f'Model number {ind+1} finished!')\nt_finish=time()\nprint(f'Total time for training and calculating thresholds is {t_finish-t_start:.2f} seconds')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13e811d2e9e58c53d54c2464de6539b18a04ec52"},"cell_type":"code","source":"# predict on test set\ntest_set=pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"183da27169957b7476608e1d730730f4e802e07c"},"cell_type":"code","source":"predictions={}\nfor key, (model, threshold) in model_stack.items():\n    predictions[key]=(get_predictions(test_set,model)>threshold).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35dd6f086e53866c5d2b3c284a9426ab4138279d"},"cell_type":"code","source":"preds=np.stack([value for value in predictions.values()],axis=1).reshape(-1,stack_size)\npreds=np.average(preds,axis=1)\npreds=np.around(preds,decimals=0).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd26e7526efd46c1df1c53c48674599db8179df8"},"cell_type":"code","source":"# test data\ntest_set['prediction']=preds\nto_submit=test_set[['qid','prediction']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"449d537fc06e6047ca7d9e0b58f018dbf89e572a"},"cell_type":"code","source":"to_submit.to_csv(\"submission.csv\", index=False)\nprint('Submissions saved to file!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6761f767b99749639d95109bd2a0f23afdfe0974"},"cell_type":"code","source":"print(f'Finishing at {datetime.datetime.now()}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e897d51cf7472911c05752cbe86b883a18a79be9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}