{"cells":[{"metadata":{"_uuid":"24ffd642a0eb9c993fdba4014345e4331c03a943"},"cell_type":"markdown","source":"Some pointers - \n1) https://www.kaggle.com/mihaskalic/lstm-is-all-you-need-well-maybe-embeddings-also \n2) https://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork\n\nAdded RL model with text input as state and prediction quality as reward\n"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import History, ModelCheckpoint\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras import backend as K\n\nfrom keras.models import Sequential\nfrom keras.layers import CuDNNLSTM, Dense, Dropout, Input, Bidirectional, concatenate\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nimport os\nimport csv\nimport random\nimport pandas\nfrom pandas import Series\nfrom collections import deque\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt \n\n%matplotlib inline\n\n\n\nGAMMA = 0.95\nLEARNING_RATE = 0.001\nMAXLEN = 30\nMEMORY_SIZE = 1000000\n\nMAX_TICKS=30\nMEMORY_BATCH_SIZE = 20\nGEN_BATCH_SIZE = MAX_TICKS\n\nEXPLORATION_MAX = 1.0\nEXPLORATION_MIN = 0.01\nEXPLORATION_DECAY = 0.995","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"21f85dfd5b9bf3d8b27ba29149d52253e5d64049"},"cell_type":"markdown","source":"# Setup"},{"metadata":{"trusted":true,"_uuid":"78578eab64a477d0a5ad6b1c917ae154868a44df"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\n#train_df, val_df = train_test_split(train_df, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92ffbf2ef35d2dc5863ee27c61eadd1869ca5440"},"cell_type":"code","source":"# embdedding setup\n# Source https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html\nembeddings_index = {}\nf = open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\nfor line in tqdm(f):\n    values = line.split(\" \")\n    word = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    embeddings_index[word] = coefs\nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cc8b0bcd285225ce651d024f506615d4656b9f7"},"cell_type":"code","source":"# Convert values to embeddings\ndef text_to_array(text):\n    empyt_emb = np.zeros(300)\n    text = text[:-1].split()[:MAXLEN]\n    embeds = [embeddings_index.get(x, empyt_emb) for x in text]\n    embeds+= [empyt_emb] * (MAXLEN - len(embeds))\n    return np.array(embeds)\n\n# train_vects = [text_to_array(X_text) for X_text in tqdm(train_df[\"question_text\"])]\n#val_vects = np.array([text_to_array(X_text) for X_text in tqdm(val_df[\"question_text\"][:10000])])\n#val_y = np.array(val_df[\"target\"][:10000])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c950448e0717eaebb920d93cfdc6b4561e21853"},"cell_type":"code","source":"# Data providers\n\n\n\ndef batch_gen(train_df, batch_size):\n    n_batches = math.ceil(len(train_df) / batch_size)\n    while True: \n        train_df = train_df.sample(frac=1.)  # Shuffle the data.\n        for i in range(n_batches):\n            texts = train_df.iloc[i*batch_size:(i+1)*batch_size, 1]\n            text_arr = np.array([text_to_array(text) for text in texts])\n            yield text_arr, np.array(train_df[\"target\"][i*batch_size:(i+1)*batch_size])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"706c0224b6112e5a8f00ad25f35e90fbb9519a5f"},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true,"_uuid":"7f41072f819ab9354830be0843236f5ea65329dc"},"cell_type":"code","source":"# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n        \n    def build(self, input_shape):\n        #print ('input_shape', input_shape)\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e7919b6a143c9e111be60bad39523f7d3582ef9"},"cell_type":"code","source":"class Env:\n    \"\"\"\n    Environment for agent to play with \n    \"\"\"\n\n    def __init__(self, gen, max_ticks):\n        self.action_space = 2\n\n        # episode over \n        self.episode_over = False\n        self.info = {} \n\n        # Store what the agent tried\n        self.curr_episode = -1\n        self.action_episode_memory = []\n        self.gen = gen\n        self.max_ticks = max_ticks\n\n    def step(self, action):\n        \"\"\"\n        The agent takes a step in the environment.\n        Parameters\n        ----------\n        action : int\n        Returns\n        -------\n        ob, reward, episode_over, info : tuple\n            ob (object) :\n                state of system\n               \n            reward (float) :\n               \n            episode_over (bool) :\n                \n            info (dict) :\n                optional info\n                 \n        \"\"\"\n        # ground_truth = self.gen_state[1]\n        self.reward =  1 if  self.gen_state[1][self.ticks] == action else -1\n        \n        self.ticks += 1\n        print('Tick', self.ticks, 'reward', self.reward) \n\n        # check if end of input batch \n        if self.ticks == self.max_ticks:\n            self.episode_over = True\n        else:\n            self.state = self.gen_state[0][self.ticks]\n            \n        \n        return self.state, self.reward, self.episode_over, self.info \n          \n\n    def get_reward(self):\n        return self.reward\n\n\n    def reset(self):\n        \"\"\"\n        Reset the state of the environment and returns an initial observation.\n        Returns\n        -------\n        observation (object): the initial observation of the space.\n        \"\"\"\n        self.curr_episode += 1\n        self.ticks = 0\n        self.action_episode_memory.append([])\n        \n        self.episode_over = False\n        self.gen_state = next(self.gen)\n        self.state = self.gen_state[0][0]\n        return self.state\n\n    def get_state(self):\n        \"\"\"Get the observation. It is the gen_state[0]\"\"\"\n        return self.state\n    \n    def cleanup(self):\n        pass\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"389aac47f6056c67f195066ea417d4afcbf65821"},"cell_type":"code","source":"# text_input = Input(shape=(MAXLEN ,300))\n# x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(text_input)\n# x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n# x = Attention(MAXLEN)(x)\n# x = Dense(64, activation=\"relu\")(x)\n# x = Dropout(0.1)(x)\n# x = Dense(64, activation=\"relu\")(x)\n# x = Dropout(0.1)(x)\n# x = Dense(24, activation=\"relu\")(x)\n# model_output = Dense(2, activation=\"linear\")(x)\n# model = Model(inputs=text_input, outputs=model_output)\n# model.compile(loss='mse',\n#               optimizer=Adam(lr=1e-3),\n#               metrics=['accuracy'])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b553f340bcd703d083c7c614ee24af2da55c59a0"},"cell_type":"code","source":"# gen = batch_gen(train_df)\n# state = next(gen)\n# print (state[0].shape)\n# print (state[0][0].shape)\n# print (state[1].shape)\n# text_input = np.reshape(state[0], [state[0].shape[0], 30, 300])\n\n# output = model.predict(text_input)\n# print (output.shape)\n# action = np.argmax(output, axis = -1)\n# sum(action == state[1]) + sum (action != state[1])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8daed20acb4fddae9dab5e233f64c432aaa1597"},"cell_type":"code","source":"class DQNSolver:\n\n    def __init__(self, action_space):\n        self.exploration_rate = EXPLORATION_MAX\n        self.action_space = action_space\n        self.memory = deque(maxlen=MEMORY_SIZE)\n        \n        text_input = Input(shape=(MAXLEN ,300))\n        x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(text_input)\n        x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n        x = Attention(MAXLEN)(x)\n        x = Dense(64, activation=\"relu\")(x)\n        #x = Dropout(0.1)(x)\n        x = Dense(64, activation=\"relu\")(x)\n        #x = Dropout(0.1)(x)\n        x = Dense(24, activation=\"relu\")(x)\n        model_output = Dense(self.action_space, activation=\"linear\")(x)\n        self.model = Model(inputs=text_input, outputs=model_output)\n        self.model.compile(loss='mse',\n                      optimizer=Adam(lr=1e-3),\n                      metrics=['accuracy'])\n\n    def remember(self, state, action, reward, next_state, done):\n        self.memory.append((state, action, reward, next_state, done))\n\n    def act(self, state):\n        if np.random.rand() < self.exploration_rate:\n            action = random.randrange(self.action_space)\n            #print (\"Taking random action\", action)\n            return action\n    \n        q_values = self.predict(state)\n        #print ('q_values', q_values)\n        action = np.argmax(q_values[0])\n        #print (\"Taking predicted  action\", action)\n        return action\n\n\n    def predict(self, state):\n        state = np.reshape(state, [1, MAXLEN, 300])\n        return self.model.predict(state)\n    \n    def fit(self, state, q_values, verbose=0):\n        state = np.reshape(state, [1, MAXLEN, 300])\n        self.model.fit(state, q_values, verbose=0)\n        \n    def experience_replay(self):\n        if len(self.memory) < MEMORY_BATCH_SIZE:\n            return\n        batch = random.sample(self.memory, MEMORY_BATCH_SIZE)\n        for state, action, reward, state_next, terminal in batch:\n            q_update = reward\n            if not terminal:                \n                q_update = (reward + GAMMA * np.amax(self.predict(state_next)[0]))\n            q_values = self.predict(state)\n            q_values[0][action] = q_update\n            self.fit(state, q_values, verbose=0)\n        self.exploration_rate *= EXPLORATION_DECAY\n        self.exploration_rate = max(EXPLORATION_MIN, self.exploration_rate)\n\n    def save_model(self, model_name='model.h5'):\n        # serialize model to JSON\n        model_json = self.model.to_json()\n        with open(\"model.json\", \"w\") as json_file:\n            json_file.write(model_json)\n        # serialize weights to HDF5\n        self.model.save_weights(model_name)\n        print(\"Saved model to disk\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce204286f79f783f5a1a84bf5a76038dedd41774"},"cell_type":"code","source":"    random.seed(100)\n    env = Env(batch_gen(train_df, GEN_BATCH_SIZE), MAX_TICKS)\n    dqn_solver = DQNSolver(env.action_space)\n    \n    run = 0\n    MAX_RUN = 50\n    score_card = []\n    while run < MAX_RUN:\n        run += 1\n        state = env.reset()\n        step = 0\n        score = 0\n        while True:\n            step += 1\n            action = dqn_solver.act(state)\n            state_next, reward, terminal, info = env.step(action)\n            score += reward\n            dqn_solver.remember(state, action, reward, state_next, terminal)\n            state = state_next\n            if terminal:\n                print (\"Run: \" + str(run) + \", exploration: \" + str(dqn_solver.exploration_rate) + \", score: \" + str(score))\n                score_card.append((run, score, step))\n                break\n            dqn_solver.experience_replay()\n\n    with open('dqn_stat_score_card_{0}.csv'.format(MAX_RUN), 'w') as f:\n        writer = csv.writer(f)\n        writer.writerows(score_card)\n\n    dqn_solver.save_model('dqn_stat_model_{0}_run.h5'.format(MAX_RUN))\n\n\n    env.cleanup()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bddd479642ae6d9bd93ec7f3ff39bd5a3f82a617","scrolled":false},"cell_type":"code","source":"# random.seed(100)\n# env = Env(batch_gen(train_df, GEN_BATCH_SIZE), MAX_TICKS)\n# dqn_solver = DQNSolver(env.action_space)\n# score_card = []\n# state = env.reset()\n# step = 0\n# while True:\n#     step += 1\n#     #env.render()\n#     action = dqn_solver.act(state)\n#     state_next, reward, terminal, info = env.step(action)        \n#     print (\"Step: \" + str(step) + \", exploration: \" + str(dqn_solver.exploration_rate) + \", score: \" + str(reward))\n#     score_card.append((step, reward))\n#     dqn_solver.remember(state, action, reward, state_next, terminal)\n#     state = state_next\n#     if terminal:\n#         print ('Training ended after {} steps'.format(step))\n#         break\n#     dqn_solver.experience_replay()\n\n# with open('dqn_nlp_score_card_{0}.csv'.format(MAX_TICKS), 'w') as f:\n#     writer = csv.writer(f)\n#     writer.writerows(score_card)\n\n# dqn_solver.save_model('dqn_nlp_model_{0}_run.h5'.format(MAX_TICKS))\n\n# env.cleanup()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f140956debf300717229b71bff737668dbb6fd46"},"cell_type":"code","source":"def draw_cumulative_score(filename,title):\n    df = pandas.read_csv(filename, header=None)\n    data = df.values\n    # x axis values \n    x =  [float(x[0]) for x in data]\n    #print (x)\n    # corresponding y axis values \n    y = [float(x[1]) for x in data]\n    #print (y)\n    series = Series(y)\n    print ('Min', series.min(), 'Max', series.max(), 'mean', series.mean())\n    # plotting the points  \n    plt.plot(x, y) \n\n    # naming the x axis \n    plt.xlabel('Steps') \n    # naming the y axis \n    plt.ylabel('Score per step') \n\n    # giving a title to my graph \n    plt.title(title) \n\n    # show a legend on the plot \n    # plt.legend() \n\n    # function to show the plot \n    plt.show() \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d81217fa9d758674560c74f414443064b62b51d8"},"cell_type":"code","source":"draw_cumulative_score('dqn_stat_score_card_50.csv', 'DQN Agent')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d6cdda0e301be0b84e826e29f0f3a85c41aa6da9"},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true,"_uuid":"49c2c424b74fdc6fda7790d7eb6957414f9c2d52"},"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# plt.plot(hist.history['loss'])\n# plt.plot(hist.history['val_loss'])\n# plt.title('model loss')\n# plt.ylabel('loss')\n# plt.xlabel('epoch')\n# plt.legend(['train', 'test'], loc='upper left')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b58cd95254f41e5002a17de0c3feab54a5fc3c67"},"cell_type":"code","source":"model = dqn_solver.model\nmodel.load_weights('dqn_stat_model_50_run.h5')\n# prediction part\nbatch_size = 128\ndef batch_gen_test(test_df):\n    n_batches = math.ceil(len(test_df) / batch_size)\n    for i in range(n_batches):\n        texts = test_df.iloc[i*batch_size:(i+1)*batch_size, 1]\n        text_arr = np.array([text_to_array(text) for text in texts])\n        yield text_arr\n\ntest_df = pd.read_csv(\"../input/test.csv\")\n\nall_preds = []\nfor x in tqdm(batch_gen_test(test_df)):\n    all_preds.extend(model.predict(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac338852c64b7a4ea7cfc71a23657bfea28635bf"},"cell_type":"code","source":"# Do Not Submit - predicts all zeroes **************************************\n# action = np.argmax(all_preds, axis = -1).astype(np.int)\n# submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": action})\n# submit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f2a59090ca95588c4f2be197a089a38671a6a7c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}