{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport re\nimport string\nimport os\nos.environ[\"KMP_SETTINGS\"] = \"false\"\n\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom tensorflow import keras\n\nraw_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\n\nraw_df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-21T12:49:04.384043Z","iopub.execute_input":"2021-12-21T12:49:04.384553Z","iopub.status.idle":"2021-12-21T12:49:07.660967Z","shell.execute_reply.started":"2021-12-21T12:49:04.384519Z","shell.execute_reply":"2021-12-21T12:49:07.660313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 512\n\nraw_df = raw_df[['question_text', 'target']]#.sample(frac=1).reset_index(drop=True)\n\ndataset = tf.data.Dataset.from_tensor_slices((raw_df['question_text'], tf.convert_to_tensor(raw_df['target'], dtype=tf.int64)))\ndataset = dataset.shuffle(10000).batch(batch_size) ","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:49:07.66221Z","iopub.execute_input":"2021-12-21T12:49:07.662449Z","iopub.status.idle":"2021-12-21T12:49:07.84802Z","shell.execute_reply.started":"2021-12-21T12:49:07.662416Z","shell.execute_reply":"2021-12-21T12:49:07.847303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split dataset\nds_batches = tf.data.experimental.cardinality(dataset)\nval_ds = dataset.take(ds_batches // 5)\ntrain_ds = dataset.skip(ds_batches // 5)\n\n#train_ds_batches = tf.data.experimental.cardinality(train_ds)\n#test_ds = dataset.take(train_ds_batches // 5)\n#train_ds = dataset.skip(train_ds_batches // 5)\n\nprint('Number of train batches: %d' % tf.data.experimental.cardinality(train_ds))\nprint('Number of validation batches: %d' % tf.data.experimental.cardinality(val_ds))\n#print('Number of test batches: %d' % tf.data.experimental.cardinality(test_ds))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:49:07.849099Z","iopub.execute_input":"2021-12-21T12:49:07.849774Z","iopub.status.idle":"2021-12-21T12:49:07.972839Z","shell.execute_reply.started":"2021-12-21T12:49:07.849735Z","shell.execute_reply":"2021-12-21T12:49:07.972125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def custom_standardization(input_data):\n    s = tf.strings.lower(input_data)\n    s = tf.strings.regex_replace(s, '<.*?>','') # html\n    s = tf.strings.regex_replace(s, 'http\\S+','') # links\n    s = tf.strings.regex_replace(s, '[^\\x00-\\x7F]+','') # non-ascii\n    s = tf.strings.regex_replace(s, '[%s]' % re.escape(string.punctuation),' ')\n    s = tf.strings.regex_replace(s, '\\n',' ')\n    s = tf.strings.regex_replace(s, '\\W+',' ') # non-word characters\n    s = tf.strings.regex_replace(s, '\\s+',' ')\n    return s\n\n# show example preprocessed\nfor row in dataset.take(1).map(lambda x, y: custom_standardization(x)):\n    print(str(row[0].numpy().decode('ascii')))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:49:07.974119Z","iopub.execute_input":"2021-12-21T12:49:07.974495Z","iopub.status.idle":"2021-12-21T12:49:08.24011Z","shell.execute_reply.started":"2021-12-21T12:49:07.974438Z","shell.execute_reply":"2021-12-21T12:49:08.239008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_features = 30000#len(vocab)\nsequence_length = 64\n\nvectorize_layer = keras.layers.experimental.preprocessing.TextVectorization(\n    standardize=custom_standardization,\n    max_tokens=max_features,\n    output_mode='int',\n    output_sequence_length=sequence_length)\n\ntext = train_ds.map(lambda x, y: x)\nvectorize_layer.adapt(next(iter(text)))","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:49:08.241567Z","iopub.execute_input":"2021-12-21T12:49:08.241826Z","iopub.status.idle":"2021-12-21T12:49:09.187869Z","shell.execute_reply.started":"2021-12-21T12:49:08.24179Z","shell.execute_reply":"2021-12-21T12:49:09.18711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\nlocal_zip = \"/kaggle/input/quora-insincere-questions-classification/embeddings.zip\"\nzip_ref = zipfile.ZipFile(local_zip, 'r')\n#zip_ref.extractall('/kaggle/temp/')\nzip_ref.namelist()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:49:09.189358Z","iopub.execute_input":"2021-12-21T12:49:09.189616Z","iopub.status.idle":"2021-12-21T12:49:09.206028Z","shell.execute_reply.started":"2021-12-21T12:49:09.189579Z","shell.execute_reply":"2021-12-21T12:49:09.205334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#zip_ref.extract(_glove)\nzip_ref.extractall()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T11:18:44.555756Z","iopub.execute_input":"2021-12-21T11:18:44.555996Z","iopub.status.idle":"2021-12-21T11:22:13.080987Z","shell.execute_reply.started":"2021-12-21T11:18:44.555972Z","shell.execute_reply":"2021-12-21T11:22:13.080008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_glove = './glove.840B.300d/glove.840B.300d.txt'\n_paragram =  './paragram_300_sl999/paragram_300_sl999.txt'\n_wiki_news = './wiki-news-300d-1M/wiki-news-300d-1M.vec'\n#_google_news = './GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n\nembeddings = [{'name': 'glove', 'path': _glove},\n              {'name': 'paragram', 'path': _paragram},\n              {'name': 'fasttext', 'path': _wiki_news}]\n\ndef load_embed(file):\n    def get_coefs(word, *arr):\n        return word, np.asarray(arr, dtype='float32')\n    \n    if file.split('/')[-1] == 'wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o) > 100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:52:12.004875Z","iopub.execute_input":"2021-12-21T12:52:12.00515Z","iopub.status.idle":"2021-12-21T12:52:12.013146Z","shell.execute_reply.started":"2021-12-21T12:52:12.005113Z","shell.execute_reply":"2021-12-21T12:52:12.012515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_emb_matrix(vocab_size, embed_size):\n    return np.zeros((vocab_size, embed_size), dtype=np.float32)\n        \ndef fill_emb_matrix(word_idx, emb_matrix, emb_index):\n    for word, i in word_idx:\n        emb_vector = emb_index.get(word)\n        if emb_vector is not None:\n            emb_matrix[i] = emb_vector\n    return emb_matrix\n\ndef add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:52:12.014631Z","iopub.execute_input":"2021-12-21T12:52:12.015083Z","iopub.status.idle":"2021-12-21T12:52:12.027168Z","shell.execute_reply.started":"2021-12-21T12:52:12.015041Z","shell.execute_reply":"2021-12-21T12:52:12.026444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorize_layer.vocabulary_size()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:52:12.030492Z","iopub.execute_input":"2021-12-21T12:52:12.030735Z","iopub.status.idle":"2021-12-21T12:52:12.037015Z","shell.execute_reply.started":"2021-12-21T12:52:12.030709Z","shell.execute_reply":"2021-12-21T12:52:12.036277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def vocab_to_integer(vocab):\n    return {word: ii for ii, word in enumerate(vocab, 0)}\n\nvocab = vectorize_layer.get_vocabulary()\nvocab_index = vocab_to_integer(vocab)\nvocab_len = len(vocab) #+ 1 \n\nconc_embedding = None\nembedding_dim = 0\n\nfor embedding in embeddings:\n    emb_name = embedding['name']\n    emb_path = embedding['path']\n    print(\"Running procedure on {}\".format(emb_name))\n    \n    print(\"Loading {}\".format(emb_name))\n    emb_index = load_embed(emb_path)\n\n    emb_size = 300\n    embedding_dim += emb_size\n    \n    emb_matrix = create_emb_matrix(vocab_len, emb_size)\n    print(emb_matrix.size)\n    print(emb_matrix.shape)\n    emb_matrix = fill_emb_matrix(vocab_index.items(), emb_matrix, emb_index)\n    \n    if conc_embedding is not None:\n        conc_embedding = np.concatenate((conc_embedding, emb_matrix), axis=1)\n        print(\"Concatenated! New shape: {}\".format(conc_embedding.shape))\n    else:\n        conc_embedding = emb_matrix\n    print(\"=================================================\")","metadata":{"execution":{"iopub.status.busy":"2021-12-21T12:52:12.038427Z","iopub.execute_input":"2021-12-21T12:52:12.03906Z","iopub.status.idle":"2021-12-21T12:56:23.29607Z","shell.execute_reply.started":"2021-12-21T12:52:12.038992Z","shell.execute_reply":"2021-12-21T12:56:23.294123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.Sequential([\n    keras.layers.Input(shape=(1,), dtype=tf.string),\n    vectorize_layer,\n    keras.layers.Embedding(vocab_len, embedding_dim, input_length=sequence_length, weights=[conc_embedding], trainable=False),\n    keras.layers.SpatialDropout1D(0.3),\n    keras.layers.Bidirectional(keras.layers.LSTM(156, return_sequences=True)),\n    keras.layers.Bidirectional(keras.layers.GRU(156, return_sequences=True)),\n    keras.layers.GlobalMaxPooling1D(),\n    #keras.layers.Dense(128, activation='relu'),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(1, activation='sigmoid')\n])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:09:33.924078Z","iopub.execute_input":"2021-12-21T13:09:33.924327Z","iopub.status.idle":"2021-12-21T13:09:34.722086Z","shell.execute_reply.started":"2021-12-21T13:09:33.924299Z","shell.execute_reply":"2021-12-21T13:09:34.721375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])\n\nearly_stop = keras.callbacks.EarlyStopping(patience=1, restore_best_weights=True)\n\nmodel.fit(train_ds, validation_data=val_ds, epochs=5, callbacks=[early_stop])","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:09:39.482448Z","iopub.execute_input":"2021-12-21T13:09:39.482882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_ds = tf.data.Dataset.from_tensor_slices(test_df['question_text'])\ny_pred = model.predict(test_df['question_text'])","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:02:06.575841Z","iopub.execute_input":"2021-12-21T13:02:06.576104Z","iopub.status.idle":"2021-12-21T13:03:50.218712Z","shell.execute_reply.started":"2021-12-21T13:02:06.576077Z","shell.execute_reply":"2021-12-21T13:03:50.217872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:03:50.223971Z","iopub.execute_input":"2021-12-21T13:03:50.224185Z","iopub.status.idle":"2021-12-21T13:03:50.233627Z","shell.execute_reply.started":"2021-12-21T13:03:50.22416Z","shell.execute_reply":"2021-12-21T13:03:50.232758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['prediction'] = np.where(y_pred >= 0.5, 1, 0)\npredictions = test_df[['qid', 'prediction']]\npredictions.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-21T13:04:00.661024Z","iopub.execute_input":"2021-12-21T13:04:00.661274Z","iopub.status.idle":"2021-12-21T13:04:00.684555Z","shell.execute_reply.started":"2021-12-21T13:04:00.661247Z","shell.execute_reply":"2021-12-21T13:04:00.683874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-20T14:23:28.88338Z","iopub.status.idle":"2021-12-20T14:23:28.883785Z","shell.execute_reply.started":"2021-12-20T14:23:28.883555Z","shell.execute_reply":"2021-12-20T14:23:28.883576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}