{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport string\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nimport matplotlib.pyplot as plt\n\nfrom statistics import *\n\nfrom sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nimport lightgbm as lgb\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\nfrom collections import defaultdict\n\n# spaCy based imports\nimport spacy\nfrom spacy.lang.en.stop_words import STOP_WORDS\nfrom spacy.lang.en import English\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# SpaCy Parser for questions\npunctuations = string.punctuation\nstopwords = list(STOP_WORDS)\nparser = English()\n\nfrom tqdm import tqdm\n\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, GRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","metadata":{"_uuid":"3af70b2f-af9f-433b-8f61-034c323cf4c7","_cell_guid":"c012b999-e694-4af9-9e1f-0f143432b957","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:13:17.810999Z","iopub.execute_input":"2021-06-10T08:13:17.811360Z","iopub.status.idle":"2021-06-10T08:13:27.570759Z","shell.execute_reply.started":"2021-06-10T08:13:17.811280Z","shell.execute_reply":"2021-06-10T08:13:27.569987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\nprint(\"Train shape: \", train_df.shape)\nprint(\"Test shape: \", test_df.shape)","metadata":{"_uuid":"34ef09f0-6950-4b7f-8f84-63ed41aefb1a","_cell_guid":"9445389f-0b22-4588-a1bc-2e9c6d5e7095","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:13:27.572375Z","iopub.execute_input":"2021-06-10T08:13:27.572718Z","iopub.status.idle":"2021-06-10T08:13:31.396017Z","shell.execute_reply.started":"2021-06-10T08:13:27.572682Z","shell.execute_reply":"2021-06-10T08:13:31.395031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","metadata":{"_uuid":"5fade0f0-a8a5-489a-abf1-3506dd74d078","_cell_guid":"8909e3e7-cc1e-4c89-9eb2-223385346ce1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:13:31.401000Z","iopub.execute_input":"2021-06-10T08:13:31.401347Z","iopub.status.idle":"2021-06-10T08:14:31.222648Z","shell.execute_reply.started":"2021-06-10T08:13:31.401312Z","shell.execute_reply":"2021-06-10T08:14:31.221728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(GRU(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())","metadata":{"_uuid":"45efadac-f3e1-48a7-ab52-d08f26c5c3f5","_cell_guid":"6c9c3acd-d92c-4a7e-a5c9-bccfd3a4186e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:14:31.226167Z","iopub.execute_input":"2021-06-10T08:14:31.226490Z","iopub.status.idle":"2021-06-10T08:14:34.322817Z","shell.execute_reply.started":"2021-06-10T08:14:31.226449Z","shell.execute_reply":"2021-06-10T08:14:34.321264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"_uuid":"d7c5d5fc-b3aa-4165-9f23-5ed104333631","_cell_guid":"b3064e81-9b83-46e1-a962-379bca3b8187","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:14:34.325945Z","iopub.execute_input":"2021-06-10T08:14:34.326185Z","iopub.status.idle":"2021-06-10T08:14:35.025665Z","shell.execute_reply.started":"2021-06-10T08:14:34.326160Z","shell.execute_reply":"2021-06-10T08:14:35.024789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\ndef get_available_devices():\n    local_device_protos = device_lib.list_local_devices()\n    return [x.name for x in local_device_protos]\nprint(get_available_devices())","metadata":{"_uuid":"2f5ade66-ce10-498e-8091-246283f87361","_cell_guid":"77c6fb0e-4c82-46a4-b660-5129f2b4fcbf","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:14:35.030077Z","iopub.execute_input":"2021-06-10T08:14:35.030357Z","iopub.status.idle":"2021-06-10T08:14:35.044452Z","shell.execute_reply.started":"2021-06-10T08:14:35.030328Z","shell.execute_reply":"2021-06-10T08:14:35.043596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.python.client import device_lib\ndevice_lib.list_local_devices()","metadata":{"_uuid":"cc745278-95b7-447b-81aa-1d377dcbaec8","_cell_guid":"5279d9ae-81c1-46b4-a7c6-36d7baf56b45","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:14:35.045676Z","iopub.execute_input":"2021-06-10T08:14:35.046147Z","iopub.status.idle":"2021-06-10T08:14:35.058500Z","shell.execute_reply.started":"2021-06-10T08:14:35.046111Z","shell.execute_reply":"2021-06-10T08:14:35.057628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\ntf.test.is_gpu_available()","metadata":{"_uuid":"3fe733c6-084e-4e23-b411-09e005ec39bd","_cell_guid":"2f59aae3-ece4-4f96-ad8d-e9893e551c20","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:14:35.060164Z","iopub.execute_input":"2021-06-10T08:14:35.060506Z","iopub.status.idle":"2021-06-10T08:14:35.074682Z","shell.execute_reply.started":"2021-06-10T08:14:35.060459Z","shell.execute_reply":"2021-06-10T08:14:35.073732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n# tf.compat.v1.disable_eager_execution()\ntf.device(\"gpu:0\")","metadata":{"_uuid":"8a145b71-2e1b-49f0-b63a-0b63eee24c63","_cell_guid":"1f713d47-e02d-49e8-a65e-7bc84b3f6258","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:14:35.076035Z","iopub.execute_input":"2021-06-10T08:14:35.076425Z","iopub.status.idle":"2021-06-10T08:14:35.084013Z","shell.execute_reply.started":"2021-06-10T08:14:35.076388Z","shell.execute_reply":"2021-06-10T08:14:35.083074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Train the model\nmodel.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","metadata":{"_uuid":"5518b85d-db4b-46a8-8a2f-1d8de7c22376","_cell_guid":"a1fb39f2-c9d7-425b-836c-6d2ed9fcf40f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-06-10T08:14:35.085616Z","iopub.execute_input":"2021-06-10T08:14:35.086211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_noemb_val_y>thresh).astype(int))))","metadata":{"_uuid":"3af7606c-dfa0-4b3f-9be2-4c9cae019f64","_cell_guid":"a1be707d-7ec7-4ef7-afff-02ca31346d9f","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1)","metadata":{"_uuid":"69824b37-bdf3-4ae9-a638-249c9f85b4c4","_cell_guid":"704f47ae-6001-4d45-a607-5799a20851e5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model, inp, x\nimport gc; gc.collect()\nimport time\ntime.sleep(10)","metadata":{"_uuid":"00554946-d96f-4aec-9a47-6dc743a7cdea","_cell_guid":"e671039f-4758-4fc5-b0d0-6ccc2a41e8a3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unzip file.zip -d destination_folder\n!unzip /kaggle/input/quora-insincere-questions-classification/embeddings.zip -d /kaggle/working/embeddings","metadata":{"_uuid":"79024105-48e8-49df-aa3a-d40e0a17c6bb","_cell_guid":"d899f19b-b40c-43b7-915e-3f780ee58780","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMBEDDING_GLOVE = './embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_GLOVE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(GRU(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())","metadata":{"_uuid":"fb2d808f-4bcc-4e1c-b26e-8fc62037f5ed","_cell_guid":"67c14a41-7d4c-424d-b863-87e7ab9437d6","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","metadata":{"_uuid":"461112bd-7d10-44ee-855c-72f0d84ae40e","_cell_guid":"794f4912-31ba-4465-b530-858299618ed9","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","metadata":{"_uuid":"e724a0f6-5696-4dd3-9a9d-971b56b5339b","_cell_guid":"09e3dc63-5ae3-4344-b99f-6b462411e233","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","metadata":{"_uuid":"b6e4c524-4476-4673-8f6d-0f97677aa6da","_cell_guid":"810b2396-ad99-4524-821d-096a66f96aa1","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\nimport time\ntime.sleep(10)","metadata":{"_uuid":"6c0d9dd3-947a-4bac-a8d3-0ff38942c69c","_cell_guid":"986cc8cf-fcd6-429b-8b71-9c995101c444","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMBEDDING_WIKINEWS = './embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_WIKINEWS) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(GRU(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"_uuid":"9d5e4089-7d65-4291-9b2b-f5bce4327f16","_cell_guid":"9f518189-2291-41fc-a51e-4bc26298ceb3","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_fasttext_val_y>thresh).astype(int))))","metadata":{"_uuid":"6cc12a12-de95-46f7-a2d1-38c2ccc7a4a7","_cell_guid":"91b1c0c9-af78-4b69-8cd0-3b6d11732c69","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)","metadata":{"_uuid":"33ff1be1-26af-47fa-ae29-fb5a70f7513f","_cell_guid":"56f7a081-76b5-40cb-ba2b-75e4e3070190","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\nimport time\ntime.sleep(10)","metadata":{"_uuid":"b99904c7-9513-4bc6-8af9-122abf726fcf","_cell_guid":"0e41da97-1f9d-4e82-a07c-bc2ae8855989","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EMBEDDING_PARAGRAM = './embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_PARAGRAM, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(GRU(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","metadata":{"_uuid":"5d8d6674-9bea-400c-af3d-6733309f130d","_cell_guid":"b9ff55e3-d3f2-489f-9c14-a59cccec7898","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","metadata":{"_uuid":"2a13a834-c761-40ee-850c-83c7338cfc59","_cell_guid":"b9bbf4d2-d1be-4c33-9bcd-e202018bd808","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_paragram_val_y>thresh).astype(int))))","metadata":{"_uuid":"b8e3e193-3f9f-4e12-8211-1748bdc0dd42","_cell_guid":"4a0928e2-59e0-4ae9-abd1-41e56cd9da8d","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1)","metadata":{"_uuid":"98e33d92-2731-4a16-9779-3ce56eddc501","_cell_guid":"10d67ec6-15c8-467d-988f-86fbb3a77e55","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","metadata":{"_uuid":"5b743eba-c70f-4601-8d2e-0170ea7c5b12","_cell_guid":"7d7f89bd-1d16-45aa-b50f-185a3122d580","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_test_y = (pred_glove_test_y + pred_fasttext_test_y + pred_paragram_test_y)/3\npred_test_y = (pred_test_y>0.35).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","metadata":{"_uuid":"769e7970-79c0-45a6-ad34-7ccab1765adc","_cell_guid":"79f6ba90-6dba-4044-bca7-2215f22dcec5","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}