{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-23T15:47:40.347922Z","iopub.execute_input":"2023-08-23T15:47:40.349285Z","iopub.status.idle":"2023-08-23T15:47:40.377123Z","shell.execute_reply.started":"2023-08-23T15:47:40.349233Z","shell.execute_reply":"2023-08-23T15:47:40.376003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! unzip /kaggle/input/quora-insincere-questions-classification/embeddings.zip","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:47:40.378986Z","iopub.execute_input":"2023-08-23T15:47:40.379395Z","iopub.status.idle":"2023-08-23T15:51:06.865022Z","shell.execute_reply.started":"2023-08-23T15:47:40.379361Z","shell.execute_reply":"2023-08-23T15:51:06.862532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:51:06.868406Z","iopub.execute_input":"2023-08-23T15:51:06.868950Z","iopub.status.idle":"2023-08-23T15:51:07.986312Z","shell.execute_reply.started":"2023-08-23T15:51:06.868895Z","shell.execute_reply":"2023-08-23T15:51:07.985027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls glove.840B.300d/","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:51:07.991633Z","iopub.execute_input":"2023-08-23T15:51:07.992074Z","iopub.status.idle":"2023-08-23T15:51:09.125035Z","shell.execute_reply.started":"2023-08-23T15:51:07.992035Z","shell.execute_reply":"2023-08-23T15:51:09.123540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! ls paragram_300_sl999/","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:51:09.127203Z","iopub.execute_input":"2023-08-23T15:51:09.127609Z","iopub.status.idle":"2023-08-23T15:51:10.226862Z","shell.execute_reply.started":"2023-08-23T15:51:09.127570Z","shell.execute_reply":"2023-08-23T15:51:10.225583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glove_emb_dict = dict()\nf = open('glove.840B.300d/glove.840B.300d.txt')\nfor line in f:\n    tokens = line.split(' ')\n    word = tokens[0]\n    embedding = np.array(tokens[1:],dtype='float')\n    glove_emb_dict[word] = embedding","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:51:10.228838Z","iopub.execute_input":"2023-08-23T15:51:10.229272Z","iopub.status.idle":"2023-08-23T15:54:49.187178Z","shell.execute_reply.started":"2023-08-23T15:51:10.229233Z","shell.execute_reply":"2023-08-23T15:54:49.185902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(glove_emb_dict['world'])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:54:49.188709Z","iopub.execute_input":"2023-08-23T15:54:49.189133Z","iopub.status.idle":"2023-08-23T15:54:49.200492Z","shell.execute_reply.started":"2023-08-23T15:54:49.189099Z","shell.execute_reply":"2023-08-23T15:54:49.198978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_emb_dict = dict()\nf = open('paragram_300_sl999/paragram_300_sl999.txt',encoding='unicode_escape')\nfor line in f:\n    tokens = line.split(' ')\n    word = tokens[0]\n    embedding = np.array(tokens[1:],dtype='float')\n    param_emb_dict[word] = embedding","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:54:49.203763Z","iopub.execute_input":"2023-08-23T15:54:49.205597Z","iopub.status.idle":"2023-08-23T15:57:54.063231Z","shell.execute_reply.started":"2023-08-23T15:54:49.205545Z","shell.execute_reply":"2023-08-23T15:57:54.061883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(param_emb_dict['world'])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:57:54.068919Z","iopub.execute_input":"2023-08-23T15:57:54.069334Z","iopub.status.idle":"2023-08-23T15:57:54.076765Z","shell.execute_reply.started":"2023-08-23T15:57:54.069300Z","shell.execute_reply":"2023-08-23T15:57:54.075518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Some Statistical Features**","metadata":{}},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:57:54.078299Z","iopub.execute_input":"2023-08-23T15:57:54.078801Z","iopub.status.idle":"2023-08-23T15:57:59.841134Z","shell.execute_reply.started":"2023-08-23T15:57:54.078766Z","shell.execute_reply":"2023-08-23T15:57:59.839850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:57:59.842805Z","iopub.execute_input":"2023-08-23T15:57:59.843654Z","iopub.status.idle":"2023-08-23T15:57:59.849731Z","shell.execute_reply.started":"2023-08-23T15:57:59.843618Z","shell.execute_reply":"2023-08-23T15:57:59.848829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:57:59.851164Z","iopub.execute_input":"2023-08-23T15:57:59.852269Z","iopub.status.idle":"2023-08-23T15:58:01.787753Z","shell.execute_reply.started":"2023-08-23T15:57:59.852222Z","shell.execute_reply":"2023-08-23T15:58:01.786567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize\ntokenizer = word_tokenize","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:58:01.789403Z","iopub.execute_input":"2023-08-23T15:58:01.789744Z","iopub.status.idle":"2023-08-23T15:58:01.795486Z","shell.execute_reply.started":"2023-08-23T15:58:01.789716Z","shell.execute_reply":"2023-08-23T15:58:01.794199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_len = data['question_text'].apply(lambda x: len(tokenizer(x)))\nlen(text_len)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T15:58:01.797223Z","iopub.execute_input":"2023-08-23T15:58:01.797656Z","iopub.status.idle":"2023-08-23T16:03:14.971991Z","shell.execute_reply.started":"2023-08-23T15:58:01.797613Z","shell.execute_reply":"2023-08-23T16:03:14.970707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport string\n\ncapitals = data['question_text'].apply(lambda x: len(re.findall(r'[A-Z]',x)))\npunctuations = data['question_text'].apply(lambda x: sum([x.count(p) for p in string.punctuation]))\nquestions = data['question_text'].apply(lambda x: x.count('?'))\nexclamations = data['question_text'].apply(lambda x: x.count('!'))","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:03:14.973807Z","iopub.execute_input":"2023-08-23T16:03:14.974303Z","iopub.status.idle":"2023-08-23T16:03:37.360326Z","shell.execute_reply.started":"2023-08-23T16:03:14.974268Z","shell.execute_reply":"2023-08-23T16:03:37.358547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"special_symbols = data['question_text'].apply(lambda x:x.count('\\\\'))\nunique_words = data['question_text'].apply(lambda x:len(set(tokenizer(x))))","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:03:37.362704Z","iopub.execute_input":"2023-08-23T16:03:37.363285Z","iopub.status.idle":"2023-08-23T16:08:56.398399Z","shell.execute_reply.started":"2023-08-23T16:03:37.363234Z","shell.execute_reply":"2023-08-23T16:08:56.396996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = pd.DataFrame.from_dict(dict({'Capitals':capitals,'Punctuations':punctuations,\n                                        'Questions':questions,'Exclaimations':exclamations,\n                                       'Special_symbols':special_symbols, 'Uniue_Words':unique_words}))","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:08:56.399883Z","iopub.execute_input":"2023-08-23T16:08:56.400251Z","iopub.status.idle":"2023-08-23T16:08:56.439026Z","shell.execute_reply.started":"2023-08-23T16:08:56.400220Z","shell.execute_reply":"2023-08-23T16:08:56.438022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:08:56.440816Z","iopub.execute_input":"2023-08-23T16:08:56.441261Z","iopub.status.idle":"2023-08-23T16:08:56.448769Z","shell.execute_reply.started":"2023-08-23T16:08:56.441231Z","shell.execute_reply":"2023-08-23T16:08:56.447591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:08:56.450301Z","iopub.execute_input":"2023-08-23T16:08:56.450627Z","iopub.status.idle":"2023-08-23T16:09:08.561023Z","shell.execute_reply.started":"2023-08-23T16:08:56.450599Z","shell.execute_reply":"2023-08-23T16:09:08.559735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding = dict()\nwords = list(set(glove_emb_dict.keys()) & set(param_emb_dict.keys()))\nfor w in words:\n    embedding[w] = 0.7*glove_emb_dict[w] + 0.3*param_emb_dict[w]\n    ","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:09:08.562850Z","iopub.execute_input":"2023-08-23T16:09:08.563876Z","iopub.status.idle":"2023-08-23T16:09:39.185813Z","shell.execute_reply.started":"2023-08-23T16:09:08.563833Z","shell.execute_reply":"2023-08-23T16:09:39.184498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words_xor = list(set(glove_emb_dict.keys()) ^ set(param_emb_dict.keys()))\nfor w in words_xor:\n    if w in glove_emb_dict.keys():\n        embedding[w] = glove_emb_dict[w]\n    else:\n        embedding[w] = param_emb_dict[w]","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:09:39.187366Z","iopub.execute_input":"2023-08-23T16:09:39.187676Z","iopub.status.idle":"2023-08-23T16:09:44.629314Z","shell.execute_reply.started":"2023-08-23T16:09:39.187649Z","shell.execute_reply":"2023-08-23T16:09:44.628053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nmax_len = 100\ntokenizer = Tokenizer()\ntokenizer.fit_on_texts(data['question_text'].tolist())\ntext_seq = tokenizer.texts_to_sequences(data['question_text'])\ntext_seq = pad_sequences(text_seq,maxlen=max_len)\n\nvocab = tokenizer.word_index\nnum_tokens = len(vocab) + 2\nfinal_embed = np.zeros((num_tokens,300))\n\nfor i,word in enumerate(vocab):\n    if word in embedding.keys():\n        final_embed[i] = embedding[word]","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:51:19.776602Z","iopub.execute_input":"2023-08-23T16:51:19.777134Z","iopub.status.idle":"2023-08-23T16:52:35.428168Z","shell.execute_reply.started":"2023-08-23T16:51:19.777095Z","shell.execute_reply":"2023-08-23T16:52:35.426715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_embed.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:35.430604Z","iopub.execute_input":"2023-08-23T16:52:35.431094Z","iopub.status.idle":"2023-08-23T16:52:35.439438Z","shell.execute_reply.started":"2023-08-23T16:52:35.431048Z","shell.execute_reply":"2023-08-23T16:52:35.438099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_seq.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:35.441084Z","iopub.execute_input":"2023-08-23T16:52:35.441570Z","iopub.status.idle":"2023-08-23T16:52:35.462434Z","shell.execute_reply.started":"2023-08-23T16:52:35.441528Z","shell.execute_reply":"2023-08-23T16:52:35.461430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_embed.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:35.465393Z","iopub.execute_input":"2023-08-23T16:52:35.465778Z","iopub.status.idle":"2023-08-23T16:52:35.479451Z","shell.execute_reply.started":"2023-08-23T16:52:35.465747Z","shell.execute_reply":"2023-08-23T16:52:35.478240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(vocab)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:35.480987Z","iopub.execute_input":"2023-08-23T16:52:35.481453Z","iopub.status.idle":"2023-08-23T16:52:35.494763Z","shell.execute_reply.started":"2023-08-23T16:52:35.481409Z","shell.execute_reply":"2023-08-23T16:52:35.493656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_input = keras.layers.Input(shape=(features.shape[1],))\nembed_input = keras.layers.Input(shape=(text_seq.shape[1],))\nembed = keras.layers.Embedding(num_tokens,300,\n                               embeddings_initializer=keras.initializers.Constant(final_embed),\n                               #weights=[final_embed],\n                               trainable=False)(embed_input)\nspatialDropout = keras.layers.SpatialDropout1D(0.2)(embed)\nbilstm = keras.layers.Bidirectional(keras.layers.LSTM(128, return_sequences=True))(spatialDropout)\nconv1d = keras.layers.Convolution1D(64, kernel_size=1)(bilstm)\nmax_pool = keras.layers.GlobalMaxPooling1D()(conv1d)\ndense_64 = keras.layers.Dense(64)(feat_input)\nconcat = keras.layers.Concatenate()([max_pool,dense_64])\ndense_128 = keras.layers.Dense(128)(concat)\ndropout = keras.layers.Dropout(0.1)(dense_128)\nbatch_normalization = keras.layers.BatchNormalization()(dropout)\n\noutputs = keras.layers.Dense(2,activation='softmax')(batch_normalization)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:35.496536Z","iopub.execute_input":"2023-08-23T16:52:35.496912Z","iopub.status.idle":"2023-08-23T16:52:37.180114Z","shell.execute_reply.started":"2023-08-23T16:52:35.496883Z","shell.execute_reply":"2023-08-23T16:52:37.178846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Model(inputs=[embed_input,feat_input],outputs=outputs)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:37.182038Z","iopub.execute_input":"2023-08-23T16:52:37.182734Z","iopub.status.idle":"2023-08-23T16:52:37.240041Z","shell.execute_reply.started":"2023-08-23T16:52:37.182698Z","shell.execute_reply":"2023-08-23T16:52:37.238780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='nadam', loss='sparse_categorical_crossentropy',metrics=[\"acc\"]) ","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:37.241752Z","iopub.execute_input":"2023-08-23T16:52:37.242263Z","iopub.status.idle":"2023-08-23T16:52:37.271715Z","shell.execute_reply.started":"2023-08-23T16:52:37.242219Z","shell.execute_reply":"2023-08-23T16:52:37.270524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nshape1 = 1306122\ntest_X = [text_seq[:math.ceil(shape1*0.1)],features[:math.ceil(shape1*0.1)]],\ntest_y = data.iloc[:math.ceil(shape1*0.1),:]['target']\ntrain_X = [text_seq[math.ceil(shape1*0.1):],features[math.ceil(shape1*0.1):]],\ntrain_y = data.iloc[math.ceil(shape1*0.1):,:]['target']","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:37.273348Z","iopub.execute_input":"2023-08-23T16:52:37.273763Z","iopub.status.idle":"2023-08-23T16:52:37.283383Z","shell.execute_reply.started":"2023-08-23T16:52:37.273727Z","shell.execute_reply":"2023-08-23T16:52:37.282086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_y.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:37.287304Z","iopub.execute_input":"2023-08-23T16:52:37.287732Z","iopub.status.idle":"2023-08-23T16:52:37.299170Z","shell.execute_reply.started":"2023-08-23T16:52:37.287699Z","shell.execute_reply":"2023-08-23T16:52:37.298218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"math.ceil(shape1*0.1)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:37.300376Z","iopub.execute_input":"2023-08-23T16:52:37.300938Z","iopub.status.idle":"2023-08-23T16:52:37.318487Z","shell.execute_reply.started":"2023-08-23T16:52:37.300906Z","shell.execute_reply":"2023-08-23T16:52:37.317227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.iloc[:math.ceil(shape1*0.1),:]['target'].shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:37.320264Z","iopub.execute_input":"2023-08-23T16:52:37.320646Z","iopub.status.idle":"2023-08-23T16:52:37.334786Z","shell.execute_reply.started":"2023-08-23T16:52:37.320612Z","shell.execute_reply":"2023-08-23T16:52:37.333517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_X,train_y)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T16:52:37.336381Z","iopub.execute_input":"2023-08-23T16:52:37.336877Z","iopub.status.idle":"2023-08-23T18:59:51.763430Z","shell.execute_reply.started":"2023-08-23T16:52:37.336832Z","shell.execute_reply":"2023-08-23T18:59:51.762390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(test_X)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T18:59:51.765701Z","iopub.execute_input":"2023-08-23T18:59:51.766527Z","iopub.status.idle":"2023-08-23T19:05:17.648953Z","shell.execute_reply.started":"2023-08-23T18:59:51.766476Z","shell.execute_reply":"2023-08-23T19:05:17.647707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model.evaluate(test_X, test_y, verbose = 0) \n\nprint('Test loss:', score[0]) \nprint('Test accuracy:', score[1])","metadata":{"execution":{"iopub.status.busy":"2023-08-23T19:11:37.333248Z","iopub.execute_input":"2023-08-23T19:11:37.333964Z"},"trusted":true},"execution_count":null,"outputs":[]}]}