{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport string\nimport numpy as np\nimport pandas as pd\nimport en_core_web_sm\nnlp = en_core_web_sm.load()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-07T22:54:59.025387Z","iopub.execute_input":"2022-08-07T22:54:59.026222Z","iopub.status.idle":"2022-08-07T22:55:11.831117Z","shell.execute_reply.started":"2022-08-07T22:54:59.026107Z","shell.execute_reply":"2022-08-07T22:55:11.829572Z"},"trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:06:29.404691Z","iopub.execute_input":"2022-08-07T23:06:29.405302Z","iopub.status.idle":"2022-08-07T23:06:33.701937Z","shell.execute_reply.started":"2022-08-07T23:06:29.405269Z","shell.execute_reply":"2022-08-07T23:06:33.70076Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"train = train[:20000]\ntest = test[:6000]","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:06:46.906025Z","iopub.execute_input":"2022-08-07T23:06:46.9065Z","iopub.status.idle":"2022-08-07T23:06:46.913285Z","shell.execute_reply.started":"2022-08-07T23:06:46.906456Z","shell.execute_reply":"2022-08-07T23:06:46.9119Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"import re\nimport html\nimport string\n\ndef contains_consonant(search_string):\n lower = search_string.lower()\n if 'a' in lower or 'e' in lower or 'i' in lower or 'o' in lower or 'u' in lower:\n return True\n return False\n\ndef transform_string(text):\n return str(text).lower().replace('.', ' ')\n\ntrain[\"Filtered\"] = train[\"question_text\"].apply(transform_string)\ntrain[\"Filtered\"] = train['Filtered'].apply(lambda x: [char for char in x if char not in string.punctuation and not char.isnumeric()])\ntrain['Filtered'] = train['Filtered'].apply(lambda x: ''.join(x))\ntrain[\"Filtered\"] = train[\"Filtered\"].apply(lambda x: re.sub(' +', ' ', x))\ntrain['Filtered'] = train['Filtered'].apply(lambda x: [token.lemma_ for token in nlp(x) if len(token.lemma_)>2])\ntrain['Filtered'] = train['Filtered'].apply(lambda x: ' '.join(x))\ntrain['Filtered'] = train['Filtered'].apply(lambda x: [str(token) for token in nlp(x) if contains_consonant(str(token))])\ntrain['Filtered'] = train['Filtered'].apply(lambda x: ' '.join(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:06:48.534367Z","iopub.execute_input":"2022-08-07T23:06:48.534799Z","iopub.status.idle":"2022-08-07T23:11:42.319322Z","shell.execute_reply.started":"2022-08-07T23:06:48.534762Z","shell.execute_reply":"2022-08-07T23:11:42.318166Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"test[\"Filtered\"] = test[\"question_text\"].apply(transform_string)\ntest[\"Filtered\"] = test['Filtered'].apply(lambda x: [char for char in x if char not in string.punctuation and not char.isnumeric()])\ntest['Filtered'] = test['Filtered'].apply(lambda x: ''.join(x))\ntest[\"Filtered\"] = test[\"Filtered\"].apply(lambda x: re.sub(' +', ' ', x))\ntest['Filtered'] = test['Filtered'].apply(lambda x: [token.lemma_ for token in nlp(x) if len(token.lemma_)>2])\ntest['Filtered'] = test['Filtered'].apply(lambda x: ' '.join(x))\ntest['Filtered'] = test['Filtered'].apply(lambda x: [str(token) for token in nlp(x) if contains_consonant(str(token))])\ntest['Filtered'] = test['Filtered'].apply(lambda x: ' '.join(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:11:42.321147Z","iopub.execute_input":"2022-08-07T23:11:42.321528Z","iopub.status.idle":"2022-08-07T23:13:11.017625Z","shell.execute_reply.started":"2022-08-07T23:11:42.321495Z","shell.execute_reply":"2022-08-07T23:13:11.016295Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:17:49.181109Z","iopub.execute_input":"2022-08-07T23:17:49.181524Z","iopub.status.idle":"2022-08-07T23:17:49.19546Z","shell.execute_reply.started":"2022-08-07T23:17:49.181488Z","shell.execute_reply":"2022-08-07T23:17:49.194538Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":" qid question_text \\\n0 0000163e3ea7c7a74cd7 Why do so many women become so rude and arroga... \n1 00002bd4fb5d505b9161 When should I apply for RV college of engineer... \n2 00007756b4a147d2b0b3 What is it really like to be a nurse practitio... \n3 000086e4b7e1c7146103 Who are entrepreneurs? \n4 0000c4c3fbe8785a3090 Is education really making good people nowadays? \n... ... ... \n5995 04188e468e43aaa16751 How can I use 4G data in my 3G phone? \n5996 0418a037f3f8cfbae874 On Excel, how do you restrict individual acess... \n5997 0418e8cde343fc23491d Is it possible to find which atoms are connected? \n5998 0418fd72f5f4a8955574 How should I address this current bullying by ... \n5999 0419394bc516efdc113f How would be career after bsc mlt in India? \n\n Filtered \n0 many woman become rude and arrogant when they ... \n1 when should apply for college engineering and ... \n2 what really like nurse practitioner \n3 who entrepreneur \n4 education really make good people nowadays \n... ... \n5995 how can use datum phone \n5996 excel how you restrict individual acesss speci... \n5997 possible find which atom connect \n5998 how should address this current bullying some ... \n5999 how would career after india \n\n[6000 rows x 3 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
qidquestion_textFiltered
00000163e3ea7c7a74cd7Why do so many women become so rude and arroga...many woman become rude and arrogant when they ...
100002bd4fb5d505b9161When should I apply for RV college of engineer...when should apply for college engineering and ...
200007756b4a147d2b0b3What is it really like to be a nurse practitio...what really like nurse practitioner
3000086e4b7e1c7146103Who are entrepreneurs?who entrepreneur
40000c4c3fbe8785a3090Is education really making good people nowadays?education really make good people nowadays
............
599504188e468e43aaa16751How can I use 4G data in my 3G phone?how can use datum phone
59960418a037f3f8cfbae874On Excel, how do you restrict individual acess...excel how you restrict individual acesss speci...
59970418e8cde343fc23491dIs it possible to find which atoms are connected?possible find which atom connect
59980418fd72f5f4a8955574How should I address this current bullying by ...how should address this current bullying some ...
59990419394bc516efdc113fHow would be career after bsc mlt in India?how would career after india
\n

6000 rows × 3 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nencoder = OneHotEncoder(sparse=False)\n\n_labels = train['target'].values.reshape((len(train['target']), 1))\nX_train_txt = train['Filtered']\ny_train = encoder.fit_transform(_labels)\nX_test_txt = test['Filtered']","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:17:56.138509Z","iopub.execute_input":"2022-08-07T23:17:56.139682Z","iopub.status.idle":"2022-08-07T23:17:56.233242Z","shell.execute_reply.started":"2022-08-07T23:17:56.139629Z","shell.execute_reply":"2022-08-07T23:17:56.232062Z"},"trusted":true},"execution_count":14,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing.text import Tokenizer\n\ntokenizer = Tokenizer(num_words=5000)\ntokenizer.fit_on_texts(X_train_txt)\nX_train = tokenizer.texts_to_sequences(X_train_txt)\nX_test = tokenizer.texts_to_sequences(X_test_txt)\n\nvocab_size = len(tokenizer.word_index) + 1\n\nprint(vocab_size)\nprint(X_train_txt.iloc[2])\nprint(X_train[2])\n\n\"\"\"\nmaxlen = 0\nfor row in X_train:\n if len(row) > maxlen:\n maxlen = len(row)\n \nmaxlen\n\"\"\"\n\nmaxlen = 40\nX_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\nX_test = pad_sequences(X_test, padding='post', maxlen=maxlen)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:17:57.765037Z","iopub.execute_input":"2022-08-07T23:17:57.765469Z","iopub.status.idle":"2022-08-07T23:17:59.259155Z","shell.execute_reply.started":"2022-08-07T23:17:57.765413Z","shell.execute_reply":"2022-08-07T23:17:59.258029Z"},"trusted":true},"execution_count":15,"outputs":[{"name":"stdout","text":"17834\nvelocity affect time velocity affect space geometry\n[1575, 243, 41, 1575, 243, 403, 3744]\n","output_type":"stream"}]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\nfrom tensorflow.keras import backend as K\n\n# Para datasets desbalanceados\ndef f1_score(y_true, y_pred):\n true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n precision = true_positives / (predicted_positives + K.epsilon())\n recall = true_positives / (possible_positives + K.epsilon())\n f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon())\n return f1_val\n\nembedding_dim = 300\nnum_classes = y_train.shape[1]\n\nmodel = Sequential()\nmodel.add(layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=maxlen))\nmodel.add(layers.Conv1D(128, 5, activation='relu'))\n#model.add(layers.Conv1D(256, 5, activation='relu'))\nmodel.add(layers.GlobalMaxPooling1D())\nmodel.add(layers.Dense(10, activation='relu'))\nmodel.add(layers.Dense(num_classes, activation='sigmoid'))\n#model.compile(optimizer='adam', loss='hinge', metrics=['acc'])\n#model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc', f1_score])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:18:03.167632Z","iopub.execute_input":"2022-08-07T23:18:03.168058Z","iopub.status.idle":"2022-08-07T23:18:04.101008Z","shell.execute_reply.started":"2022-08-07T23:18:03.168025Z","shell.execute_reply":"2022-08-07T23:18:04.099781Z"},"trusted":true},"execution_count":16,"outputs":[{"name":"stderr","text":"2022-08-07 23:18:03.663475: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n","output_type":"stream"},{"name":"stdout","text":"Model: \"sequential\"\n_________________________________________________________________\nLayer (type) Output Shape Param # \n=================================================================\nembedding (Embedding) (None, 40, 300) 5350200 \n_________________________________________________________________\nconv1d (Conv1D) (None, 36, 128) 192128 \n_________________________________________________________________\nglobal_max_pooling1d (Global (None, 128) 0 \n_________________________________________________________________\ndense (Dense) (None, 10) 1290 \n_________________________________________________________________\ndense_1 (Dense) (None, 2) 22 \n=================================================================\nTotal params: 5,543,640\nTrainable params: 5,543,640\nNon-trainable params: 0\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nmodel.fit(X_train, y_train, epochs=30, callbacks=EarlyStopping(monitor='acc', patience=5), verbose=True)","metadata":{"execution":{"iopub.status.idle":"2022-08-07T23:29:51.184023Z","shell.execute_reply.started":"2022-08-07T23:18:07.218176Z","shell.execute_reply":"2022-08-07T23:29:51.182751Z"},"trusted":true},"execution_count":17,"outputs":[{"name":"stdout","text":"625/625 [==============================] - 35s 56ms/step - loss: 0.0012 - acc: 0.9998 - f1_score: 0.9998\nEpoch 16/30\n625/625 [==============================] - 35s 56ms/step - loss: 0.0022 - acc: 0.9994 - f1_score: 0.9993\nEpoch 17/30\n625/625 [==============================] - 35s 56ms/step - loss: 0.0031 - acc: 0.9990 - f1_score: 0.9990\nEpoch 18/30\n625/625 [==============================] - 35s 57ms/step - loss: 0.0031 - acc: 0.9992 - f1_score: 0.9992\nEpoch 19/30\n625/625 [==============================] - 35s 57ms/step - loss: 0.0019 - acc: 0.9995 - f1_score: 0.9993\nEpoch 20/30\n625/625 [==============================] - 35s 55ms/step - loss: 0.0029 - acc: 0.9990 - f1_score: 0.9990\n","output_type":"stream"},{"execution_count":17,"output_type":"execute_result","data":{"text/plain":""},"metadata":{}}]},{"cell_type":"code","source":"loss, accuracy, f1_score = model.evaluate(X_train, y_train, verbose=False)\nprint(\"Training Accuracy: {:.4f}\".format(accuracy))","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:32:34.155719Z","iopub.execute_input":"2022-08-07T23:32:34.156178Z","iopub.status.idle":"2022-08-07T23:33:34.662336Z","shell.execute_reply.started":"2022-08-07T23:32:34.156144Z","shell.execute_reply":"2022-08-07T23:33:34.661426Z"},"trusted":true},"execution_count":18,"outputs":[{"name":"stdout","text":"Training Accuracy: 0.9994\n","output_type":"stream"}]},{"cell_type":"code","source":"result = model.predict(X_test)\npredictions = []\n\nfor tupla in result:\n if tupla[0] > tupla[1]:\n predictions.append(0)\n else:\n predictions.append(1)\ntest['prediction'] = predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:33:34.663851Z","iopub.execute_input":"2022-08-07T23:33:34.664248Z","iopub.status.idle":"2022-08-07T23:33:35.854402Z","shell.execute_reply.started":"2022-08-07T23:33:34.664214Z","shell.execute_reply":"2022-08-07T23:33:35.853188Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"submission = test\nsubmission.pop('question_text')\nsubmission.pop('Filtered')","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:33:35.856151Z","iopub.execute_input":"2022-08-07T23:33:35.856645Z","iopub.status.idle":"2022-08-07T23:33:35.871207Z","shell.execute_reply.started":"2022-08-07T23:33:35.856598Z","shell.execute_reply":"2022-08-07T23:33:35.87Z"},"trusted":true},"execution_count":20,"outputs":[{"execution_count":20,"output_type":"execute_result","data":{"text/plain":"0 many woman become rude and arrogant when they ...\n1 when should apply for college engineering and ...\n2 what really like nurse practitioner\n3 who entrepreneur\n4 education really make good people nowadays\n ... \n5995 how can use datum phone\n5996 excel how you restrict individual acesss speci...\n5997 possible find which atom connect\n5998 how should address this current bullying some ...\n5999 how would career after india\nName: Filtered, Length: 6000, dtype: object"},"metadata":{}}]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:33:35.874278Z","iopub.execute_input":"2022-08-07T23:33:35.874769Z","iopub.status.idle":"2022-08-07T23:33:35.890515Z","shell.execute_reply.started":"2022-08-07T23:33:35.874716Z","shell.execute_reply":"2022-08-07T23:33:35.889536Z"},"trusted":true},"execution_count":21,"outputs":[{"execution_count":21,"output_type":"execute_result","data":{"text/plain":" qid prediction\n0 0000163e3ea7c7a74cd7 1\n1 00002bd4fb5d505b9161 0\n2 00007756b4a147d2b0b3 0\n3 000086e4b7e1c7146103 0\n4 0000c4c3fbe8785a3090 0\n... ... ...\n5995 04188e468e43aaa16751 0\n5996 0418a037f3f8cfbae874 0\n5997 0418e8cde343fc23491d 0\n5998 0418fd72f5f4a8955574 0\n5999 0419394bc516efdc113f 0\n\n[6000 rows x 2 columns]","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
qidprediction
00000163e3ea7c7a74cd71
100002bd4fb5d505b91610
200007756b4a147d2b0b30
3000086e4b7e1c71461030
40000c4c3fbe8785a30900
.........
599504188e468e43aaa167510
59960418a037f3f8cfbae8740
59970418e8cde343fc23491d0
59980418fd72f5f4a89555740
59990419394bc516efdc113f0
\n

6000 rows × 2 columns

\n
"},"metadata":{}}]},{"cell_type":"code","source":"submission.to_csv('submission.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:33:35.892159Z","iopub.execute_input":"2022-08-07T23:33:35.893297Z","iopub.status.idle":"2022-08-07T23:33:35.911951Z","shell.execute_reply.started":"2022-08-07T23:33:35.893265Z","shell.execute_reply":"2022-08-07T23:33:35.910946Z"},"trusted":true},"execution_count":22,"outputs":[]}]}