{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import re\nimport string\nimport numpy as np\nimport pandas as pd\nimport en_core_web_sm\nnlp = en_core_web_sm.load()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-08T01:39:14.252273Z","iopub.execute_input":"2022-08-08T01:39:14.253201Z","iopub.status.idle":"2022-08-08T01:39:28.525634Z","shell.execute_reply.started":"2022-08-08T01:39:14.253167Z","shell.execute_reply":"2022-08-08T01:39:28.524329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/comptt/train.csv')\ntest = pd.read_csv('../input/comptt/test.csv')\n#test_labels = pd.read_csv('../input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:39:28.528515Z","iopub.execute_input":"2022-08-08T01:39:28.530956Z","iopub.status.idle":"2022-08-08T01:39:32.995917Z","shell.execute_reply.started":"2022-08-08T01:39:28.530909Z","shell.execute_reply":"2022-08-08T01:39:32.994617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train = train[:2000]\n#test = test[:600]\n#test_labels = test_labels[:600]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:39:32.998603Z","iopub.execute_input":"2022-08-08T01:39:32.998966Z","iopub.status.idle":"2022-08-08T01:39:33.035343Z","shell.execute_reply.started":"2022-08-08T01:39:32.998935Z","shell.execute_reply":"2022-08-08T01:39:33.034097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport html\nimport string\n\ndef contains_consonant(search_string):\n    lower = search_string.lower()\n    if 'a' in lower or 'e' in lower or 'i' in lower or 'o' in lower or 'u' in lower:\n        return True\n    return False\n\ndef remove_special_chars(text):\n    re1 = re.compile(r'  +')\n    x1 = text.lower().replace('#39;', \" \").replace('amp;', ' ').replace('#146;', \" \").replace(\n        'nbsp;', ' ').replace('#36;', ' ').replace('\\\\n', \" \").replace('quot;', \" \").replace(\n        '<br />', \" \").replace('\\\\\"', ' ').replace('<unk>', ' ').replace(' @.@ ', ' ').replace(\n        ' @-@ ', ' ').replace('\\\\', ' ').replace('\\n',' ')\n    return re1.sub(' ', html.unescape(x1))\n\ndef transform_string(text):\n    return str(text).lower().replace('.', ' ')\n\ntrain[\"Filtered\"] = train[\"comment_text\"].apply(transform_string)\ntrain[\"Filtered\"] = train[\"Filtered\"].apply(lambda x: remove_special_chars(x))\ntrain[\"Filtered\"] = train[\"Filtered\"].apply(lambda x: [str(token) for token in nlp(x) if token.is_stop == False and len(token)>2 and (\"_\" not in token.text) and (\"/\" not in token.text)])\ntrain[\"Filtered\"] = train['Filtered'].apply(lambda x: ' '.join(x))\ntrain[\"Filtered\"] = train['Filtered'].apply(lambda x: [char for char in x if char not in string.punctuation and not char.isnumeric()])\ntrain['Filtered'] = train['Filtered'].apply(lambda x: ''.join(x))\ntrain[\"Filtered\"] = train[\"Filtered\"].apply(lambda x: re.sub(' +', ' ', x))\ntrain['Filtered'] = train['Filtered'].apply(lambda x: [token.lemma_ for token in nlp(x)])\ntrain['Filtered'] = train['Filtered'].apply(lambda x: ' '.join(x))\ntrain['Filtered'] = train['Filtered'].apply(lambda x: [str(token) for token in nlp(x) if contains_consonant(str(token))])\ntrain['Filtered'] = train['Filtered'].apply(lambda x: ' '.join(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-07T21:59:02.831794Z","iopub.execute_input":"2022-08-07T21:59:02.833134Z","iopub.status.idle":"2022-08-07T23:35:25.493853Z","shell.execute_reply.started":"2022-08-07T21:59:02.833059Z","shell.execute_reply":"2022-08-07T23:35:25.492463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['label'] = [toxic+severe+obscene+threat+insult+hate for txt, toxic, severe, obscene, threat, insult, hate in zip(train['Filtered'], train['toxic'], train['severe_toxic'], train['obscene'], train['threat'], train['insult'], train['identity_hate'])]\ntrain['label'] = train['label'].apply(lambda x: 1 if (x > 0) else 0)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:35:25.495931Z","iopub.execute_input":"2022-08-07T23:35:25.496360Z","iopub.status.idle":"2022-08-07T23:35:25.760872Z","shell.execute_reply.started":"2022-08-07T23:35:25.496316Z","shell.execute_reply":"2022-08-07T23:35:25.759532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"Filtered\"] = test[\"comment_text\"].apply(transform_string)\ntest[\"Filtered\"] = test[\"Filtered\"].apply(lambda x: remove_special_chars(x))\ntest[\"Filtered\"] = test[\"Filtered\"].apply(lambda x: [str(token) for token in nlp(x) if token.is_stop == False and len(token)>2 and (\"_\" not in token.text) and (\"/\" not in token.text)])\ntest[\"Filtered\"] = test['Filtered'].apply(lambda x: ' '.join(x))\ntest[\"Filtered\"] = test['Filtered'].apply(lambda x: [char for char in x if char not in string.punctuation and not char.isnumeric()])\ntest['Filtered'] = test['Filtered'].apply(lambda x: ''.join(x))\ntest[\"Filtered\"] = test[\"Filtered\"].apply(lambda x: re.sub(' +', ' ', x))\ntest['Filtered'] = test['Filtered'].apply(lambda x: [token.lemma_ for token in nlp(x)])\ntest['Filtered'] = test['Filtered'].apply(lambda x: ' '.join(x))\ntest['Filtered'] = test['Filtered'].apply(lambda x: [str(token) for token in nlp(x) if contains_consonant(str(token))])\ntest['Filtered'] = test['Filtered'].apply(lambda x: ' '.join(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-07T23:35:25.762962Z","iopub.execute_input":"2022-08-07T23:35:25.763392Z","iopub.status.idle":"2022-08-08T01:07:55.294850Z","shell.execute_reply.started":"2022-08-07T23:35:25.763316Z","shell.execute_reply":"2022-08-08T01:07:55.293534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform_string(text):\n    return str(text)\n\ntrain[\"Filtered\"] = train[\"Filtered\"].apply(transform_string)\ntest[\"Filtered\"] = test[\"Filtered\"].apply(transform_string)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:40:21.830982Z","iopub.execute_input":"2022-08-08T01:40:21.831419Z","iopub.status.idle":"2022-08-08T01:40:21.942866Z","shell.execute_reply.started":"2022-08-08T01:40:21.831383Z","shell.execute_reply":"2022-08-08T01:40:21.941508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:40:23.994325Z","iopub.execute_input":"2022-08-08T01:40:23.994814Z","iopub.status.idle":"2022-08-08T01:40:24.011910Z","shell.execute_reply.started":"2022-08-08T01:40:23.994776Z","shell.execute_reply":"2022-08-08T01:40:24.010284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\nencoder = OneHotEncoder(sparse=False)\n\n_labels = train['label'].values.reshape((len(train['label']), 1))\nX_train_txt = train['Filtered']\ny_train = encoder.fit_transform(_labels)\nX_test_txt = test['Filtered']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:40:32.957250Z","iopub.execute_input":"2022-08-08T01:40:32.958153Z","iopub.status.idle":"2022-08-08T01:40:32.980095Z","shell.execute_reply.started":"2022-08-08T01:40:32.958117Z","shell.execute_reply":"2022-08-08T01:40:32.978694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\n\ntokenizer = Tokenizer(num_words=5000)\ntokenizer.fit_on_texts(X_train_txt)\nX_train   = tokenizer.texts_to_sequences(X_train_txt)\nX_test    = tokenizer.texts_to_sequences(X_test_txt)\n\nvocab_size = len(tokenizer.word_index) + 1\n\nprint(vocab_size)\nprint(X_train_txt.iloc[2])\nprint(X_train[2])","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:40:36.001793Z","iopub.execute_input":"2022-08-08T01:40:36.003037Z","iopub.status.idle":"2022-08-08T01:40:51.152445Z","shell.execute_reply.started":"2022-08-08T01:40:36.002986Z","shell.execute_reply":"2022-08-08T01:40:51.150221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\n\n\"\"\"\nmaxlen = 0\nfor row in X_train:\n    if len(row) > maxlen:\n        maxlen = len(row)\n        \nmaxlen\n\"\"\"\n\nmaxlen = 340\nX_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\nX_test = pad_sequences(X_test, padding='post', maxlen=maxlen)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:40:51.155393Z","iopub.execute_input":"2022-08-08T01:40:51.156294Z","iopub.status.idle":"2022-08-08T01:40:52.931532Z","shell.execute_reply.started":"2022-08-08T01:40:51.156249Z","shell.execute_reply":"2022-08-08T01:40:52.930151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train[0, :], y_train[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-08-08T02:02:41.261507Z","iopub.execute_input":"2022-08-08T02:02:41.261932Z","iopub.status.idle":"2022-08-08T02:02:41.268600Z","shell.execute_reply.started":"2022-08-08T02:02:41.261887Z","shell.execute_reply":"2022-08-08T02:02:41.266691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\nfrom tensorflow.keras import backend as K\n\n# Para datasets desbalanceados\ndef f1_score(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    recall = true_positives / (possible_positives + K.epsilon())\n    f1_val = 2 * (precision * recall) / (precision + recall + K.epsilon())\n    return f1_val\n\nembedding_dim = 300\nnum_classes = y_train.shape[1]\n\nmodel = Sequential()\nmodel.add(layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=maxlen))\nmodel.add(layers.Conv1D(128, 5, activation='relu'))\n#model.add(layers.Conv1D(256, 5, activation='relu'))\nmodel.add(layers.GlobalMaxPooling1D())\nmodel.add(layers.Dense(10, activation='relu'))\nmodel.add(layers.Dense(num_classes, activation='sigmoid'))\n#model.compile(optimizer='adam', loss='hinge', metrics=['acc'])\n#model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc', f1_score])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-08-08T02:01:23.388958Z","iopub.execute_input":"2022-08-08T02:01:23.390420Z","iopub.status.idle":"2022-08-08T02:01:34.765480Z","shell.execute_reply.started":"2022-08-08T02:01:23.390334Z","shell.execute_reply":"2022-08-08T02:01:34.762692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping\n\nmodel.fit(X_train, y_train, epochs=30, callbacks=EarlyStopping(monitor='val_loss', patience=5), verbose=True)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss, accuracy, f1_score = model.evaluate(X_train, y_train, verbose=False)\nprint(\"Training Accuracy: {:.4f}\".format(accuracy))","metadata":{"execution":{"iopub.status.busy":"2022-08-08T02:49:25.310723Z","iopub.execute_input":"2022-08-08T02:49:25.311836Z","iopub.status.idle":"2022-08-08T02:49:39.006931Z","shell.execute_reply.started":"2022-08-08T02:49:25.311777Z","shell.execute_reply":"2022-08-08T02:49:39.005543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Exemplo\nresult = model.predict(np.array([X_train[12]]))[0]\nresult[0], result[1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = model.predict(X_test)\npredictions = []\n\nfor tupla in result:\n    if tupla[0] > tupla[1]:\n        predictions.append(0)\n    else:\n        predictions.append(1)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T02:49:39.008863Z","iopub.execute_input":"2022-08-08T02:49:39.009264Z","iopub.status.idle":"2022-08-08T02:49:46.331486Z","shell.execute_reply.started":"2022-08-08T02:49:39.009234Z","shell.execute_reply":"2022-08-08T02:49:46.330109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['prediction'] = predictions\ntest.to_csv('submission-toxic.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T02:50:43.932656Z","iopub.execute_input":"2022-08-08T02:50:43.933576Z","iopub.status.idle":"2022-08-08T02:50:45.950498Z","shell.execute_reply.started":"2022-08-08T02:50:43.933523Z","shell.execute_reply":"2022-08-08T02:50:45.949019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['prediction']","metadata":{"execution":{"iopub.status.busy":"2022-08-08T02:56:15.756479Z","iopub.execute_input":"2022-08-08T02:56:15.756979Z","iopub.status.idle":"2022-08-08T02:56:15.767995Z","shell.execute_reply.started":"2022-08-08T02:56:15.756936Z","shell.execute_reply":"2022-08-08T02:56:15.766504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.to_csv('test.csv' , index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-08T01:22:59.742999Z","iopub.execute_input":"2022-08-08T01:22:59.743421Z","iopub.status.idle":"2022-08-08T01:23:02.098580Z","shell.execute_reply.started":"2022-08-08T01:22:59.743373Z","shell.execute_reply":"2022-08-08T01:23:02.097175Z"},"trusted":true},"execution_count":null,"outputs":[]}]}