{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os \nimport sys\n\n#data processing library\nimport numpy as np\nimport pandas as pd\n\n#data visualization library\nimport matplotlib.pyplot as plt\n\n#deep learning\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, GlobalMaxPooling1D\nfrom keras.layers import Conv1D, MaxPooling1D, Embedding\nfrom keras.models import Model\n\n#metrics\nfrom sklearn.metrics import roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:08:32.757807Z","iopub.execute_input":"2021-10-17T14:08:32.75805Z","iopub.status.idle":"2021-10-17T14:08:37.565825Z","shell.execute_reply.started":"2021-10-17T14:08:32.758023Z","shell.execute_reply":"2021-10-17T14:08:37.565119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Download The Word Vector","metadata":{}},{"cell_type":"code","source":"!wget http://nlp.stanford.edu/data/glove.6B.zip","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:13:32.047682Z","iopub.execute_input":"2021-10-17T14:13:32.047984Z","iopub.status.idle":"2021-10-17T14:16:12.696396Z","shell.execute_reply.started":"2021-10-17T14:13:32.047952Z","shell.execute_reply":"2021-10-17T14:16:12.695582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip glove.6B.zip ","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:20:28.566161Z","iopub.execute_input":"2021-10-17T14:20:28.566958Z","iopub.status.idle":"2021-10-17T14:20:50.024184Z","shell.execute_reply.started":"2021-10-17T14:20:28.566909Z","shell.execute_reply":"2021-10-17T14:20:50.023239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ls -l ","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:21:02.174071Z","iopub.execute_input":"2021-10-17T14:21:02.174396Z","iopub.status.idle":"2021-10-17T14:21:02.902439Z","shell.execute_reply.started":"2021-10-17T14:21:02.174362Z","shell.execute_reply":"2021-10-17T14:21:02.901588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## choosing some hyperparameter. ","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:23:00.923152Z","iopub.execute_input":"2021-10-17T14:23:00.92376Z","iopub.status.idle":"2021-10-17T14:23:09.831535Z","shell.execute_reply.started":"2021-10-17T14:23:00.923723Z","shell.execute_reply":"2021-10-17T14:23:09.830049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:24:21.675942Z","iopub.execute_input":"2021-10-17T14:24:21.676232Z","iopub.status.idle":"2021-10-17T14:24:21.70274Z","shell.execute_reply.started":"2021-10-17T14:24:21.676203Z","shell.execute_reply":"2021-10-17T14:24:21.701943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"sentence_length\"] = train[\"comment_text\"].str.len()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:42:30.042182Z","iopub.execute_input":"2021-10-17T14:42:30.042914Z","iopub.status.idle":"2021-10-17T14:42:30.236674Z","shell.execute_reply.started":"2021-10-17T14:42:30.042876Z","shell.execute_reply":"2021-10-17T14:42:30.235969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:42:36.369781Z","iopub.execute_input":"2021-10-17T14:42:36.370035Z","iopub.status.idle":"2021-10-17T14:42:36.383452Z","shell.execute_reply.started":"2021-10-17T14:42:36.370007Z","shell.execute_reply":"2021-10-17T14:42:36.38253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"comment_text\"][0]","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:43:30.96234Z","iopub.execute_input":"2021-10-17T14:43:30.962635Z","iopub.status.idle":"2021-10-17T14:43:30.968844Z","shell.execute_reply.started":"2021-10-17T14:43:30.962604Z","shell.execute_reply":"2021-10-17T14:43:30.968188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train[\"comment_text\"][0])","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:43:53.262134Z","iopub.execute_input":"2021-10-17T14:43:53.262677Z","iopub.status.idle":"2021-10-17T14:43:53.267417Z","shell.execute_reply.started":"2021-10-17T14:43:53.262639Z","shell.execute_reply":"2021-10-17T14:43:53.266769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize =(12, 8))\nax.hist(train[\"sentence_length\"],bins=20)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:45:15.552422Z","iopub.execute_input":"2021-10-17T14:45:15.552711Z","iopub.status.idle":"2021-10-17T14:45:15.80133Z","shell.execute_reply.started":"2021-10-17T14:45:15.552679Z","shell.execute_reply":"2021-10-17T14:45:15.800619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some configuration\nMAX_SEQUENCE_LENGTH = 100\nMAX_VOCAB_SIZE = 20000\nEMBEDDING_DIM = 100\nVALIDATION_SPLIT = 0.2\nBATCH_SIZE = 128\nEPOCHS = 10","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:47:11.56104Z","iopub.execute_input":"2021-10-17T14:47:11.561333Z","iopub.status.idle":"2021-10-17T14:47:11.565933Z","shell.execute_reply.started":"2021-10-17T14:47:11.561297Z","shell.execute_reply":"2021-10-17T14:47:11.565055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"word2vec = {}\nwith open(os.path.join('./glove.6B.100d.txt')) as f:\n    \n  # is just a space-separated text file in the format:\n  # word vec[0] vec[1] vec[2] ...\n    for line in f:\n        \n        values = line.split()\n        word = values[0]\n        vec = np.asarray(values[1:], dtype='float32')\n        word2vec[word] = vec\nprint('Found %s word vectors.' % len(word2vec))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:50:04.430171Z","iopub.execute_input":"2021-10-17T14:50:04.430718Z","iopub.status.idle":"2021-10-17T14:50:20.587608Z","shell.execute_reply.started":"2021-10-17T14:50:04.430682Z","shell.execute_reply":"2021-10-17T14:50:20.586851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentences = train[\"comment_text\"].fillna(\"DUMMY_VALUE\").values\npossible_labels = [\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]\ntargets = train[possible_labels].values","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:51:08.605653Z","iopub.execute_input":"2021-10-17T14:51:08.606199Z","iopub.status.idle":"2021-10-17T14:51:08.668302Z","shell.execute_reply.started":"2021-10-17T14:51:08.606163Z","shell.execute_reply":"2021-10-17T14:51:08.667591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert the sentences (strings) into integers\ntokenizer = Tokenizer(num_words=MAX_VOCAB_SIZE)\ntokenizer.fit_on_texts(sentences)\nsequences = tokenizer.texts_to_sequences(sentences)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:51:25.4677Z","iopub.execute_input":"2021-10-17T14:51:25.468524Z","iopub.status.idle":"2021-10-17T14:51:51.556318Z","shell.execute_reply.started":"2021-10-17T14:51:25.46848Z","shell.execute_reply":"2021-10-17T14:51:51.555559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"max sequence length:\", max(len(s) for s in sequences))\nprint(\"min sequence length:\", min(len(s) for s in sequences))\ns = sorted(len(s) for s in sequences)\nprint(\"median sequence length:\", s[len(s) // 2])\n\nprint(\"max word index:\", max(max(seq) for seq in sequences if len(seq) > 0))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:51:51.557851Z","iopub.execute_input":"2021-10-17T14:51:51.558099Z","iopub.status.idle":"2021-10-17T14:51:51.965301Z","shell.execute_reply.started":"2021-10-17T14:51:51.558065Z","shell.execute_reply":"2021-10-17T14:51:51.964599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get word -> integer mapping\nword2idx = tokenizer.word_index\nprint('Found %s unique tokens.' % len(word2idx))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:52:26.707993Z","iopub.execute_input":"2021-10-17T14:52:26.70831Z","iopub.status.idle":"2021-10-17T14:52:26.713485Z","shell.execute_reply.started":"2021-10-17T14:52:26.70826Z","shell.execute_reply":"2021-10-17T14:52:26.712743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pad sequences so that we get a N x T matrix\ndata = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\nprint('Shape of data tensor:', data.shape)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:52:38.6265Z","iopub.execute_input":"2021-10-17T14:52:38.626951Z","iopub.status.idle":"2021-10-17T14:52:41.964043Z","shell.execute_reply.started":"2021-10-17T14:52:38.626893Z","shell.execute_reply":"2021-10-17T14:52:41.963347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prepare embedding matrix\nprint('Filling pre-trained embeddings...')\nnum_words = min(MAX_VOCAB_SIZE, len(word2idx) + 1)\nembedding_matrix = np.zeros((num_words, EMBEDDING_DIM))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:52:56.020893Z","iopub.execute_input":"2021-10-17T14:52:56.021654Z","iopub.status.idle":"2021-10-17T14:52:56.027683Z","shell.execute_reply.started":"2021-10-17T14:52:56.021609Z","shell.execute_reply":"2021-10-17T14:52:56.026873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for word, i in word2idx.items():\n    \n    if i < MAX_VOCAB_SIZE:\n        embedding_vector = word2vec.get(word)\n        if embedding_vector is not None:\n      # words not found in embedding index will be all zeros.\n            embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:53:40.018422Z","iopub.execute_input":"2021-10-17T14:53:40.018889Z","iopub.status.idle":"2021-10-17T14:53:40.116051Z","shell.execute_reply.started":"2021-10-17T14:53:40.018852Z","shell.execute_reply":"2021-10-17T14:53:40.115323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load pre-trained word embeddings into an Embedding layer\n# note that we set trainable = False so as to keep the embeddings fixed\nembedding_layer = Embedding(\n  num_words,\n  EMBEDDING_DIM,\n  weights=[embedding_matrix],\n  input_length=MAX_SEQUENCE_LENGTH,\n  trainable=False\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:53:42.607695Z","iopub.execute_input":"2021-10-17T14:53:42.607944Z","iopub.status.idle":"2021-10-17T14:53:42.978329Z","shell.execute_reply.started":"2021-10-17T14:53:42.607917Z","shell.execute_reply":"2021-10-17T14:53:42.977571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Building model...')\n\n# train a 1D convnet with global maxpooling\ninput_ = Input(shape=(MAX_SEQUENCE_LENGTH,))\nx = embedding_layer(input_)\nx = Conv1D(128, 3, activation='relu')(x)\nx = MaxPooling1D(3)(x)\nx = Conv1D(128, 3, activation='relu')(x)\nx = MaxPooling1D(3)(x)\nx = Conv1D(128, 3, activation='relu')(x)\nx = GlobalMaxPooling1D()(x)\nx = Dense(128, activation='relu')(x)\noutput = Dense(len(possible_labels), activation='sigmoid')(x)\n\nmodel = Model(input_, output)\nmodel.compile(\n  loss='binary_crossentropy',\n  optimizer='rmsprop',\n  metrics=['accuracy']\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:53:56.041069Z","iopub.execute_input":"2021-10-17T14:53:56.041886Z","iopub.status.idle":"2021-10-17T14:53:58.539958Z","shell.execute_reply.started":"2021-10-17T14:53:56.041837Z","shell.execute_reply":"2021-10-17T14:53:58.539263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training model...')\nr = model.fit(\n  data,\n  targets,\n  batch_size=BATCH_SIZE,\n  epochs=EPOCHS,\n  validation_split=VALIDATION_SPLIT\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:54:14.852162Z","iopub.execute_input":"2021-10-17T14:54:14.852782Z","iopub.status.idle":"2021-10-17T14:56:37.75197Z","shell.execute_reply.started":"2021-10-17T14:54:14.852743Z","shell.execute_reply":"2021-10-17T14:56:37.751123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot some data\nplt.plot(r.history['loss'], label='loss')\nplt.plot(r.history['val_loss'], label='val_loss')\nplt.legend()\nplt.show()\n\n# accuracies\nplt.plot(r.history['accuracy'], label='acc')\nplt.plot(r.history['val_accuracy'], label='val_acc')\nplt.legend()\nplt.show()\n\n# plot the mean AUC over each label\np = model.predict(data)\naucs = []\nfor j in range(6):\n    auc = roc_auc_score(targets[:,j], p[:,j])\n    aucs.append(auc)\nprint(np.mean(aucs))","metadata":{"execution":{"iopub.status.busy":"2021-10-17T14:56:37.754016Z","iopub.execute_input":"2021-10-17T14:56:37.754298Z","iopub.status.idle":"2021-10-17T14:56:47.514991Z","shell.execute_reply.started":"2021-10-17T14:56:37.754243Z","shell.execute_reply":"2021-10-17T14:56:47.514167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}