{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Text classification\n\nThis notebook shows how to do text classification starting from raw text (as a set of text files on disk). ","metadata":{}},{"cell_type":"code","source":"import os\nimport tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-07T18:20:06.552257Z","iopub.execute_input":"2023-02-07T18:20:06.552633Z","iopub.status.idle":"2023-02-07T18:20:11.182670Z","shell.execute_reply.started":"2023-02-07T18:20:06.552581Z","shell.execute_reply":"2023-02-07T18:20:11.181555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load the data: IMDB movie review sentiment classification","metadata":{}},{"cell_type":"code","source":"!curl -O https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n!tar -xf aclImdb_v1.tar.gz","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-02-07T18:20:11.188635Z","iopub.execute_input":"2023-02-07T18:20:11.191798Z","iopub.status.idle":"2023-02-07T18:20:23.463304Z","shell.execute_reply.started":"2023-02-07T18:20:11.191755Z","shell.execute_reply":"2023-02-07T18:20:23.461987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:23.465515Z","iopub.execute_input":"2023-02-07T18:20:23.465917Z","iopub.status.idle":"2023-02-07T18:20:24.451580Z","shell.execute_reply.started":"2023-02-07T18:20:23.465877Z","shell.execute_reply":"2023-02-07T18:20:24.450161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `aclImdb` folder contains a train and test subfolder:","metadata":{}},{"cell_type":"code","source":"!ls aclImdb/","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:24.454967Z","iopub.execute_input":"2023-02-07T18:20:24.455751Z","iopub.status.idle":"2023-02-07T18:20:25.464907Z","shell.execute_reply.started":"2023-02-07T18:20:24.455709Z","shell.execute_reply":"2023-02-07T18:20:25.463687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls aclImdb/test","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:25.467109Z","iopub.execute_input":"2023-02-07T18:20:25.467550Z","iopub.status.idle":"2023-02-07T18:20:26.440249Z","shell.execute_reply.started":"2023-02-07T18:20:25.467509Z","shell.execute_reply":"2023-02-07T18:20:26.438937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls aclImdb/train","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:26.442536Z","iopub.execute_input":"2023-02-07T18:20:26.442965Z","iopub.status.idle":"2023-02-07T18:20:27.422577Z","shell.execute_reply.started":"2023-02-07T18:20:26.442924Z","shell.execute_reply":"2023-02-07T18:20:27.421331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are only interested on  `pos` and `neg` subfolders, so let's delete the rest:","metadata":{}},{"cell_type":"code","source":"!rm -r aclImdb/train/unsup","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:27.424543Z","iopub.execute_input":"2023-02-07T18:20:27.425326Z","iopub.status.idle":"2023-02-07T18:20:29.391198Z","shell.execute_reply.started":"2023-02-07T18:20:27.425282Z","shell.execute_reply":"2023-02-07T18:20:29.389819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `aclImdb/train/pos` and `aclImdb/test/neg` folders contain text files, each of which represents on review (either positive or negative).","metadata":{}},{"cell_type":"code","source":"!cat aclImdb/train/neg/5003_4.txt","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:29.393303Z","iopub.execute_input":"2023-02-07T18:20:29.393734Z","iopub.status.idle":"2023-02-07T18:20:30.382945Z","shell.execute_reply.started":"2023-02-07T18:20:29.393693Z","shell.execute_reply":"2023-02-07T18:20:30.381219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cat aclImdb/train/pos/4002_8.txt","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:30.385154Z","iopub.execute_input":"2023-02-07T18:20:30.385680Z","iopub.status.idle":"2023-02-07T18:20:31.657207Z","shell.execute_reply.started":"2023-02-07T18:20:30.385606Z","shell.execute_reply":"2023-02-07T18:20:31.655914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of positive and negative reviews on train and test set\n\ntrain_path = \"aclImdb/train\"\ntest_path = \"aclImdb/test\"\n\nprint(\"====================Train Set====================\")\nprint(f\"We have: {len(os.listdir(train_path+'/pos'))} positive reviews.\")\nprint(f\"We have: {len(os.listdir(train_path+'/neg'))} negative reviews.\")\n\nprint(\"====================Test Set====================\")\nprint(f\"We have: {len(os.listdir(test_path+'/pos'))} positive reviews.\")\nprint(f\"We have: {len(os.listdir(test_path+'/neg'))} negative reviews.\")","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:31.662145Z","iopub.execute_input":"2023-02-07T18:20:31.662477Z","iopub.status.idle":"2023-02-07T18:20:31.696900Z","shell.execute_reply.started":"2023-02-07T18:20:31.662445Z","shell.execute_reply":"2023-02-07T18:20:31.695697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You can use the utility `tf.keras.preprocessing.text_dataset_from_directory` to generate a labeled `tf.data.Dataset` object from a set of text files on disk filed into class-specific folders.\n\nLet's use it to generate the training, validation, and test datasets. The validation and training datasets are generated from two subsets of the `train` directory, with 20% samples going to the validation dataset and 80% going to the training dataset.\n\nHaving a validation dataset in addition to the test dataset is useful for tuning hyperparameters, such as the model architecture, for which the test dataset should not be used.\n\nBefore putting the model out into the real world however, it should be retrained using all available training data (without creating a validation dataset), so its performance is maximized.\n\nWhen using the `validation_split & subset` arguments, make sure to either specify a random seed, or set `shuffle=False`, so that the validation & training splits you get have overlap.","metadata":{}},{"cell_type":"code","source":"batch_size = 8\nraw_train_ds = tf.keras.preprocessing.text_dataset_from_directory(\n    \"aclImdb/train\",\n    batch_size=batch_size,\n    validation_split=0.5,\n    subset=\"training\",\n    seed=42\n)\n\nraw_val_ds = tf.keras.preprocessing.text_dataset_from_directory(\n    \"aclImdb/test\",\n    batch_size=batch_size,\n    validation_split=0.1,\n    subset=\"validation\",\n    seed=42\n)\n\nraw_test_ds = tf.keras.preprocessing.text_dataset_from_directory(\n    \"aclImdb/test\",\n    batch_size=batch_size\n)\n\nprint(f\"Number of batches in raw_train_ds: {tf.data.experimental.cardinality(raw_train_ds)}\")\nprint(f\"Number of batches in raw_val_ds: {tf.data.experimental.cardinality(raw_val_ds)}\")\nprint(f\"Number of batches in raw_test_ds: {tf.data.experimental.cardinality(raw_test_ds)}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:31.698661Z","iopub.execute_input":"2023-02-07T18:20:31.699063Z","iopub.status.idle":"2023-02-07T18:20:39.395174Z","shell.execute_reply.started":"2023-02-07T18:20:31.699024Z","shell.execute_reply":"2023-02-07T18:20:39.394110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It's important to take a look at your raw data to ensure your normalization and tokenization will work as expected. We can do that by taking a few examples from the training set and looking at them.","metadata":{}},{"cell_type":"code","source":"for text_batch, label_batch in raw_train_ds.take(1):\n    for i in range(5):\n        print(text_batch.numpy()[i])\n        print(label_batch.numpy()[i])","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:39.396855Z","iopub.execute_input":"2023-02-07T18:20:39.397502Z","iopub.status.idle":"2023-02-07T18:20:39.472874Z","shell.execute_reply.started":"2023-02-07T18:20:39.397462Z","shell.execute_reply":"2023-02-07T18:20:39.471878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare the data\n\nHaving looked at our data above, we see that the raw text contains HTML break tags of the form `<br />`. These tags will not be removed by the default standardizer (which doesn't strip HTML). Because of this, we will need to create a custom standardization function.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\nimport string\nimport re\n\n\n# Removing <br/>\ndef custom_standardization(input_data):\n    lowercase = tf.strings.lower(input_data)\n    stripped_html = tf.strings.regex_replace(lowercase, \"<br />\", \" \")\n    return tf.strings.regex_replace(\n        stripped_html, \"[%s] % re.escape(string.punctuation)\", \"\"\n    )\n\n# Model constants.\nmax_features = 60000\nembedding_dim = 512\nsequence_length = 1000","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:39.474342Z","iopub.execute_input":"2023-02-07T18:20:39.477887Z","iopub.status.idle":"2023-02-07T18:20:39.486123Z","shell.execute_reply.started":"2023-02-07T18:20:39.477848Z","shell.execute_reply":"2023-02-07T18:20:39.485121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now that we have our custom standardization, we can instantiate our text vectorization layer. We are using this layer to normalize, split, and map strings to integers, so we set our `'output_mode'` to `int`. Note that we are using the default split function, and the custom standardization defined above. We also set an explicit maximum sequence length, since the CNNs later in our model won't support ragged sequences.","metadata":{}},{"cell_type":"code","source":"vectorize_layer = TextVectorization(\n    standardize=custom_standardization,\n    max_tokens=max_features,\n    output_mode=\"int\",\n    output_sequence_length=sequence_length,\n)\n\n# Let's make a text-only dataset (no labels)\ntext_ds = raw_train_ds.map(lambda x, y: x)\n# Let's call `adapt`\nvectorize_layer.adapt(text_ds)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:39.487661Z","iopub.execute_input":"2023-02-07T18:20:39.488042Z","iopub.status.idle":"2023-02-07T18:20:42.733869Z","shell.execute_reply.started":"2023-02-07T18:20:39.488006Z","shell.execute_reply":"2023-02-07T18:20:42.732809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After the vocab layer has been created, we called `adapt` on a text-only dataset to create the vocabulary. You don't have to batch, but for very large datasets this means you are not keeping spare copies of the dataset in memory.\n\n# Two options to vectorize the data\n\nThere are 2 ways we can use our text vectorization layer:\n\n- **Option 1: Make it part of the model,** so as to obtain a model that processes raw strings.\n- **Option 2: Apply it to the text dataset** to obtain a dataset of word indices, then feed it into a model that expects integer sequences as inputs.\n\nAn importance difference between the two is that option 2 enables you to do `asynchronous CPU processing and buffering` of your data when training on GPU. So if you are training the model on GPU, you probably want to go with this option to get best performance. This is what we will do below.\n\nIf we were to export our model to production, we'd ship a model that accepts raw strings as input. This can be done after training.","metadata":{}},{"cell_type":"code","source":"def vectorize_text(text, label):\n    text = tf.expand_dims(text, -1)\n    return vectorize_layer(text), label\n\n\n# Vectorize the data\ntrain_ds = raw_train_ds.map(vectorize_text)\nval_ds = raw_val_ds.map(vectorize_text)\ntest_ds = raw_test_ds.map(vectorize_text)\n\n# Do async prefetching / buffering of the data for best performace on GPU\ntrain_ds = train_ds.cache().prefetch(buffer_size=10)\nval_ds = val_ds.cache().prefetch(buffer_size=10)\ntest_ds = test_ds.cache().prefetch(buffer_size=10)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:42.735245Z","iopub.execute_input":"2023-02-07T18:20:42.736296Z","iopub.status.idle":"2023-02-07T18:20:42.871336Z","shell.execute_reply.started":"2023-02-07T18:20:42.736257Z","shell.execute_reply":"2023-02-07T18:20:42.870379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build a model\n\nWe choose a simple 1D convnet starting with an `Embedding` layer.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers\n\n# An integer input for vocab indices.\ninputs = tf.keras.Input(shape=(None, ), dtype=\"int64\")\n\n# Add a layer to map those vocab indices into a space of dimensionality\n# 'embedding_dim'.\nx = layers.Embedding(max_features, embedding_dim)(inputs)\n# x = layers.Dropout(0.5)(x)\n\n# Conv1D + global max pooling\nx = layers.Conv1D(512, 7, padding=\"valid\", activation=\"relu\", strides=3)(x)\nx = layers.Dropout(0.5)(x)\nx = layers.Conv1D(256, 7, padding=\"valid\", activation=\"relu\", strides=3)(x)\nx = layers.Dropout(0.5)(x)\nx = layers.Conv1D(128, 7, padding=\"valid\", activation=\"relu\", strides=3)(x)\nx = layers.GlobalMaxPooling1D()(x)\n\n# Add a vanilla hidden layer\nx = layers.Dense(128, activation=\"relu\")(x)\nx = layers.Dropout(0.5)(x)\n\n# We project onto a single unit output layer, and squash it with a sigmoid:\npredictions = layers.Dense(1, activation=\"sigmoid\", name=\"predictions\")(x)\n\nmodel = tf.keras.Model(inputs, predictions)\n\n# Compile the model with binary crossentropy loss and an adam optimizer.\nmodel.compile(loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:42.872867Z","iopub.execute_input":"2023-02-07T18:20:42.873199Z","iopub.status.idle":"2023-02-07T18:20:43.004787Z","shell.execute_reply.started":"2023-02-07T18:20:42.873165Z","shell.execute_reply":"2023-02-07T18:20:43.003335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"code","source":"epochs = 10\n\n# Fit the model using the train and test datasets.\nr = model.fit(train_ds, validation_data=val_ds, epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:20:43.006273Z","iopub.execute_input":"2023-02-07T18:20:43.006979Z","iopub.status.idle":"2023-02-07T18:26:48.544470Z","shell.execute_reply.started":"2023-02-07T18:20:43.006941Z","shell.execute_reply":"2023-02-07T18:26:48.543399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate the model on the test set","metadata":{}},{"cell_type":"code","source":"model.evaluate(train_ds)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:26:48.546731Z","iopub.execute_input":"2023-02-07T18:26:48.547352Z","iopub.status.idle":"2023-02-07T18:26:58.811320Z","shell.execute_reply.started":"2023-02-07T18:26:48.547314Z","shell.execute_reply":"2023-02-07T18:26:58.810401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(val_ds)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:26:58.812881Z","iopub.execute_input":"2023-02-07T18:26:58.813538Z","iopub.status.idle":"2023-02-07T18:27:00.438321Z","shell.execute_reply.started":"2023-02-07T18:26:58.813501Z","shell.execute_reply":"2023-02-07T18:27:00.437312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(test_ds)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:27:00.439973Z","iopub.execute_input":"2023-02-07T18:27:00.440404Z","iopub.status.idle":"2023-02-07T18:27:16.220770Z","shell.execute_reply.started":"2023-02-07T18:27:00.440364Z","shell.execute_reply":"2023-02-07T18:27:16.219550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12, 7))\n\nplt.subplot(2, 2, 1)\nplt.plot(r.history['loss'], label='Loss')\nplt.plot(r.history['val_loss'], label='val_loss')\nplt.legend()\nplt.title('Loss function evolution')\n\nplt.subplot(2, 2, 2)\nplt.plot(r.history['accuracy'], label='accuracy')\nplt.plot(r.history['val_accuracy'], label='val_accuracy')\nplt.legend()\nplt.title('Loss function evolution')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:27:16.222597Z","iopub.execute_input":"2023-02-07T18:27:16.223006Z","iopub.status.idle":"2023-02-07T18:27:16.576484Z","shell.execute_reply.started":"2023-02-07T18:27:16.222966Z","shell.execute_reply":"2023-02-07T18:27:16.575563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import confusion_matrix, classification_report\n\n# pred = model.predict(test_ds)\n\n# print(confusion_matrix(raw_test_ds., pred > 0.5))","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:27:16.577820Z","iopub.execute_input":"2023-02-07T18:27:16.581286Z","iopub.status.idle":"2023-02-07T18:27:16.585882Z","shell.execute_reply.started":"2023-02-07T18:27:16.581255Z","shell.execute_reply":"2023-02-07T18:27:16.584635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make an end-to-end model\n\nIf you want to obtain a model capable of processing raw strings, you can simply create a new model (using the weights we just trained)","metadata":{}},{"cell_type":"code","source":"# A string input\ninputs = tf.keras.Input(shape=(1, ), dtype=\"string\")\n\n# Turn strings into vocab indices\nindices = vectorize_layer(inputs)\n\n# Turn vocab indices into predictions\noutputs = model(indices)\n\n# Our end to end model\nend_to_end_model = tf.keras.Model(inputs, outputs)\nend_to_end_model.compile(\n    loss=\"binary_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"]\n)\n\n# Test it with `raw_test_ds`, which yields raw strings\nend_to_end_model.evaluate(raw_test_ds)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T18:27:16.587439Z","iopub.execute_input":"2023-02-07T18:27:16.588061Z","iopub.status.idle":"2023-02-07T18:27:36.275507Z","shell.execute_reply.started":"2023-02-07T18:27:16.588021Z","shell.execute_reply":"2023-02-07T18:27:36.274505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}