{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n\n![](\"https://i.ibb.co/zFn12DK/Screen-Shot-2021-12-27-at-12-40-55-PM.png\")
\n\nFeedback Prize - Exhaustive Learning
\n\nEDA SURVEY
\n\nCREATED BY: DARIEN SCHETTLER
\n\n
\n\n---\n\n
\n\n\n ๐ WARNING:
THIS IS A WORK IN PROGRESS
\n
\n\n\n\n\n ๐ IF YOU FORK THIS OR FIND THIS HELPFUL ๐
PLEASE UPVOTE! THE ORIGINAL NOTEBOOKS TOO!\n
\n\n\n","metadata":{"papermill":{"duration":0.120561,"end_time":"2021-11-06T21:15:09.611563","exception":false,"start_time":"2021-11-06T21:15:09.491002","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"\n\n
\n\nTABLE OF CONTENTS
\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---\n\n\n\n---","metadata":{"papermill":{"duration":0.085591,"end_time":"2021-11-06T21:15:09.78303","exception":false,"start_time":"2021-11-06T21:15:09.697439","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n0 IMPORTS ⤒
","metadata":{"papermill":{"duration":0.050527,"end_time":"2021-11-06T21:15:09.894476","exception":false,"start_time":"2021-11-06T21:15:09.843949","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"0.1 PIP INSTALLS
\n\n---\n","metadata":{}},{"cell_type":"code","source":"print(\"\\n... PIP/APT INSTALLS AND DOWNLOADS/ZIP STARTING ...\")\n!pip install -q --upgrade tensorflow-model-optimization\n!pip install -q --upgrade neural-structured-learning\n!pip install -q --upgrade tensorflow_datasets\n!pip install -q --upgrade plotly\n\n## Only to be used if we can figure out that TPU bug\n# !pip install -q tf-nightly\n\n# Try to skip and disable so we can submit w/o internet\n# !pip install -q ../input/tensorflow-model-optimization/numpy-1.21.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n# !pip install -q ../input/tensorflow-model-optimization/dm_tree-0.1.6-cp37-cp37m-manylinux_2_24_x86_64.whl\n# !pip install -q ../input/tensorflow-model-optimization/six-1.16.0-py2.py3-none-any.whl\n# !pip install -q ../input/tensorflow-model-optimization/tensorflow_model_optimization-0.7.0-py2.py3-none-any.whl\n# !pip install -q ../input/neural-structued-learning/neural_structured_learning-1.3.1-py2.py3-none-any.whl\n\n\nprint(\"... PIP/APT INSTALLS COMPLETE ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:28:55.814858Z","iopub.execute_input":"2022-01-10T15:28:55.815199Z","iopub.status.idle":"2022-01-10T15:30:03.824355Z","shell.execute_reply.started":"2022-01-10T15:28:55.815118Z","shell.execute_reply":"2022-01-10T15:30:03.823483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"0.2 IMPORTS & SEED
\n\n---","metadata":{}},{"cell_type":"code","source":"print(\"\\n... IMPORTS STARTING ...\\n\")\n\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\tโ TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\tโ TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\tโ NUMPY VERSION: {np.__version__}\");\nimport sklearn; print(f\"\\t\\tโ SKLEARN VERSION: {sklearn.__version__}\");\nfrom sklearn.preprocessing import RobustScaler, PolynomialFeatures\nfrom pandarallel import pandarallel; pandarallel.initialize();\nfrom sklearn.model_selection import GroupKFold;\n\n# # This is necessary to force the TPU client to have the same TF version as TF-Nightly\n# from cloud_tpu_client import Client\n# c = Client(tpu=''); c.configure_tpu_version(tf.__version__, restart_type='ifNeeded')\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance\nimport matplotlib; print(f\"\\t\\tโ MATPLOTLIB VERSION: {matplotlib.__version__}\");\nfrom matplotlib import animation, rc; rc('animation', html='jshtml')\nimport plotly\nimport PIL\nimport cv2\n\ndef seed_it_all(seed=7):\n \"\"\" Attempt to be Reproducible \"\"\"\n os.environ['PYTHONHASHSEED'] = str(seed)\n random.seed(seed)\n np.random.seed(seed)\n tf.random.set_seed(seed)\n\n \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")\n \nprint(\"\\n... SEEDING FOR DETERMINISTIC BEHAVIOUR ...\\n\")\nseed_it_all()\nprint(\"\\n... SEEDING COMPLETE ...\\n\")","metadata":{"papermill":{"duration":162.144149,"end_time":"2021-11-06T21:17:52.087371","exception":false,"start_time":"2021-11-06T21:15:09.943222","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T15:30:03.828018Z","iopub.execute_input":"2022-01-10T15:30:03.828246Z","iopub.status.idle":"2022-01-10T15:30:13.787290Z","shell.execute_reply.started":"2022-01-10T15:30:03.828219Z","shell.execute_reply":"2022-01-10T15:30:13.786488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n---\n\nAll of the competition background information is contained within my solutioning document.
To access this document please click the link below...\n\n
>>>> SOLUTIONING DOCUMENT <<<<
","metadata":{"papermill":{"duration":0.05019,"end_time":"2021-11-06T21:17:52.231372","exception":false,"start_time":"2021-11-06T21:17:52.181182","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"
\n\n\n\n2 SETUP ⤒
\n\n---\n","metadata":{"papermill":{"duration":0.052944,"end_time":"2021-11-06T21:17:52.65284","exception":false,"start_time":"2021-11-06T21:17:52.599896","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"2.1 ACCELERATOR DETECTION
\n\n---\n\nIn order to use **`TPU`**, we use **`TPUClusterResolver`** for the initialization which is necessary to connect to the remote cluster and initialize cloud TPUs. Let's go over two important points\n\n1. When using TPU on Kaggle, you don't need to specify arguments for **`TPUClusterResolver`**\n2. However, on **G**oogle **C**ompute **E**ngine (**GCE**), you will need to do the following:\n\n
\n\n```python\n# The name you gave to the TPU to use\nTPU_WORKER = 'my-tpu-name'\n\n# or you can also specify the grpc path directly\n# TPU_WORKER = 'grpc://xxx.xxx.xxx.xxx:8470'\n\n# The zone you chose when you created the TPU to use on GCP.\nZONE = 'us-east1-b'\n\n# The name of the GCP project where you created the TPU to use on GCP.\nPROJECT = 'my-tpu-project'\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=TPU_WORKER, zone=ZONE, project=PROJECT)\n```\n\n\n ๐ WARNING:
- Although the Tensorflow documentation says it is the project name that should be provided for the argument `project`, it is actually the Project ID, that you should provide. This can be found on the GCP project dashboard page.
\n
\n\n","metadata":{"papermill":{"duration":0.053821,"end_time":"2021-11-06T21:17:52.761303","exception":false,"start_time":"2021-11-06T21:17:52.707482","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"\\n... ACCELERATOR SETUP STARTING ...\\n\")\n\n# Detect hardware, return appropriate distribution strategy\ntry:\n # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n TPU = tf.distribute.cluster_resolver.TPUClusterResolver() \nexcept ValueError:\n TPU = None\n\nif TPU:\n print(f\"\\n... RUNNING ON TPU - {TPU.master()}...\")\n tf.config.experimental_connect_to_cluster(TPU)\n tf.tpu.experimental.initialize_tpu_system(TPU)\n strategy = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n print(f\"\\n... RUNNING ON CPU/GPU ...\")\n # Yield the default distribution strategy in Tensorflow\n # --> Works on CPU and single GPU.\n strategy = tf.distribute.get_strategy() \n\n# What Is a Replica?\n# --> A single Cloud TPU device consists of FOUR chips, each of which has TWO TPU cores. \n# --> Therefore, for efficient utilization of Cloud TPU, a program should make use of each of the EIGHT (4x2) cores. \n# --> Each replica is essentially a copy of the training graph that is run on each core and \n# trains a mini-batch containing 1/8th of the overall batch size\nN_REPLICAS = strategy.num_replicas_in_sync\n \nprint(f\"... # OF REPLICAS: {N_REPLICAS} ...\\n\")\n\nprint(f\"\\n... ACCELERATOR SETUP COMPLTED ...\\n\")","metadata":{"papermill":{"duration":0.07574,"end_time":"2021-11-06T21:17:52.892074","exception":false,"start_time":"2021-11-06T21:17:52.816334","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T15:30:13.791230Z","iopub.execute_input":"2022-01-10T15:30:13.793371Z","iopub.status.idle":"2022-01-10T15:30:13.821168Z","shell.execute_reply.started":"2022-01-10T15:30:13.793332Z","shell.execute_reply":"2022-01-10T15:30:13.820421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.2 COMPETITION DATA ACCESS
\n\n---\n\nTPUs read data must be read directly from **G**oogle **C**loud **S**torage **(GCS)**. Kaggle provides a utility library โย **`KaggleDatasets`** โ which has a utility function **`.get_gcs_path`** that will allow us to access the location of our input datasets within **GCS**.
\n\n\n ๐ TIPS:
- If you have multiple datasets attached to the notebook, you should pass the name of a specific dataset to the `get_gcs_path()` function. In our case, the name of the dataset is the name of the directory the dataset is mounted within.
\n
","metadata":{"papermill":{"duration":0.053551,"end_time":"2021-11-06T21:17:52.999073","exception":false,"start_time":"2021-11-06T21:17:52.945522","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... DATA ACCESS SETUP STARTED ...\\n\")\n\nif TPU:\n # Google Cloud Dataset path to training and validation images\n DATA_DIR = KaggleDatasets().get_gcs_path('feedback-prize-2021')\n save_locally = tf.saved_model.SaveOptions(experimental_io_device='/job:localhost')\n load_locally = tf.saved_model.LoadOptions(experimental_io_device='/job:localhost')\nelse:\n # Local path to training and validation images\n DATA_DIR = \"/kaggle/input/feedback-prize-2021\"\n save_locally = load_locally = None\n \n \nprint(f\"\\n... DATA DIRECTORY PATH IS:\\n\\t--> {DATA_DIR}\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF DATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(DATA_DIR, \"*\")): print(f\"\\t--> {file}\")\n\n \nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"papermill":{"duration":0.0797,"end_time":"2021-11-06T21:17:53.133972","exception":false,"start_time":"2021-11-06T21:17:53.054272","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T15:30:13.825162Z","iopub.execute_input":"2022-01-10T15:30:13.827904Z","iopub.status.idle":"2022-01-10T15:30:13.873239Z","shell.execute_reply.started":"2022-01-10T15:30:13.827443Z","shell.execute_reply":"2022-01-10T15:30:13.872490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.3 LEVERAGING XLA OPTIMIZATIONS
\n\n---\n\n\n**XLA** (Accelerated Linear Algebra) is a domain-specific compiler for linear algebra that can accelerate TensorFlow models with potentially no source code changes. **The results are improvements in speed and memory usage**.\n\n
\n\nWhen a TensorFlow program is run, all of the operations are executed individually by the TensorFlow executor. Each TensorFlow operation has a precompiled GPU/TPU kernel implementation that the executor dispatches to.\n\nXLA provides us with an alternative mode of running models: it compiles the TensorFlow graph into a sequence of computation kernels generated specifically for the given model. Because these kernels are unique to the model, they can exploit model-specific information for optimization.
\n\n\n ๐ WARNING:
- XLA can not currently compile functions where dimensions are not inferrable: that is, if it's not possible to infer the dimensions of all tensors without running the entire computation
\n
\n\n\n ๐ NOTE:
- XLA compilation is only applied to code that is compiled into a graph (in TF2 that's only a code inside tf.function).
- The jit_compile API has must-compile semantics, i.e. either the entire function is compiled with XLA, or an errors.InvalidArgumentError exception is thrown)\n
\n\n","metadata":{"papermill":{"duration":0.051494,"end_time":"2021-11-06T21:17:53.236017","exception":false,"start_time":"2021-11-06T21:17:53.184523","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(f\"\\n... XLA OPTIMIZATIONS STARTING ...\\n\")\n\nprint(f\"\\n... CONFIGURE JIT (JUST IN TIME) COMPILATION ...\\n\")\n# enable XLA optmizations (10% speedup when using @tf.function calls)\ntf.config.optimizer.set_jit(True)\n\nprint(f\"\\n... XLA OPTIMIZATIONS COMPLETED ...\\n\")","metadata":{"papermill":{"duration":0.128128,"end_time":"2021-11-06T21:17:53.442803","exception":false,"start_time":"2021-11-06T21:17:53.314675","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T15:30:13.874248Z","iopub.execute_input":"2022-01-10T15:30:13.874511Z","iopub.status.idle":"2022-01-10T15:30:13.883816Z","shell.execute_reply.started":"2022-01-10T15:30:13.874476Z","shell.execute_reply":"2022-01-10T15:30:13.883093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"2.4 BASIC DATA DEFINITIONS & INITIALIZATIONS
\n\n---\n","metadata":{"papermill":{"duration":0.090507,"end_time":"2021-11-06T21:17:53.624612","exception":false,"start_time":"2021-11-06T21:17:53.534105","status":"completed"},"tags":[]}},{"cell_type":"code","source":"print(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_df[['discourse_id', 'discourse_start', 'discourse_end']] = train_df[['discourse_id', 'discourse_start', 'discourse_end']].astype(int)\ntrain_df[\"txt_path\"] = train_df[\"id\"].apply(lambda x: os.path.join(DATA_DIR, \"train\", f\"{x}.txt\"))\nprint(\"\\n... TRAIN DATAFRAME ...\\n\")\ndisplay(train_df)\n\nprint(\"\\n... SAMPLE SUBMISSION DATAFRAME ..\\n\")\nSS_CSV = os.path.join(DATA_DIR, \"sample_submission.csv\")\nss_df = pd.read_csv(SS_CSV)\nss_df[\"txt_path\"] = ss_df[\"id\"].apply(lambda x: os.path.join(DATA_DIR, \"test\", f\"{x}.txt\"))\ndisplay(ss_df)\n\nDISCOURSE_CLASSES = sorted(train_df.discourse_type.unique())\n\nprint(\"\\n\\n... BASIC DATA SETUP FINISHING ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:30:13.885279Z","iopub.execute_input":"2022-01-10T15:30:13.885852Z","iopub.status.idle":"2022-01-10T15:30:16.280569Z","shell.execute_reply.started":"2022-01-10T15:30:13.885818Z","shell.execute_reply":"2022-01-10T15:30:16.279678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n 3 HELPER FUNCTION & CLASSES ⤒\n
\n\n---\n\nWe will collect the essay details at this step. We will need these details for later EDAs\n* Information such as unique words, length of essay (character and word count), etc.","metadata":{"papermill":{"duration":0.054893,"end_time":"2021-11-06T21:17:54.695576","exception":false,"start_time":"2021-11-06T21:17:54.640683","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n \"\"\" Flatten a list of lists \"\"\"\n return [item for sublist in nested_list for item in sublist]\n\ndef get_essay_details(df, essay_paths):\n \"\"\"\n \n This function will take a list of paths to essays and return the\n lengths of the respective essays as well as all unique words contained within\n that essay\n \n this code chunk is heavily inspired from Rob Mulla's code\n https://www.kaggle.com/robikscube/student-writing-competition-twitch\n \n \n Args:\n df (pd.DataFrame): The dataframe to update\n txt_paths (list of strings): The paths to all of the text files to probe\n \n Returns:\n Updated pd.DataFrame\n \"\"\"\n # Copy to prevent unwanted changes\n _df = df.copy()\n \n char_len_map, word_len_map, unique_word_map = dict(), dict(), dict()\n \n for essay_path in tqdm(essay_paths, total=len(essay_paths)):\n with open(essay_path, \"r\") as essay:\n \n # Get ID to use for mapping\n _id = essay_path[:-4].rsplit(\"/\", 1)[-1]\n \n # Get words and character information\n data = essay.read()\n words = [x.strip() for x in data.strip().split()]\n c_len = len(data.strip())\n w_len = len(words)\n \n # Update respective id maps\n char_len_map[_id] = c_len\n word_len_map[_id] = w_len\n unique_word_map[_id] = set(words)\n \n _df[\"essay_c_len\"] = _df[\"id\"].map(char_len_map)\n _df[\"essay_w_len\"] = _df[\"id\"].map(word_len_map)\n _df[\"essay_unique_words\"] = _df[\"id\"].map(unique_word_map)\n \n return _df\n\ntrain_df = get_essay_details(train_df, train_df.txt_path.unique())\ntrain_df","metadata":{"papermill":{"duration":0.098071,"end_time":"2021-11-06T21:17:54.848036","exception":false,"start_time":"2021-11-06T21:17:54.749965","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-10T15:30:16.281990Z","iopub.execute_input":"2022-01-10T15:30:16.282347Z","iopub.status.idle":"2022-01-10T15:31:05.056449Z","shell.execute_reply.started":"2022-01-10T15:30:16.282305Z","shell.execute_reply":"2022-01-10T15:31:05.055759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n\n---\n\nThis excellent EDA was written by [**Darek Kลeczek (thedrcat)**](https://www.kaggle.com/thedrcat) a Data Scientist at P&G. I will sample certain sections, comments, or code from his EDA to contribute to this survey. This will be contained within the below section","metadata":{"papermill":{"duration":0.056443,"end_time":"2021-11-06T21:17:54.960518","exception":false,"start_time":"2021-11-06T21:17:54.904075","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"4.1 EDA SPECIFIC SETUP
\n\n---","metadata":{}},{"cell_type":"code","source":"# EDA SPECIFIC IMPORTS - I generally import again even if I did it above just\n# for visual consistency with the source notebook\nimport spacy\nfrom spacy import displacy\nfrom pylab import cm, matplotlib\n\n# EDA SPECIFIC COLOURING\nd2c_map = {\n 'Lead': '#8000ff',\n 'Position': '#2b7ff6',\n 'Evidence': '#2adddd',\n 'Claim': '#80ffb4',\n 'Concluding Statement': '#d4dd80',\n 'Counterclaim': '#ff8042',\n 'Rebuttal': '#ff0000',\n 'Gap Text': '#808080', # FOR LATER \n }\nc2d_map = {v:k for k,v in d2c_map.items()}\n\n# EDA SPECIFIC FUNCTIONS - SOME REWRITING MOSTLY @THEDRCAT\ndef visualize_1(df, _id=None):\n \"\"\" Function to visualize a single discourse with displacy\n \n Args:\n df (pd.DataFrame): The dataframe to pull examples from.\n _id (str, optional): The string index used to identify the relevant body text\n to pull the discourse from. If no id is provided than a random one from the\n provided df will be selected.\n \n Returns:\n None; An example will be displayed.\n \"\"\"\n _df = df.copy()\n \n # Pick a random id if none is passed to the function\n if _id is None: _id = _df.sample(1).id.values[0]\n \n # Create the entity list which is a collection of discourse segments\n # where each discourse segment is described by it's start and end character\n # position as well as the discourse type label for that respective segment.\n entities = []\n for i, row in _df[_df['id']==_id].iterrows():\n entities.append({\n 'start': row['discourse_start'], \n 'end': row['discourse_end'], \n 'label': row['discourse_type']\n })\n \n # Open the correct text file\n with open(row[\"txt_path\"], 'r') as file: raw_txt_file = file.read()\n \n # Provide the required document details as a single dictionary\n # - \"text\": The Raw Text File\n # - \"ents\": A list of dictionaries describing the relevant discourse chunks\n # - \"title\": The title to ascribe to the visualization\n doc_details = {\n \"text\": raw_txt_file,\n \"ents\": entities,\n \"title\": f\"ID: {_id}\"\n }\n \n # A mapping of options that will allow us to customize the rendering that displacy will\n # generate. We leverage our previously defined colour map and class list.\n options = {\"ents\": DISCOURSE_CLASSES, \"colors\": d2c_map}\n \n # Displacy's render function that we control with specific arguments\n displacy.render(docs=doc_details, style=\"ent\", options=options, manual=True, jupyter=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:05.057870Z","iopub.execute_input":"2022-01-10T15:31:05.058307Z","iopub.status.idle":"2022-01-10T15:31:09.061251Z","shell.execute_reply.started":"2022-01-10T15:31:05.058268Z","shell.execute_reply":"2022-01-10T15:31:09.060504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"4.2 VISUALIZE WITH DISPLACY
\n\n---\n\n[**Darek Kลeczek (thedrcat)**](https://www.kaggle.com/thedrcat)\n\n> \"I think it will work best if we can overlay the spans from the csv file onto the text files. Spacy has a great visualizer for this: displacy. Let's use it to see what a single example looks like!\"","metadata":{}},{"cell_type":"code","source":"for i in range(3):\n visualize_1(train_df, _id=None)\n print('\\n\\n')","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:09.062546Z","iopub.execute_input":"2022-01-10T15:31:09.062813Z","iopub.status.idle":"2022-01-10T15:31:09.354866Z","shell.execute_reply.started":"2022-01-10T15:31:09.062779Z","shell.execute_reply":"2022-01-10T15:31:09.354028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n---\n\nThis excellent EDA was written by [**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin) a Data Scientist. I will sample certain sections, comments, or code from his EDA to contribute to this survey. This will be contained within the below section.","metadata":{}},{"cell_type":"markdown","source":"5.1 EDA SPECIFIC SETUP
\n\n---","metadata":{}},{"cell_type":"code","source":"from matplotlib.ticker import FuncFormatter\nfrom nltk.corpus import stopwords\nfrom tqdm.notebook import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\nimport spacy\nfrom sklearn.feature_extraction.text import CountVectorizer","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:09.357791Z","iopub.execute_input":"2022-01-10T15:31:09.358060Z","iopub.status.idle":"2022-01-10T15:31:09.705154Z","shell.execute_reply.started":"2022-01-10T15:31:09.358022Z","shell.execute_reply":"2022-01-10T15:31:09.704433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2 LOOK AT A SINGLE FULL ESSAY
\n\n---\n\n[**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin)\n\n> \"*Let's look at the full text of one essay first. [...] The train dataset gives us the following human annotations that are extracted from this essay. [...]* ***Kaggle gives us the following field descriptions:***\n>\n>* ***`id`*** *- ID code for essay response*\n>* ***`discourse_id`*** *- ID code for discourse element*\n>* ***`discourse_start`*** *- character position where discourse element begins in the essay response*\n>* ***`discourse_end`*** *- character position where discourse element ends in the essay response*\n>* ***`discourse_text`*** *- text of discourse element*\n>* ***`discourse_type`*** *- classification of discourse element*\n>* ***`discourse_type_num`*** *- enumerated class label of discourse element*\n>* ***`predictionstring`*** *- the word indices of the training sample, as required for predictions*\n>\n>*The Ground Truth here is a combination of the discourse type and the prediction string. The* ***`predictionstring`*** *corresponds to the index of the words in the essay and the predicted discourse type for this sequence of words should be correct. There can be partial matches, if the correct discourse type is predicted but on a longer or shorter sequence of words than specified in the Ground Truth.*\n>\n>*As we can see, not necessarily all text of an essay is part of a discourse. In this case, the title is not part of any discourse.\"*","metadata":{}},{"cell_type":"code","source":"txt_f_path = train_df.txt_path.sample(1).values[0]\n\nprint(\"\\n... FULL ESSAY - RANDOM EXAMPLE ...\\n\")\n!cat {txt_f_path}\n\nprint(\"\\n\\n\\n... HUMAN ANNOTATIONS - RANDOM EXAMPLE ...\\n\\n\")\ndisplay(train_df[train_df.txt_path==txt_f_path])\n\nprint(\"\\n\\n... DISCOURSE TYPE COUNTS - RANDOM EXAMPLE ...\\n\")\nfor k,v in train_df[train_df.txt_path==txt_f_path].discourse_type.value_counts().items():\n print(f\"\\tDiscourse Type : {k}\\n\\tDiscourse Count : {v}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:09.706461Z","iopub.execute_input":"2022-01-10T15:31:09.706719Z","iopub.status.idle":"2022-01-10T15:31:10.540749Z","shell.execute_reply.started":"2022-01-10T15:31:09.706684Z","shell.execute_reply":"2022-01-10T15:31:10.539830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.2 LENGTH OF THE DISCOURSE_TEXT AND PREDICTIONSTRING
\n\n---\n\n[**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin)\n\n> \"First, I would like to check if the discourse_text and the predictionstring always have the same number of words (as they should). [...] Is this always correct? No, I find **468** discourses where this goes wrong (by one word)\"\n> ","metadata":{}},{"cell_type":"code","source":"#add columns\ntrain_df[\"discourse_len\"] = train_df[\"discourse_text\"].apply(lambda x: len(x.split()))\ntrain_df[\"pred_len\"] = train_df[\"predictionstring\"].apply(lambda x: len(x.split()))\ntrain_df[\"chunk_gap_length\"] = (train_df[\"discourse_start\"].shift(-1)-train_df[\"discourse_end\"]-1).fillna(0).astype(int)\ntrain_df[\"chunk_gap_length\"] = train_df[\"chunk_gap_length\"].apply(lambda x: 0 if x<0 else x)\n\nprint(\"\\n\\n... DATAFRAME HEAD ...\\n\\n\\n\")\ndisplay(train_df.head())\n\nprint(\"\\n\\n\\n... ROWS WHERE DISCOURSE LENGTH != PRED LENGTH ...\\n\\n\\n\")\ndisplay(train_df[train_df.discourse_len!=train_df.pred_len])\n\nprint(f\"\\n\\n\\n# OF TOTAL ROWS : {len(train_df)}\")\nprint(f\"# OF TOTAL ROWS WHERE DISCOURSE LENGTH != PRED LENGTH : {len(train_df[train_df.discourse_len!=train_df.pred_len])}\")\nprint(f\"# OF TOTAL ROWS WHERE DISCOURSE LENGTH == PRED LENGTH : {len(train_df[train_df.discourse_len==train_df.pred_len])}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:10.542431Z","iopub.execute_input":"2022-01-10T15:31:10.542811Z","iopub.status.idle":"2022-01-10T15:31:11.457478Z","shell.execute_reply.started":"2022-01-10T15:31:10.542768Z","shell.execute_reply":"2022-01-10T15:31:11.456712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.3 INVESTIGATE A DISCOURSE WITH UNEQUAL LENGTHS AS DISCUSSED
\n\n---\n\n[**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin)\n\n> \"Let's check the first one.\"","metadata":{}},{"cell_type":"code","source":"unequal_ex_1 = train_df[train_df.discourse_len!=train_df.pred_len].head(1)\n\nprint(\"\\n\\n... RAW STRINGS ...\")\nprint(f\"\\n --> DISCOURSE STRING :\\n - {unequal_ex_1['discourse_text'].values[0]}\")\nprint(f\"\\n --> PREDICTION STRING :\\n - {unequal_ex_1['predictionstring'].values[0]}\")\n\nprint(\"\\n\\n\\n... RAW TOKENS ...\")\nprint(f\"\\n --> DISCOURSE TOKENS :\\n - {unequal_ex_1['discourse_text'].values[0].split()}\")\nprint(f\"\\n --> PREDICTION TOKENS :\\n - {unequal_ex_1['predictionstring'].values[0].split()}\")\n\nprint(\"\\n\\n\\n... NUMBER OF TOKENS ...\")\nprint(f\"\\n --> # OF DISCOURSE TOKENS :\\n - {len(unequal_ex_1['discourse_text'].values[0].split())}\")\nprint(f\"\\n --> # OF PREDICTION TOKENS :\\n - {len(unequal_ex_1['predictionstring'].values[0].split())}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:11.458869Z","iopub.execute_input":"2022-01-10T15:31:11.459287Z","iopub.status.idle":"2022-01-10T15:31:11.469926Z","shell.execute_reply.started":"2022-01-10T15:31:11.459250Z","shell.execute_reply":"2022-01-10T15:31:11.469142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.4 LENGTH, FREQUENCY & POSITION PER DISCOURSE TYPE
\n\n---\n\n[**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin)\n\n> \"Is there a correlation between the length of a discourse and the class (discourse_type)? Yes, there is. Evidence is the longest discount type on average. When looking at the frequencies of occurence, we see that Counterclaim and Rebuttal are relatively rare [...] We do have the field discourse_type_num. We see that Evidence1, Position1 and Claim1 are almost always there in an essay. Most students also had at least one Concluding Statement. What's surprising to me is that a Lead is missing in about 40% of the essays (Lead 1 is found in almost 60% of the essays).
The graph only plots `discourse_type_nums` which are found in at least 5% of the essays.
[...] Below that you can see a plot with the average positions of the discourse start and end. [...] We also know that a Lead is missing in around 40% of the essays. Below you can see that if there is a Lead, it's almost always the first discourse identified in an essay (Lead 2 is very rare anyway).\"\n\n
\n\n**NOTE: I expanded upon the last idea and plotted both the location of the chunks as an absolute position**\n* chunk 1 of 100 would be 1, chunk 100 of 100 would be 1, chunk 29 of 58 would be 29\n\n**AND as a relative position**\n* chunk 1 of 100 would be 0.01, chunk 100 of 100 would be 1.00, chunk 29 of 58 would be 0.50\n\n
","metadata":{}},{"cell_type":"code","source":"# Important new information\ntrain_df[\"discourse_absolute_position\"] = train_df.groupby(['id']).cumcount()\ntrain_df[\"total_discourse_chunks\"] = train_df.groupby(\"id\")[\"discourse_absolute_position\"].transform(max)\ntrain_df[\"discourse_relative_position\"] = train_df[\"discourse_absolute_position\"]/train_df[\"total_discourse_chunks\"]\n\n# Mappings for discourse types and names\ndt_n_map = {k:round(100*v/train_df.id.nunique(), 2) for k,v in train_df.discourse_type_num.value_counts().to_dict().items()}\ndt_n_keys = [k for k,v in dt_n_map.items() if v>5.0]\ndt_n_values = [dt_n_map[x] for x in dt_n_keys]\n\n# ########################### #\n# ######### Figures ######### #\n# ########################### #\nfig=px.bar(y=dt_n_keys, x=dt_n_values, text_auto=True, color=[x.split()[0] for x in dt_n_keys])\nfig.update_layout(title_text='Percentage of Examples Containing Each Discourse Type',\n xaxis_title_text='Percentage Of Examples', yaxis_title_text='Discourse Type',\n legend=dict(yanchor=\"top\", y=0.99, xanchor=\"right\", x=0.99), legend_title_text='Discourse Type').update_yaxes(categoryorder='total descending')\nfig.show()\n\nfig = px.histogram(train_df, y=\"discourse_type\", x=\"discourse_len\", histfunc='avg', color=\"discourse_type\", text_auto=True)\nfig.update_layout(showlegend=False, title_text='Average Number of Tokens For Each Discourse Type',\n xaxis_title_text='Average Number Of Tokens', yaxis_title_text='Discourse Type').update_yaxes(categoryorder='total descending')\nfig.show()\n\nfig = px.histogram(train_df, y=\"discourse_type\", x=\"discourse_len\", histfunc='count', color=\"discourse_type\", text_auto=True)\nfig.update_layout(showlegend=False, title_text='Total Token Count For Each Discoure Type',\n xaxis_title_text='Total Token Count', yaxis_title_text='Discourse Type', ).update_yaxes(categoryorder='total descending')\nfig.show()\n\ntmp_df = train_df.groupby(\"discourse_type\")[['discourse_start', 'discourse_end']].mean().sort_values(by='discourse_start', ascending = False).round()\nfig = px.bar(tmp_df, x=[\"discourse_end\", \"discourse_start\"], y=tmp_df.index, barmode=\"group\", text_auto=True)\nfig.update_layout(title_text='Absolute Average Start and End Position For Each Discourse Type', legend_traceorder=\"reversed\",\n xaxis_title_text='Absolute Average Position', yaxis_title_text='Discourse Type',\n legend=dict(yanchor=\"top\", y=0.99, xanchor=\"right\", x=0.99), legend_title_text='Start vs. End Position')\nfig.show()\n\nfig = px.histogram(train_df, x=\"discourse_absolute_position\", nbins=50, color=\"discourse_type\")\nfig.update_layout(title_text='Total Counts For ABSOLUTE Position Of Discourse Chunks Within Respective Paragraphs',\n xaxis_title_text='ABSOLUTE Position Of Discourse Chunk In Paragraph', yaxis_title_text='Number Of Discourse Chunks',\n legend=dict(yanchor=\"top\", y=0.99, xanchor=\"right\", x=0.99), legend_title_text='Discourse Type')\nfig.show()\n\nfig = px.histogram(train_df, x=\"discourse_relative_position\", nbins=50, color=\"discourse_type\")\nfig.update_layout(title_text='Total Counts For RELATIVE Position Of Discourse Chunks Within Respective Paragraphs',\n xaxis_title_text='RELATIVE Position Of Discourse Chunk In Paragraph', yaxis_title_text='Number Of Discourse Chunks',\n legend_title_text='Discourse Type')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:11.471313Z","iopub.execute_input":"2022-01-10T15:31:11.471880Z","iopub.status.idle":"2022-01-10T15:31:17.153825Z","shell.execute_reply.started":"2022-01-10T15:31:11.471845Z","shell.execute_reply":"2022-01-10T15:31:17.153224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.5 INVESTIGATING UNCLASSIFIED TEXT
\n\n---\n\n[**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin)\n\n> \"Just taking the last discourse_end in train is not entirely correct as a last piece of text may not have been used as a discourse. Therefore, I will go through the essays to find the real ends. Eh....until I remebered that Rob Mulla already did that in the excellent EDA (https://www.kaggle.com/robikscube/student-writing-competition-twitch) ;-). Please upvote his notebook!\"\n\n**NOTE: I will be covering Rob's notebook next... so I will skip the section that re-covers what he investigates to not duplicate any work (if possible).**\n\n[**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin)\n\n> \"Are there many really bad essays (large percentage of text not classified)?\nYes, we do have those. Some have around 90% of text not classified as one of the discourse types.\n>Regarding the one with gap_end_length 7348: I found out that this student just copied and pasted the same texts multiple times in his/her essay. See discussion topic: https://www.kaggle.com/c/feedback-prize-2021/discussion/298193\"\n\n**Following this Erik makes a function to display the text passages similar to the previous EDA conducted by Darek. However, Erik includes functionality to plot the gap text iin his code. I will update our previous function to include this functionality as well.**","metadata":{}},{"cell_type":"code","source":"# EDA SPECIFIC FUNCTIONS - SOME REWRITING MOSTLY @THEDRCAT AND @ERIKBRUIN\ndef visualize_2(df, _id=None):\n \"\"\" Function to visualize a single discourse with displacy and include gaps\n \n Args:\n df (pd.DataFrame): The dataframe to pull examples from.\n _id (str, optional): The string index used to identify the relevant body text\n to pull the discourse from. If no id is provided than a random one from the\n provided df will be selected.\n \n Returns:\n None; An example will be displayed.\n \"\"\"\n _df = df.copy()\n \n # Pick a random id if none is passed to the function\n if _id is None: _id = _df.sample(1).id.values[0]\n \n # Create the entity list which is a collection of discourse segments\n # where each discourse segment is described by it's start and end character\n # position as well as the discourse type label for that respective segment.\n entities = []\n for i, row in _df[_df['id']==_id].iterrows():\n \n if (row.discourse_absolute_position==0 and row.discourse_start>0):\n entities.append({\n 'start': 0, \n 'end': row['discourse_start'],\n 'label': \"Gap Text\"\n })\n \n entities.append({\n 'start': row['discourse_start'], \n 'end': row['discourse_end'], \n 'label': row['discourse_type']\n })\n \n if row.chunk_gap_length!=0:\n entities.append({\n 'start': row['discourse_end'], \n 'end': row['discourse_end']+row.chunk_gap_length, \n 'label': \"Gap Text\"\n })\n elif (row.discourse_relative_position>=1.0 and row.essay_c_len>row.discourse_end):\n entities.append({\n 'start': row['discourse_end'], \n 'end': row['essay_c_len'],\n 'label': \"Gap Text\"\n })\n \n # Open the correct text file\n with open(row[\"txt_path\"], 'r') as file: raw_txt_file = file.read()\n \n # Provide the required document details as a single dictionary\n # - \"text\": The Raw Text File\n # - \"ents\": A list of dictionaries describing the relevant discourse chunks\n # - \"title\": The title to ascribe to the visualization\n doc_details = {\n \"text\": raw_txt_file,\n \"ents\": entities,\n \"title\": f\"ID: {_id}\"\n }\n \n # A mapping of options that will allow us to customize the rendering that displacy will\n # generate. We leverage our previously defined colour map and class list.\n options = {\"ents\": DISCOURSE_CLASSES+[\"Gap Text\",], \"colors\": d2c_map}\n \n # Displacy's render function that we control with specific arguments\n displacy.render(docs=doc_details, style=\"ent\", options=options, manual=True, jupyter=True)\n\n# Create a dataframe with information about the gaps in the essays and display\ngap_df = train_df.groupby(\"id\").agg({'essay_c_len': 'first', 'chunk_gap_length': 'sum', 'discourse_end':'last'})\ngap_df[\"gap_at_end\"] = gap_df.apply(lambda row: 0 if row.discourse_end>=row.essay_c_len else row.essay_c_len-row.discourse_end, axis=1)\ngap_df[\"total_unclassified\"] = gap_df[\"gap_at_end\"]+gap_df[\"chunk_gap_length\"]\ngap_df[\"percent_unclassified\"] = (100*gap_df[\"total_unclassified\"]/gap_df[[\"essay_c_len\", \"discourse_end\"]].max(axis=1)).round(3)\ngap_df = gap_df.sort_values(by=\"percent_unclassified\", ascending=False)\n\nprint(\"\\n... INFORMATION ABOUT THE GAPS (NON-CLASSIFIED TEXT) IN THE STUDENT ESSAYS ...\\n\\n\")\ndisplay(gap_df.head(10)); print(\"\\n\\n\")\n\nvisualize_2(train_df, \"C278EDC82048\"); print(\"\\n\\n\")\nvisualize_2(train_df, \"D753B3C6BDDA\"); print(\"\\n\\n\")\nvisualize_2(train_df, \"129497C3E0FC\"); print(\"\\n\\n\")\nvisualize_2(train_df, \"BBE0D2A103B3\"); print(\"\\n\\n\")\nvisualize_2(train_df); print(\"\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:17.155241Z","iopub.execute_input":"2022-01-10T15:31:17.155639Z","iopub.status.idle":"2022-01-10T15:31:17.871505Z","shell.execute_reply.started":"2022-01-10T15:31:17.155599Z","shell.execute_reply":"2022-01-10T15:31:17.870683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"5.6 INVESTIGATE NGRAMS WITHIN THE ESSAYS
\n\n---\n\n[**Erik Bruin (erikbruin)**](https://www.kaggle.com/erikbruin)\n\n> \"Making n_grams for each discourse type. [...] I wanted to make a function to compose Top-10 n_grams per discount type by using **`CountVectorizer()`**. This function should also work for the single words (just run it with **`n_grams=1`**).\"\n\n","metadata":{}},{"cell_type":"code","source":"def get_n_grams(df, n_items, top_n=10):\n \"\"\"\n \n Retrieve the most common n-grams for each respective discourse type\n \n Original Function:\n Created By : Erik Bruin (https://www.kaggle.com/erikbruin)\n Created Within : NLP On Student Writing EDA (https://www.kaggle.com/erikbruin/nlp-on-student-writing-eda)\n \n I will simply be providing some extra annotation to help my understanding.\n \n \n \"\"\"\n df_words = pd.DataFrame()\n for dt in tqdm(DISCOURSE_CLASSES):\n # Get the relevant texts and create our count vectorizer\n texts = df[df.discourse_type==dt][\"discourse_text\"].tolist()\n count_vec = CountVectorizer(\n lowercase=True, stop_words='english', ngram_range=(n_items, n_items)\n ).fit(texts)\n \n # Use the count vectorizer to transform the text\n bag_of_words = count_vec.transform(texts)\n \n # Get statistics and relationships from bag of words\n sum_of_words = bag_of_words.sum(axis=0)\n words_freq = [(word, sum_of_words[0, idx]) for word, idx in count_vec.vocabulary_.items()]\n \n cvec_df = pd.DataFrame.from_records(words_freq, columns= ['words', 'counts']).sort_values(by=\"counts\", ascending=False)\n cvec_df.insert(0, \"discourse_type\", dt)\n cvec_df = cvec_df.iloc[:top_n,:]\n \n df_words = df_words.append(cvec_df)\n return df_words\n\nN_GRAMS_TO_GO_THROUGH=5\nTOP_N = 3\nfor i in range(N_GRAMS_TO_GO_THROUGH):\n train_ngram_df = get_n_grams(train_df, n_items=i, top_n=TOP_N)\n display(train_ngram_df.head(N_GRAMS_TO_GO_THROUGH*TOP_N))","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:31:17.873236Z","iopub.execute_input":"2022-01-10T15:31:17.873525Z","iopub.status.idle":"2022-01-10T15:32:51.732713Z","shell.execute_reply.started":"2022-01-10T15:31:17.873488Z","shell.execute_reply":"2022-01-10T15:32:51.731927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n\n---\n\nThis excellent EDA was written by [**Rob Mulla (robikscube)**](https://www.kaggle.com/robikscube) a Senior Data Scientist at [BioCore](https://biocorellc.com/). I will sample certain sections, comments, functions and/or code from his EDA to contribute to this survey. This will be contained within the below sections.","metadata":{}},{"cell_type":"markdown","source":"6.1 EDA SPECIFIC SETUP
\n\n---","metadata":{}},{"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:32:51.736293Z","iopub.execute_input":"2022-01-10T15:32:51.736616Z","iopub.status.idle":"2022-01-10T15:32:51.789512Z","shell.execute_reply.started":"2022-01-10T15:32:51.736580Z","shell.execute_reply":"2022-01-10T15:32:51.788669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6.2 THE TXT FILES
\n\n---\n\n[**Rob Mulla (robikscube)**](https://www.kaggle.com/robikscube)\n\n> \"What is the lengnth of each text file? We add info about the essays to training dataset [The length of the essay, The number of words in each essay].\"\n\n**I already retrieved this information above and credit Rob as the author.**\n\n[**Rob Mulla (robikscube)**](https://www.kaggle.com/robikscube)\n\n> \"Interestingly, most essays end around 6,000 words (I'm guessing that's the max length allowed for the essay)\"\n\n**I plot this and add colour in based on the discourse type to add an extra piece of information**","metadata":{}},{"cell_type":"code","source":"# .groupby(\"id\").first() is a clever way from Rob to simply get \n# the id specific information (repeated across all entries for that id)\nfig = px.scatter(train_df.groupby(\"id\").first(), x=\"essay_c_len\", y=\"essay_w_len\", color=\"discourse_type\")\nfig.update_layout(title_text='Essay Length - Character Count vs. Word Count',\n xaxis_title_text='Number Of Characters', yaxis_title_text='Number Of Words',\n legend=dict(yanchor=\"top\", y=0.99, xanchor=\"left\", x=0.01), legend_title_text='Discourse Type')\nfig.update_traces(marker={'size': 5})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:32:51.790701Z","iopub.execute_input":"2022-01-10T15:32:51.790926Z","iopub.status.idle":"2022-01-10T15:32:52.195956Z","shell.execute_reply.started":"2022-01-10T15:32:51.790894Z","shell.execute_reply":"2022-01-10T15:32:52.195069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6.3 WORDCLOUDS BY DISCOURSE TYPE
\n\n---\n\n**The code below is an exact copy from Rob's notebook. I couldn't figure out how to make it better or get further insights... Here's the [link to his notebook](https://www.kaggle.com/robikscube/student-writing-competition-twitch-stream#Wordclouds-by-Discourse-Type) showing the same thing**","metadata":{}},{"cell_type":"code","source":"# Create the figure and axes we will use (7 subplots for 7 classes)\nfig, axs = plt.subplots(7, 1, figsize=(20, 40))\nplt_idx = 0\n\nfor discourse_type, d in train_df.groupby(\"discourse_type\"):\n # Concatenate all the discourse texts into a single string\n discourse_text = \" \".join(d[\"discourse_text\"].values.tolist())\n \n # Generate the word cloud\n wordcloud = WordCloud(\n max_font_size=200,\n max_words=200,\n width=1400,\n height=800,\n colormap=\"copper\",\n background_color=\"white\",\n ).generate(discourse_text)\n \n # Manage the axes\n axs = axs.flatten()\n axs[plt_idx].imshow(wordcloud, interpolation=\"bilinear\")\n axs[plt_idx].set_title(discourse_type.upper(), fontsize=18)\n axs[plt_idx].axis(\"off\")\n plt_idx += 1\n\n# Plot\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:32:52.198826Z","iopub.execute_input":"2022-01-10T15:32:52.199146Z","iopub.status.idle":"2022-01-10T15:33:24.225775Z","shell.execute_reply.started":"2022-01-10T15:32:52.199114Z","shell.execute_reply":"2022-01-10T15:33:24.223477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"6.4 THE COMPETITION METRIC
\n\n---\n\n[**Rob Mulla (robikscube)**](https://www.kaggle.com/robikscube)\n\n> \"The metric is a version of micro f1 where matches are made between the predicted text and the ground truth when the overlap is >= 0.5. The metric page states that this happens in the process:\n> * all ground truths and predictions for a given class are compared.\n> * overlap between the ground truth and prediction is >= 0.5, and the overlap between the prediction and the ground truth >= 0.5 is a true positive. \n> * If multiple matches exist, the match with the highest pair of overlaps is taken.\n> * Any unmatched ground truths are false negatives and any unmatched predictions are false positives.\n> \n>
[...] Example scoring a single label.\n> * We will purposefully make the label not overlap and show how it will be a false positive.\n\n","metadata":{}},{"cell_type":"code","source":"def get_discourse_ex(df, _id=None, style=\"pred\"):\n \"\"\" Simple function to return predictions for a given id \n \n Adapted from code written by @robikscube\n \n Args:\n df (pd.DataFrame): The dataframe to query for the given id\n _id (str, optional): The string index used to identify the relevant essay\n to pull the discourse chunks. If no id is provided than a random one from the\n provided df will be selected.\n style (str,optional): One of ['pred'|'gt'|'both']. \n Used to identify whether to return the prediction, ground truth or both\n \n Returns:\n A pd.DataFrame object containing rows each discourse type\n for the respective `id` where each row contains the id, \n discourse type and prediction string.\n \n Raises:\n NotImplemnetedError() if 'style' other than 'pred' is passed as only 'pred' \n is supported currently\n \"\"\"\n if style!=\"pred\":\n raise NotImplementedError(\"\\n\\n... Only style=='pred' is currently implemented ...\\n\\n\")\n \n _df = df.copy()\n \n if _id is None:\n _id = _df.id.sample(1).values[0]\n \n _df = _df[[\"id\", \"discourse_type\", \"predictionstring\"]].query('id == @_id')\n _df = _df.rename(columns={\"discourse_type\": \"class\"}).reset_index(drop=True)\n \n return _df\n\n# Create pred dataframe for a random example\nex_pred_df = get_discourse_ex(train_df, \"423A1CA112E2\")\n\n# Copy pred dataframe to create ground truth dataframe\nex_gt_df = get_discourse_ex(train_df, \"423A1CA112E2\").rename(columns={\"class\": \"discourse_type\"})\n\n# Change a single prediction string in the pred dataframe so the score isn't perfect\nex_pred_df.loc[0, \"predictionstring\"] = \" \".join(ex_pred_df[\"predictionstring\"].values[0].split(\" \")[:10])\n\n# We also change the class for a different prediction to introduce that error as well\nex_pred_df.loc[5, \"class\"] = \"Lead\" if ex_gt_df.loc[1, \"discourse_type\"]!=\"Lead\" else \"Position\"\n\njoined_example = ex_pred_df.merge(\n ex_gt_df,\n left_on=[\"id\", \"class\"],\n right_on=[\"id\", \"discourse_type\"],\n how=\"outer\",\n suffixes=(\"_pred\", \"_gt\"))\n\ndisplay(joined_example.head(5))\n\n# Let's look at the first discourse example (the one we truncated to size 10)\nset_pred = set(joined_example[\"predictionstring_pred\"][0].split(\" \"))\nset_gt = set(joined_example[\"predictionstring_gt\"][0].split(\" \"))\n\n# Find the lengths of the sets for this example\nlen_gt = len(set_gt)\nlen_pred = len(set_pred)\n\n# Find size of intersection\ninter = len(set_gt.intersection(set_pred))\n\n# Find overlap\noverlap_1 = inter / len_gt\noverlap_2 = inter / len_pred\n\nprint(\"\\n\\n... FIRST DISCOURSE PRED v. GT DETAILS ...\\n\")\nprint(f\"GROUND TRUTH LENGTH : {len_gt}\")\nprint(f\"PREDICTION LENGTH : {len_pred}\")\nprint(f\"INTERSECTION LENGTH : {inter}\\n\")\n\n# In this example both overlap percentages are not >= 0.5 so it is not a true positive\nprint(f\"OVERLAP 1 LENGTH : {round(overlap_1, 3)}\")\nprint(f\"OVERLAP 2 LENGTH : {round(overlap_2, 3)}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:33:24.227196Z","iopub.execute_input":"2022-01-10T15:33:24.227639Z","iopub.status.idle":"2022-01-10T15:33:24.331239Z","shell.execute_reply.started":"2022-01-10T15:33:24.227606Z","shell.execute_reply":"2022-01-10T15:33:24.330362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[**Rob Mulla (robikscube)**](https://www.kaggle.com/robikscube)\n\n> \"The metric is a version of micro f1 where matches are made between the predicted text and the ground truth when the overlap is >= 0.5. The metric page states that this happens in the process:\n> * all ground truths and predictions for a given class are compared.\n> * overlap between the ground truth and prediction is >= 0.5, and the overlap between the prediction and the ground truth >= 0.5 is a true positive. \n> * If multiple matches exist, the match with the highest pair of overlaps is taken.\n> * Any unmatched ground truths are false negatives and any unmatched predictions are false positives.\n> \n>
[...] Example scoring a single label.\n> * We will purposefully make the label not overlap and show how it will be a false positive.\n> \n>
[...] Note `@cdeotte` noted in the comments. The scoring metric for this competition is **macro_f1** and **`thescore_feedback_comp`** function now computes the **macro f1 score to follow the evaluation page**.\"\n\n**I will go through the functions and double check my understanding and potentially modify or add details/things as I see fit... all credit goes to Rob. I will also share a snippet from a StackExchange Question/Answer that explains a bit more about Macro v. Micro scores...**\n\n> \"A macro-average will compute the metric independently for each class and then take the average (hence treating all classes equally), whereas a micro-average will aggregate the contributions of all classes to compute the average metric. In a multi-class classification setup, micro-average is preferable if you suspect there might be class imbalance (i.e you may have many more examples of one class than of other classes).\"","metadata":{}},{"cell_type":"code","source":"def calc_overlap(row):\n \"\"\"\n \n Calculates the overlap between prediction and\n ground truth and overlap percentages used for determining\n true positives.\n \n Args:\n row (pd.Series): A single row from a dataframe containing\n both the prediction and ground truth string\n \n Returns:\n A list containing the two overlap lengths\n \n \"\"\"\n \n # Get unique words in the prediction string and ground truth string\n set_pred, set_gt = set(row.predictionstring_pred.split(\" \")), set(row.predictionstring_gt.split(\" \"))\n \n # Length of each set as well as the intersection length\n len_gt, len_pred = len(set_gt), len(set_pred)\n inter = len(set_gt.intersection(set_pred))\n overlap_1, overlap_2 = inter/len_gt, inter/len_pred\n \n return [overlap_1, overlap_2]\n\n\ndef score_feedback_comp_micro(pred_df, gt_df):\n \"\"\"\n A function that scores for the kaggle Student Writing Competition\n\n Uses the steps in the evaluation page here:\n https://www.kaggle.com/c/feedback-prize-2021/overview/evaluation\n \n Args:\n pred_df (pd.DataFrame): Dataframe containing prediction values\n gt_df (pd.DataFrame): Dataframe containing ground truth values\n \n Returns:\n The value of the micro F1 score comparing the two dataframes\n \"\"\"\n \n # Reduce pandas dataframe to the relevant columns, reset the indices and prevent shallow copying\n gt_df = (gt_df[[\"id\", \"discourse_type\", \"predictionstring\"]].reset_index(drop=True).copy())\n pred_df = pred_df[[\"id\", \"class\", \"predictionstring\"]].reset_index(drop=True).copy()\n \n # Create a column to hold the original index\n gt_df[\"gt_id\"], pred_df[\"pred_id\"] = gt_df.index, pred_df.index\n\n ######################################################################\n ######################################################################\n ############ MAIN STEPS TO CALCULATE MICRO F1 ############\n ######################################################################\n ######################################################################\n \n ######################################################################\n # STEP 1. Compare Ground Truth w/ Predicted Value For A Given Class #\n ######################################################################\n \n # Simple outer merge to bring together the two dataframes\n joined = pred_df.merge(\n gt_df,\n left_on=[\"id\", \"class\"],\n right_on=[\"id\", \"discourse_type\"],\n how=\"outer\",\n suffixes=(\"_pred\", \"_gt\"),\n )\n \n # Fill the nan values with a blank to prevent errors\n joined[\"predictionstring_gt\"] = joined[\"predictionstring_gt\"].fillna(\" \")\n joined[\"predictionstring_pred\"] = joined[\"predictionstring_pred\"].fillna(\" \")\n\n # Utilize our calc overlap function to determine the respective overlap\n # values for each discourse chunk (row) in the pred/gt dataframes\n joined[\"overlaps\"] = joined.apply(calc_overlap, axis=1)\n\n ######################################################################\n # STEP 2. Conditional Handling For Overlap and Multiple Matches #\n ######################################################################\n # - If the overlap between the ground truth and prediction is # \n # >= 0.5, and the overlap between the prediction and the #\n # ground truth >= 0.5, the prediction is a match and #\n # considered a true positive. #\n # #\n # - If multiple matches exist, the match with the highest #\n # pair of overlaps is taken. #\n ######################################################################\n \n # Get the percentage overlap and save it to two new columns (respectively)\n joined[\"overlap1\"] = joined[\"overlaps\"].apply(lambda x: eval(str(x))[0])\n joined[\"overlap2\"] = joined[\"overlaps\"].apply(lambda x: eval(str(x))[1])\n \n # Look for potential true positives by identifying rows where\n # the overlap value in both overlap columns is greater (or equal) to 50%\n # We also capture the maximum overlap between the two respective overlap columns \n # and save that to a new column\n joined[\"potential_TP\"] = (joined[\"overlap1\"] >= 0.5) & (joined[\"overlap2\"] >= 0.5)\n joined[\"max_overlap\"] = joined[[\"overlap1\", \"overlap2\"]].max(axis=1)\n \n # Now we capture the ids for which we have succesfully predicted with a\n # high enough overlap. If more than one prediction we take the `.first()` one.\n tp_pred_ids = (\n joined.query(\"potential_TP\")\n .sort_values(\"max_overlap\", ascending=False)\n .groupby([\"id\", \"predictionstring_gt\"])\n .first()[\"pred_id\"]\n .values\n )\n\n ######################################################################\n # STEP 3. Handle False Positive and False Negatives #\n ######################################################################\n # - Any unmatched ground truths are false negatives # \n # - Any unmatched predictions are false positives #\n ######################################################################\n\n # Assume all submitted predictions which are not TPs are FP\n # i.e. predicted incorrectly\n fp_pred_ids = [p for p in joined[\"pred_id\"].unique() if p not in tp_pred_ids]\n\n # All remaining unmatched ground truth segments are now classified as FN\n # i.e. failed to predict\n matched_gt_ids = joined.query(\"potential_TP\")[\"gt_id\"].unique()\n unmatched_gt_ids = [c for c in joined[\"gt_id\"].unique() if c not in matched_gt_ids]\n\n # Get the numbers found for each type\n TP = len(tp_pred_ids)\n FP = len(fp_pred_ids)\n FN = len(unmatched_gt_ids)\n \n \n ######################################################################\n # STEP 4. Calculate MicroF1 #\n ######################################################################\n \n # Calculate the F1 score (micro)\n my_f1_score = TP / (TP + 0.5 * (FP + FN))\n \n ######################################################################\n ######################################################################\n ####################### MAIN STEPS FINISHED ##########################\n ######################################################################\n ######################################################################\n \n return my_f1_score\n\n\ndef score_feedback_comp(pred_df, gt_df, return_class_scores=False):\n \"\"\" Function to calculate the `score` used in this Feedback Competition \n \n Args:\n pred_df (pd.DataFrame): Dataframe containing prediction values\n gt_df (pd.DataFrame): Dataframe containing ground truth values\n return_class_scores (bool, optional): Whether or not to \n return the individual classwise scores\n \n Returns:\n Either returns only the Macro F1 Score or returns both\n the individual Classwise Macro F1 Scores as well as the Overall\n Macro F1 Score\n \n \"\"\"\n \n # Initialize a Classwise Scoring Dictionary\n class_scores = {}\n \n # Grab only the part of the dataframe that is relevant to calculating the score\n pred_df = pred_df[[\"id\", \"class\", \"predictionstring\"]].reset_index(drop=True).copy()\n \n # Iterate over a given discourse type and the respective ground truth df subset \n # - tqdm for visual progressbar\n for discourse_type, gt_subset in tqdm(gt_df.groupby(\"discourse_type\"), total=len(gt_df.groupby(\"discourse_type\"))):\n # Get the pred subset to compare to the ground truth \n # subset for the respective discourse type\n pred_subset = (\n pred_df.loc[pred_df[\"class\"] == discourse_type]\n .reset_index(drop=True)\n .copy()\n )\n \n # Calculate the classwise score (Micro F1) and store in dictionary\n class_score = score_feedback_comp_micro(pred_subset, gt_subset)\n class_scores[discourse_type] = class_score\n \n # To calculate Macro we simply average the classwise Micro scores.\n f1 = np.mean([v for v in class_scores.values()])\n \n # Only return the classwise information if argument is passed\n if return_class_scores:\n return f1, class_scores\n else:\n return f1\n \ngt_df = train_df.copy()\npred_df = gt_df.copy()\npred_df = pred_df.rename(columns={\"discourse_type\": \"class\"})\n\nmicrof1_score = score_feedback_comp_micro(pred_df, gt_df)\nmacrof1_score, class_scores = score_feedback_comp(\n pred_df, gt_df, return_class_scores=True\n)\n\nprint(\n f\"Using the ground truth to predict the micro f1 score is {microf1_score:0.4f} and macro f1 score {macrof1_score:0.4f}\"\n)\nprint(\"The individual class scores are:\")\nprint(class_scores)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:33:24.332981Z","iopub.execute_input":"2022-01-10T15:33:24.333461Z","iopub.status.idle":"2022-01-10T15:34:33.326129Z","shell.execute_reply.started":"2022-01-10T15:33:24.333423Z","shell.execute_reply":"2022-01-10T15:34:33.325336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n\n---\n\n","metadata":{}},{"cell_type":"markdown","source":"7.1 EDA SPECIFIC IMPORTS
\n\n---","metadata":{}},{"cell_type":"code","source":"from transformers import *","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:34:33.327709Z","iopub.execute_input":"2022-01-10T15:34:33.328200Z","iopub.status.idle":"2022-01-10T15:34:36.636919Z","shell.execute_reply.started":"2022-01-10T15:34:33.328162Z","shell.execute_reply":"2022-01-10T15:34:36.636026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"7.2 How to Convert Characters, Tokens, and Words
\n\n---\n\n[**Chris Deotte (cdeotte)**](https://www.kaggle.com/cdeotte)\n\n> \"How to Create Train Labels and Infer Test Labels
Models train with tokens and output tokens. Kaggle provides train data as characters and words. The LB requires words. Below I show how to use a character array to help convert between characters, tokens, and words.\"\n>\n> Let's consider the example text \"Hello, my name is Chris Deotte. How are you?\". This text contains 44 characters, 13 tokens, and 9 words. (When I say \"words\", i'm referring to the result of Python's **`split()`** function). Notice that words are not tokens.Also note the result will be tokens (where the funny character ฤ represents a space):\n\n**I replace the funny ฤ below for ease of visualization. Chris created the figure below as well to illustrate the different ways characters, tokens and words are demarcated.**\n\n","metadata":{}},{"cell_type":"code","source":"# Setup\ntokenizer = AutoTokenizer.from_pretrained(\"allenai/longformer-base-4096\")\ndemo_str = \"Hello, my name is Chris Deotte. How are you?\"\n\n# ################################################### #\n# ############### STEP 1 - CHARACTERS ############### #\n# ################################################### #\n\n# Get Characters\nchars = list(demo_str)\n\n# Visualize the Characters\nprint(f\"\\n[{len(chars)}] CHARACTERS IN THE STRING --> '{demo_str}'\")\nfor i, x in enumerate(chars): print(f\"\\tCharacter #{i+1} --> {x}\")\n\n\n# ################################################### #\n# ############### STEP 2 - TOKENS ############### #\n# ################################################### #\n\n# Tokenize\ntokens = tokenizer.tokenize(demo_str)\n\n# Replace ฤ With A Space\ntokens = [x.replace(\"\\u0120\", \" \") for x in tokens]\n\n# Visualize the Tokens\nprint(f\"\\n[{len(tokens)}] TOKENS IN THE STRING --> '{demo_str}'\")\nfor i, x in enumerate(tokens): print(f\"\\tToken #{i+1} --> {x}\")\n\n# ################################################### #\n# ############### STEP 3 - WORDS ############### #\n# ################################################### #\n\n# Get Words\nwords = demo_str.split()\n\n# Visualize the Words\nprint(f\"\\n[{len(words)}] WORDS IN THE STRING --> '{demo_str}'\")\nfor i, x in enumerate(words): print(f\"\\tWord #{i+1} --> {x}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:34:36.638681Z","iopub.execute_input":"2022-01-10T15:34:36.638940Z","iopub.status.idle":"2022-01-10T15:34:39.765906Z","shell.execute_reply.started":"2022-01-10T15:34:36.638904Z","shell.execute_reply":"2022-01-10T15:34:39.765163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n[**Chris Deotte (cdeotte)**](https://www.kaggle.com/cdeotte)\n\n> \"The Secret of Convert is Use Char Array
In this competition, **Kaggle provides LABELS as CHARACTERS (`discourse_start`, `discourse_end`) AND WORDS (`predictionstring`).** Let's imagine that the target (**`discourse_text`**) is **`\"my name is Chris Deotte.\"`**. Therefore we need to convert these labels to token labels. **The easiest way to convert to tokens or from tokens is to use an intermediate character array.**\"\n\n\n","metadata":{}},{"cell_type":"code","source":"MAX_LEN = 10\ndemo_d_text = \"my name is Chris Deotte.\"\ndemo_d_type = \"Claim\"\ndemo_d_start = 7\ndemo_d_end = 31\ndemo_pred_str = \"1 2 3 4 5\"\n\nprint(f\"\\n... DEMO STRING ...\\n\\t--> '{demo_d_text}'\\n\")\n\n'''\nFrom @cdeotte - reformatted for code comment to improve inline readability\n\n - Let's say that we know our target's discourse_start and discourse_end \n and we want to create tokens with targets. \n - First create an array of what we want, i.e token array. \n - In this example, 0 will designate no target\n - In this example 1 will be a positive target\n - We can then replace 1's with our target class numbers\n - Next we create a character array and place Kaggle's target into it\n - Next, iterate over the token array and grab the respective \n target from character array!\n - Now token_array has a 1 where there exists a target and 0 otherwise. \n - We can train our model with this MAX_LEN sized token array.\n'''\n\n# STEP 1 - CREATE DESTINATION TOKEN ARRAY\ndemo_token_array = np.zeros(MAX_LEN)\n\n# STEP 2 - PUT TARGET INTO ORIGIN CHARACTER ARRAY\ndemo_char_array = np.zeros((len(demo_d_text)))\ndemo_char_array[demo_d_start:demo_d_end] = 1\nprint(f\"\\n... DEMO CHARACTER ARRAY ...\\n\\t--> {demo_char_array}\\n\")\n\n# STEP 3 - FILL TOKEN ARRAY FROM CHARACTER ARRAY\ndemo_encoding = tokenizer(demo_d_text, max_length=MAX_LEN, padding='max_length', \n truncation=True, return_offsets_mapping=True)\nprint(f\"\\n... DEMO ENCODING ...\\n\\t--> {demo_encoding}\\n\")\n\ndemo_map_token_to_char = demo_encoding['offset_mapping']\nprint(f\"\\n... DEMO TOKEN TO CHAR MAP ...\\n\\t--> {demo_map_token_to_char}\\n\")\n\nfor k in range(MAX_LEN):\n demo_token_array[k] = demo_char_array[demo_map_token_to_char[k][0]]\nprint(f\"\\n... DEMO TOKEN ARRAY ...\\n\\t--> {demo_token_array}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-10T15:35:45.234000Z","iopub.execute_input":"2022-01-10T15:35:45.234581Z","iopub.status.idle":"2022-01-10T15:35:45.247341Z","shell.execute_reply.started":"2022-01-10T15:35:45.234545Z","shell.execute_reply":"2022-01-10T15:35:45.246483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n[**Chris Deotte (cdeotte)**](https://www.kaggle.com/cdeotte)\n\n> \"Inference - Convert Tokens to Words
After inference, our predictions are in a token array. We want to convert them to a word array to submit to LB. First create an array of what you want, i.e. word array. We now have a word_array which has 1's where the target is and 0's otherwise. We then generate the `predictionstring`. That's it! Using an intermediate character array makes converting from tokens to words and words to tokens easy! The only missing code is making the map_word_to_char. We [show that below too]\"\n\n","metadata":{}},{"cell_type":"code","source":"# For testing purposes...\nclass DumbModel:\n def __init__(self,):\n pass\n def predict(self, text):\n return list(np.round(np.random.random(len(text))))\n \ndef create_word2char_map(text, blank_chars=(' ','\\n','\\xa0','\\x85',)):\n \"\"\" Function to create word to char mapping\n \n From @cdeotte - reformatted as a function to improve readability/funcitonality\n \n Args:\n text (str): String to create the mapping from\n blank_chars (tuple of strings): The chars to consider as blank space\n \n Returns:\n The prediction formatted correctly as a word array\n \n \"\"\"\n \n # Initialize\n word_start = True\n word2char_map = []\n \n # Loop to create mapping\n for i, char in enumerate(text):\n if (char not in blank_chars)&(word_start==True):\n word2char_map.append(i)\n word_start=False\n elif char in blank_chars:\n word_start=True\n return word2char_map\n\ndef text_inference(model, text, tokenizer, max_len=MAX_LEN, return_pred_str=True):\n \"\"\" Function to perform inference and format appropriately\n \n From @cdeotte - reformatted as a function to improve readability/funcitonality\n \n Args:\n model (): The model to use for inference\n text (str): String to perform inference on\n tokenizer (PreTrainedTokenizerFast): The tokenizer that was\n fit on the respective text corpus for this task\n max_len (int, optional): The maximum allowed number of tokens\n return_pred_str (bool, optional): Whether to return the mask\n array or the respective prediction string.\n \n Returns:\n The prediction formatted correctly as a word array\n \n \"\"\"\n\n # STEP 1 - CREATE DESTINATION WORD ARRAY\n word_array = np.zeros((len(text.split())), dtype=np.int32)\n\n # STEP 2 - CREATE INTERMEDIATE CHARACTER ARRAY\n char_array = np.zeros((len(text)), dtype=np.int32)\n\n # STEP 3 - FILL CHAR ARRAY FROM PREDICTION TOKEN ARRAY\n preds = model.predict(text) \n \n _encoding = tokenizer(text, max_length=max_len, padding='max_length', truncation=True, return_offsets_mapping=True)\n _map_token_to_char = _encoding['offset_mapping']\n _word2char_map = create_word2char_map(text)\n \n for k in range(max_len): \n char_array[_map_token_to_char[k][0]:_map_token_to_char[k][1]] = preds[k]\n\n # STEP 4 - FILL WORD ARRAY FROM CHARACTER ARRAY\n word_array = np.array([char_array[_word2char_map[k]] for k in range(len(word_array))], dtype=np.int32)\n \n if not return_pred_str:\n return word_array\n else:\n return ' '.join([str(n) for n in np.where(word_array==1)[0]])\n \n\n_input_string = \"my name is Chris Deotte.\"\nprint(f\"\\n... INPUT STRING : {_input_string}\")\n\npred_raw = text_inference(DumbModel(), _input_string, tokenizer, return_pred_str=False)\nprint(f\"... RAW BINARY PREDICTION : {pred_raw}\")\n\npred_str = ' '.join([str(n) for n in np.where(pred_raw==1)[0]])\nprint(f\"... PREDICTION STRING : '{pred_str}'\")\nprint()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T16:08:11.776570Z","iopub.execute_input":"2022-01-10T16:08:11.776877Z","iopub.status.idle":"2022-01-10T16:08:11.793164Z","shell.execute_reply.started":"2022-01-10T16:08:11.776846Z","shell.execute_reply":"2022-01-10T16:08:11.792212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n---\n\n[**Chris Deotte (cdeotte)**](https://www.kaggle.com/cdeotte)\n\n> \"
Kaggle's Preprocess Algorithm
After brainstorming in the forums with others (especially helpful was [@hkaggleqrdl](https://www.kaggle.com/hkaggleqrdl) ), we have discovered Kaggle's preprocess function for creating the **`predictionstring`** from discourse text using Python's **`split()`**.\n>\n>The algorithm below explains the **mystery of why we are seeing 2% of `predictionstring` that do not match discourse text in the train data**. The **main cause** of the discrepancy is that **the first \"word\" (which may be a single punctuation mark like a comma) of discourse text is sometimes a portion of a longer \"word\" in the full text**. (Here \"word\" means connected sequence of characters without blank space).
Mystery Algorithm
The following algorithm produces 100% of train's predictionstring's perfectly when using discourse_start and discourse_end. So the mystery is solved! ๐","metadata":{}},{"cell_type":"code","source":"def mystery_algorithm(full_text, discourse_start, discourse_end):\n \"\"\" Algorithm reverse engineered by @cdeotte and team \"\"\"\n \n char_start = discourse_start\n char_end = discourse_end\n word_start = len(full_text[:char_start].split())\n word_end = word_start + len(full_text[char_start:char_end].split())\n word_end = min( word_end, len(full_text.split()) )\n predictionstring = \" \".join( [str(x) for x in range(word_start,word_end)] )\n \n return predictionstring\n\ndef calc_word_indices(full_text, discourse_start, discourse_end):\n \"\"\" Algorithm shared by host \"\"\"\n start_index = len(full_text[:discourse_start].split())\n token_len = len(full_text[discourse_start:discourse_end].split())\n output = list(range(start_index, start_index + token_len))\n if output[-1] >= len(full_text.split()):\n output = list(range(start_index, start_index + token_len-1))\n \n return output\n\nexample_full_text_1 = \"If you want to learn data science, then you should join Kaggle. There are opportunities to discuss, code, and compete.\"\ndiscourse_text = \", then you should join Kaggle.\"\n\nprint(discourse_text)\nprint(mystery_algorithm(example_full_text_1, example_full_text_1.index(discourse_text), example_full_text_1.index(discourse_text)+len(discourse_text)))\nprint(calc_word_indices(example_full_text_1, example_full_text_1.index(discourse_text), example_full_text_1.index(discourse_text)+len(discourse_text)))\nprint(np.array(example_full_text_1.split())[np.array(calc_word_indices(example_full_text_1, example_full_text_1.index(discourse_text), example_full_text_1.index(discourse_text)+len(discourse_text)))])","metadata":{"execution":{"iopub.status.busy":"2022-01-10T16:28:15.226931Z","iopub.execute_input":"2022-01-10T16:28:15.227413Z","iopub.status.idle":"2022-01-10T16:28:15.240597Z","shell.execute_reply.started":"2022-01-10T16:28:15.227355Z","shell.execute_reply":"2022-01-10T16:28:15.239835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n\n\n\n\n\n\n 8 MY EDA! ⤒\n
\n\n---\n\nHmmm, I'm not 100% sure what there is left to contribute but I wanted to give it a shot!","metadata":{}},{"cell_type":"markdown","source":"8.1 EDA SPECIFIC IMPORTS
\n\n---","metadata":{}},{"cell_type":"code","source":"import cudf, cuml, cupy\nfrom cuml.feature_extraction.text import TfidfVectorizer\nfrom cuml.cluster import KMeans\nfrom cuml.manifold import TSNE","metadata":{"execution":{"iopub.status.busy":"2022-01-10T16:28:53.116793Z","iopub.execute_input":"2022-01-10T16:28:53.117060Z","iopub.status.idle":"2022-01-10T16:28:55.226863Z","shell.execute_reply.started":"2022-01-10T16:28:53.117030Z","shell.execute_reply":"2022-01-10T16:28:55.226144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"8.2 TFIDF
\n\n---\n\n
\n\n![](\"https://miro.medium.com/max/1400/0*SbYJ91S927H-xbmd\")
\n\n
\n\nImage and Quote From [**This Blog Post**](https://medium.com/rapids-ai/accelerating-tf-idf-for-natural-language-processing-with-dask-and-rapids-6f6e416429df)\n\n> Term frequency-inverse document frequency (TF-IDF) is a scoring measure widely used in information retrieval (IR) or summarization used to reflect how relevant a term is in a given document. TF-IDF is a widely used Natural Language Processing (NLP) technique in document retrieval, summarization, text classification, and document clustering. (For more details check out this blog post).\n>\n> [Below...] we will show how to use NVIDIA GPUs to accelerate TF-IDF end-to-end pipelines using RAPIDS.\n\n**I wrote the functions below because initially I was going to preprocess manually... however, I will simply use the built in TFIDF vetorizer within CuML. As such, I have hidden the cell below. If you'd like to see the previous code feel free to expand it and examine it.**\n\n
\n\nKMeans clustering is an algorithm that aims to partition n observations into k clusters in which each observation belongs to the cluster with the nearest center. Cluster centers (or cluster centroid) are simply the clusterโs mean. KMeans require us to provide the number of cluster k beforehand. Ideally, we want to choose the number of clusters carefully. One technique to find a good number of clusters is the Elbow Method. Essentially, the method consists of plotting the explained variation as a function of the number of clusters, and picking the elbow of the curve as the number of clusters. However, for simplicity, Iโll arbitrarily set k = 10 as it seems to be decent for this dataset.\n\nโElbowโ Method\nWith the Elbow method, we plot the within-cluster sum of squares (RSS) for different K values and try to see what looks like an elbow. Letโs test K being 1 to 8.","metadata":{}},{"cell_type":"code","source":"def preprocess_text(input_strs , filters=None , stopwords=None):\n \"\"\"\n \n Function to preprocess a column of text from a CuDF dataframe\n \n Code from NVIDIA blog post \n --> https://developer.nvidia.com/blog/nlp-and-text-precessing-with-rapids-now-simpler-and-faster/\n \n --------------------------------------------\n STEPS:\n --------------------------------------------\n * filter punctuation\n * to_lower\n * remove stop words (from nltk corpus)\n * remove multiple spaces with one\n * remove leading spaces \n ----------------------------------------\n \n Args:\n input_strs (CuDF Series): Strings to be preprocessed\n filters (list of strs, optional): Which symbols to remove\n - If no list is provided a default list will be used.\n stopwords (list of strs, optional): Which words to remove\n - If no list is provided a default from nltk will be used.\n \n Returns:\n CuDF series of strings (i.e. a single column) after preprocessing\n conducted on the GPU\n \n \"\"\"\n \n # Defaults\n _STOPWORDS = nltk.corpus.stopwords.words('english') \n _FILTERS = [ '!', '\"', '#', '$', '%', '&', '(', ')', '*', \n '+', '-', '.', '/', '\\\\', ':', ';', '<', '=', \n '>', '?', '@', '[', ']', '^', '_', '`', '{', \n '|', '}', '\\~', '\\t','\\\\n',\"'\",\",\",'~' , 'โ' ]\n \n # Assign default if no specific arg is passed passed\n if stopwords is None:\n stopwords=_STOPWORDS\n if filters is None:\n filters=_FILTERS\n \n # filter punctuation and case conversion\n input_strs = input_strs.str.lower().replace_multi(FILTERS, ' ', regex=False)\n \n # place stopwords on device to accelerate stopword removal\n stopwords_gpu = nvstrings.to_device(stopwords)\n input_strs = nvtext.replace_tokens(input_strs.data, stopwords_gpu, ' ')\n input_strs = cudf.Series(input_strs)\n \n # replace multiple spaces with single one and strip leading/trailing spaces\n input_strs = input_strs.str.replace(r\"\\s+\", ' ', regex=True)\n input_strs = input_strs.str.strip(' ')\n \n return input_strs\n\ndef preprocess_text_df(df, text_cols=('discourse_text',), **kwargs):\n \"\"\"\n \n Function to preprocess a CuDF dataframe and the respective text columns\n \n Code from NVIDIA blog post \n --> https://developer.nvidia.com/blog/nlp-and-text-precessing-with-rapids-now-simpler-and-faster/\n \n Args:\n df (CuDF Dataframe): The dataframe to preprocess\n text_cols (tuple, optional): The columns containing text that\n we want to be preprocessed.\n \n Returns:\n The preprocessed CuDF dataframe\n \n \"\"\"\n _df = df.copy()\n \n for col in text_cols:\n _df[col] = preprocess_text(_df[col], **kwargs)\n return _df\n\n# preprocess_text_df(train_df)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-01-06T16:28:48.372109Z","iopub.execute_input":"2022-01-06T16:28:48.372485Z","iopub.status.idle":"2022-01-06T16:28:48.384288Z","shell.execute_reply.started":"2022-01-06T16:28:48.372447Z","shell.execute_reply":"2022-01-06T16:28:48.383494Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create TFIDF vectorizer from the RAPIDS CuML library\n# - N_FEATURES refers to the number of dimensions we will use\n# to express the datacontained within our corpus of text\nN_FEATURES = 5_000\ntfidf_vec = TfidfVectorizer(stop_words='english', max_features=N_FEATURES)\ncudf_corpus = cudf.Series(train_df[\"discourse_text\"])\ntfidf_matrix = tfidf_vec.fit_transform(cudf_corpus)\n\n# This is to convert our sparse matrix to a dense one and subset if necessary\n# - Change SUBSAMPLE_LEN to a smaller constant for a smaller dense subset\nSUBSAMPLE_LEN = tfidf_matrix.shape[0] # 100_000\ntfidf_dense_sample = tfidf_matrix[:SUBSAMPLE_LEN].todense()\n\nprint(f\"\\n... TFIDF MATRIX SHAPE: {tfidf_matrix.shape} ...\\n\")\n\n# KMEANS Variables\nnum_clusters = 25 # arbitrary\nkmeans_model = KMeans(n_clusters=num_clusters, n_init=1, max_iter=1000)\n\n# Calculate clusters and representative distances\nkmeans = kmeans_model.fit(tfidf_dense_sample)\nkmeans_clusters = kmeans.predict(tfidf_dense_sample)\nkmeans_distances = kmeans.transform(tfidf_dense_sample)\n\n# Using the previously computed TF-IDF, we can look at the most important words \n# for each of our clusters (which are the words with the highest tf-idf value \n# in each of the cluster centers). We filter terms that are also present in \n# other clusters as they do not help us distinguish different topics. \nsorted_centroids = kmeans.cluster_centers_.argsort()[:, ::-1]\nterms = tfidf_vec.get_feature_names()\n\nTOP_N = 100\nclusters_terms = sorted_centroids[:, :TOP_N].get()\n\nfor i, c1 in enumerate(clusters_terms):\n cluster = set(c1)\n for j, c2 in enumerate(clusters_terms):\n if i == j:\n continue\n cluster -= set(c2)\n cluster = c1[np.isin(c1, list(cluster))][:10]\n \n k_cluster = train_df[(kmeans_clusters==i).get()]\n print(f\"\\n\\n--------------- CLUSTER {i+1} ---------------\\n\" \\\n f\"\\n\\tCommon Terms: \" \\\n f\"{', '.join(terms[cluster].to_arrow().to_pylist())}\")\n \n print(f\"\\n\\tLabel Distribution: {k_cluster.discourse_type.value_counts().to_dict()}\")\n \n for j in range(3):\n d_ex = k_cluster.sample(1)\n print(f\"\\n\\tCluster Example #{j+1}\" \\\n f\"\\n\\t\\tDiscourse TYPE: --> {d_ex['discourse_type'].values[0]}\" \\\n f\"\\n\\t\\tDiscourse TEXT: --> {d_ex['discourse_text'].values[0]}\")\n \n# ######################################################### #\n# ---###################=== t-SNE ===###################--- #\n# ######################################################### #\n# KMeans helps us find clusters for our data, \n# but the resulting matrix is highly-dimensional with \n# 30 components which is hard to represent on a 2D plane. \n# In order to visualize those clusters, we will use the t-SNE \n# algorithm to reduce the dimensionality of our data. \n# t-SNE models each high-dimensional object by a n-dimensional \n# point (for instance n=2) in such a way that similar objects \n# are modeled by nearby points and dissimilar objects are modeled \n# by distant points with high probability. \n# In our case, t-SNE will help us find a low dimensional data representation \n# that keeps close points together in the 2D space. \n# With this 2D representation, we will see more clearly if our KMeans worked \n# as clusters should be visible on the plot.\n# ######################################################### #\ntsne_model = TSNE(n_components=2, verbose=1, n_iter=1000)\ntsne_kmeans = tsne_model.fit_transform(kmeans_distances)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T16:28:55.228361Z","iopub.execute_input":"2022-01-10T16:28:55.228608Z","iopub.status.idle":"2022-01-10T16:29:29.945981Z","shell.execute_reply.started":"2022-01-10T16:28:55.228576Z","shell.execute_reply":"2022-01-10T16:29:29.945262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.scatter(\n tsne_kmeans, x=0, y=1,\n color=train_df.iloc[:SUBSAMPLE_LEN].discourse_type, labels={'color': 'Discourse Type'}\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T16:29:29.947741Z","iopub.execute_input":"2022-01-10T16:29:29.947983Z","iopub.status.idle":"2022-01-10T16:30:42.369796Z","shell.execute_reply.started":"2022-01-10T16:29:29.947946Z","shell.execute_reply":"2022-01-10T16:30:42.368937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3D TSNE is not supported in RAPIDS so import it from SKLearn\nfrom sklearn.manifold import TSNE as sk_TSNE\n\ntsne_model = sk_TSNE(n_components=3, verbose=1, n_iter=1000)\ntsne_kmeans = tsne_model.fit_transform(kmeans_distances.get())\n\nfig = px.scatter_3d(\n tsne_kmeans, x=0, y=1, z=2,\n color=train_df.iloc[:SUBSAMPLE_LEN].discourse_type, labels={'color': 'Discourse Type'},\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-10T16:33:44.626671Z","iopub.execute_input":"2022-01-10T16:33:44.626924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"7.2 PCA VISUALIZATION [TSNE ABOVE FOR NOW...]
\n\n---\n\n\nWe will create a scattertext visualization.\n* First we create a corpus for the dataset. \n* Second we will aggregate all the text for a given discourse type together into a dictionary mapping the discourse type to the entire corpus of text (and back into a dataframe).\n* Third we set the **`discourse_type`** column as the **`category_col`** as this is what we are trying to predict and want insight into.\n\nFor creating this corpus we have used the en_core_web_sm as the English model which we downloaded via SpaCy.\n\n**Understanding The Scoring System**\n\nScattertext uses scaled f-score, which takes into account the category-specific precision and term frequency. While a term may appear frequently in multiple categories, the scaled f-score determines whether the term is more characteristic of a particular category compared with others. For example, while the term `park` is frequent in both `Lead` and `Evidence` categories, the scaled f-score concludes `park` is more associated with `Lead` than `Evidence` (or another category). Thus, when a discourse includes the term `park` it is more characteristic of a `Lead` category type.\n\n**Stop Words**\n\nNotice how some terms are not expected in the list, such as in the, of the, to the, it was, these are some examples of stop words that can be removed. While doing NLP, stop words are some extremely common words that would appear to be of little value in helping select documents are excluded from the vocabulary entirely such as the, them and they.\n","metadata":{}},{"cell_type":"code","source":"# Import and load spacy nlp model\nimport scattertext as st\nfrom scattertext import produce_scattertext_explorer\nnlp_ecws = spacy.load(\"en_core_web_sm\")\n\n# ############################################################# #\n# ##### THOUGHT THIS WAS NECESSARY... I DON'T THINK IT IS ##### #\n# ############################################################# #\n# # Create The Dataframe For Visualization That Maps Discourse Type To The Respective Corpus Of Text\n# st_train_df = train_df.copy()\n# st_train_df[\"dt_all_text\"] = st_train_df.groupby(\"discourse_type\")[\"discourse_text\"].transform(lambda x: ' '.join(x))\n# st_train_df=st_train_df.groupby(\"discourse_type\").first()[[\"dt_all_text\"]]\n# st_train_df[\"len_c_dt\"] = st_train_df[\"dt_all_text\"].apply(len)\n# st_train_df[\"len_w_dt\"] = st_train_df[\"dt_all_text\"].apply(lambda x: len(x.split()))\n# st_train_df.reset_index(inplace=True)\n# display(st_train_df)\n# ############################################################# #\n\n# Build scattertext corpus\ncorpus = st.CorpusFromPandas(train_df.iloc[:1000], category_col='discourse_type', text_col='discourse_text', nlp=nlp_ecws).build().remove_terms(nlp_ecws.Defaults.stop_words, ignore_absences=True)\nfeat_builder = st.FeatsFromGeneralInquirer()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:23:41.398719Z","iopub.execute_input":"2022-01-05T22:23:41.398987Z","iopub.status.idle":"2022-01-05T22:23:57.371511Z","shell.execute_reply.started":"2022-01-05T22:23:41.398956Z","shell.execute_reply":"2022-01-05T22:23:57.370801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"st_df = corpus.get_term_freq_df()\nfor dt in DISCOURSE_CLASSES:\n st_df[f'{dt}_Score'] = corpus.get_scaled_f_scores(dt).round(2)\n\nst_df.sort_values(by='Lead freq', ascending = False).reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:27:37.012302Z","iopub.execute_input":"2022-01-05T22:27:37.012934Z","iopub.status.idle":"2022-01-05T22:27:37.083513Z","shell.execute_reply.started":"2022-01-05T22:27:37.012878Z","shell.execute_reply":"2022-01-05T22:27:37.082853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"st_df","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:26:23.988928Z","iopub.execute_input":"2022-01-05T22:26:23.989745Z","iopub.status.idle":"2022-01-05T22:26:24.027876Z","shell.execute_reply.started":"2022-01-05T22:26:23.989702Z","shell.execute_reply":"2022-01-05T22:26:24.026996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"7.2 SPELLING!
\n\n---\n","metadata":{}},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-01-05T22:19:37.006152Z","iopub.execute_input":"2022-01-05T22:19:37.006884Z","iopub.status.idle":"2022-01-05T22:19:37.010127Z","shell.execute_reply.started":"2022-01-05T22:19:37.006845Z","shell.execute_reply":"2022-01-05T22:19:37.009443Z"},"trusted":true},"execution_count":null,"outputs":[]}]}