{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os.path\nimport re  # regular expressions\nimport random\nfrom tqdm import tqdm  # progress bars\nfrom collections import namedtuple\nfrom collections import defaultdict\n# multi-threading for reading input files faster:\nfrom threading import Lock\nfrom concurrent.futures import ThreadPoolExecutor\n# sklearn and ML stuff:\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import accuracy_score, balanced_accuracy_score\nfrom sklearn.neighbors import BallTree\n# viz:\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-12T19:23:55.781066Z","iopub.execute_input":"2023-07-12T19:23:55.781431Z","iopub.status.idle":"2023-07-12T19:23:57.047031Z","shell.execute_reply.started":"2023-07-12T19:23:55.781401Z","shell.execute_reply":"2023-07-12T19:23:57.046202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = os.path.join(\"/kaggle\", \"input\", \"colie\", \"train\", \"train\")\ndata_dir_valid = os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid\", \"valid\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:23:57.049224Z","iopub.execute_input":"2023-07-12T19:23:57.049980Z","iopub.status.idle":"2023-07-12T19:23:57.055770Z","shell.execute_reply.started":"2023-07-12T19:23:57.049940Z","shell.execute_reply":"2023-07-12T19:23:57.054836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_fname_parser = re.compile(r\"(\\d+)_(\\d+)\\.txt\")\n\n# train\nchunk_count = {}\nunparseable = []\nfor fname in os.listdir(data_dir):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count:\n        chunk_count[book] = []\n    chunk_count[book].append(chunk)\nprint(f\"Total books train set: {len(chunk_count)}\")\nprint(f\"{len(unparseable)} chunk names could not be parsed.\")\nprint(\"Work in another notebook suggests that \\\"<BOOK_ID> (1)\\\" and \\\"<BOOK_ID>\\\" are duplicates\")\n    \n# valid\nchunk_count_valid = {}\nunparseable = []\nfor fname in os.listdir(data_dir_valid):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count_valid:\n        chunk_count_valid[book] = []\n    chunk_count_valid[book].append(chunk)\nprint(f\"Total books valid set: {len(chunk_count_valid)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:23:57.057047Z","iopub.execute_input":"2023-07-12T19:23:57.057832Z","iopub.status.idle":"2023-07-12T19:24:03.039992Z","shell.execute_reply.started":"2023-07-12T19:23:57.057764Z","shell.execute_reply":"2023-07-12T19:24:03.038983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_corpus = {}\n\nlock = Lock()\n\ndef read_book(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE TRAIN SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            train_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:24:03.041983Z","iopub.execute_input":"2023-07-12T19:24:03.042924Z","iopub.status.idle":"2023-07-12T19:24:03.052196Z","shell.execute_reply.started":"2023-07-12T19:24:03.042889Z","shell.execute_reply":"2023-07-12T19:24:03.051252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter():\n    for book in chunk_count.keys():\n        yield book, chunk_count[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book(*t), thread_args_iter()), total=len(chunk_count)))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:24:03.053514Z","iopub.execute_input":"2023-07-12T19:24:03.053840Z","iopub.status.idle":"2023-07-12T19:26:17.665326Z","shell.execute_reply.started":"2023-07-12T19:24:03.053808Z","shell.execute_reply":"2023-07-12T19:26:17.664576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(train_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:17.666420Z","iopub.execute_input":"2023-07-12T19:26:17.666695Z","iopub.status.idle":"2023-07-12T19:26:17.672632Z","shell.execute_reply.started":"2023-07-12T19:26:17.666670Z","shell.execute_reply":"2023-07-12T19:26:17.671617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_corpus = {}\n\nlock = Lock()\n\ndef read_book_valid(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE VALID SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_valid, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            valid_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:17.674390Z","iopub.execute_input":"2023-07-12T19:26:17.674796Z","iopub.status.idle":"2023-07-12T19:26:17.684815Z","shell.execute_reply.started":"2023-07-12T19:26:17.674741Z","shell.execute_reply":"2023-07-12T19:26:17.683964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_valid():\n    for book in chunk_count_valid.keys():\n        yield book, chunk_count_valid[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_valid(*t), thread_args_iter_valid()), total=len(chunk_count_valid)))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:17.686089Z","iopub.execute_input":"2023-07-12T19:26:17.686387Z","iopub.status.idle":"2023-07-12T19:26:26.730293Z","shell.execute_reply.started":"2023-07-12T19:26:17.686354Z","shell.execute_reply":"2023-07-12T19:26:26.729576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(valid_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:26.731140Z","iopub.execute_input":"2023-07-12T19:26:26.731432Z","iopub.status.idle":"2023-07-12T19:26:26.736325Z","shell.execute_reply.started":"2023-07-12T19:26:26.731404Z","shell.execute_reply":"2023-07-12T19:26:26.735315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_train = list(train_corpus.keys())\nrow2book_valid = list(valid_corpus.keys())\ny_train = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"train.csv\"))\ny_valid = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid.csv\"))\n\nremoved_chunks_train = y_train.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_train = removed_chunks_train.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\nremoved_chunks_valid = y_valid.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_valid = removed_chunks_valid.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\n\nrow2class_train = []\nfor b in tqdm(row2book_train):\n    book_class = removed_chunks_train[removed_chunks_train[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_train.append(book_class)\nrow2class_valid = []\nfor b in tqdm(row2book_valid):\n    book_class = removed_chunks_valid[removed_chunks_valid[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_valid.append(book_class)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:26.741751Z","iopub.execute_input":"2023-07-12T19:26:26.742266Z","iopub.status.idle":"2023-07-12T19:26:41.962299Z","shell.execute_reply.started":"2023-07-12T19:26:26.742229Z","shell.execute_reply":"2023-07-12T19:26:41.961372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSNAME2NUMB = {\n    \"Viktorian\": 0,\n    \"Romantici\": 1,\n    \"Modernism\": 2,\n    \"PostModer\": 3,\n    \"OurDays\": 4\n}\n\nNUMB2CLASSNAME = {v: k for k, v in CLASSNAME2NUMB.items()}","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:41.963380Z","iopub.execute_input":"2023-07-12T19:26:41.963655Z","iopub.status.idle":"2023-07-12T19:26:41.968973Z","shell.execute_reply.started":"2023-07-12T19:26:41.963630Z","shell.execute_reply":"2023-07-12T19:26:41.967979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import Word2Vec model and compute Gaussian of Words embedding of train corpus\n\n* See [Multivariate Gaussian Document Representation from Word Embeddings for Text Categorization](https://aclanthology.org/E17-2072) (Nikolentzos et al., EACL 2017)","metadata":{}},{"cell_type":"code","source":"import gensim.downloader\n# from gensim.models import Word2Vec\n# from gensim.models import KeyedVectors","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:41.970086Z","iopub.execute_input":"2023-07-12T19:26:41.970371Z","iopub.status.idle":"2023-07-12T19:26:42.198557Z","shell.execute_reply.started":"2023-07-12T19:26:41.970346Z","shell.execute_reply":"2023-07-12T19:26:42.197678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glove_vectors = gensim.downloader.load(\"glove-wiki-gigaword-200\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:26:42.199698Z","iopub.execute_input":"2023-07-12T19:26:42.200023Z","iopub.status.idle":"2023-07-12T19:28:10.809748Z","shell.execute_reply.started":"2023-07-12T19:26:42.199995Z","shell.execute_reply":"2023-07-12T19:28:10.808745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* See [GloVe: Global Vectors for Word Representation](https://nlp.stanford.edu/pubs/glove.pdf) (Jeffrey Pennington, Richard Socher, and Christopher D. Manning. 2014)","metadata":{}},{"cell_type":"markdown","source":"We will not build a data structure associating each book to the mean and covariance matrix of the GloVe embeds of some of its chunks","metadata":{}},{"cell_type":"code","source":"# glove_vectors.get_vector(\"potatoe\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:28:10.811124Z","iopub.execute_input":"2023-07-12T19:28:10.811421Z","iopub.status.idle":"2023-07-12T19:28:10.816968Z","shell.execute_reply.started":"2023-07-12T19:28:10.811394Z","shell.execute_reply":"2023-07-12T19:28:10.815872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Preprocessing:","metadata":{}},{"cell_type":"code","source":"from gensim.parsing.preprocessing import preprocess_string, remove_stopwords, strip_non_alphanum, strip_multiple_whitespaces","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:28:10.818326Z","iopub.execute_input":"2023-07-12T19:28:10.818687Z","iopub.status.idle":"2023-07-12T19:28:10.828489Z","shell.execute_reply.started":"2023-07-12T19:28:10.818659Z","shell.execute_reply":"2023-07-12T19:28:10.827806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREPROC_LIST = [remove_stopwords, strip_non_alphanum, strip_multiple_whitespaces, lambda s: s.lower()]","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:28:10.829629Z","iopub.execute_input":"2023-07-12T19:28:10.830259Z","iopub.status.idle":"2023-07-12T19:28:10.840588Z","shell.execute_reply.started":"2023-07-12T19:28:10.830220Z","shell.execute_reply":"2023-07-12T19:28:10.839673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_book = random.choices(list(train_corpus.keys()), k=1)[0]\ns = train_corpus[sample_book][:150]\nprint(f\"Book {sample_book}\")\nprint(\"Raw:\", s)\nprint(\"Preprocessed:\", preprocess_string(s, PREPROC_LIST))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:28:10.841754Z","iopub.execute_input":"2023-07-12T19:28:10.842402Z","iopub.status.idle":"2023-07-12T19:28:10.854982Z","shell.execute_reply.started":"2023-07-12T19:28:10.842363Z","shell.execute_reply":"2023-07-12T19:28:10.854022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_vec_if_present(s):\n    try:\n        return glove_vectors.get_vector(s)\n    except KeyError:\n        return None","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:28:10.856225Z","iopub.execute_input":"2023-07-12T19:28:10.856514Z","iopub.status.idle":"2023-07-12T19:28:10.866354Z","shell.execute_reply.started":"2023-07-12T19:28:10.856489Z","shell.execute_reply":"2023-07-12T19:28:10.865410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### embeds dict:","metadata":{}},{"cell_type":"code","source":"train_vectors = {}\nfor book, text in tqdm(train_corpus.items()):\n    tokenized_text = preprocess_string(text, PREPROC_LIST)\n    vecs = [v for token in tokenized_text if (v := get_vec_if_present(token)) is not None]\n    if len(vecs) > 0:\n        mean_vec = np.mean(vecs, axis=0)\n        mean_vec /= np.linalg.norm(mean_vec)\n        covar_matrix_flat = np.cov(vecs, rowvar=False).flatten()\n        covar_matrix_flat /= np.linalg.norm(covar_matrix_flat)\n    else:\n        mean_vec = np.zeros(glove_vectors.vector_size)\n        covar_matrix_flat = np.zeros(glove_vectors.vector_size ** 2)\n    train_vectors[book] = np.concatenate((mean_vec, covar_matrix_flat))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T19:28:10.867737Z","iopub.execute_input":"2023-07-12T19:28:10.868244Z","iopub.status.idle":"2023-07-12T20:08:32.577469Z","shell.execute_reply.started":"2023-07-12T19:28:10.868205Z","shell.execute_reply":"2023-07-12T20:08:32.574984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.array([train_vectors[k] for k in train_corpus.keys()])  # same insertion order as the variables row2book_train and row2class_train, so we know which book corresponds to each row of X\nX.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-12T20:08:32.583241Z","iopub.execute_input":"2023-07-12T20:08:32.584135Z","iopub.status.idle":"2023-07-12T20:08:33.852860Z","shell.execute_reply.started":"2023-07-12T20:08:32.584070Z","shell.execute_reply":"2023-07-12T20:08:33.852105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree = BallTree(X)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T20:08:33.854282Z","iopub.execute_input":"2023-07-12T20:08:33.854840Z","iopub.status.idle":"2023-07-12T20:09:04.076734Z","shell.execute_reply.started":"2023-07-12T20:08:33.854809Z","shell.execute_reply":"2023-07-12T20:09:04.075845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Proceed similarly to valid set","metadata":{}},{"cell_type":"code","source":"valid_vectors = {}\nfor book, text in tqdm(valid_corpus.items()):\n    tokenized_text = preprocess_string(text, PREPROC_LIST)\n    vecs = [v for token in tokenized_text if (v := get_vec_if_present(token)) is not None]\n    if len(vecs) > 0:\n        mean_vec = np.mean(vecs, axis=0)\n        mean_vec /= np.linalg.norm(mean_vec)\n        covar_matrix_flat = np.cov(vecs, rowvar=False).flatten()\n        covar_matrix_flat /= np.linalg.norm(covar_matrix_flat)\n    else:\n        mean_vec = np.zeros(glove_vectors.vector_size)\n        covar_matrix_flat = np.zeros(glove_vectors.vector_size ** 2)\n    valid_vectors[book] = np.concatenate((mean_vec, covar_matrix_flat))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T20:09:04.078115Z","iopub.execute_input":"2023-07-12T20:09:04.078496Z","iopub.status.idle":"2023-07-12T20:11:42.419993Z","shell.execute_reply.started":"2023-07-12T20:09:04.078467Z","shell.execute_reply":"2023-07-12T20:11:42.418908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### K-NN classification using gaussians of words","metadata":{}},{"cell_type":"markdown","source":"First, the function to predict the class of a given (valid) book, given a hyperparam k (number of neighbors to use)","metadata":{}},{"cell_type":"code","source":"def predict_valid(book, k, compare2target=False):\n    v = valid_vectors[book]\n    closest_neigh_indices = tree.query(np.array([v]), k=k, return_distance=False, sort_results=False)[0]\n    closest_neigh_classes = [CLASSNAME2NUMB[row2class_train[i]] for i in closest_neigh_indices]\n    voted_class = NUMB2CLASSNAME[np.argmax(np.bincount(closest_neigh_classes))]\n    if compare2target:\n        target = removed_chunks_valid[removed_chunks_valid[\"BOOK_id\"] == book].iat[0, 1]\n        return voted_class, (target == voted_class)\n    else:\n        return voted_class","metadata":{"execution":{"iopub.status.busy":"2023-07-12T20:11:42.421809Z","iopub.execute_input":"2023-07-12T20:11:42.422507Z","iopub.status.idle":"2023-07-12T20:11:42.433017Z","shell.execute_reply.started":"2023-07-12T20:11:42.422465Z","shell.execute_reply":"2023-07-12T20:11:42.431493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Best value of k","metadata":{}},{"cell_type":"code","source":"k_candidates = [1, 5, 15, 25, 50]\naccuracies = []\nfor k in k_candidates:\n    cum_score = 0\n    total = 0\n    for book in tqdm(valid_corpus.keys()):\n        _, score = predict_valid(book, k, True)\n        cum_score += score\n        total += 1\n    accuracies.append(cum_score / total)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T20:11:42.434290Z","iopub.execute_input":"2023-07-12T20:11:42.434736Z","iopub.status.idle":"2023-07-12T20:37:08.181436Z","shell.execute_reply.started":"2023-07-12T20:11:42.434700Z","shell.execute_reply":"2023-07-12T20:37:08.180350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracies # best: 15 and 25, but less than 70% acc","metadata":{"execution":{"iopub.status.busy":"2023-07-12T20:37:08.182981Z","iopub.execute_input":"2023-07-12T20:37:08.183319Z","iopub.status.idle":"2023-07-12T20:37:08.189888Z","shell.execute_reply.started":"2023-07-12T20:37:08.183279Z","shell.execute_reply":"2023-07-12T20:37:08.189074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}