{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os.path\nimport re  # regular expressions\nimport random\nfrom tqdm import tqdm  # progress bars\nfrom collections import namedtuple\nfrom collections import defaultdict\n# multi-threading for reading input files faster:\nfrom threading import Lock\nfrom concurrent.futures import ThreadPoolExecutor\n# sklearn and ML stuff:\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.metrics import accuracy_score, balanced_accuracy_score\nfrom xgboost import XGBClassifier\n# viz:\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-30T13:16:52.297111Z","iopub.execute_input":"2023-07-30T13:16:52.297635Z","iopub.status.idle":"2023-07-30T13:16:54.334124Z","shell.execute_reply.started":"2023-07-30T13:16:52.297593Z","shell.execute_reply":"2023-07-30T13:16:54.333037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = os.path.join(\"/kaggle\", \"input\", \"colie\", \"train\", \"train\")\ndata_dir_valid = os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid\", \"valid\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:16:54.336519Z","iopub.execute_input":"2023-07-30T13:16:54.337068Z","iopub.status.idle":"2023-07-30T13:16:54.342688Z","shell.execute_reply.started":"2023-07-30T13:16:54.337026Z","shell.execute_reply":"2023-07-30T13:16:54.341800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_fname_parser = re.compile(r\"(\\d+)_(\\d+)\\.txt\")\n\n# train\nchunk_count = {}\nunparseable = []\nfor fname in os.listdir(data_dir):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count:\n        chunk_count[book] = []\n    chunk_count[book].append(chunk)\nprint(f\"Total books train set: {len(chunk_count)}\")\nprint(f\"{len(unparseable)} chunk names could not be parsed.\")\nprint(\"Work in another notebook suggests that \\\"<BOOK_ID> (1)\\\" and \\\"<BOOK_ID>\\\" are duplicates\")\n    \n# valid\nchunk_count_valid = {}\nunparseable = []\nfor fname in os.listdir(data_dir_valid):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count_valid:\n        chunk_count_valid[book] = []\n    chunk_count_valid[book].append(chunk)\nprint(f\"Total books valid set: {len(chunk_count_valid)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:16:54.344515Z","iopub.execute_input":"2023-07-30T13:16:54.345422Z","iopub.status.idle":"2023-07-30T13:17:00.878896Z","shell.execute_reply.started":"2023-07-30T13:16:54.345380Z","shell.execute_reply":"2023-07-30T13:17:00.877487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_corpus = {}\n\nlock = Lock()\n\ndef read_book(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE TRAIN SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            train_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:17:00.882068Z","iopub.execute_input":"2023-07-30T13:17:00.882489Z","iopub.status.idle":"2023-07-30T13:17:00.892110Z","shell.execute_reply.started":"2023-07-30T13:17:00.882432Z","shell.execute_reply":"2023-07-30T13:17:00.891205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter():\n    for book in chunk_count.keys():\n        yield book, chunk_count[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book(*t), thread_args_iter()), total=len(chunk_count)))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:17:00.893836Z","iopub.execute_input":"2023-07-30T13:17:00.894450Z","iopub.status.idle":"2023-07-30T13:19:06.595551Z","shell.execute_reply.started":"2023-07-30T13:17:00.894417Z","shell.execute_reply":"2023-07-30T13:19:06.594362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(train_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:06.597263Z","iopub.execute_input":"2023-07-30T13:19:06.598013Z","iopub.status.idle":"2023-07-30T13:19:06.603771Z","shell.execute_reply.started":"2023-07-30T13:19:06.597970Z","shell.execute_reply":"2023-07-30T13:19:06.602936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_corpus = {}\n\nlock = Lock()\n\ndef read_book_valid(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE VALID SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_valid, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            valid_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:06.604732Z","iopub.execute_input":"2023-07-30T13:19:06.605127Z","iopub.status.idle":"2023-07-30T13:19:06.622766Z","shell.execute_reply.started":"2023-07-30T13:19:06.605098Z","shell.execute_reply":"2023-07-30T13:19:06.620273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_valid():\n    for book in chunk_count_valid.keys():\n        yield book, chunk_count_valid[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_valid(*t), thread_args_iter_valid()), total=len(chunk_count_valid)))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:06.624529Z","iopub.execute_input":"2023-07-30T13:19:06.625489Z","iopub.status.idle":"2023-07-30T13:19:14.717592Z","shell.execute_reply.started":"2023-07-30T13:19:06.625430Z","shell.execute_reply":"2023-07-30T13:19:14.716396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(valid_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:14.719630Z","iopub.execute_input":"2023-07-30T13:19:14.720079Z","iopub.status.idle":"2023-07-30T13:19:14.726550Z","shell.execute_reply.started":"2023-07-30T13:19:14.720037Z","shell.execute_reply":"2023-07-30T13:19:14.725353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_train = list(train_corpus.keys())\nrow2book_valid = list(valid_corpus.keys())\ny_train = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"train.csv\"))\ny_valid = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid.csv\"))\n\nremoved_chunks_train = y_train.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_train = removed_chunks_train.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\nremoved_chunks_valid = y_valid.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_valid = removed_chunks_valid.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\n\nrow2class_train = []\nfor b in tqdm(row2book_train):\n    book_class = removed_chunks_train[removed_chunks_train[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_train.append(book_class)\nrow2class_valid = []\nfor b in tqdm(row2book_valid):\n    book_class = removed_chunks_valid[removed_chunks_valid[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_valid.append(book_class)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:14.733575Z","iopub.execute_input":"2023-07-30T13:19:14.734624Z","iopub.status.idle":"2023-07-30T13:19:37.322491Z","shell.execute_reply.started":"2023-07-30T13:19:14.734580Z","shell.execute_reply":"2023-07-30T13:19:37.321329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSNAME2NUMB = {\n    \"Viktorian\": 0,\n    \"Romantici\": 1,\n    \"Modernism\": 2,\n    \"PostModer\": 3,\n    \"OurDays\": 4\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:37.324020Z","iopub.execute_input":"2023-07-30T13:19:37.324696Z","iopub.status.idle":"2023-07-30T13:19:37.330689Z","shell.execute_reply.started":"2023-07-30T13:19:37.324657Z","shell.execute_reply":"2023-07-30T13:19:37.329886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train XGBoost Model","metadata":{}},{"cell_type":"code","source":"tfidf_model = TfidfVectorizer(\n    input=\"content\",\n    decode_error=\"ignore\",\n    # stop_words=\"english\",\n    max_df=0.7,  # CHOOSE EITHER ENGLISH STOP WORDS OR USE A FREQUENCY-BASED THRESHOLD\n    min_df=2,  # ignore words that appear in only 1 book (min_df is used in a strict commparison, so \"< 2\" is the cutoff)\n    norm=\"l2\",\n    sublinear_tf=True # recommended by sklearn to apply TruncatedSVD/LSA on results since it makes TF-IDF look more \"gaussian\"\n)\n\nsvd_model = TruncatedSVD(n_components=128, random_state=42)\n\nxgboost_model = XGBClassifier(\n    n_estimators=1000,\n    learning_rate=0.05,\n    verbosity=1,\n    objective=\"multi:softmax\",\n    random_state=4242,\n    n_jobs=-1\n)\n\npipeline = Pipeline(\n    steps = [\n        (\"preprocess1_tfidf\", tfidf_model),\n        (\"preprocess2_svd\", svd_model),\n        (\"classifier\", xgboost_model)\n    ],\n    verbose=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:37.331858Z","iopub.execute_input":"2023-07-30T13:19:37.332140Z","iopub.status.idle":"2023-07-30T13:19:37.348962Z","shell.execute_reply.started":"2023-07-30T13:19:37.332116Z","shell.execute_reply":"2023-07-30T13:19:37.347732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pipeline.fit(train_corpus.values(), np.array([CLASSNAME2NUMB[s] for s in row2class_train]))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:37.350633Z","iopub.execute_input":"2023-07-30T13:19:37.351134Z","iopub.status.idle":"2023-07-30T13:19:37.362692Z","shell.execute_reply.started":"2023-07-30T13:19:37.351088Z","shell.execute_reply":"2023-07-30T13:19:37.361714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that we built row2class_xxx using xxx_corpus.keys(), and that we fed the pipeline with xxx_corpus.values(). This is safe because a dict's keys() and values() return in the same order :)","metadata":{}},{"cell_type":"code","source":"# preds_valid = pipeline.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:37.364117Z","iopub.execute_input":"2023-07-30T13:19:37.364441Z","iopub.status.idle":"2023-07-30T13:19:37.380167Z","shell.execute_reply.started":"2023-07-30T13:19:37.364414Z","shell.execute_reply":"2023-07-30T13:19:37.378717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid)}\")\n# print(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:37.381734Z","iopub.execute_input":"2023-07-30T13:19:37.382142Z","iopub.status.idle":"2023-07-30T13:19:37.392558Z","shell.execute_reply.started":"2023-07-30T13:19:37.382110Z","shell.execute_reply":"2023-07-30T13:19:37.391464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Trying again with more SVD dimensions","metadata":{}},{"cell_type":"code","source":"svd_model_2 = TruncatedSVD(n_components=384, random_state=42)\n\npipeline_2 = Pipeline(\n    steps = [\n        (\"preprocess1_tfidf\", tfidf_model),\n        (\"preprocess2_svd\", svd_model_2),\n        (\"classifier\", xgboost_model)\n    ],\n    verbose=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:37.393924Z","iopub.execute_input":"2023-07-30T13:19:37.394264Z","iopub.status.idle":"2023-07-30T13:19:37.405298Z","shell.execute_reply.started":"2023-07-30T13:19:37.394236Z","shell.execute_reply":"2023-07-30T13:19:37.404530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline_2.fit(train_corpus.values(), np.array([CLASSNAME2NUMB[s] for s in row2class_train]))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T13:19:37.406361Z","iopub.execute_input":"2023-07-30T13:19:37.406727Z","iopub.status.idle":"2023-07-30T14:03:31.830617Z","shell.execute_reply.started":"2023-07-30T13:19:37.406693Z","shell.execute_reply":"2023-07-30T14:03:31.828587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_valid_2 = pipeline_2.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:03:31.833181Z","iopub.execute_input":"2023-07-30T14:03:31.834099Z","iopub.status.idle":"2023-07-30T14:04:19.957197Z","shell.execute_reply.started":"2023-07-30T14:03:31.834063Z","shell.execute_reply":"2023-07-30T14:04:19.956113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")\nprint(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:19.961336Z","iopub.execute_input":"2023-07-30T14:04:19.962100Z","iopub.status.idle":"2023-07-30T14:04:19.975728Z","shell.execute_reply.started":"2023-07-30T14:04:19.962065Z","shell.execute_reply":"2023-07-30T14:04:19.974527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion matrix","metadata":{}},{"cell_type":"code","source":"chunk_fname_parser = re.compile(\"((\\d+)(\\s\\(\\d\\))?)_(\\d+)\\.txt\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:19.977411Z","iopub.execute_input":"2023-07-30T14:04:19.977740Z","iopub.status.idle":"2023-07-30T14:04:19.988570Z","shell.execute_reply.started":"2023-07-30T14:04:19.977714Z","shell.execute_reply":"2023-07-30T14:04:19.987696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# valid\nchunk_count_valid = {}\nunparseable = []\nfor fname in os.listdir(data_dir_valid):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(4)\n    if book not in chunk_count_valid:\n        chunk_count_valid[book] = set()\n    chunk_count_valid[book].add(chunk)\nprint(f\"Total books valid set: {len(chunk_count_valid)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:19.989914Z","iopub.execute_input":"2023-07-30T14:04:19.990422Z","iopub.status.idle":"2023-07-30T14:04:20.123321Z","shell.execute_reply.started":"2023-07-30T14:04:19.990394Z","shell.execute_reply":"2023-07-30T14:04:20.122065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_corpus = {}\n\nlock = Lock()\n\ndef read_book_valid(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE VALID SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_valid, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n        except FileNotFoundError:\n            try:\n                with open(os.path.join(data_dir_valid, f\"{book} (1)_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                    content += f.read()\n            except UnicodeDecodeError:\n                print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            valid_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:20.124892Z","iopub.execute_input":"2023-07-30T14:04:20.125229Z","iopub.status.idle":"2023-07-30T14:04:20.143978Z","shell.execute_reply.started":"2023-07-30T14:04:20.125201Z","shell.execute_reply":"2023-07-30T14:04:20.142695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_valid():\n    for book in chunk_count_valid.keys():\n        yield book, chunk_count_valid[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_valid(*t), thread_args_iter_valid()), total=len(chunk_count_valid)))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:20.146117Z","iopub.execute_input":"2023-07-30T14:04:20.146839Z","iopub.status.idle":"2023-07-30T14:04:26.666801Z","shell.execute_reply.started":"2023-07-30T14:04:20.146799Z","shell.execute_reply":"2023-07-30T14:04:26.665649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(valid_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:26.668454Z","iopub.execute_input":"2023-07-30T14:04:26.668880Z","iopub.status.idle":"2023-07-30T14:04:26.674376Z","shell.execute_reply.started":"2023-07-30T14:04:26.668849Z","shell.execute_reply":"2023-07-30T14:04:26.673551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_valid = list(valid_corpus.keys())\ny_valid = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid.csv\"))\n\nremoved_chunks_valid = y_valid.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_valid = removed_chunks_valid.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\n\nrow2class_valid = []\nfor b in tqdm(row2book_valid):\n    book_class = removed_chunks_valid[removed_chunks_valid[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_valid.append(book_class)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:26.675424Z","iopub.execute_input":"2023-07-30T14:04:26.675788Z","iopub.status.idle":"2023-07-30T14:04:27.282661Z","shell.execute_reply.started":"2023-07-30T14:04:26.675760Z","shell.execute_reply":"2023-07-30T14:04:27.281531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_valid_2 = pipeline_2.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:04:27.284609Z","iopub.execute_input":"2023-07-30T14:04:27.285092Z","iopub.status.idle":"2023-07-30T14:05:14.683151Z","shell.execute_reply.started":"2023-07-30T14:04:27.285044Z","shell.execute_reply":"2023-07-30T14:05:14.682199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")\nprint(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:14.684725Z","iopub.execute_input":"2023-07-30T14:05:14.685330Z","iopub.status.idle":"2023-07-30T14:05:14.694883Z","shell.execute_reply.started":"2023-07-30T14:05:14.685297Z","shell.execute_reply":"2023-07-30T14:05:14.693686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMB2CLASSNAME = {v: k for k, v in CLASSNAME2NUMB.items()}\nbook2pred = {book: NUMB2CLASSNAME[pred] for book, pred in zip(row2book_valid, preds_valid_2)}","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:14.704442Z","iopub.execute_input":"2023-07-30T14:05:14.705034Z","iopub.status.idle":"2023-07-30T14:05:14.714088Z","shell.execute_reply.started":"2023-07-30T14:05:14.705000Z","shell.execute_reply":"2023-07-30T14:05:14.712754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(book2pred)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:14.716027Z","iopub.execute_input":"2023-07-30T14:05:14.716593Z","iopub.status.idle":"2023-07-30T14:05:14.730975Z","shell.execute_reply.started":"2023-07-30T14:05:14.716548Z","shell.execute_reply":"2023-07-30T14:05:14.729955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_valid[[\"BOOK_id\"]].query('BOOK_id.str.startswith(\"10005\")', engine=\"python\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:14.732565Z","iopub.execute_input":"2023-07-30T14:05:14.733609Z","iopub.status.idle":"2023-07-30T14:05:14.744639Z","shell.execute_reply.started":"2023-07-30T14:05:14.733574Z","shell.execute_reply":"2023-07-30T14:05:14.743593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid = y_valid.copy()\ndf_valid.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:14.746140Z","iopub.execute_input":"2023-07-30T14:05:14.747061Z","iopub.status.idle":"2023-07-30T14:05:14.782989Z","shell.execute_reply.started":"2023-07-30T14:05:14.747024Z","shell.execute_reply":"2023-07-30T14:05:14.781888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern = re.compile(\"((\\d+)(\\s\\(\\d\\))?)_\\d+\\.txt\")\ndf_valid[\"Predicted Epoch\"] = df_valid.apply(lambda row: book2pred[pattern.fullmatch(row[\"BOOK_id\"]).group(1)], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:14.784716Z","iopub.execute_input":"2023-07-30T14:05:14.785348Z","iopub.status.idle":"2023-07-30T14:05:15.266014Z","shell.execute_reply.started":"2023-07-30T14:05:14.785315Z","shell.execute_reply":"2023-07-30T14:05:15.264711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid[[\"Epoch\"]].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:15.267603Z","iopub.execute_input":"2023-07-30T14:05:15.267981Z","iopub.status.idle":"2023-07-30T14:05:15.290309Z","shell.execute_reply.started":"2023-07-30T14:05:15.267950Z","shell.execute_reply":"2023-07-30T14:05:15.289003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid[[\"Predicted Epoch\"]].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:15.292186Z","iopub.execute_input":"2023-07-30T14:05:15.292588Z","iopub.status.idle":"2023-07-30T14:05:15.309046Z","shell.execute_reply.started":"2023-07-30T14:05:15.292557Z","shell.execute_reply":"2023-07-30T14:05:15.307387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Fix names of classes Romanticism and PostModernism","metadata":{}},{"cell_type":"markdown","source":"block below probably wrong!","metadata":{}},{"cell_type":"code","source":"df_valid.loc[df_valid[\"Predicted Epoch\"] == \"Romantici\", \"Predicted Epoch\"] = \"Romanticism\"\ndf_valid.loc[df_valid[\"Predicted Epoch\"] == \"PostModer\", \"Predicted Epoch\"] = \"PostModernism\"","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:15.310568Z","iopub.execute_input":"2023-07-30T14:05:15.310986Z","iopub.status.idle":"2023-07-30T14:05:15.333737Z","shell.execute_reply.started":"2023-07-30T14:05:15.310937Z","shell.execute_reply":"2023-07-30T14:05:15.332323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid[[\"Predicted Epoch\"]].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:15.335629Z","iopub.execute_input":"2023-07-30T14:05:15.336700Z","iopub.status.idle":"2023-07-30T14:05:15.356860Z","shell.execute_reply.started":"2023-07-30T14:05:15.336654Z","shell.execute_reply":"2023-07-30T14:05:15.356026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_valid[df_valid[\"Predicted Epoch\"] == \"Romantici\"] = \"Romanticism\"\n# df_valid[df_valid[\"Predicted Epoch\"] == \"PostModer\"] = \"PostModernism\"","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:15.358445Z","iopub.execute_input":"2023-07-30T14:05:15.358895Z","iopub.status.idle":"2023-07-30T14:05:15.363992Z","shell.execute_reply.started":"2023-07-30T14:05:15.358864Z","shell.execute_reply":"2023-07-30T14:05:15.362740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_valid[[\"Predicted Epoch\"]].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:15.366023Z","iopub.execute_input":"2023-07-30T14:05:15.366511Z","iopub.status.idle":"2023-07-30T14:05:15.378453Z","shell.execute_reply.started":"2023-07-30T14:05:15.366456Z","shell.execute_reply":"2023-07-30T14:05:15.377502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:05:15.380344Z","iopub.execute_input":"2023-07-30T14:05:15.380690Z","iopub.status.idle":"2023-07-30T14:05:15.392628Z","shell.execute_reply.started":"2023-07-30T14:05:15.380662Z","shell.execute_reply":"2023-07-30T14:05:15.391767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm = confusion_matrix(df_valid[\"Epoch\"], df_valid[\"Predicted Epoch\"], labels=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'])\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'])\ndisp.plot()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:18:10.389800Z","iopub.execute_input":"2023-07-30T14:18:10.390914Z","iopub.status.idle":"2023-07-30T14:18:10.969720Z","shell.execute_reply.started":"2023-07-30T14:18:10.390874Z","shell.execute_reply":"2023-07-30T14:18:10.968606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(np.diag(cm)) / np.sum(cm.flatten())","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:28:59.698078Z","iopub.execute_input":"2023-07-30T14:28:59.699098Z","iopub.status.idle":"2023-07-30T14:28:59.706196Z","shell.execute_reply.started":"2023-07-30T14:28:59.699060Z","shell.execute_reply":"2023-07-30T14:28:59.705082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:24:31.483443Z","iopub.execute_input":"2023-07-30T14:24:31.483960Z","iopub.status.idle":"2023-07-30T14:24:31.842700Z","shell.execute_reply.started":"2023-07-30T14:24:31.483927Z","shell.execute_reply":"2023-07-30T14:24:31.841655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm_df = pd.DataFrame(data=cm, columns=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'], index=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'])\ncm_df.index.name = \"True label\"\ncm_df.columns.name = \"Predicted label\"\n","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:24:31.845162Z","iopub.execute_input":"2023-07-30T14:24:31.845616Z","iopub.status.idle":"2023-07-30T14:24:31.852404Z","shell.execute_reply.started":"2023-07-30T14:24:31.845576Z","shell.execute_reply":"2023-07-30T14:24:31.851196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(cm_df,cmap=\"viridis\", annot=True,annot_kws={\"size\": 9}, fmt=\"d\")","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:24:31.993030Z","iopub.execute_input":"2023-07-30T14:24:31.993439Z","iopub.status.idle":"2023-07-30T14:24:32.466868Z","shell.execute_reply.started":"2023-07-30T14:24:31.993409Z","shell.execute_reply":"2023-07-30T14:24:32.465879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### test set","metadata":{}},{"cell_type":"code","source":"data_dir_test = os.path.join(\"/kaggle\", \"input\", \"colie\", \"test\", \"test\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:32:59.895835Z","iopub.execute_input":"2023-07-24T00:32:59.896339Z","iopub.status.idle":"2023-07-24T00:32:59.902993Z","shell.execute_reply.started":"2023-07-24T00:32:59.896306Z","shell.execute_reply":"2023-07-24T00:32:59.901558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test\nchunk_count_test = {}\nunparseable = []\nfor fname in os.listdir(data_dir_test):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(4)\n    if book not in chunk_count_test:\n        chunk_count_test[book] = set()\n    chunk_count_test[book].add(chunk)\nprint(f\"Total books test set: {len(chunk_count_test)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:33:01.051331Z","iopub.execute_input":"2023-07-24T00:33:01.051773Z","iopub.status.idle":"2023-07-24T00:33:05.373494Z","shell.execute_reply.started":"2023-07-24T00:33:01.05174Z","shell.execute_reply":"2023-07-24T00:33:05.372317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_corpus = {}\n\nlock = Lock()\n\ndef read_book_test(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE test SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_test, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n            break\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            test_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:33:25.078499Z","iopub.execute_input":"2023-07-24T00:33:25.079016Z","iopub.status.idle":"2023-07-24T00:33:25.09147Z","shell.execute_reply.started":"2023-07-24T00:33:25.07898Z","shell.execute_reply":"2023-07-24T00:33:25.089757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_test():\n    for book in chunk_count_test.keys():\n        yield book, chunk_count_test[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_test(*t), thread_args_iter_test()), total=len(chunk_count_test)))","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:33:33.745502Z","iopub.execute_input":"2023-07-24T00:33:33.746044Z","iopub.status.idle":"2023-07-24T00:34:20.382218Z","shell.execute_reply.started":"2023-07-24T00:33:33.746006Z","shell.execute_reply":"2023-07-24T00:34:20.38083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(test_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:34:20.38549Z","iopub.execute_input":"2023-07-24T00:34:20.386051Z","iopub.status.idle":"2023-07-24T00:34:20.392974Z","shell.execute_reply.started":"2023-07-24T00:34:20.386005Z","shell.execute_reply":"2023-07-24T00:34:20.391354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_test = list(test_corpus.keys())","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:34:20.394917Z","iopub.execute_input":"2023-07-24T00:34:20.395354Z","iopub.status.idle":"2023-07-24T00:34:20.409312Z","shell.execute_reply.started":"2023-07-24T00:34:20.39532Z","shell.execute_reply":"2023-07-24T00:34:20.408018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test_2 = pipeline_2.predict(test_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:34:20.418197Z","iopub.execute_input":"2023-07-24T00:34:20.419188Z","iopub.status.idle":"2023-07-24T00:37:23.080032Z","shell.execute_reply.started":"2023-07-24T00:34:20.419107Z","shell.execute_reply":"2023-07-24T00:37:23.078943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSNAME2NUMB_ = {\n    \"Viktorian\": 0,\n    \"Romanticism\": 1,\n    \"Modernism\": 2,\n    \"PostModernism\": 3,\n    \"OurDays\": 4\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.085518Z","iopub.execute_input":"2023-07-24T00:37:23.08638Z","iopub.status.idle":"2023-07-24T00:37:23.094916Z","shell.execute_reply.started":"2023-07-24T00:37:23.086339Z","shell.execute_reply":"2023-07-24T00:37:23.094046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMB2CLASSNAME_ = {v: k for k, v in CLASSNAME2NUMB_.items()}\nbook2pred = {book: NUMB2CLASSNAME_[pred] for book, pred in zip(row2book_test, preds_test_2)}","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.096686Z","iopub.execute_input":"2023-07-24T00:37:23.097147Z","iopub.status.idle":"2023-07-24T00:37:23.121905Z","shell.execute_reply.started":"2023-07-24T00:37:23.097107Z","shell.execute_reply":"2023-07-24T00:37:23.120918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"test.csv\"))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.124958Z","iopub.execute_input":"2023-07-24T00:37:23.125465Z","iopub.status.idle":"2023-07-24T00:37:23.259869Z","shell.execute_reply.started":"2023-07-24T00:37:23.125421Z","shell.execute_reply":"2023-07-24T00:37:23.258338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern = re.compile(\"(\\d+)(\\s\\(\\d\\))?_\\d+\\.txt\")\ndf_test[\"Epoch\"] = df_test.apply(lambda row: book2pred[pattern.fullmatch(row[\"BOOK_id\"]).group(1)], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.261244Z","iopub.execute_input":"2023-07-24T00:37:23.261643Z","iopub.status.idle":"2023-07-24T00:37:25.102411Z","shell.execute_reply.started":"2023-07-24T00:37:23.261607Z","shell.execute_reply":"2023-07-24T00:37:25.101028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:25.104318Z","iopub.execute_input":"2023-07-24T00:37:25.10483Z","iopub.status.idle":"2023-07-24T00:37:25.118792Z","shell.execute_reply.started":"2023-07-24T00:37:25.104784Z","shell.execute_reply":"2023-07-24T00:37:25.116934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:25.121038Z","iopub.execute_input":"2023-07-24T00:37:25.12158Z","iopub.status.idle":"2023-07-24T00:37:25.603968Z","shell.execute_reply.started":"2023-07-24T00:37:25.121516Z","shell.execute_reply":"2023-07-24T00:37:25.602598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}