{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os.path\nimport re  # regular expressions\nimport random\nfrom tqdm import tqdm  # progress bars\nfrom collections import namedtuple\nfrom collections import defaultdict\n# multi-threading for reading input files faster:\nfrom threading import Lock\nfrom concurrent.futures import ThreadPoolExecutor\n# sklearn and ML stuff:\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.metrics import accuracy_score, balanced_accuracy_score\nfrom xgboost import XGBClassifier\n# viz:\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-23T21:28:14.466990Z","iopub.execute_input":"2023-07-23T21:28:14.467473Z","iopub.status.idle":"2023-07-23T21:28:15.731922Z","shell.execute_reply.started":"2023-07-23T21:28:14.467440Z","shell.execute_reply":"2023-07-23T21:28:15.730465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = os.path.join(\"/kaggle\", \"input\", \"colie\", \"train\", \"train\")\ndata_dir_valid = os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid\", \"valid\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:28:17.293135Z","iopub.execute_input":"2023-07-23T21:28:17.293683Z","iopub.status.idle":"2023-07-23T21:28:17.299647Z","shell.execute_reply.started":"2023-07-23T21:28:17.293634Z","shell.execute_reply":"2023-07-23T21:28:17.298769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_fname_parser = re.compile(r\"(\\d+)_(\\d+)\\.txt\")\n\n# train\nchunk_count = {}\nunparseable = []\nfor fname in os.listdir(data_dir):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count:\n        chunk_count[book] = []\n    chunk_count[book].append(chunk)\nprint(f\"Total books train set: {len(chunk_count)}\")\nprint(f\"{len(unparseable)} chunk names could not be parsed.\")\nprint(\"Work in another notebook suggests that \\\"<BOOK_ID> (1)\\\" and \\\"<BOOK_ID>\\\" are duplicates\")\n    \n# valid\nchunk_count_valid = {}\nunparseable = []\nfor fname in os.listdir(data_dir_valid):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count_valid:\n        chunk_count_valid[book] = []\n    chunk_count_valid[book].append(chunk)\nprint(f\"Total books valid set: {len(chunk_count_valid)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:28:19.165366Z","iopub.execute_input":"2023-07-23T21:28:19.165794Z","iopub.status.idle":"2023-07-23T21:28:25.723813Z","shell.execute_reply.started":"2023-07-23T21:28:19.165764Z","shell.execute_reply":"2023-07-23T21:28:25.722331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_corpus = {}\n\nlock = Lock()\n\ndef read_book(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE TRAIN SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            train_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:28:25.726582Z","iopub.execute_input":"2023-07-23T21:28:25.727039Z","iopub.status.idle":"2023-07-23T21:28:25.738334Z","shell.execute_reply.started":"2023-07-23T21:28:25.726998Z","shell.execute_reply":"2023-07-23T21:28:25.736606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter():\n    for book in chunk_count.keys():\n        yield book, chunk_count[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book(*t), thread_args_iter()), total=len(chunk_count)))","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:28:25.741413Z","iopub.execute_input":"2023-07-23T21:28:25.741932Z","iopub.status.idle":"2023-07-23T21:31:10.835358Z","shell.execute_reply.started":"2023-07-23T21:28:25.741888Z","shell.execute_reply":"2023-07-23T21:31:10.834137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(train_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:10.838380Z","iopub.execute_input":"2023-07-23T21:31:10.839889Z","iopub.status.idle":"2023-07-23T21:31:10.845635Z","shell.execute_reply.started":"2023-07-23T21:31:10.839844Z","shell.execute_reply":"2023-07-23T21:31:10.844746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_corpus = {}\n\nlock = Lock()\n\ndef read_book_valid(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE VALID SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_valid, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            valid_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:10.847145Z","iopub.execute_input":"2023-07-23T21:31:10.848457Z","iopub.status.idle":"2023-07-23T21:31:10.860088Z","shell.execute_reply.started":"2023-07-23T21:31:10.848421Z","shell.execute_reply":"2023-07-23T21:31:10.859012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_valid():\n    for book in chunk_count_valid.keys():\n        yield book, chunk_count_valid[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_valid(*t), thread_args_iter_valid()), total=len(chunk_count_valid)))","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:10.861584Z","iopub.execute_input":"2023-07-23T21:31:10.862761Z","iopub.status.idle":"2023-07-23T21:31:21.535461Z","shell.execute_reply.started":"2023-07-23T21:31:10.862718Z","shell.execute_reply":"2023-07-23T21:31:21.533952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(valid_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:21.537025Z","iopub.execute_input":"2023-07-23T21:31:21.537558Z","iopub.status.idle":"2023-07-23T21:31:21.545149Z","shell.execute_reply.started":"2023-07-23T21:31:21.537485Z","shell.execute_reply":"2023-07-23T21:31:21.543576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_train = list(train_corpus.keys())\nrow2book_valid = list(valid_corpus.keys())\ny_train = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"train.csv\"))\ny_valid = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid.csv\"))\n\nremoved_chunks_train = y_train.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_train = removed_chunks_train.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\nremoved_chunks_valid = y_valid.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_valid = removed_chunks_valid.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\n\nrow2class_train = []\nfor b in tqdm(row2book_train):\n    book_class = removed_chunks_train[removed_chunks_train[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_train.append(book_class)\nrow2class_valid = []\nfor b in tqdm(row2book_valid):\n    book_class = removed_chunks_valid[removed_chunks_valid[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_valid.append(book_class)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:21.547701Z","iopub.execute_input":"2023-07-23T21:31:21.548204Z","iopub.status.idle":"2023-07-23T21:31:46.827258Z","shell.execute_reply.started":"2023-07-23T21:31:21.548160Z","shell.execute_reply":"2023-07-23T21:31:46.825456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSNAME2NUMB = {\n    \"Viktorian\": 0,\n    \"Romantici\": 1,\n    \"Modernism\": 2,\n    \"PostModer\": 3,\n    \"OurDays\": 4\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:04:13.306589Z","iopub.execute_input":"2023-07-24T00:04:13.307033Z","iopub.status.idle":"2023-07-24T00:04:13.312881Z","shell.execute_reply.started":"2023-07-24T00:04:13.307002Z","shell.execute_reply":"2023-07-24T00:04:13.311409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train XGBoost Model","metadata":{}},{"cell_type":"code","source":"tfidf_model = TfidfVectorizer(\n    input=\"content\",\n    decode_error=\"ignore\",\n    # stop_words=\"english\",\n    max_df=0.7,  # CHOOSE EITHER ENGLISH STOP WORDS OR USE A FREQUENCY-BASED THRESHOLD\n    min_df=2,  # ignore words that appear in only 1 book (min_df is used in a strict commparison, so \"< 2\" is the cutoff)\n    norm=\"l2\",\n    sublinear_tf=True # recommended by sklearn to apply TruncatedSVD/LSA on results since it makes TF-IDF look more \"gaussian\"\n)\n\nsvd_model = TruncatedSVD(n_components=128, random_state=42)\n\nxgboost_model = XGBClassifier(\n    n_estimators=1000,\n    learning_rate=0.05,\n    verbosity=1,\n    objective=\"multi:softmax\",\n    random_state=4242,\n    n_jobs=-1\n)\n\npipeline = Pipeline(\n    steps = [\n        (\"preprocess1_tfidf\", tfidf_model),\n        (\"preprocess2_svd\", svd_model),\n        (\"classifier\", xgboost_model)\n    ],\n    verbose=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:46.841110Z","iopub.execute_input":"2023-07-23T21:31:46.841643Z","iopub.status.idle":"2023-07-23T21:31:46.853748Z","shell.execute_reply.started":"2023-07-23T21:31:46.841607Z","shell.execute_reply":"2023-07-23T21:31:46.851945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pipeline.fit(train_corpus.values(), np.array([CLASSNAME2NUMB[s] for s in row2class_train]))","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:46.856002Z","iopub.execute_input":"2023-07-23T21:31:46.856836Z","iopub.status.idle":"2023-07-23T21:31:46.866621Z","shell.execute_reply.started":"2023-07-23T21:31:46.856778Z","shell.execute_reply":"2023-07-23T21:31:46.865554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that we built row2class_xxx using xxx_corpus.keys(), and that we fed the pipeline with xxx_corpus.values(). This is safe because a dict's keys() and values() return in the same order :)","metadata":{}},{"cell_type":"code","source":"# preds_valid = pipeline.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:46.868046Z","iopub.execute_input":"2023-07-23T21:31:46.869731Z","iopub.status.idle":"2023-07-23T21:31:46.887184Z","shell.execute_reply.started":"2023-07-23T21:31:46.869676Z","shell.execute_reply":"2023-07-23T21:31:46.885495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid)}\")\n# print(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:46.888640Z","iopub.execute_input":"2023-07-23T21:31:46.889170Z","iopub.status.idle":"2023-07-23T21:31:46.902112Z","shell.execute_reply.started":"2023-07-23T21:31:46.889133Z","shell.execute_reply":"2023-07-23T21:31:46.900415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Trying again with more SVD dimensions","metadata":{}},{"cell_type":"code","source":"svd_model_2 = TruncatedSVD(n_components=384, random_state=42)\n\npipeline_2 = Pipeline(\n    steps = [\n        (\"preprocess1_tfidf\", tfidf_model),\n        (\"preprocess2_svd\", svd_model_2),\n        (\"classifier\", xgboost_model)\n    ],\n    verbose=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:46.903508Z","iopub.execute_input":"2023-07-23T21:31:46.904022Z","iopub.status.idle":"2023-07-23T21:31:46.916984Z","shell.execute_reply.started":"2023-07-23T21:31:46.903986Z","shell.execute_reply":"2023-07-23T21:31:46.915605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline_2.fit(train_corpus.values(), np.array([CLASSNAME2NUMB[s] for s in row2class_train]))","metadata":{"execution":{"iopub.status.busy":"2023-07-23T21:31:46.918963Z","iopub.execute_input":"2023-07-23T21:31:46.919519Z","iopub.status.idle":"2023-07-23T22:17:01.200009Z","shell.execute_reply.started":"2023-07-23T21:31:46.919470Z","shell.execute_reply":"2023-07-23T22:17:01.197250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_valid_2 = pipeline_2.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-23T22:17:01.203065Z","iopub.execute_input":"2023-07-23T22:17:01.203572Z","iopub.status.idle":"2023-07-23T22:17:50.085757Z","shell.execute_reply.started":"2023-07-23T22:17:01.203516Z","shell.execute_reply":"2023-07-23T22:17:50.084572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")\nprint(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T22:59:28.611004Z","iopub.execute_input":"2023-07-23T22:59:28.611658Z","iopub.status.idle":"2023-07-23T22:59:28.624183Z","shell.execute_reply.started":"2023-07-23T22:59:28.611618Z","shell.execute_reply":"2023-07-23T22:59:28.622469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion matrix","metadata":{}},{"cell_type":"code","source":"chunk_fname_parser = re.compile(\"((\\d+)(\\s\\(\\d\\))?)_(\\d+)\\.txt\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:41:01.807423Z","iopub.execute_input":"2023-07-23T23:41:01.808931Z","iopub.status.idle":"2023-07-23T23:41:01.814608Z","shell.execute_reply.started":"2023-07-23T23:41:01.808889Z","shell.execute_reply":"2023-07-23T23:41:01.813113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# valid\nchunk_count_valid = {}\nunparseable = []\nfor fname in os.listdir(data_dir_valid):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(4)\n    if book not in chunk_count_valid:\n        chunk_count_valid[book] = set()\n    chunk_count_valid[book].add(chunk)\nprint(f\"Total books valid set: {len(chunk_count_valid)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:41:39.006076Z","iopub.execute_input":"2023-07-23T23:41:39.007503Z","iopub.status.idle":"2023-07-23T23:41:39.118740Z","shell.execute_reply.started":"2023-07-23T23:41:39.007465Z","shell.execute_reply":"2023-07-23T23:41:39.117569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_corpus = {}\n\nlock = Lock()\n\ndef read_book_valid(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE VALID SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_valid, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n        except FileNotFoundError:\n            try:\n                with open(os.path.join(data_dir_valid, f\"{book} (1)_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                    content += f.read()\n            except UnicodeDecodeError:\n                print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            valid_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:41:40.020866Z","iopub.execute_input":"2023-07-23T23:41:40.021301Z","iopub.status.idle":"2023-07-23T23:41:40.038333Z","shell.execute_reply.started":"2023-07-23T23:41:40.021268Z","shell.execute_reply":"2023-07-23T23:41:40.036935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_valid():\n    for book in chunk_count_valid.keys():\n        yield book, chunk_count_valid[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_valid(*t), thread_args_iter_valid()), total=len(chunk_count_valid)))","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:41:41.437393Z","iopub.execute_input":"2023-07-23T23:41:41.437813Z","iopub.status.idle":"2023-07-23T23:41:49.541513Z","shell.execute_reply.started":"2023-07-23T23:41:41.437783Z","shell.execute_reply":"2023-07-23T23:41:49.539264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(valid_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:41:52.548147Z","iopub.execute_input":"2023-07-23T23:41:52.548690Z","iopub.status.idle":"2023-07-23T23:41:52.556911Z","shell.execute_reply.started":"2023-07-23T23:41:52.548653Z","shell.execute_reply":"2023-07-23T23:41:52.555142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_valid = list(valid_corpus.keys())\ny_valid = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid.csv\"))\n\nremoved_chunks_valid = y_valid.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_valid = removed_chunks_valid.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\n\nrow2class_valid = []\nfor b in tqdm(row2book_valid):\n    book_class = removed_chunks_valid[removed_chunks_valid[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_valid.append(book_class)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:41:54.348938Z","iopub.execute_input":"2023-07-23T23:41:54.349483Z","iopub.status.idle":"2023-07-23T23:41:54.972487Z","shell.execute_reply.started":"2023-07-23T23:41:54.349447Z","shell.execute_reply":"2023-07-23T23:41:54.971381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_valid_2 = pipeline_2.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:41:59.259715Z","iopub.execute_input":"2023-07-23T23:41:59.260706Z","iopub.status.idle":"2023-07-23T23:42:45.634734Z","shell.execute_reply.started":"2023-07-23T23:41:59.260657Z","shell.execute_reply":"2023-07-23T23:42:45.633149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")\nprint(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:42:59.237105Z","iopub.execute_input":"2023-07-23T23:42:59.237621Z","iopub.status.idle":"2023-07-23T23:42:59.249271Z","shell.execute_reply.started":"2023-07-23T23:42:59.237586Z","shell.execute_reply":"2023-07-23T23:42:59.248281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMB2CLASSNAME = {v: k for k, v in CLASSNAME2NUMB.items()}\nbook2pred = {book: NUMB2CLASSNAME[pred] for book, pred in zip(row2book_valid, preds_valid_2)}","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:43:34.109331Z","iopub.execute_input":"2023-07-23T23:43:34.109847Z","iopub.status.idle":"2023-07-23T23:43:34.119197Z","shell.execute_reply.started":"2023-07-23T23:43:34.109812Z","shell.execute_reply":"2023-07-23T23:43:34.117479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(book2pred)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:44:07.395123Z","iopub.execute_input":"2023-07-23T23:44:07.396175Z","iopub.status.idle":"2023-07-23T23:44:07.405652Z","shell.execute_reply.started":"2023-07-23T23:44:07.396118Z","shell.execute_reply":"2023-07-23T23:44:07.404421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_valid[[\"BOOK_id\"]].query('BOOK_id.str.startswith(\"10005\")', engine=\"python\")","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:44:21.328636Z","iopub.execute_input":"2023-07-23T23:44:21.329102Z","iopub.status.idle":"2023-07-23T23:44:21.335065Z","shell.execute_reply.started":"2023-07-23T23:44:21.329068Z","shell.execute_reply":"2023-07-23T23:44:21.333164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid = y_valid.copy()\ndf_valid.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:44:26.337014Z","iopub.execute_input":"2023-07-23T23:44:26.340793Z","iopub.status.idle":"2023-07-23T23:44:26.357648Z","shell.execute_reply.started":"2023-07-23T23:44:26.340729Z","shell.execute_reply":"2023-07-23T23:44:26.356465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern = re.compile(\"((\\d+)(\\s\\(\\d\\))?)_\\d+\\.txt\")\ndf_valid[\"Predicted Epoch\"] = df_valid.apply(lambda row: book2pred[pattern.fullmatch(row[\"BOOK_id\"]).group(1)], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:44:43.512503Z","iopub.execute_input":"2023-07-23T23:44:43.513066Z","iopub.status.idle":"2023-07-23T23:44:43.997864Z","shell.execute_reply.started":"2023-07-23T23:44:43.513031Z","shell.execute_reply":"2023-07-23T23:44:43.996434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid[[\"Epoch\"]].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:02:08.364207Z","iopub.execute_input":"2023-07-24T00:02:08.364691Z","iopub.status.idle":"2023-07-24T00:02:08.387579Z","shell.execute_reply.started":"2023-07-24T00:02:08.364656Z","shell.execute_reply":"2023-07-24T00:02:08.385752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid[[\"Predicted Epoch\"]].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:03:54.437025Z","iopub.execute_input":"2023-07-24T00:03:54.437485Z","iopub.status.idle":"2023-07-24T00:03:54.457291Z","shell.execute_reply.started":"2023-07-24T00:03:54.437454Z","shell.execute_reply":"2023-07-24T00:03:54.455752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Fix names of classes Romanticism and PostModernism","metadata":{}},{"cell_type":"code","source":"df_valid[df_valid[\"Predicted Epoch\"] == \"Romantici\"] = \"Romanticism\"\ndf_valid[df_valid[\"Predicted Epoch\"] == \"PostModer\"] = \"PostModernism\"","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:08:49.219915Z","iopub.execute_input":"2023-07-24T00:08:49.220424Z","iopub.status.idle":"2023-07-24T00:08:49.245126Z","shell.execute_reply.started":"2023-07-24T00:08:49.220391Z","shell.execute_reply":"2023-07-24T00:08:49.243361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid[[\"Predicted Epoch\"]].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:09:02.583166Z","iopub.execute_input":"2023-07-24T00:09:02.583676Z","iopub.status.idle":"2023-07-24T00:09:02.603007Z","shell.execute_reply.started":"2023-07-24T00:09:02.583640Z","shell.execute_reply":"2023-07-24T00:09:02.601642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay","metadata":{"execution":{"iopub.status.busy":"2023-07-23T23:56:25.124720Z","iopub.execute_input":"2023-07-23T23:56:25.125777Z","iopub.status.idle":"2023-07-23T23:56:25.133479Z","shell.execute_reply.started":"2023-07-23T23:56:25.125718Z","shell.execute_reply":"2023-07-23T23:56:25.132228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm = confusion_matrix(df_valid[\"Epoch\"], df_valid[\"Predicted Epoch\"], labels=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'])\ndisp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'])\ndisp.plot()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:19:55.670021Z","iopub.execute_input":"2023-07-24T00:19:55.670441Z","iopub.status.idle":"2023-07-24T00:19:56.251899Z","shell.execute_reply.started":"2023-07-24T00:19:55.670409Z","shell.execute_reply":"2023-07-24T00:19:56.250453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(np.diag(cm)) / np.sum(cm.flatten())","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:10:44.843351Z","iopub.execute_input":"2023-07-24T00:10:44.843803Z","iopub.status.idle":"2023-07-24T00:10:44.852204Z","shell.execute_reply.started":"2023-07-24T00:10:44.843771Z","shell.execute_reply":"2023-07-24T00:10:44.850549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:22:54.100410Z","iopub.execute_input":"2023-07-24T00:22:54.100991Z","iopub.status.idle":"2023-07-24T00:22:54.477218Z","shell.execute_reply.started":"2023-07-24T00:22:54.100934Z","shell.execute_reply":"2023-07-24T00:22:54.475875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cm_df = pd.DataFrame(data=cm, columns=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'], index=['Viktorian', 'Romanticism', 'Modernism', 'PostModernism', 'OurDays'])\ncm_df.index.name = \"True label\"\ncm_df.columns.name = \"Predicted label\"\n","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:23:45.972675Z","iopub.execute_input":"2023-07-24T00:23:45.973199Z","iopub.status.idle":"2023-07-24T00:23:45.981328Z","shell.execute_reply.started":"2023-07-24T00:23:45.973160Z","shell.execute_reply":"2023-07-24T00:23:45.979920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(cm_df,cmap=\"viridis\", annot=True,annot_kws={\"size\": 9}, fmt=\"d\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:25:56.614484Z","iopub.execute_input":"2023-07-24T00:25:56.614968Z","iopub.status.idle":"2023-07-24T00:25:57.097399Z","shell.execute_reply.started":"2023-07-24T00:25:56.614937Z","shell.execute_reply":"2023-07-24T00:25:57.095983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### test set","metadata":{}},{"cell_type":"code","source":"data_dir_test = os.path.join(\"/kaggle\", \"input\", \"colie\", \"test\", \"test\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:32:59.895835Z","iopub.execute_input":"2023-07-24T00:32:59.896339Z","iopub.status.idle":"2023-07-24T00:32:59.902993Z","shell.execute_reply.started":"2023-07-24T00:32:59.896306Z","shell.execute_reply":"2023-07-24T00:32:59.901558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test\nchunk_count_test = {}\nunparseable = []\nfor fname in os.listdir(data_dir_test):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(4)\n    if book not in chunk_count_test:\n        chunk_count_test[book] = set()\n    chunk_count_test[book].add(chunk)\nprint(f\"Total books test set: {len(chunk_count_test)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:33:01.051331Z","iopub.execute_input":"2023-07-24T00:33:01.051773Z","iopub.status.idle":"2023-07-24T00:33:05.373494Z","shell.execute_reply.started":"2023-07-24T00:33:01.051740Z","shell.execute_reply":"2023-07-24T00:33:05.372317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_corpus = {}\n\nlock = Lock()\n\ndef read_book_test(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE test SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_test, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n            break\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            test_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:33:25.078499Z","iopub.execute_input":"2023-07-24T00:33:25.079016Z","iopub.status.idle":"2023-07-24T00:33:25.091470Z","shell.execute_reply.started":"2023-07-24T00:33:25.078980Z","shell.execute_reply":"2023-07-24T00:33:25.089757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_test():\n    for book in chunk_count_test.keys():\n        yield book, chunk_count_test[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_test(*t), thread_args_iter_test()), total=len(chunk_count_test)))","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:33:33.745502Z","iopub.execute_input":"2023-07-24T00:33:33.746044Z","iopub.status.idle":"2023-07-24T00:34:20.382218Z","shell.execute_reply.started":"2023-07-24T00:33:33.746006Z","shell.execute_reply":"2023-07-24T00:34:20.380830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(test_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:34:20.385490Z","iopub.execute_input":"2023-07-24T00:34:20.386051Z","iopub.status.idle":"2023-07-24T00:34:20.392974Z","shell.execute_reply.started":"2023-07-24T00:34:20.386005Z","shell.execute_reply":"2023-07-24T00:34:20.391354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_test = list(test_corpus.keys())","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:34:20.394917Z","iopub.execute_input":"2023-07-24T00:34:20.395354Z","iopub.status.idle":"2023-07-24T00:34:20.409312Z","shell.execute_reply.started":"2023-07-24T00:34:20.395320Z","shell.execute_reply":"2023-07-24T00:34:20.408018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test_2 = pipeline_2.predict(test_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:34:20.418197Z","iopub.execute_input":"2023-07-24T00:34:20.419188Z","iopub.status.idle":"2023-07-24T00:37:23.080032Z","shell.execute_reply.started":"2023-07-24T00:34:20.419107Z","shell.execute_reply":"2023-07-24T00:37:23.078943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSNAME2NUMB_ = {\n    \"Viktorian\": 0,\n    \"Romanticism\": 1,\n    \"Modernism\": 2,\n    \"PostModernism\": 3,\n    \"OurDays\": 4\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.085518Z","iopub.execute_input":"2023-07-24T00:37:23.086380Z","iopub.status.idle":"2023-07-24T00:37:23.094916Z","shell.execute_reply.started":"2023-07-24T00:37:23.086339Z","shell.execute_reply":"2023-07-24T00:37:23.094046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMB2CLASSNAME_ = {v: k for k, v in CLASSNAME2NUMB_.items()}\nbook2pred = {book: NUMB2CLASSNAME_[pred] for book, pred in zip(row2book_test, preds_test_2)}","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.096686Z","iopub.execute_input":"2023-07-24T00:37:23.097147Z","iopub.status.idle":"2023-07-24T00:37:23.121905Z","shell.execute_reply.started":"2023-07-24T00:37:23.097107Z","shell.execute_reply":"2023-07-24T00:37:23.120918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"test.csv\"))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.124958Z","iopub.execute_input":"2023-07-24T00:37:23.125465Z","iopub.status.idle":"2023-07-24T00:37:23.259869Z","shell.execute_reply.started":"2023-07-24T00:37:23.125421Z","shell.execute_reply":"2023-07-24T00:37:23.258338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern = re.compile(\"(\\d+)(\\s\\(\\d\\))?_\\d+\\.txt\")\ndf_test[\"Epoch\"] = df_test.apply(lambda row: book2pred[pattern.fullmatch(row[\"BOOK_id\"]).group(1)], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:23.261244Z","iopub.execute_input":"2023-07-24T00:37:23.261643Z","iopub.status.idle":"2023-07-24T00:37:25.102411Z","shell.execute_reply.started":"2023-07-24T00:37:23.261607Z","shell.execute_reply":"2023-07-24T00:37:25.101028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:25.104318Z","iopub.execute_input":"2023-07-24T00:37:25.104830Z","iopub.status.idle":"2023-07-24T00:37:25.118792Z","shell.execute_reply.started":"2023-07-24T00:37:25.104784Z","shell.execute_reply":"2023-07-24T00:37:25.116934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-24T00:37:25.121038Z","iopub.execute_input":"2023-07-24T00:37:25.121580Z","iopub.status.idle":"2023-07-24T00:37:25.603968Z","shell.execute_reply.started":"2023-07-24T00:37:25.121516Z","shell.execute_reply":"2023-07-24T00:37:25.602598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}