{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os.path\nimport re  # regular expressions\nimport random\nfrom tqdm import tqdm  # progress bars\nfrom collections import namedtuple\nfrom collections import defaultdict\n# multi-threading for reading input files faster:\nfrom threading import Lock\nfrom concurrent.futures import ThreadPoolExecutor\n# sklearn and ML stuff:\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.metrics import accuracy_score, balanced_accuracy_score\nfrom xgboost import XGBClassifier\n# viz:\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-12T10:35:49.188401Z","iopub.execute_input":"2023-07-12T10:35:49.18875Z","iopub.status.idle":"2023-07-12T10:35:50.528324Z","shell.execute_reply.started":"2023-07-12T10:35:49.188696Z","shell.execute_reply":"2023-07-12T10:35:50.527164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = os.path.join(\"/kaggle\", \"input\", \"colie\", \"train\", \"train\")\ndata_dir_valid = os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid\", \"valid\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:35:55.086249Z","iopub.execute_input":"2023-07-12T10:35:55.086615Z","iopub.status.idle":"2023-07-12T10:35:55.092423Z","shell.execute_reply.started":"2023-07-12T10:35:55.086586Z","shell.execute_reply":"2023-07-12T10:35:55.091025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_fname_parser = re.compile(r\"(\\d+)_(\\d+)\\.txt\")\n\n# train\nchunk_count = {}\nunparseable = []\nfor fname in os.listdir(data_dir):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count:\n        chunk_count[book] = []\n    chunk_count[book].append(chunk)\nprint(f\"Total books train set: {len(chunk_count)}\")\nprint(f\"{len(unparseable)} chunk names could not be parsed.\")\nprint(\"Work in another notebook suggests that \\\"<BOOK_ID> (1)\\\" and \\\"<BOOK_ID>\\\" are duplicates\")\n    \n# valid\nchunk_count_valid = {}\nunparseable = []\nfor fname in os.listdir(data_dir_valid):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count_valid:\n        chunk_count_valid[book] = []\n    chunk_count_valid[book].append(chunk)\nprint(f\"Total books valid set: {len(chunk_count_valid)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:35:57.933977Z","iopub.execute_input":"2023-07-12T10:35:57.934333Z","iopub.status.idle":"2023-07-12T10:36:31.245023Z","shell.execute_reply.started":"2023-07-12T10:35:57.934304Z","shell.execute_reply":"2023-07-12T10:36:31.24412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_corpus = {}\n\nlock = Lock()\n\ndef read_book(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE TRAIN SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            train_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:41:28.640528Z","iopub.execute_input":"2023-07-12T10:41:28.640911Z","iopub.status.idle":"2023-07-12T10:41:28.722028Z","shell.execute_reply.started":"2023-07-12T10:41:28.640884Z","shell.execute_reply":"2023-07-12T10:41:28.721206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter():\n    for book in chunk_count.keys():\n        yield book, chunk_count[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book(*t), thread_args_iter()), total=len(chunk_count)))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:41:33.422514Z","iopub.execute_input":"2023-07-12T10:41:33.422869Z","iopub.status.idle":"2023-07-12T10:47:18.73788Z","shell.execute_reply.started":"2023-07-12T10:41:33.422841Z","shell.execute_reply":"2023-07-12T10:47:18.73654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(train_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:48:41.60747Z","iopub.execute_input":"2023-07-12T10:48:41.607851Z","iopub.status.idle":"2023-07-12T10:48:41.613471Z","shell.execute_reply.started":"2023-07-12T10:48:41.607822Z","shell.execute_reply":"2023-07-12T10:48:41.612241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_corpus = {}\n\nlock = Lock()\n\ndef read_book_valid(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE VALID SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_valid, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            valid_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:48:45.487316Z","iopub.execute_input":"2023-07-12T10:48:45.487673Z","iopub.status.idle":"2023-07-12T10:48:45.495997Z","shell.execute_reply.started":"2023-07-12T10:48:45.487645Z","shell.execute_reply":"2023-07-12T10:48:45.494657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_valid():\n    for book in chunk_count_valid.keys():\n        yield book, chunk_count_valid[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_valid(*t), thread_args_iter_valid()), total=len(chunk_count_valid)))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:48:52.2984Z","iopub.execute_input":"2023-07-12T10:48:52.298773Z","iopub.status.idle":"2023-07-12T10:49:19.828506Z","shell.execute_reply.started":"2023-07-12T10:48:52.298741Z","shell.execute_reply":"2023-07-12T10:49:19.826851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(valid_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:49:23.266109Z","iopub.execute_input":"2023-07-12T10:49:23.26646Z","iopub.status.idle":"2023-07-12T10:49:23.272492Z","shell.execute_reply.started":"2023-07-12T10:49:23.266433Z","shell.execute_reply":"2023-07-12T10:49:23.271223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_train = list(train_corpus.keys())\nrow2book_valid = list(valid_corpus.keys())\ny_train = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"train.csv\"))\ny_valid = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"valid.csv\"))\n\nremoved_chunks_train = y_train.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_train = removed_chunks_train.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\nremoved_chunks_valid = y_valid.apply(lambda row: (row[0].split(\"_\")[0], row[1]), axis=1, raw=True)\nremoved_chunks_valid = removed_chunks_valid.groupby(by=[\"BOOK_id\", \"Epoch\"], as_index=False).size()\n\nrow2class_train = []\nfor b in tqdm(row2book_train):\n    book_class = removed_chunks_train[removed_chunks_train[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_train.append(book_class)\nrow2class_valid = []\nfor b in tqdm(row2book_valid):\n    book_class = removed_chunks_valid[removed_chunks_valid[\"BOOK_id\"] == b].iat[0, 1]\n    row2class_valid.append(book_class)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:49:25.641066Z","iopub.execute_input":"2023-07-12T10:49:25.641409Z","iopub.status.idle":"2023-07-12T10:49:38.623422Z","shell.execute_reply.started":"2023-07-12T10:49:25.64138Z","shell.execute_reply":"2023-07-12T10:49:38.621471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASSNAME2NUMB = {\n    \"Viktorian\": 0,\n    \"Romantici\": 1,\n    \"Modernism\": 2,\n    \"PostModer\": 3,\n    \"OurDays\": 4\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:49:38.625307Z","iopub.execute_input":"2023-07-12T10:49:38.625591Z","iopub.status.idle":"2023-07-12T10:49:38.632407Z","shell.execute_reply.started":"2023-07-12T10:49:38.625565Z","shell.execute_reply":"2023-07-12T10:49:38.631093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train XGBoost Model","metadata":{}},{"cell_type":"code","source":"tfidf_model = TfidfVectorizer(\n    input=\"content\",\n    decode_error=\"ignore\",\n    # stop_words=\"english\",\n    max_df=0.7,  # CHOOSE EITHER ENGLISH STOP WORDS OR USE A FREQUENCY-BASED THRESHOLD\n    min_df=2,  # ignore words that appear in only 1 book (min_df is used in a strict commparison, so \"< 2\" is the cutoff)\n    norm=\"l2\",\n    sublinear_tf=True # recommended by sklearn to apply TruncatedSVD/LSA on results since it makes TF-IDF look more \"gaussian\"\n)\n\nsvd_model = TruncatedSVD(n_components=128, random_state=42)\n\nxgboost_model = XGBClassifier(\n    n_estimators=1000,\n    learning_rate=0.05,\n    verbosity=1,\n    objective=\"multi:softmax\",\n    random_state=4242,\n    n_jobs=-1\n)\n\npipeline = Pipeline(\n    steps = [\n        (\"preprocess1_tfidf\", tfidf_model),\n        (\"preprocess2_svd\", svd_model),\n        (\"classifier\", xgboost_model)\n    ],\n    verbose=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:49:38.634218Z","iopub.execute_input":"2023-07-12T10:49:38.634544Z","iopub.status.idle":"2023-07-12T10:49:38.644551Z","shell.execute_reply.started":"2023-07-12T10:49:38.634517Z","shell.execute_reply":"2023-07-12T10:49:38.643656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pipeline.fit(train_corpus.values(), np.array([CLASSNAME2NUMB[s] for s in row2class_train]))","metadata":{"execution":{"iopub.status.busy":"2023-07-10T15:17:08.6423Z","iopub.execute_input":"2023-07-10T15:17:08.642662Z","iopub.status.idle":"2023-07-10T15:32:44.019594Z","shell.execute_reply.started":"2023-07-10T15:17:08.642632Z","shell.execute_reply":"2023-07-10T15:32:44.017885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that we built row2class_xxx using xxx_corpus.keys(), and that we fed the pipeline with xxx_corpus.values(). This is safe because a dict's keys() and values() return in the same order :)","metadata":{}},{"cell_type":"code","source":"# preds_valid = pipeline.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:46:27.527967Z","iopub.execute_input":"2023-07-10T16:46:27.528364Z","iopub.status.idle":"2023-07-10T16:46:58.649148Z","shell.execute_reply.started":"2023-07-10T16:46:27.528339Z","shell.execute_reply":"2023-07-10T16:46:58.648279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid)}\")\n# print(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-10T16:48:51.392713Z","iopub.execute_input":"2023-07-10T16:48:51.393151Z","iopub.status.idle":"2023-07-10T16:48:51.405765Z","shell.execute_reply.started":"2023-07-10T16:48:51.393124Z","shell.execute_reply":"2023-07-10T16:48:51.404396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Trying again with more SVD dimensions","metadata":{}},{"cell_type":"code","source":"svd_model_2 = TruncatedSVD(n_components=384, random_state=42)\n\npipeline_2 = Pipeline(\n    steps = [\n        (\"preprocess1_tfidf\", tfidf_model),\n        (\"preprocess2_svd\", svd_model_2),\n        (\"classifier\", xgboost_model)\n    ],\n    verbose=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:49:45.068744Z","iopub.execute_input":"2023-07-12T10:49:45.069122Z","iopub.status.idle":"2023-07-12T10:49:45.075134Z","shell.execute_reply.started":"2023-07-12T10:49:45.069091Z","shell.execute_reply":"2023-07-12T10:49:45.073529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipeline_2.fit(train_corpus.values(), np.array([CLASSNAME2NUMB[s] for s in row2class_train]))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T10:49:51.610909Z","iopub.execute_input":"2023-07-12T10:49:51.611261Z","iopub.status.idle":"2023-07-12T11:12:27.932486Z","shell.execute_reply.started":"2023-07-12T10:49:51.611234Z","shell.execute_reply":"2023-07-12T11:12:27.929372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_valid_2 = pipeline_2.predict(valid_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:12:27.935853Z","iopub.execute_input":"2023-07-12T11:12:27.936354Z","iopub.status.idle":"2023-07-12T11:12:59.832979Z","shell.execute_reply.started":"2023-07-12T11:12:27.936308Z","shell.execute_reply":"2023-07-12T11:12:59.832124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Accuracy (= challenge chosen metric) on valid set: {accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")\nprint(f\"Balanced accuracy on valid set: {balanced_accuracy_score(np.array([CLASSNAME2NUMB[s] for s in row2class_valid]), preds_valid_2)}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:12:59.834223Z","iopub.execute_input":"2023-07-12T11:12:59.834558Z","iopub.status.idle":"2023-07-12T11:12:59.846761Z","shell.execute_reply.started":"2023-07-12T11:12:59.834522Z","shell.execute_reply":"2023-07-12T11:12:59.845128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test data","metadata":{}},{"cell_type":"code","source":"data_dir_test = os.path.join(\"/kaggle\", \"input\", \"colie\", \"test\", \"test\")\n\nchunk_count_test = {}\nunparseable = []\nfor fname in os.listdir(data_dir_test):\n    match = chunk_fname_parser.fullmatch(fname)\n    if match is None:\n        unparseable.append(fname)\n        continue\n    book, chunk = match.group(1), match.group(2)\n    if book not in chunk_count_test:\n        chunk_count_test[book] = []\n    chunk_count_test[book].append(chunk)\nprint(f\"Total books test set: {len(chunk_count_test)}\")\nprint(f\"{len(unparseable)} chunks could not be parsed.\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:14:03.992599Z","iopub.execute_input":"2023-07-12T11:14:03.993911Z","iopub.status.idle":"2023-07-12T11:14:13.931181Z","shell.execute_reply.started":"2023-07-12T11:14:03.993854Z","shell.execute_reply":"2023-07-12T11:14:13.930114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_corpus = {}\n\nlock = Lock()\n\ndef read_book_test(book, book_chunks, k=-1):\n    \"\"\"function for reading chunks belonging to a same book OF THE test SET and addind them to the respective dict (k=book id, v=content of some chunks)\n    \n    This function was conceived to be called by a Thread object in a separated thread\n\n    Args:\n        book (str): book id\n        book_chunks (list[str]): list of chunk numbers (e.g. chunk_count[book])\n        k (int): how many chunks per book (use all chunks if -1)\n    \"\"\"\n    content = \"\"\n    # it is unuseful to reconstruct the book in the right order since TF-IDF is a bag method\n    # but doing things properly can avoid a headache if we reuse this code later\n    sorted_chunks = sorted(book_chunks, key=lambda x: int(x))\n    if k != -1:\n        sorted_chunks = random.choices(sorted_chunks, k=k)\n    # Try to read it with Windows-1252 aka CP-1252\n    for chunk in sorted_chunks:\n        try:\n            with open(os.path.join(data_dir_test, f\"{book}_{chunk}.txt\"), \"r\", encoding=\"Windows-1252\") as f:\n                content += f.read()\n        except UnicodeDecodeError:\n            print(f\"UnicodeDecodeError with {book}_{chunk}.txt\")\n            break\n    # 4. Add to train_corpus\n    if content != \"\":\n        with lock:\n            test_corpus[book] = content","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:14:13.932968Z","iopub.execute_input":"2023-07-12T11:14:13.934305Z","iopub.status.idle":"2023-07-12T11:14:13.944404Z","shell.execute_reply.started":"2023-07-12T11:14:13.934267Z","shell.execute_reply":"2023-07-12T11:14:13.943017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def thread_args_iter_test():\n    for book in chunk_count_test.keys():\n        yield book, chunk_count_test[book]\n\nwith ThreadPoolExecutor(max_workers=32) as e:\n    _ = list(tqdm(e.map(lambda t: read_book_test(*t), thread_args_iter_test()), total=len(chunk_count_test)))","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:14:13.945921Z","iopub.execute_input":"2023-07-12T11:14:13.946284Z","iopub.status.idle":"2023-07-12T11:16:36.18125Z","shell.execute_reply.started":"2023-07-12T11:14:13.946252Z","shell.execute_reply":"2023-07-12T11:16:36.179787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"finished with {len(test_corpus)} books on corpus\")","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:17:14.362612Z","iopub.execute_input":"2023-07-12T11:17:14.363016Z","iopub.status.idle":"2023-07-12T11:17:14.36976Z","shell.execute_reply.started":"2023-07-12T11:17:14.362987Z","shell.execute_reply":"2023-07-12T11:17:14.368757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row2book_test = list(test_corpus.keys())","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:17:15.997181Z","iopub.execute_input":"2023-07-12T11:17:15.99851Z","iopub.status.idle":"2023-07-12T11:17:16.00438Z","shell.execute_reply.started":"2023-07-12T11:17:15.998447Z","shell.execute_reply":"2023-07-12T11:17:16.003524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_test_2 = pipeline_2.predict(test_corpus.values())","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:17:16.849187Z","iopub.execute_input":"2023-07-12T11:17:16.849602Z","iopub.status.idle":"2023-07-12T11:19:20.602281Z","shell.execute_reply.started":"2023-07-12T11:17:16.849568Z","shell.execute_reply":"2023-07-12T11:19:20.601255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMB2CLASSNAME = {v: k for k, v in CLASSNAME2NUMB.items()}\nbook2pred = {book: NUMB2CLASSNAME[pred] for book, pred in zip(row2book_test, preds_test_2)}","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:19:20.603819Z","iopub.execute_input":"2023-07-12T11:19:20.604511Z","iopub.status.idle":"2023-07-12T11:19:20.613331Z","shell.execute_reply.started":"2023-07-12T11:19:20.604487Z","shell.execute_reply":"2023-07-12T11:19:20.612136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(os.path.join(\"/kaggle\", \"input\", \"colie\", \"test.csv\"))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:19:20.614602Z","iopub.execute_input":"2023-07-12T11:19:20.615055Z","iopub.status.idle":"2023-07-12T11:19:20.740568Z","shell.execute_reply.started":"2023-07-12T11:19:20.615028Z","shell.execute_reply":"2023-07-12T11:19:20.739205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pattern = re.compile(\"(\\d+)(\\s\\(\\d\\))?_\\d+\\.txt\")\ndf_test[\"Epoch\"] = df_test.apply(lambda row: book2pred[pattern.fullmatch(row[\"BOOK_id\"]).group(1)], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:19:20.74302Z","iopub.execute_input":"2023-07-12T11:19:20.743357Z","iopub.status.idle":"2023-07-12T11:19:21.733906Z","shell.execute_reply.started":"2023-07-12T11:19:20.743327Z","shell.execute_reply":"2023-07-12T11:19:21.732892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:19:21.735015Z","iopub.execute_input":"2023-07-12T11:19:21.735301Z","iopub.status.idle":"2023-07-12T11:19:21.746183Z","shell.execute_reply.started":"2023-07-12T11:19:21.735277Z","shell.execute_reply":"2023-07-12T11:19:21.744654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-12T11:19:21.747311Z","iopub.execute_input":"2023-07-12T11:19:21.747595Z","iopub.status.idle":"2023-07-12T11:19:21.964924Z","shell.execute_reply.started":"2023-07-12T11:19:21.747571Z","shell.execute_reply":"2023-07-12T11:19:21.962748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}