{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! mkdir /kaggle/processed_data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# 1. Process data\n# -*- coding: utf-8 -*- #\n\"\"\"Preprocess Meta Information\"\"\"\n\nimport sys\nsys.path.append('/kaggle/input/configutil')\nsys.path.append('/kaggle/input/competitionutils')\n\nimport os\nimport gc\nfrom PIL import Image\nfrom pathlib import PosixPath\nfrom argparse import ArgumentParser\n\nimport pandas as pd\nimport utils\nimport config\n\n\ndef argparse():\n    \"\"\"Parse Comndline Args.\"\"\"\n    usage_msg = \"\"\"\n\\n  python {0} [-tr] [-te]\\n\n\"\"\".format(__file__,)\n    parser = ArgumentParser(\n        prog=\"preprocess.py\", usage=usage_msg)\n\n    parser.add_argument('-tr', \"--train\", dest=\"train\", action='store_true')\n    parser.add_argument('-te', \"--test\", dest=\"test\", action='store_true')\n    argvs = parser.parse_args()\n    return argvs\n\n\ndef convert_parquet_to_images(\n    parquet_file_path: PosixPath, image_dir_path: PosixPath\n) -> bool:\n    \"\"\"Convert a parquet file to image files and save them\"\"\"\n    df = pd.read_parquet(parquet_file_path)\n    image_ids = df.image_id.values\n    arrs = df.iloc[:, 1:].values\n    del df\n    gc.collect()\n\n    for i, image_id in enumerate(image_ids):\n        Image.fromarray(\n            arrs[i, :].reshape(config.ORIGINAL_IMAGE_HIGHT, config.ORIGINAL_IMAGE_WIDTH)\n        ).save(image_dir_path / \"{}.png\".format(image_id))\n    del arrs\n    del image_ids\n    gc.collect()\n\n\ndef preprocess_parquet_files(process_train: bool, process_test: bool) -> None:\n    \"\"\"Read parquet files and convert them into images.\"\"\"\n    if process_train:\n        # # train\n        print(\"preprocess train parquet files\")\n        if os.path.isdir(config.TRAIN_IMAGES_DIR):\n            print(\"train images dir already exists!\")\n            pass\n        else:\n            os.mkdir(config.TRAIN_IMAGES_DIR)\n            for i in range(config.PARUET_FILE_NUM):\n                pqt_file_name = \"train_image_data_{}.parquet\".format(i)\n                parquet_file_path = config.RAW_DATA / pqt_file_name\n                with utils.timer(\"convert {} to png files.\".format(pqt_file_name)):\n                    convert_parquet_to_images(parquet_file_path, config.TRAIN_IMAGES_DIR)\n\n    if process_test:\n        # # test\n        print(\"preprocess test parquet files\")\n        if os.path.isdir(config.TEST_IMAGES_DIR):\n            print(\"test images dir already exists!\")\n            pass\n        else:\n            os.mkdir(config.TEST_IMAGES_DIR)\n            for i in range(config.PARUET_FILE_NUM):\n                pqt_file_name = \"test_image_data_{}.parquet\".format(i)\n                parquet_file_path = config.RAW_DATA / pqt_file_name\n                with utils.timer(\"convert {} to png files.\".format(pqt_file_name)):\n                    convert_parquet_to_images(parquet_file_path, config.TEST_IMAGES_DIR)\n\n\ndef preprocess_meta_info_files(process_train: bool, process_test: bool) -> None:\n    \"\"\"Preprocess Train and Test Meta Info.\"\"\"\n    if process_train:\n        with utils.timer(\"preprocess train meta file\"):\n            train = pd.read_csv(config.RAW_DATA / \"train.csv\")\n            # # K-fold split.\n            train[\"character_id\"] = train.apply(\n                lambda row: \"{:0>3}_{:0>2}_{}\".format(\n                    row[\"grapheme_root\"], row[\"vowel_diacritic\"], row[\"consonant_diacritic\"]), axis=1)\n\n            labels_arr = pd.get_dummies(\n                train[[\"grapheme_root\", \"vowel_diacritic\", \"consonant_diacritic\"]],\n                columns=[\"grapheme_root\", \"vowel_diacritic\", \"consonant_diacritic\"]).values\n\n            train[\"fold\"] = -1\n            for fold_id, (train_idx, valid_idx) in enumerate(\n                utils.multi_label_stratified_group_k_fold(\n                    train.character_id.values, labels_arr, config.FOLD_NUM, config.RANDAM_SEED)\n            ):\n                train.loc[valid_idx, \"fold\"] = fold_id\n\n            train.to_csv(config.PROC_DATA / \"train_add-{}fold-index.csv\".format(config.FOLD_NUM), index=False)\n\n    if process_test:\n        with utils.timer(\"preprocess test meta file\"):\n            test = pd.read_csv(config.RAW_DATA / \"test.csv\")\n            test_proc = pd.DataFrame({\"image_id\": test.image_id.drop_duplicates().values})\n            test_proc[\"grapheme_root\"] = 0\n            test_proc[\"vowel_diacritic\"] = 0\n            test_proc[\"consonant_diacritic\"] = 0\n            test_proc.to_csv(config.PROC_DATA / \"test_reshaped.csv\", index=False)\n\n\ndef main_process():\n    \"\"\"Main.\"\"\"\n    preprocess_parquet_files(False, True)\n    preprocess_meta_info_files(False, True)\n\n\nif __name__ == \"__main__\":\n    main_process()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make prediction\n# -*- coding: utf-8 -*- #\nimport sys\nsys.path.append('/kaggle/input')\nsys.path.append('/kaggle/input/configutil')\nsys.path.append('/kaggle/input/competitionutils')\n\"\"\"script for inference validation and test.\"\"\"\nimport gc\nimport os\nimport shutil\nfrom pathlib import Path, PosixPath\nfrom argparse import ArgumentParser\n\nimport yaml\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import recall_score\n\nfrom chainer import serializers, datasets\n\nimport utils\nimport backbonechain\nimport globalpoolingchain \nimport classifierchain\nimport nntraining\n\nimport config\n\n\ndef argparse():\n    \"\"\"Parse Comndline Args.\"\"\"\n    usage_msg = \"\"\"\n\\n  python {0} --trained_path <str> --output_path <str> --epoch_of_model <int> --gpu_device <int> --batch_size <int> [-va]\\n\n\"\"\".format(__file__,)\n    parser = ArgumentParser(prog=\"nn_inference.py\", usage=usage_msg)\n\n    parser.add_argument(\"-t\", \"--trained_path\", dest=\"trained_path\", required=True)\n    parser.add_argument(\"-o\", \"--output_path\", dest=\"output_path\", default=\"\")\n    parser.add_argument(\"-e\", \"--epoch_of_model\", dest=\"epoch_of_model\", default=-1, type=int)\n    parser.add_argument(\"-g\", \"--gpu_device\", dest=\"gpu_device\", default=-1, type=int)\n    parser.add_argument(\"-bs\", \"--batch_size\", dest=\"batch_size\", default=64, type=int)\n    parser.add_argument('-va', \"--valid\", dest=\"valid\", action='store_true')\n    argvs = parser.parse_args()\n    return argvs\n\n\ndef inference(\n    trained_path: PosixPath, output_path: PosixPath, epoch_of_model: int =-1,\n    gpu_device: int=-1, batch_size: int=64, inference_valid: bool=False\n):\n    \"\"\"Inference function for kernel.\"\"\"\n    # # read settings from training outputs directory.\n    with open((trained_path / \"settings.yml\").as_posix(), \"r\") as fr:\n        settings = yaml.safe_load(fr)\n\n    # # make dataset\n    # # # read meta info.\n    with utils.timer(\"make val dataset\"):\n        val_dataset = test_dataset = None\n        if inference_valid:\n            train_df = pd.read_csv(config.PROC_DATA / \"train_add-{}fold-index.csv\".format(settings[\"n_folds\"]))\n            # # # # make label arr\n            train_labels_arr = train_df[config.COMP_NAMES].values.astype(\"i\")\n\n            # # # # make chainer dataset\n            val_dataset = datasets.LabeledImageDataset(\n                pairs=list(zip(\n                    (train_df[train_df[\"fold\"] == settings[\"val_fold\"]][\"image_id\"] + \".png\").tolist(),\n                    train_labels_arr[train_df[\"fold\"] == settings[\"val_fold\"], ...])),\n                root=config.TRAIN_IMAGES_DIR.as_posix())\n            # # # # set transform\n            val_dataset = datasets.TransformDataset(\n                val_dataset, nntraining.ImageTransformer(settings[\"inference_transforms\"]))\n\n    # # # test set\n    with utils.timer(\"make test dataset\"):\n        test_df = pd.read_csv(config.PROC_DATA / \"test_reshaped.csv\")\n        sample_sub = pd.read_csv(config.RAW_DATA / \"sample_submission.csv\")\n\n        # # # # make chainer dataset\n        test_dataset = datasets.LabeledImageDataset(\n            pairs=list(zip((test_df[\"image_id\"] + \".png\").tolist(), ([-1] * len(test_df)))),\n            root=config.TEST_IMAGES_DIR.as_posix())\n        # # # # set transform\n        test_dataset = datasets.TransformDataset(\n            test_dataset, nntraining.ImageTransformer(settings[\"inference_transforms\"]))\n\n    with utils.timer(\"init and load model\"):\n        # # initialize model.\n        settings[\"backbone_kwargs\"][\"pretrained_model_path\"] = None\n        model = nntraining.ImageClassificationModel(\n            extractor=getattr(\n                backbonechain, settings[\"backbone_class\"])(**settings[\"backbone_kwargs\"]),\n            global_pooling=None if settings[\"pooling_class\"] is None else getattr(\n                globalpoolingchain, settings[\"pooling_class\"])(**settings[\"pooling_kwargs\"]),\n            classifier=getattr(\n                classifierchain, settings[\"head_class\"])(**settings[\"head_kwargs\"])\n        )\n        # # load model.\n        model_path = trained_path / \"model_snapshot_{}.npz\".format(epoch_of_model)\n        print(model_path)\n        if not (epoch_of_model != -1 and os.path.isfile(model_path)):\n            model_path = trained_path / \"model_snapshot_last_epoch.npz\"\n\n        print(\"use model: {}\".format(model_path))\n\n        serializers.load_npz(model_path, model)\n        if gpu_device != -1:\n            model.to_gpu(gpu_device)\n        gc.collect()\n\n    settings[\"batch_size\"] = batch_size\n    _, val_iter, test_iter = nntraining.create_iterator(settings, None, val_dataset, test_dataset)\n\n    if inference_valid:\n        with utils.timer(\"inference validation set\"):\n            val_pred, val_label = nntraining.inference_test_data(model, val_iter, gpu_device=gpu_device)\n            np.save(output_path / \"val_pred_arr_fold{}\".format(settings[\"val_fold\"]), val_pred)\n            # # calc score\n            score_list = [[] for i in range(2)]\n\n            for i in range(len(config.N_CLASSES)):\n                y_pred_subset = val_pred[:, config.COMP_INDEXS[i]:config.COMP_INDEXS[i + 1]].argmax(axis=1)\n                y_true_subset = val_label[:, i]\n                score_list[0].append(\n                    recall_score(y_true_subset, y_pred_subset, average='macro', zero_division=0))\n                score_list[1].append(\n                    recall_score(y_true_subset, y_pred_subset, average='macro', zero_division=1))\n\n            del val_dataset\n            del val_iter\n            del val_pred\n            del val_label\n            del y_pred_subset\n            del y_true_subset\n\n            gc.collect()\n            score_list[0].append(np.average(score_list[0], weights=[2, 1, 1]))\n            score_list[1].append(np.average(score_list[1], weights=[2, 1, 1]))\n\n            score_df = pd.DataFrame(\n                score_list, columns=config.COMP_NAMES + [\"score\"])\n\n            print(\"[score for validation set]\")\n            print(score_df)\n            score_df.to_csv(output_path / \"score.csv\", index=False)\n\n    with utils.timer(\"inference test set\"):\n        test_pred, test_label = nntraining.inference_test_data(model, test_iter, gpu_device=gpu_device)\n        del test_label\n\n        np.save(output_path / \"test_pred_arr_fold{}\".format(settings[\"val_fold\"]), test_pred)\n\n    with utils.timer(\"make submission\"):\n        # # # arg max for each component.\n        for i, c_name in enumerate(config.COMP_NAMES):\n            test_pred_subset = test_pred[:, config.COMP_INDEXS[i]:config.COMP_INDEXS[i + 1]].argmax(axis=1)\n            test_df[c_name] = test_pred_subset\n\n        del test_pred\n        gc.collect()\n\n        # # # reshape test_df to submisson format.\n        melt_df = pd.melt(test_df, id_vars=\"image_id\", value_vars=config.COMP_NAMES, value_name=\"target\")\n        melt_df[\"row_id\"] = melt_df[\"image_id\"] + \"_\" + melt_df[\"variable\"]\n\n        submission_df = pd.merge(\n            sample_sub[[\"row_id\"]], melt_df[[\"row_id\", \"target\"]], on=\"row_id\", how=\"left\")\n\n        submission_df.to_csv(output_path / \"submission.csv\", index=False)\n\n\ndef main_pred():\n    \"\"\"Main.\"\"\"\n#     if argvs.output_path != \"\":\n#         output_path = Path(argvs.output_path).resolve()\n#         if os.path.isdir(output_path):\n#             print(\"Directory `{}` already exists. \".format(output_path))\n#             print(\"You must remove it or specify the other directory.\")\n#             quit()\n\n#         os.mkdir(output_path)\n\n#     shutil.copyfile(Path(\".\") / \"nn_inference.py\", output_path / \"nn_inference.py\")\n\n    inference(\n        Path('/kaggle/input/trainpath'), Path('.'), 40,\n        0, 128, False)\n\n\nif __name__ == \"__main__\":\n    main_pred()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}