{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load up data ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2 = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import transformers\nimport json\nimport logging\nimport os\nimport time\nfrom dataclasses import dataclass, field\nfrom enum import Enum\nfrom pathlib import Path\nfrom typing import List, Optional, Union\nfrom collections import Counter\nimport nltk\nimport torch\nfrom filelock import FileLock\nfrom torch.utils.data.dataset import Dataset\nfrom transformers.data.processors.utils import InputFeatures, DataProcessor, InputExample\nfrom transformers.tokenization_utils import PreTrainedTokenizer\nfrom dataclasses import dataclass, field","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass Split(Enum):\n    train = \"train\"\n    dev = \"dev\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Setup classifier arguments","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"@dataclass\nclass CsvClassifierDataTrainingArguments:\n    \"\"\"\n    Arguments pertaining to what data we are going to input our model for training and eval.\n\n    Using `HfArgumentParser` we can turn this class\n    into argparse arguments to be able to specify them on\n    the command line.\n    \"\"\"\n\n    data_dir: str = field(\n        default='/kaggle/input/quora-insincere-questions-classification/',\n        metadata={\"help\": \"The input data dir. Should contain the .csv files (or other data files) for the task.\"}\n    )\n    cache_dir: str = field(\n        default='/kaggle/temp/',\n        metadata={\"help\": \"The cache data dir. Can be used to write the .lock files\"}\n    )\n    source_key: str = field(\n        default='question_text',\n        metadata={\n            \"help\": \"The source key in the csv.\"\n        }\n    )\n    source_key_b: Optional[str] = field(\n        default=None,\n        metadata={\n            \"help\": \"The source key b in the csv.\"\n        }\n    )\n    target_key: str = field(\n        default='target',\n        metadata={\n            \"help\": \"The target key in the csv.\"\n        }\n    )\n\n    max_seq_length: int = field(\n        default=100,\n        metadata={\n            \"help\": \"The maximum total input sequence length after tokenization. Sequences longer \"\n                    \"than this will be truncated, sequences shorter will be padded.\"\n        },\n    )\n    overwrite_cache: bool = field(\n        default=False, metadata={\"help\": \"Overwrite the cached training and evaluation sets\"}\n    )\n    num_labels: int = field(\n        default=2,\n        metadata={\n            \"help\": \"The number of classification labels.\"\n        },\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass CsvClassifierProcessor(DataProcessor):\n    \"\"\"Base class for data converters for sequence classification data sets.\"\"\"\n\n    def __init__(self, data_dir, train_file, dev_file, source_key, label_key, source_key_b=None):\n        self.label_key = label_key\n        self.source_key = source_key\n        self.dev_file = dev_file\n        self.train_file = train_file\n        self.data_dir = data_dir\n        self.source_key_b = source_key_b\n        self.map = {}\n        \n    def _get_examples_from_file(self, filepath: Path, split: str):\n        print(f'Opening {filepath}')\n        examples = []\n        data = pd.read_csv(filepath)\n        for i, row in tqdm.tqdm(data.iterrows()):\n            if type(row[self.source_key]) == str:\n                text_b = row.get(self.source_key_b, None)\n                if text_b is not None:\n                    text_b = text_b.strip()\n                label_ = row.get(self.label_key, None)\n                ie = InputExample(guid=f\"{split}-{i}\", text_a=row[self.source_key], text_b=text_b,\n                                      label=label_)\n                examples.append(ie)\n        self.map[filepath] = examples\n        return examples\n\n    def get_train_examples(self):\n        \"\"\"Gets a collection of `InputExample`s for the train set.\"\"\"\n        filepath = Path(self.data_dir) / self.train_file\n        print('Getting train examples')\n        if filepath in self.map:\n            return self.map[filepath]\n\n        data = self._get_examples_from_file(filepath, 'train')\n        print('Done')\n\n        return data\n\n    def get_dev_examples(self):\n        \"\"\"Gets a collection of `InputExample`s for the test set.\"\"\"\n        filepath = Path(self.data_dir) / self.dev_file\n        print('Getting dev examples')\n        if filepath in self.map:\n            return self.map[filepath]\n\n        return self._get_examples_from_file(filepath, 'test')\n\n\n\nclass CsvClassifierDataset(Dataset):\n    \"\"\"\n    This will be superseded by a framework-agnostic approach\n    soon.\n    \"\"\"\n\n    args: CsvClassifierDataTrainingArguments\n    features: List[InputFeatures]\n\n    def __init__(\n            self,\n            processor : CsvClassifierProcessor,\n            args: CsvClassifierDataTrainingArguments,\n            tokenizer: PreTrainedTokenizer,\n            limit_length: Optional[int] = None,\n            start_length: Optional[int] = 0,\n            mode: Union[str, Split] = Split.train,\n            cache_dir: Optional[str] = None,\n        \n    ):\n        self.args = args\n        print(f'Getting for {args.data_dir}')\n\n        self.processor = processor\n        if isinstance(mode, str):\n            try:\n                mode = Split[mode]\n            except KeyError:\n                raise KeyError(f\"mode {mode} is not a valid split name\")\n\n\n        logger.info(f\"Creating features from dataset file at {args.data_dir}\")\n\n        if mode == Split.dev:\n            examples = self.processor.get_dev_examples()\n        else:\n            examples = self.processor.get_train_examples()\n        if limit_length is not None:\n            examples = examples[start_length:limit_length]\n        self.features = convert_examples_to_features(\n            examples,\n            tokenizer,\n            max_length=100,\n        )\n\n    def __len__(self):\n        return len(self.features)\n\n    def __getitem__(self, i) -> InputFeatures:\n        return self.features[i]\n\n\ndef convert_examples_to_features(\n        examples,\n        tokenizer,\n        max_length=256,\n        pad_on_left=False,\n        pad_token=0,\n        pad_token_segment_id=0,\n        mask_padding_with_zero=True,\n):\n    print('Running convert_examples_to_features')\n    features = []\n    for (ex_index, example) in tqdm.tqdm(enumerate(examples)):\n        len_examples = len(examples)\n        if ex_index % 1000 == 0:\n            logger.info(\"Writing example %d/%d\" % (ex_index, len_examples))\n        inputs = tokenizer.encode_plus(text=example.text_a, text_pair=example.text_b, add_special_tokens=True,\n                                       max_length=max_length, return_token_type_ids=True,\n                                       )\n\n        input_ids, token_type_ids = inputs[\"input_ids\"], inputs[\"token_type_ids\"]\n\n        # The mask has 1 for real tokens and 0 for padding tokens. Only real\n        # tokens are attended to.\n        attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n\n        # Zero-pad up to the sequence length.\n        padding_length = max_length - len(input_ids)\n        if pad_on_left:\n            input_ids = ([pad_token] * padding_length) + input_ids\n            attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask\n            token_type_ids = ([pad_token_segment_id] * padding_length) + token_type_ids\n        else:\n            input_ids = input_ids + ([pad_token] * padding_length)\n            attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)\n            token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length)\n\n        assert len(input_ids) == max_length, \"Error with input length {} vs {}\".format(len(input_ids), max_length)\n        assert len(attention_mask) == max_length, \"Error with input length {} vs {}\".format(\n            len(attention_mask), max_length\n        )\n        assert len(token_type_ids) == max_length, \"Error with input length {} vs {}\".format(\n            len(token_type_ids), max_length\n        )\n\n        if ex_index < 5:\n            logger.info(\"*** Example ***\")\n            logger.info(\"guid: %s\" % (example.guid))\n            logger.info(\"input_ids: %s\" % \" \".join([str(x) for x in input_ids]))\n            logger.info(f'{tokenizer.decode(input_ids)}')\n            logger.info(\"attention_mask: %s\" % \" \".join([str(x) for x in attention_mask]))\n            logger.info(\"token_type_ids: %s\" % \" \".join([str(x) for x in token_type_ids]))\n            logger.info(\"label: %s \" % (example.label))\n            logger.info(f'{tokenizer.decode(input_ids)}')\n        features.append(\n            InputFeatures(\n                input_ids=input_ids, attention_mask=attention_mask, #token_type_ids=token_type_ids,\n                label=example.label\n            )\n        )\n    print('Done')\n    return features\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import logging\nimport os\nimport sys\nfrom dataclasses import dataclass, field\nfrom typing import Optional\nimport numpy as np\nfrom sklearn.metrics import precision_recall_fscore_support\nfrom sklearn.utils import compute_class_weight\n\nfrom transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer, EvalPrediction, AutoModel\nfrom transformers import (\n    HfArgumentParser,\n    Trainer,\n    TrainingArguments,\n    set_seed,\n)\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclass\nclass ModelArguments:\n    \"\"\"\n    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n    \"\"\"\n\n    model_name_or_path: str = field(\n        default='distilbert-base-uncased',\n        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n    )\n    config_name: Optional[str] = field(\n        default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n    )\n    tokenizer_name: Optional[str] = field(\n        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n    )\n    model_type: str = field(\n        default=\"distilbert\", metadata={\"help\": \"Type of the model (e.g. bert-base-uncased)\"}\n    )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Setup configuration for the training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"parser = HfArgumentParser((ModelArguments, CsvClassifierDataTrainingArguments, TrainingArguments))\n\n\ntraining_args = TrainingArguments('/kaggle/working', save_total_limit=1)\n\nmodel_args = ModelArguments()\ndata_args = CsvClassifierDataTrainingArguments()\n\n\n\n# Set seed\nset_seed(training_args.seed)\n\nconfig = AutoConfig.from_pretrained(\n    model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n    num_labels=data_args.num_labels,\n    cache_dir=data_args.cache_dir,\n)\ntokenizer = AutoTokenizer.from_pretrained(\n    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n    cache_dir=data_args.cache_dir,\n)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load up datasets for training and evaluation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"processor = CsvClassifierProcessor(data_args.data_dir, train_file='train.csv', dev_file='test.csv',\n                                                 source_key=data_args.source_key, label_key=data_args.target_key,\n                                                 source_key_b=data_args.source_key_b)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get datasets\ntrain_dataset = (\n    CsvClassifierDataset(processor, data_args, tokenizer=tokenizer, cache_dir=data_args.cache_dir, limit_length=900000)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nlabels = [feature.label for feature in train_dataset]\nclass_weights = compute_class_weight(class_weight='balanced', classes=np.unique(labels), y=labels)\n\neval_dataset = (\n    CsvClassifierDataset(processor, data_args, tokenizer=tokenizer, cache_dir=data_args.cache_dir, start_length=900000, limit_length=105000)\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#save the processed dataset \n\nimport pickle\npickle.dump(train_dataset, open('/kaggle/working/dataset.pkl', 'wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/working","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(training_args)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_args.per_device_train_batch_size = 128\ntraining_args.num_train_epochs = 1\ntraining_args.logging_steps = 100\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Configure model and run the training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel = AutoModelForSequenceClassification.from_pretrained(\n    model_args.model_name_or_path,\n    from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n    config=config,\n    cache_dir=data_args.cache_dir,\n)\nmodel = model.to('cuda')\n\ndef compute_metrics_fn(p: EvalPrediction):\n    prediction = p.predictions.argmax(axis=1)\n    breakpoint()\n    precision, recall, fbeta, *_ = precision_recall_fscore_support(y_true=p.label_ids, y_pred=prediction,\n                                                                   average='weighted'\n                                                                   )\n    return {'accuracy': (prediction == p.label_ids).mean(),\n            'precision': precision,\n            'recall': recall,\n            'f1': fbeta\n            }\n\n# Initialize our Trainer\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=eval_dataset,\n    compute_metrics=compute_metrics_fn,\n)\n\n# Training\ntrainer.train(\n    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path) else None\n)\ntrainer.save_model()\n# For convenience, we also re-save the tokenizer to the same directory,\n# so that you can share your model easily on huggingface.co/models =)\nif trainer.is_world_master():\n    tokenizer.save_pretrained(training_args.output_dir)\n\n# Evaluation\neval_results = {}\nlogger.info(\"*** Evaluate ***\")\n\ntrainer.compute_metrics = compute_metrics_fn\neval_result = trainer.evaluate(eval_dataset=eval_dataset)\n\noutput_eval_file = os.path.join(\n    training_args.output_dir, f\"eval_results.txt\"\n)\nif trainer.is_world_master():\n    with open(output_eval_file, \"w\") as writer:\n        logger.info(\"***** Eval results {} *****\")\n        for key, value in eval_result.items():\n            logger.info(\"  %s = %s\", key, value)\n            writer.write(\"%s = %s\\n\" % (key, value))\n\n    eval_results.update(eval_result)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.save_model()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load up the test dataset to do predictions on","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df3 = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\ndf3.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"processor2 = CsvClassifierProcessor(data_args.data_dir, train_file='test.csv', dev_file='test.csv',\n                                                 source_key=data_args.source_key, label_key=data_args.target_key,\n                                                 source_key_b=data_args.source_key_b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntest_dataset = (\n    CsvClassifierDataset(processor2, data_args, tokenizer=tokenizer, mode=Split.dev, cache_dir=data_args.cache_dir)\n)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = trainer.predict(test_dataset)\nprint('Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(predictions[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_softmax = torch.nn.functional.softmax(torch.tensor(predictions[0]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_softmax.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class_predictions = predictions_softmax.argmax(dim=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df3['prediction'] = class_predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df3.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del(df3['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df3.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Done')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}