{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"---\n## [Jigsaw Rate Severity of Toxic Comments][1]\n---\n**Comments1**: 'Internet' is required for this Notebook.\n\n**Comments2**: Thanks to previous great Notebooks.\n\n1. [☣️ Jigsaw - Incredibly Simple Naive Bayes [0.768]][2]\n2. [AutoNLP for toxic ratings ;)][3]\n\n\n[1]: https://www.kaggle.com/c/jigsaw-toxic-severity-rating/overview\n[2]: https://www.kaggle.com/julian3833/jigsaw-incredibly-simple-naive-bayes-0-768\n[3]: https://www.kaggle.com/abhishek/autonlp-for-toxic-ratings","metadata":{}},{"cell_type":"markdown","source":"# 0. Settings","metadata":{}},{"cell_type":"code","source":"# Import dependencies \nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \n%matplotlib inline\n\nimport os\nimport pathlib\nimport gc\nimport sys\nimport math \nimport time \nimport tqdm \nfrom tqdm import tqdm \nimport random\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.model_selection import KFold \nfrom sklearn.model_selection import StratifiedKFold \n\nimport tensorflow as tf\nimport tensorflow.keras as keras\nfrom tensorflow.keras.layers.experimental import preprocessing\n\nimport transformers \nimport datasets ","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:34:17.306720Z","iopub.execute_input":"2021-12-03T04:34:17.306984Z","iopub.status.idle":"2021-12-03T04:34:17.317039Z","shell.execute_reply.started":"2021-12-03T04:34:17.306953Z","shell.execute_reply":"2021-12-03T04:34:17.316363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# global config\nconfig = {\n    'nfolds': 10,\n    'batch_size': 32,\n    'learning_rate': 1e-4,\n    'num_epochs': 3,\n    'batch_size': 8,\n}\n\nAUTOTUNE = tf.data.experimental.AUTOTUNE\n\n# For reproducible results    \ndef seed_all(s):\n    random.seed(s)\n    np.random.seed(s)\n    tf.random.set_seed(s)\n    os.environ['TF_CUDNN_DETERMINISTIC'] = '1'\n    os.environ['PYTHONHASHSEED'] = str(s) \nglobal_seed = 42\nseed_all(global_seed)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:34:38.636636Z","iopub.execute_input":"2021-12-03T04:34:38.636918Z","iopub.status.idle":"2021-12-03T04:34:38.643770Z","shell.execute_reply.started":"2021-12-03T04:34:38.636887Z","shell.execute_reply":"2021-12-03T04:34:38.642180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":"### 1. Create train data\n\nFor training data, I used [Toxic Comment Classification Challenge][1] dataset.\n\n[1]: https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/data\n\nI turn it into a binary toxic/ no-toxic classification","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/jigsaw-toxic-comment-classification-challenge/train.csv')\ndf['y'] = (df[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].sum(axis=1) > 0 ).astype(int)\ndf = df[['comment_text', 'y']].rename(columns={'comment_text': 'text'})\ndf.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:34:50.046136Z","iopub.execute_input":"2021-12-03T04:34:50.046831Z","iopub.status.idle":"2021-12-03T04:34:51.643887Z","shell.execute_reply.started":"2021-12-03T04:34:50.046790Z","shell.execute_reply":"2021-12-03T04:34:51.643162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Undersampling\n\nThe dataset is very unbalanced. Here we undersample the majority class. Other strategies might work better.","metadata":{}},{"cell_type":"code","source":"df['y'].value_counts(normalize=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:34:54.737344Z","iopub.execute_input":"2021-12-03T04:34:54.737744Z","iopub.status.idle":"2021-12-03T04:34:54.748712Z","shell.execute_reply.started":"2021-12-03T04:34:54.737708Z","shell.execute_reply":"2021-12-03T04:34:54.747803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_len = (df['y'] == 1).sum()\ndf_y0_undersample = df[df['y'] == 0].sample(n=min_len, random_state=global_seed)\ntrain_df = pd.concat([df[df['y'] == 1], df_y0_undersample]).reset_index(drop=True)\ntrain_df['y'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:34:56.034885Z","iopub.execute_input":"2021-12-03T04:34:56.035417Z","iopub.status.idle":"2021-12-03T04:34:56.065518Z","shell.execute_reply.started":"2021-12-03T04:34:56.035377Z","shell.execute_reply":"2021-12-03T04:34:56.064805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:34:59.156115Z","iopub.execute_input":"2021-12-03T04:34:59.156411Z","iopub.status.idle":"2021-12-03T04:34:59.166381Z","shell.execute_reply.started":"2021-12-03T04:34:59.156376Z","shell.execute_reply":"2021-12-03T04:34:59.165424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 k-fold","metadata":{}},{"cell_type":"code","source":"n_folds = 10\n\nskf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=global_seed)\nfor nfold, (train_index, val_index) in enumerate(skf.split(X=train_df.index,\n                                                           y=train_df.y)):\n    train_df.loc[val_index, 'fold'] = nfold\nprint(train_df.groupby(['fold', train_df.y]).size())","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:35:02.461960Z","iopub.execute_input":"2021-12-03T04:35:02.462235Z","iopub.status.idle":"2021-12-03T04:35:02.494198Z","shell.execute_reply.started":"2021-12-03T04:35:02.462204Z","shell.execute_reply":"2021-12-03T04:35:02.493286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_fold = 0\np_train = train_df.query(f'fold != {p_fold}').reset_index(drop=True)\np_valid = train_df.query(f'fold == {p_fold}').reset_index(drop=True)\n\nprint(len(p_train))\nprint(len(p_valid))\n\np_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:35:03.990921Z","iopub.execute_input":"2021-12-03T04:35:03.991416Z","iopub.status.idle":"2021-12-03T04:35:04.018186Z","shell.execute_reply.started":"2021-12-03T04:35:03.991378Z","shell.execute_reply":"2021-12-03T04:35:04.017472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. DataSet","metadata":{}},{"cell_type":"code","source":"checkpoint = \"bert-base-uncased\"\ntokenizer = transformers.AutoTokenizer.from_pretrained(checkpoint)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:36:04.052202Z","iopub.execute_input":"2021-12-03T04:36:04.053087Z","iopub.status.idle":"2021-12-03T04:36:05.793173Z","shell.execute_reply.started":"2021-12-03T04:36:04.053045Z","shell.execute_reply":"2021-12-03T04:36:05.792309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = datasets.Dataset.from_pandas(p_train)\nvalid_ds = datasets.Dataset.from_pandas(p_valid)\n\nprint(train_ds)\nprint(valid_ds)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:36:32.074016Z","iopub.execute_input":"2021-12-03T04:36:32.074330Z","iopub.status.idle":"2021-12-03T04:36:32.131542Z","shell.execute_reply.started":"2021-12-03T04:36:32.074277Z","shell.execute_reply":"2021-12-03T04:36:32.130749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize_function(example):\n    return tokenizer(example[\"text\"], truncation=True)\n\ntokenized_train_ds = train_ds.map(tokenize_function, batched=True)\ntokenized_valid_ds = valid_ds.map(tokenize_function, batched=True)\n\nprint(tokenized_train_ds)\nprint(tokenized_valid_ds)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:44:56.672324Z","iopub.execute_input":"2021-12-03T04:44:56.673000Z","iopub.status.idle":"2021-12-03T04:45:05.548642Z","shell.execute_reply.started":"2021-12-03T04:44:56.672955Z","shell.execute_reply":"2021-12-03T04:45:05.547823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = transformers.DataCollatorWithPadding(tokenizer=tokenizer)\n\ntf_train_ds = tokenized_train_ds.to_tf_dataset(\n    columns=[\"attention_mask\", \"input_ids\", \"token_type_ids\"],\n    label_cols=[\"y\"],\n    shuffle=True,\n    collate_fn=data_collator,\n    batch_size=config['batch_size'],\n)\n\ntf_valid_ds = tokenized_valid_ds.to_tf_dataset(\n    columns=[\"attention_mask\", \"input_ids\", \"token_type_ids\"],\n    label_cols=[\"y\"],\n    shuffle=False,\n    collate_fn=data_collator,\n    batch_size=config['batch_size'],\n)\n\nprint(len(tf_train_ds))\nprint(len(tf_valid_ds))","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:48:13.625815Z","iopub.execute_input":"2021-12-03T04:48:13.626318Z","iopub.status.idle":"2021-12-03T04:48:19.188847Z","shell.execute_reply.started":"2021-12-03T04:48:13.626263Z","shell.execute_reply":"2021-12-03T04:48:19.187990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Model Training","metadata":{}},{"cell_type":"code","source":"from transformers import TFAutoModelForSequenceClassification\n\nmodel = TFAutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:49:20.104858Z","iopub.execute_input":"2021-12-03T04:49:20.105321Z","iopub.status.idle":"2021-12-03T04:49:36.862638Z","shell.execute_reply.started":"2021-12-03T04:49:20.105268Z","shell.execute_reply":"2021-12-03T04:49:36.861853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 2\nnum_train_steps = len(tf_train_ds) * num_epochs\n\nlr_scheduler = tf.keras.optimizers.schedules.PolynomialDecay(\n    initial_learning_rate=5e-5, end_learning_rate=0.0, decay_steps=num_train_steps\n)\n\nmodel.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=lr_scheduler),\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:49:42.368620Z","iopub.execute_input":"2021-12-03T04:49:42.369467Z","iopub.status.idle":"2021-12-03T04:49:42.402660Z","shell.execute_reply.started":"2021-12-03T04:49:42.369408Z","shell.execute_reply":"2021-12-03T04:49:42.401950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fit_history = model.fit(tf_train_ds,\n                        epochs=num_epochs,\n                        validation_data=tf_valid_ds,\n                        verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T04:49:46.142593Z","iopub.execute_input":"2021-12-03T04:49:46.143440Z","iopub.status.idle":"2021-12-03T05:27:55.982863Z","shell.execute_reply.started":"2021-12-03T04:49:46.143383Z","shell.execute_reply":"2021-12-03T05:27:55.982144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Prediction & Submit","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(\"../input/jigsaw-toxic-severity-rating/comments_to_score.csv\")\ntest_ds = datasets.Dataset.from_pandas(test_df)\ntokenized_test_ds = test_ds.map(tokenize_function, batched=True)\ntf_test_ds = tokenized_test_ds.to_tf_dataset(\n    columns=[\"attention_mask\", \"input_ids\", \"token_type_ids\"],\n    shuffle=False,\n    collate_fn=data_collator,\n    batch_size=config['batch_size'],\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-03T05:31:29.940174Z","iopub.execute_input":"2021-12-03T05:31:29.940810Z","iopub.status.idle":"2021-12-03T05:31:32.155510Z","shell.execute_reply.started":"2021-12-03T05:31:29.940766Z","shell.execute_reply":"2021-12-03T05:31:32.154857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_result = model.predict(tf_test_ds)\nresult = tf.sigmoid(raw_result.logits)\n\ntest_df['score'] = result.numpy()[:, 0]\nsubmission_df = test_df[['comment_id', 'score']]\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-12-03T05:38:03.064682Z","iopub.execute_input":"2021-12-03T05:38:03.064958Z","iopub.status.idle":"2021-12-03T05:40:25.039528Z","shell.execute_reply.started":"2021-12-03T05:38:03.064927Z","shell.execute_reply":"2021-12-03T05:40:25.038822Z"},"trusted":true},"execution_count":null,"outputs":[]}]}