{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# 遍历kaggle目录，检查文件是否载入正确。\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# 加载packages。\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nfrom tensorflow.keras.layers import Dense, Input,concatenate,Bidirectional, LSTM,MaxPool1D,MaxPool3D,GlobalMaxPooling1D,GlobalAveragePooling1D\nfrom tensorflow.keras.optimizers import Adam,SGD\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\n\nimport transformers\nfrom transformers import *\nfrom transformers import TFAutoModel, AutoTokenizer,AutoModel\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\n\n\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\nfrom sklearn.metrics import f1_score\nimport re\n\ndef seed_everything(seed=0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\n# 设定seed\nSEED = 36\nseed_everything(SEED)\n#from kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 调用bert tokenizer将text转化为tokens（以及mask matrix）\ndef regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=True, \n        return_token_type_ids=True,\n        pad_to_max_length=True,\n        max_length=maxlen\n     )\n    #,np.asarray(enc_di['attention_masks'],dtype=np.int32)\n    return np.array(enc_di['input_ids'],dtype=np.int32),np.array(enc_di['attention_mask'],dtype=np.int32),np.array(enc_di['token_type_ids'],dtype=np.int32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n# 将loss function换成focal，效果不明显。\n\ndef focal_loss(gamma=2., alpha=.2):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tf model structure\ndef build_model(transformer,max_len=512):\n\n    input_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")#input_word_ids\n    input_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n    segment_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n    sequence_output = transformer((input_ids,input_mask,segment_ids))[0]\n    \n    #cls_token = sequence_output[:, 0, :]\n    gp = GlobalMaxPooling1D()(sequence_output)\n    ap = GlobalAveragePooling1D()(sequence_output)\n    stack = concatenate([gp,ap],axis=1)\n    #bert的输出使用max和mean，也可以像上一个comment中一样只用CLS，但此比赛CLS效果不佳。\n    \n    out = Dense(1, activation='sigmoid')(stack)\n    #在bert的输出pooling之后下接linear layer。\n    model = Model(inputs=[input_ids,input_mask,segment_ids], outputs=out)\n    #定义模型的输入与输入。\n    model.compile(Adam(lr=0.25e-5), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC()]) \n    #设定学习率Loss function和评分metrics。\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n# 设定使用TPU需要的参数。","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\n#GCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\n# 训练的Configuration\nEPOCHS = 2\nBATCH_SIZE = 16* strategy.num_replicas_in_sync\nMAX_LEN = 256\nMODEL = 'jplu/tf-xlm-roberta-large'\n# 从hugging face的目录中选择模型型号，这里选用XLMR-large，也可以换成mbert、XLMR-base等。","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n# 载入数据","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv\")\ndf_train['lang'] = 'lang'\ndf_train = df_train[['comment_text', 'toxic', 'lang']].head(0)\n# 载入数据","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nfor lang in ['es', 'tr', 'it', 'ru', 'pt', 'fr']:\n                                                      \n    df_lang = pd.read_csv(f'../input/jigsawtanslatedgoogle/jigsaw-unintended-bias-train_{lang}_clean.csv')[['id', 'comment_text', 'toxic']]\n    df_lang = df_lang[(df_lang['toxic'] < 0.2) | (df_lang['toxic'] > 0.5)]\n    df_lang['toxic'] = df_lang['toxic'].apply(lambda x:round(x)).astype(np.int32)\n    df_lang = df_lang[~df_lang['comment_text'].isna()]\n    df_lang_sampled = pd.concat([\n        df_lang[['comment_text', 'toxic']].query('toxic==1').sample(n=35000),\n        df_lang[['comment_text', 'toxic']].query('toxic==0').sample(n=80000),])  \n    del df_lang\n#     df_lang = df_lang.drop_duplicates(subset='comment_text')                                                                \n    df_lang_sampled['lang'] = lang\n    df_train = df_train.append(df_lang_sampled) \n# 载入数据","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def text_head_tail(input_df, text_col='comment_text'):\n    \n    df = input_df.copy()\n    df['text_head'] = df[text_col].apply(lambda x:' '.join(x.split()[:50]))\n    df['text_tail'] = df[text_col].apply(lambda x:' '.join(x.split()[-70:]))\n    df['text_length'] = df[text_col].apply(lambda x:len(x.split()))\n    df[text_col] = np.where(df['text_length']>120,\n                            df['text_head'] + ' '+df['text_tail'],\n                            df[text_col])\n    df['text_length_2'] = df[text_col].apply(lambda x:len(x.split()))  \n    \n    return df\n\ndf_train = text_head_tail(df_train)\nvalid = text_head_tail(valid)\ntest = text_head_tail(test, 'content')\n\n# 修改过长的原始text，只保留头尾的部分。（根据论文保留头尾是一个处理长文本分类问题的好解决方案）    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x_train = regular_encode(train['comment_text'].values, tokenizer, maxlen=MAX_LEN)\nx_train = regular_encode(df_train['comment_text'].values, tokenizer, maxlen=MAX_LEN)\nx_valid = regular_encode(valid['comment_text'].values, tokenizer, maxlen=MAX_LEN)\nx_test = regular_encode(test['content'].values, tokenizer, maxlen=MAX_LEN)\n\n#y_train = train.toxic.values\ny_train = df_train.toxic.values\ny_valid = valid.toxic.values\n# 调用tokenizer将text转化为tokens","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 将数据导入loader。\ntrain_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .shuffle(len(y_train))\n    .batch(BATCH_SIZE)\n    .repeat()\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 载入与训练的模型，注意这里使用了之前定义的MODEL变量（模型地址）。\nwith strategy.scope():\n#     transformer_layer = TFAutoModel.from_pretrained('/kaggle/input/jigsaw-mlm-finetuned-xlm-r-large/')\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 训练training set。\nn_steps = df_train.shape[0] // BATCH_SIZE\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS,\n    shuffle=False,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 训练validation set。\n# 此次比赛training set为纯英文，而validation set中含有其他语言，在validation set上训练可以提高模型对非英文的预测表现。\nn_steps = valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=4,\n    shuffle=False,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 预测，储存模型，观察predictoin的分布。\nsub['toxic'] = model.predict(x_test, verbose=1)\nsub.to_csv('submission.csv', index=False)\nmodel.save_weights('checkpoint.h5', overwrite=True)\nsub.head(55)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}