{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os,re,gc,pickle,random,sys\n\nimport numpy as np \nimport pandas as pd \nimport tensorflow as tf\nimport transformers\nfrom transformers import TFAutoModel\nfrom tensorflow.data.experimental import sample_from_datasets\nfrom tensorflow.data import Dataset\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configurations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(strategy.num_replicas_in_sync)\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(1234)\nrandom.seed(1234)\nos.environ['TF_DETERMINISTIC_OPS'] = '1'\nwith strategy.scope():\n    tf.random.set_seed(1234)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LEN = 192\nHEAD=\"cls\"\nMODEL = 'jplu/tf-xlm-roberta-large'\n\nTRANSFER = \"../input/buffer/submission-9461.csv\"\nPATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\"\nINPATH = \"../input/jwtc-xlmroberta-encoding-192-pickle/datain/\"\nFORM = \"training/encode_{}.pkl\"\nlangs = [\"en\",\"en2\",\"es\",\"fr\",\"it\",\"pt\",\"ru\",\"tr\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trans = pd.read_csv(TRANSFER).toxic.values.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pick_load_format(path):\n    with open(INPATH+path,\"rb\") as f:\n        return pickle.load(f)\n    \ndef get_cong(n,verb=True):\n    tot = round(1+(n*2)/10_000)*10_000\n    if verb: print(\"Pos: {}, Sample neg: {}, Total: {}\".format(n,tot-n,tot))\n    return tot-n\n\ndef load_balance_shuffle_train(seed=1214):\n    train = []\n    for i in langs:\n        df1 = pick_load_format(FORM.format(i+\"_l1\"))\n        sample_size = get_cong(df1.shape[0])\n        df0 = pick_load_format(FORM.format(i+\"_l0\")).sample(n=sample_size, random_state=seed)\n        train += [df1,df0]\n    train = pd.concat(train).sample(frac=1, random_state=seed)\n    train.reset_index(inplace=True,drop=True)\n    return np.stack(train.comment_text.values, axis=0).astype(\"int32\"), train.toxic.values.astype(\"float32\")\n\ndef balance_shuffle(df,col=\"comment_text\",pos_thred=0.6,neg_thred=0.4,balance=True,seed=1214):\n    label = df.toxic.values\n    df_pos = df[label>=pos_thred]\n    df_neg = df[label<=neg_thred]\n    psize,nsize = df_pos.shape[0],df_neg.shape[0]\n    cong = get_cong(psize,balance)\n    if balance and nsize>cong: \n        df_neg = df_neg.sample(n=cong, random_state=seed)\n    df0 = pd.concat([df_pos,df_neg]).sample(frac=1, random_state=seed)\n    df0.reset_index(inplace=True,drop=True)\n    return np.stack(df0[col].values, axis=0).astype(\"int32\"), df0.toxic.values\n\ndef load_data(pos_thred=0.6,neg_thred=0.4):\n    train = load_balance_shuffle_train()\n    valid = balance_shuffle(pick_load_format(\"valid.pkl\"),balance=False)\n    \n    df_test = pick_load_format(\"test.pkl\")\n    df_test[\"toxic\"] = trans\n    quesdo = balance_shuffle(df_test,col=\"content\",balance=False,\n                             pos_thred=pos_thred,neg_thred=neg_thred)\n    \n    return train,valid,quesdo,np.stack(df_test.content.values, axis=0).astype(\"int32\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain,valid,quesdo,test =  load_data(pos_thred=0.6,neg_thred=0.4)\ntrain_size,valid_size,quesdo_size = len(train[1]),len(valid[1]),len(quesdo[1])\nprint(train_size,valid_size,quesdo_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_dataset_pipeline(dataset, cache=False,repeat_and_shuffle=False,shuffle_size=128_000,seed=386491):\n    if cache: dataset = dataset.cache()\n    if repeat_and_shuffle:\n        dataset = dataset.repeat().shuffle(shuffle_size,seed)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef build_datasets(train,valid,quesdo,test):\n    dtrain = Dataset.from_tensor_slices(train)\n    dvalid = Dataset.from_tensor_slices(valid)\n    dquesdo = Dataset.from_tensor_slices(quesdo)\n    dtest = Dataset.from_tensor_slices(test)\n\n    train_dataset = make_dataset_pipeline(dtrain,True, repeat_and_shuffle=True)\n    valid_dataset = make_dataset_pipeline(dvalid, True,repeat_and_shuffle=True) \n    quesdo_dataset = make_dataset_pipeline(dquesdo, True,repeat_and_shuffle=True) \n\n    validset = make_dataset_pipeline(dvalid) \n    testset = make_dataset_pipeline(dtest)\n    return train_dataset,valid_dataset,quesdo_dataset,validset,testset\n\ndef mix_dataset(dss,szs,weight=None,seed=1214):\n    if weight is None: weight = np.ones(len(szs))\n    prop = np.array(szs)*weight\n    return sample_from_datasets(dss,prop/np.sum(prop),seed),sum(szs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset,valid_dataset,quesdo_dataset,validset,testset = build_datasets(train,valid,quesdo,test)\ntrain_dataset,train_size = mix_dataset([train_dataset,quesdo_dataset],[train_size,quesdo_size],weight=np.array([1,3])) #55:5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the model and check summary","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Input,Dropout,Dense,GlobalAveragePooling1D,GlobalMaxPool1D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.metrics import AUC \nfrom tensorflow.keras.initializers import GlorotUniform\n\ndef get_cls(x):\n    return x[:, 0, :]\n\ndic = {\"mean\":GlobalAveragePooling1D(),\n      \"max\":GlobalMaxPool1D(),\n      \"cls\":get_cls}\n\ndef build_model(transformer,head=\"cls\" , loss='binary_crossentropy',\n                max_len=512, drop_rate=None, lr=1e-5,seed=940208):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    x = dic[head](sequence_output)\n    if drop_rate is not None: \n        x = Dropout(drop_rate)(x)\n    out = Dense(1, activation='sigmoid',kernel_initializer=GlorotUniform(seed))(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=lr), loss=loss, metrics=[AUC()])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy',\n                        max_len=MAX_LEN,lr=1e-5)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler,ReduceLROnPlateau\n\nmodel_path = \"xlm-roberta.h5\"\ncheckpoint = ModelCheckpoint(model_path, monitor='val_auc', mode=\"max\", \n                             save_best_only=True, save_weights_only=True, verbose=1)\n\n# es = EarlyStopping(monitor='val_auc', mode='max', patience=6, restore_best_weights=False, verbose=1)\nrp = ReduceLROnPlateau(monitor='val_auc', factor=0.8, patience=3, verbose=1, mode='max')\n\ncallback_list = [checkpoint,rp]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nN_STEPS = train_size // (BATCH_SIZE*4)\nEPOCHS = 8\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=N_STEPS,\n    validation_data=validset,\n    callbacks=callback_list,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model\ngc.collect()\ntf.tpu.experimental.initialize_tpu_system(tpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=5e-6)\n    model.load_weights(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nn_steps = valid_size // (BATCH_SIZE)\nEPOCHS = 1\ntrain_history_2 =model.fit(\n    valid_dataset,\n    steps_per_epoch=n_steps,\n    epochs= EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.save_weights(model_path)\n!rm xlm-roberta.h5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(PATH + \"sample_submission.csv\")\nsub['toxic'] = model.predict(testset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}