{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os,gc,pickle,random,sys\n\nimport numpy as np \nimport pandas as pd \nimport tensorflow as tf\nimport transformers\nfrom transformers import TFAutoModel\nfrom tqdm.notebook import tqdm\n\nfrom tensorflow.data import Dataset\nfrom tensorflow.data.experimental import sample_from_datasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"transformers.__version__","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configurations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LEN = 192\nHEAD = \"cls\"\n\nPATH = \"../input/jigsaw-multilingual-toxic-comment-classification/\"\nINPATH = \"../input/jmtc-monolinguish-encoding-pickle/\"\nTRANSFER = \"../input/buffer/submission-pt-9470.csv\"\n\nLANG = \"fr\"\nlang_valid = [\"it\",\"es\",\"tr\"]\nMODEL = \"flaubert/flaubert_large_cased\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(strategy.num_replicas_in_sync)\nBATCH_SIZE = 12 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(1234)\nrandom.seed(1234)\nos.environ['TF_DETERMINISTIC_OPS'] = '1'\nwith strategy.scope():\n    tf.random.set_seed(1234)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = {\"pt\":[\"neuralmind/bert-large-portuguese-cased\"],\n         \"it\":[\"dbmdz/bert-base-italian-xxl-uncased\",\n              \"m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0\"],\n         \"es\":[\"dccuchile/bert-base-spanish-wwm-cased\"],\n         \"tr\":[\"dbmdz/bert-base-turkish-128k-cased\",\n              \"dbmdz/electra-base-turkish-cased-discriminator\"],\n         \"ru\":[\"DeepPavlov/rubert-base-cased\"],\n          \"fr\":[\"flaubert/flaubert_large_cased\",\n               \"camembert/camembert-large\"]}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"trans = pd.read_csv(TRANSFER).toxic.values.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fetch_title = lambda x: x.split(\"/\")[-1]\n\ndef pick_load_format(path):\n    with open(INPATH+path,\"rb\") as f:\n        return pickle.load(f)\n    \ndef get_cong(n,verb=True):\n    tot = round(1+(n*2)/10_000)*10_000\n    if verb: print(\"Pos: {}, Sample neg: {}, Total: {}\".format(n,tot-n,tot))\n    return tot-n\n\ndef balance_shuffle(list_arr,pos_thred=0.6,neg_thred=0.4,balance=True):\n    target = list_arr[-1]\n    idx = np.arange(len(target))\n    pos_idx = idx[target>=pos_thred] \n    neg_idx = idx[target<=neg_thred]\n    psize,nsize = len(pos_idx),len(neg_idx)\n    cong = get_cong(psize,balance)\n    if balance and nsize>cong: \n        neg_idx = np.random.choice(neg_idx,cong,replace=False)\n    idx = np.concatenate((pos_idx, neg_idx))\n    np.random.shuffle(idx)\n    \n    return tuple(arr[idx] for arr in list_arr)\n\ndef load_data(model,lang,frac=.05):\n    title = fetch_title(model)\n    x_train,y_train = balance_shuffle(pick_load_format(\"train/train_%s\"%title))\n    train = x_train,y_train.astype(\"float32\")\n    if lang in lang_valid:\n        valid = balance_shuffle(pick_load_format(\"valid/valid_%s\"%title),balance=False)\n    else:\n        thres = round(len(train[-1])*frac)\n        valid = tuple(arr[:thres] for arr in train)\n        train = tuple(arr[thres:] for arr in train)\n    \n    x_test,id_test = pick_load_format(\"test/test_%s\"%title)\n    quesdo = balance_shuffle((x_test,trans[id_test]),balance=False)\n    \n    return train,valid,quesdo,x_test,id_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train,valid,quesdo,test,ids = load_data(MODEL,LANG)\ntrain_size,valid_size,quesdo_size = len(train[1]),len(valid[1]),len(quesdo[1])\nprint(train_size,valid_size,quesdo_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_dataset_pipeline(dataset, cache=False,repeat_and_shuffle=False,shuffle_size=128_000,seed=386491):\n    if cache: dataset = dataset.cache()\n    if repeat_and_shuffle:\n        dataset = dataset.repeat().shuffle(shuffle_size,seed)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef build_datasets(train,valid,quesdo,test):\n    dtrain = Dataset.from_tensor_slices(train)\n    dvalid = Dataset.from_tensor_slices(valid)\n    dquesdo = Dataset.from_tensor_slices(quesdo)\n    dtest = Dataset.from_tensor_slices(test)\n\n    train_dataset = make_dataset_pipeline(dtrain,True, repeat_and_shuffle=True)\n    valid_dataset = make_dataset_pipeline(dvalid, True,repeat_and_shuffle=True) \n    quesdo_dataset = make_dataset_pipeline(dquesdo, True,repeat_and_shuffle=True) \n\n    validset = make_dataset_pipeline(dvalid) \n    testset = make_dataset_pipeline(dtest)\n    return train_dataset,valid_dataset,quesdo_dataset,validset,testset\n\ndef mix_dataset(dss,szs,weight=None,seed=1214):\n    if weight is None: weight = np.ones(len(szs))\n    prop = np.array(szs)*weight\n    return sample_from_datasets(dss,prop/np.sum(prop),seed),sum(szs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset,valid_dataset,quesdo_dataset,validset,testset = build_datasets(train,valid,quesdo,test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the model and check summary","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Input,Dropout,Dense,GlobalAveragePooling1D,GlobalMaxPool1D\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.metrics import AUC \nfrom tensorflow.keras.initializers import GlorotUniform\n\ndef get_cls(x):\n    return x[:, 0, :]\n\ndic = {\"mean\":GlobalAveragePooling1D(),\n      \"max\":GlobalMaxPool1D(),\n      \"cls\":get_cls}\n\ndef build_model(transformer,head=\"cls\" , loss='binary_crossentropy',\n                max_len=512, drop_rate=None, lr=1e-5,seed=940208):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    x = dic[head](sequence_output)\n    if drop_rate is not None: \n        x = Dropout(drop_rate)(x)\n    out = Dense(1, activation='sigmoid',kernel_initializer=GlorotUniform(seed))(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=lr), loss=loss, metrics=[AUC()])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL, from_pt=True)\n    model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=1e-5)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n\nmodel_path = \"model1.h5\"\ncheckpoint = ModelCheckpoint(model_path, monitor='val_auc', mode='max', save_best_only=True, save_weights_only=True, verbose=1)\ncallback_list = [checkpoint]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset,train_size = mix_dataset([train_dataset,quesdo_dataset],[train_size,quesdo_size],weight=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nN_STEPS = train_size // (BATCH_SIZE*2)\nEPOCHS = 20\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=N_STEPS,\n    validation_data=validset,\n     callbacks=callback_list,\n    epochs=EPOCHS\n) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL, from_pt=True)\n    model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=5e-6)\n    model.load_weights(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nn_steps = valid_size // (BATCH_SIZE)\nEPOCHS = 1\ntrain_history_2 =model.fit(\n    valid_dataset,\n    steps_per_epoch=n_steps,\n    epochs= EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm model1.h5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(TRANSFER)\nsub.loc[ids,'toxic'] = model.predict(testset, verbose=1).reshape(-1)\nsub.to_csv('submission.csv', index=False) ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}