{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimport os, time\nimport tensorflow as tf\nimport math\nfrom transformers import TFXLMRobertaModel\nfrom tensorflow.keras.optimizers import Adam\nimport os\nfrom kaggle_datasets import KaggleDatasets\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There I will build model upon xlm-roberta using techniques to fight unbalanced classes, as I've shown there:\nhttps://www.kaggle.com/vgodie/class-balancing\n\nUsing preprocessed data from this my notebook\n\nhttps://www.kaggle.com/vgodie/data-encoding","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#set TPU coniguration\nAUTO = tf.data.experimental.AUTOTUNE\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\nprint(tpu_strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" MY_GCS_PATH=KaggleDatasets().get_gcs_path('dataencoding')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = \"../input/dataencoding/\"\n\n\ntrain_ids = np.load(os.path.join(data_path, \"ids.npy\"))\ntrain_labels = np.load(os.path.join(data_path, \"labels.npy\")).astype(int)\nval_ids = np.load(os.path.join(data_path, \"val_ids.npy\"))\nval_labels = np.load(os.path.join(data_path, \"val_labels.npy\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = \"jplu/tf-xlm-roberta-large\"\nSEQUENCE_LENGTH = 192\nBATCH_SIZE =  16 * tpu_strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos = train_ids[np.where(train_labels == 1)[0]]\nneg = train_ids[np.where(train_labels == 0)[0]]\n\npos_labels = train_labels[np.where(train_labels==1)[0]]\nneg_labels = train_labels[np.where(train_labels==0)[0]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_ds(features, labels):\n    ds = tf.data.Dataset.from_tensor_slices((features, labels))\n    ds = ds.shuffle(1500000).repeat()\n    return ds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_ids\ndel train_labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make resampled dataset to ensure that classes are balanced in training batches","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_ds = make_ds(pos, pos_labels)\nneg_ds = make_ds(neg, neg_labels)\n\nresampled_ds = tf.data.experimental.sample_from_datasets([pos_ds, neg_ds], weights=[0.5, 0.5])\nresampled_ds = resampled_ds.batch(BATCH_SIZE).prefetch(AUTO)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del pos\ndel neg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_dataset = (tf.data.Dataset.from_tensor_slices((val_ids, val_labels))\n               .shuffle(len(val_ids))\n               .repeat()\n               .batch(BATCH_SIZE)\n               .prefetch(AUTO)\n              )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Making model taking sentence emdedding as concatenation of max and average pooling","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model(embed_model):\n    \n    \n    input_ids = tf.keras.layers.Input(shape=(SEQUENCE_LENGTH,), name='input_token', dtype='int32')\n\n    embed_layer = embed_model([input_ids])[0]\n    avg_pool = tf.reduce_mean(embed_layer, axis=1)\n    max_pool = tf.reduce_max(embed_layer, axis=1)\n    X = tf.concat([avg_pool, max_pool], axis=1)\n    X = tf.keras.layers.Dropout(0.3)(X)\n    X = tf.keras.layers.Dense(1, activation=\"sigmoid\")(X)\n    model = tf.keras.Model(inputs=input_ids, outputs = X)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with tpu_strategy.scope():\n    xlm_roberta = TFXLMRobertaModel.from_pretrained(MODEL)\n    xr_model = make_model(xlm_roberta)\n    xr_model.summary()\n    xr_model.compile(optimizer=tf.keras.optimizers.Adam(lr=3e-5), loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1), metrics=[tf.keras.metrics.AUC()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_STEPS = 1200000//BATCH_SIZE\nN_STEPS\n\nVAL_STEPS = val_ids.shape[0]//BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler\n\ndef build_lrfn(lr_start=0.000001, lr_max=0.000002, \n               lr_min=0.0000001, lr_rampup_epochs=7, \n               lr_sustain_epochs=0, lr_exp_decay=.87):\n    lr_max = lr_max * tpu_strategy.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn\n\nlrfn = build_lrfn()\n\n\nes = EarlyStopping(monitor='val_accuracy', mode='max', patience=2, \n                   restore_best_weights=True, verbose=1)\nlr_callback = LearningRateScheduler(lrfn, verbose=1)\ncallbacks = [es, lr_callback]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistory = xr_model.fit(resampled_ds,\n                       validation_data=val_dataset,\n                       epochs=2,\n                       steps_per_epoch=N_STEPS,\n                       validation_steps = VAL_STEPS,\n                       callbacks=callbacks\n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_history = xr_model.fit(val_dataset,\n                       epochs=2,\n                       steps_per_epoch=VAL_STEPS,\n                       callbacks=callbacks\n                      )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xr_model.save_weights(\"weights.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(os.path.join('../input/jigsaw-multilingual-toxic-comment-classification/','sample_submission.csv'))\ntest_ids = np.load(\"../input/dataencoding/test_ids.npy\", allow_pickle=True)\nsub['toxic'] = xr_model.predict(test_ids, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}