{"cells":[{"metadata":{},"cell_type":"markdown","source":"translated train1 + train2, updated\n\n6 folds with L1 error threshold 0.36","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\n\nimport os, gc\n\nfrom tqdm.notebook import tqdm\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\n\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\nSEED = 6  # original seed from input datasets\nN_PSEUDO = 2  # N copies of test pseudo labels\ner_threshold = 0.36  # max L1 error threshold between blended predictions and ground-truth labels\n\nSUB_SAMPLE = 0.74","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ny_valid = valid.toxic = valid.toxic.astype(np.float32)\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n\ndef regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids']).astype(np.int32)\n\nMAX_LEN = 192\nMODEL = 'jplu/tf-xlm-roberta-large'\ntokenizer = AutoTokenizer.from_pretrained(MODEL)\n\nx_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)\nx_test  = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)\n\ndel test\ndel valid\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import glob\nds1 = sorted(glob.glob('../input/jigsaw20-xlm-rtt42-translations-ds*/*.parquet.gzip'))\nds2 = sorted(glob.glob('../input/jigsaw20-xlm-rtt42-translations2-ds*/*.parquet.gzip'))\nassert len(ds1) == len(ds2)\ncols = ['toxic', 'label', 'token']\n\nlangs = set()\nfor i, (t1, t2) in enumerate(zip(ds1, ds2)):\n    lang = t1[-15:-13]\n    assert lang == t2[-15:-13]\n    langs.add(lang)\n    print(i, lang, t1, t2, langs)\n    t1 = pd.read_parquet(t1)\n    t2 = pd.read_parquet(t2)\n    print(t1.shape[0] + t2.shape[0])\n\n    folds = sorted(t1.fold.unique())\n    assert t2.fold.isna().sum() == 0,  t2.fold.value_counts()\n\n    for fold in folds:\n        print(fold, t1.loc[t1.fold == fold].shape, t2.loc[t2.fold == fold].shape)\n        f = t1.loc[t1.fold == fold, cols].append(t2.loc[t2.fold == fold, cols], ignore_index=True)\n        print(f.shape, 'mean:', f.toxic.mean(), 'ratio:', (f.toxic > 0.5).mean())\n\n        # sub-sample\n        f = f.sample(frac=SUB_SAMPLE, random_state=SEED, weights=0.01+f.toxic)\n        print(f.shape, 'mean:', f.toxic.mean(), 'ratio:', (f.toxic > 0.5).mean())\n        f.to_parquet(f'fold{fold}_{lang}.parquet.gzip', compression='gzip')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del t1\ndel t2\ndel f\ngc.collect()\nlangs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pseudo labels\npseudo = pd.read_csv(\"/kaggle/input/jigsaw20-ensemble06-14-lb9491/submission.csv\")\npseudo.toxic -= pseudo.toxic.min()\npseudo.toxic /= pseudo.toxic.max()\npseudo['token'] = [x for x in x_test]\npseudo","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for fold in folds:\n    df = pd.concat([pd.read_parquet(f'fold{fold}_{lang}.parquet.gzip') for lang in langs],\n                   ignore_index=True)\n    print(fold, df.shape)\n    df['l1er'] = abs(df.toxic - df.label)\n    # use er_threshold/2 as toxic is already a 50% blend with the GT labels\n    ax = df.toxic.hist(bins=100, log=True, alpha=0.6)\n    df = df.loc[df.l1er < er_threshold/2, ['toxic', 'token']]\n    ax = df.toxic.hist(bins=100, log=True, alpha=0.4, ax=ax)\n    print(fold, df.shape)\n\n    for n in range(N_PSEUDO):\n        df = df.append(pseudo[['toxic', 'token']], ignore_index=True)\n    print(fold, df.shape)\n\n    ax = pseudo.toxic.hist(bins=100, log=True, alpha=0.4, ax=ax)\n    ax = df.toxic.hist(bins=100, log=True, alpha=0.3, ax=ax)\n    plt.legend(['all', 'cut', 'pseudo', 'final'])\n    plt.savefig(f'fold{fold}.png')\n    plt.show()\n    print(df.shape, 'mean:', df.toxic.mean(), 'ratio:', (df.toxic > 0.5).mean())\n\n    # shuffle and save\n    df = df.sample(frac=1, random_state=SEED)\n    np.savez_compressed(f'jigsaw20_ds{len(df)}tt{SEED}_fold{fold}.npz',\n                        np.array(df.token.tolist()), x_valid, x_test,\n                        df.toxic.values, y_valid)\n    del df\n    gc.collect()\n!ls -sh *.npz","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm *.parquet.gzip","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}