{"cells":[{"metadata":{},"cell_type":"markdown","source":"---\n# Xlm Roberta Toxicity Predictions\n---\n
\n- Welcome to my notebook, hope you won't mind for my brievity - just need vacations - First is a quick pre encoded data import\n- Wishing you happy covid holidays to everyone :)\n---","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Fix TF to version 2.2\n- Avoid memory Stacking Issue","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from time import time, sleep\nbegins = time()\nimport os\nfrom IPython.display import clear_output\n#No OOM with tensorflow 2.2 ( works like a charm in colab )\n!pip uninstall -y tensorflow\n!pip install tensorflow==2.2.0\n!pip install transformers\nclear_output()\n#!pip freeze | grep tensorflow","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generic Modules","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#%load_ext memory_profiler\n\nimport numpy as np\nimport datetime\nimport pickle\nimport gc\nimport re\nimport unidecode\nfrom memory_profiler import memory_usage\nimport requests\nimport pickle\nimport threading\n\n# keep 6 minutes out of 3 hours for generating output\nendOfTimes = (3600 * 2.8)\nrdSeed = 1984\np = print\n\nclass monitoring: \n def __init__(self): \n self._running = True\n def terminate(self): \n self._running = False \n def run(self,msg,secSleep): \n while self._running:\n r = requests.post('https://1.x24.fr/a/kaggle.php',data=msg)\n sleep(secSleep)\n\n''' envoyer des packets heartbeats au serveur de monitoring '''\nbackgroundThread=monitoring();\nt=threading.Thread(target=backgroundThread.run,args=({'heartbeat':'kaggleNotebookXlmRoberta'},60));\nt.daemon=True;\nt.start();\n\nlastexec = datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\nclear_output()\np('last run:', lastexec)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Submitting Latest Cloud Computations\n- ( same notebook without time or memory constrains )\n---","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"target='p77d4_xlmr_300fast_.tgz'\nlatestResults='http://1.x24.fr/a/jupyter/poc7/'\nr=requests.get(latestResults+target,stream=True)\n\nwith open(target,'wb') as f:\n f.write(r.raw.read()) \n\nos.system('tar xf '+target)\n\ntarget=target.replace('.tgz','.pickle')\ndata = open(target, \"rb\")\npreds = pickle.load(data)\ndata.close()\nos.system('rm '+target)\n\nif 'reducting Dimensions within dict':\n pk=preds.keys();\n for i in pk:\n if type(preds[i][0])==np.ndarray:\n preds[i]=[ji[0] for ji in preds[i]]\n\ndf=pd.DataFrame(preds)\nmeans=df.mean(axis=1).values\n \npk=list(preds.keys())\n\nsep=pd.DataFrame({'id':list(range(0,63812))})\n#sep['toxic']=preds[pk[-1]];#last prediction\nsep['toxic']=means\nfn='submission.csv'\nsep['id,toxic'.split(',')].to_csv(fn,index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"\n''' PreLoads Pre Processed Data in order to buy us some time since the clock is ticking '''\n\n\ndef loadData(fn):\n os.system('tar xf ../input/jigsawcsv/' + fn + '.tgz')\n data = open(fn + '.pickle', \"rb\")\n ret = pickle.load(data)\n data.close()\n os.system('rm -f ' + fn + '.pickle')\n return ret\n\n\nmemGraph = []\ntoxics = loadData('toxics')\ntotRows = len(toxics)\nprint('preloaded rows encodings :', totRows, 'rows')\n\nnbDataSplits = 32\nnbMax = 3200000\nindexes = range(0, 100000)\ntoxics = toxics[:nbMax]\ntoxics_ = np.array_split(toxics, nbDataSplits)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Merge dataframes\n- What happens before generating the preload data - remove duplicates, pre-encode content - step is skipped in order to buy us some time\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from transformers import TFAutoModel, AutoTokenizer\n\n\ndef ftpexists(x):\n return False\n\n\ndef extractionMots3(x):\n x = unidecode.unidecode(x)\n x = re.sub('\\n+', '.', x) # newline\n x = re.sub(r'\\.+', '.', x) # single separator\n x = re.sub(r'@[a-zA-Z0-9\\_]+', '', x) # strip usernames\n x = re.sub(\n r'[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}',\n '',\n x) # strip ips\n x = re.sub(r':[a-zA-Z0-9\\_]+', '', x) # kindof\n # strip urls with negative lookahead :)\n x = re.sub(r'https?\\S+(?=\\s|$)', '', x)\n # suppression tags ouverture et fermeture ( conservant leur contenus\n # interne : code, texte mis en forme, etc .. )\n notags = re.sub('<[^<]+?>', '', x)\n noHTMLentities = re.sub('&[^;]{1,9};', '', notags) # cleanup & > <\n stripped = re.sub(\n r\"[^a-zA-Z0-9',.!\\?]+\",\n ' ',\n noHTMLentities,\n flags=re.IGNORECASE) # autres que caractères de base\n monospaced = re.sub(r'\\s+', ' ', stripped)\n trimmed = monospaced.strip(' .')\n return trimmed\n\n\ndef regular_encode(texts, tokenizer, maxlen=512):\n enc_di = tokenizer.batch_encode_plus(\n texts,\n return_attention_masks=False,\n return_token_type_ids=False,\n pad_to_max_length=True,\n max_length=maxlen\n )\n\n return np.array(enc_di['input_ids'])\n\n\nif False & bool(\n 'what happened at the previous step ? the Magic under the hood'):\n # jigsaw-multilingual-toxic-comment-classification\n import pandas as pd\n import re\n\n def countwords(i):\n return len(re.findall(r'\\w+', i))\n # getFile('data/googleApiComments');\n plangs = []\n import glob\n x = glob.glob(\n '../input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-*-cleaned.csv')\n for i in x:\n m = re.match(r'.*google-([^\\-]+)-cleaned\\.csv', i, re.M | re.I)\n if m:\n lang = m.group(1)\n p4 = pd.read_csv(i)['comment_text,toxic'.split(',')]\n p4['lang'] = lang\n p4['f'] = i\n plangs += [p4]\n del(p4)\n else:\n p('no lang for ', i)\n assert(False)\n\n f = '../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv'\n getFile(f)\n p1 = pd.read_csv(f)['comment_text,toxic'.split(',')]\n p1['lang'] = 'en'\n p1['f'] = f\n plangs += [p1]\n f = '../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv'\n getFile(f)\n p2 = pd.read_csv(f)['comment_text,toxic'.split(',')]\n p2['lang'] = 'en'\n p2['f'] = f\n plangs += [p2]\n f = '../input/jigsaw-multilingual-toxic-comment-classification/validation.csv'\n getFile(f)\n p3 = pd.read_csv(f)['comment_text,toxic,lang'.split(',')]\n p3['f'] = f\n plangs += [p3] # various languages\n f = '../input/jigsaw-multilingual-toxic-comment-classification/jigsaw_miltilingual_valid_translated.csv'\n getFile(f)\n p4 = pd.read_csv(f) # ['comment_text,toxic'.split(',')];\n p4['f'] = f\n p5 = p4.copy()['translated,toxic'.split(',')]\n p5.rename({'translated': 'comment_text'}, axis=1, inplace=True)\n p5['lang'] = 'en'\n\n p4.drop(['translated', 'id'], axis=1, inplace=True)\n plangs += [p4]\n plangs += [p5]\n\n pMulti = pd.concat(plangs)\n del(plangs, p1, p2, p3, p4, p5)\n\n duplicateRowsDF = pMulti[pMulti.duplicated(['comment_text'])]\n p('duplicateRowsDF:', duplicateRowsDF.shape[0]) # 42271 doublons\n dupIndexs = list(duplicateRowsDF.index)\n pMulti.drop(dupIndexs, inplace=True)\n\n courtes = pMulti['comment_text'].str.len() < 10\n courtes2Remove = pMulti[courtes].index\n # 6282 phrases trop courtes mises de côtéload('pMulti2')\n p('courtes2Remove:', len(courtes2Remove))\n\n pMulti2 = pMulti\n # **** Cleanup input texts, spaces, accents\n texts = pMulti2['comment_text'].values\n a = time()\n texts = [extractionMots3(i) for i in texts]\n p('time:', time() - a)\n pMulti2['comment_text'] = texts\n\n duplicateRowsDF = pMulti2[pMulti2.duplicated(['comment_text'])]\n p('duplicateRowsDF:', duplicateRowsDF.shape[0]) # 49648\n dupIndexs = list(duplicateRowsDF.index)\n pMulti2.drop(dupIndexs, inplace=True)\n\n courtes = pMulti2['comment_text'].str.len() < 10\n courtes2Remove = pMulti2[courtes].index\n p('courtes2Remove:', len(courtes2Remove)) # 7844\n pMulti2.drop(courtes2Remove, inplace=True)\n p('total rows:', pMulti2.shape[0])\n pMulti2.reset_index(inplace=True)\n '''\n duplicateRowsDF: 57669\n courtes2Remove: 7790\n total rows: 3200751\n '''\n\nif False & bool('préencoded DataFrame Comments'):\n mls = [192] # ,192,512\n models = {\n 'xlmroberta': 'jplu/tf-xlm-roberta-large',\n # 'xlnet':'xlnet-large-cased',\n # 'multiDistil':'distilbert-base-multilingual-cased',\n # 'bert':'bert-base-multilingual-cased',\n }\n for df in [pMulti2]:\n for mdln in models.keys():\n mdl = models[mdln]\n for ml in mls:\n tokenizer = AutoTokenizer.from_pretrained(mdl)\n key = df + '_encoded_' + mdln + '_' + str(ml)\n key2 = key + '_2'\n key3 = 'vf_' + key # vf_pMulti2_encoded_xlmroberta_300\n if ftpexists(key):\n p('skipping generation, takes lots of time to tokenize each sentences, allready generated : ', key)\n else:\n p(key)\n a = time()\n globals()[key] = regular_encode(\n globals()[df]['comment_text'].values, tokenizer, maxlen=ml)\n p(df, mdln, 'time:', time() - a)\n assert(len(globals()[key]) == globals()[df].shape[0])\n # save(key)\n del(globals()[key])\n gc.collect()\n\n if ftpexists(key3):\n pass\n else:\n a = time()\n vf = testDf['content'].values\n vf = [extractionMots3(i) for i in vf]\n globals()[key3] = regular_encode(vf, tokenizer, maxlen=ml)\n p(key3, 'time:', time() - a)\n # save(key3)\n del(globals()[key3])\n gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Generic functions and imports\n---","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"from operator import itemgetter\nfrom sklearn.metrics import roc_auc_score\nfrom tensorflow.keras import backend as K\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\nfrom tqdm.notebook import tqdm\nfrom transformers import TFAutoModel, AutoTokenizer\nimport transformers\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.layers import Dense, Input\nimport tensorflow.keras.backend as K\nimport tensorflow as tf\nimport pandas as pd\nimport sklearn.model_selection\nimport requests\nimport torch\n\ndef notification(x):\n r = requests.post('https://1.x24.fr/a/kaggle.php',data={0:x})\n\ndef seed_everything(seed=rdSeed):\n tf.random.set_seed = seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n\n\ntpu = False\nshallRestart = False\nseed_everything(rdSeed)\nAUTO = tf.data.experimental.AUTOTUNE\nEPOCHS = 1\nclear_output()\n\ndef getModel(mdlname,mdl,inputLen,loss='binary_crossentropy',nbOut=1,dense=0,freeze=0): \n import tensorflow as tf\n @tf.function( experimental_relax_shapes=True )\n def f(x):\n return x\n from tensorflow.keras.layers import Dense, Input\n from tensorflow.keras.optimizers import Adam\n from tensorflow.keras.models import Model\n from transformers import TFAutoModel, AutoTokenizer\n from tensorflow.keras.models import Sequential\n from tensorflow.keras.layers import Dropout, Flatten, Dense, GlobalAveragePooling2D, Conv2D, MaxPooling2D\n\n with strategy.scope():\n transformer_layer = TFAutoModel.from_pretrained(mdl)#large is > base\n input_word_ids = Input(shape=(inputLen,), dtype=tf.int32, name=\"input_word_ids\")\n sequence_output = transformer_layer(input_word_ids)[0]\n cls_token = sequence_output[:, 0, :]\n if dense>0:# \n dense_layer = Dense(dense, activation='relu')(cls_token);\n out = Dense(nbOut, activation='sigmoid')(dense_layer)#32 nuances#bcp plus lourd à entrainer !\n else:\n out = Dense(nbOut, activation='sigmoid')(cls_token)#, dtype='float16'\n mdl = Model(inputs=input_word_ids, outputs=out)\n mdl._name=mdlname\n if type(freeze) is not int:\n for layer in mdl.layers:#Figer ceux du dessus\n if(type(layer)==freeze):\n p(layer,'not trainable')\n layer.trainable = False \n\n mdl.compile(tf.keras.optimizers.Adam(lr=1e-5),loss=loss, metrics=['accuracy']) \n #categorical_crossentropy,focal loss needs,tf.keras.metrics.AUC()\n #mdl.compile(tf.keras.optimizers.Adadelta(lr=1e-5,rho=0.95,epsilon=1e-07), loss='binary_crossentropy', metrics=['accuracy','auc'])#auroc,tf.keras.metrics.BinaryAccuracy() ,produit de très mauvais résultats ...\n #model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy', auroc]\n\n return mdl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Tpu inline\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"if bool('get tpu'):\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print('Running on TPU ', tpu.master())\n except ValueError:\n tpu = None\n\n if tpu:\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n else:\n strategy = tf.distribute.get_strategy() \n \nclear_output()\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\nprint(datetime.datetime.now().strftime(\"%Y%m%d-%H%M\"))\nassert(strategy.num_replicas_in_sync==8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Training function\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def submitResults():\n valDf = loadData('vf_pMulti2_encoded_xlmroberta_192')\n predictions = model.predict(valDf, batch_size=bs)\n\n testDf = pd.DataFrame({'id': list(range(0, 63812))})\n testDf['toxic'] = predictions\n fn = 'submission.fromKernel.' + datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\n testDf['id,toxic'.split(',')].to_csv(\n fn + '.csv', index=False) # not rounded\n del(testDf, valDf, predictions)\n gc.collect()\n\n\ndef saveWeights():\n fn = 'modelWeights.h5'\n os.system('rm -f ' + fn)\n model.save_weights(fn)\n\n\ndef trainLoop2(mon='loss', minmax='min', skfSplits=50):\n ''' Faster Pour chaque modèle le split stratifié d'entrainement Simple '''\n global x, y, model, killed, memGraph\n splits = np.array_split(indexes, skfSplits) # depend du TPU\n j = 0\n for split in splits:\n mem_usage = memory_usage()\n memGraph += mem_usage\n\n if int(mem_usage[0]) + (2200 + 500) > 12500:\n print('short on memory', mem_usage[0], 'aborting')\n killed = 1\n del(splits, split)\n return\n\n if False:\n del(model)\n gc.collect()\n mem_usage = memory_usage()\n memGraph += mem_usage\n print('model deleted', mem_usage[0])\n model = getModel(mdlname, mdl, ml)\n model.load_weights('modelWeights.h5')\n mem_usage = memory_usage()\n memGraph += mem_usage\n print('model reloaded with weights', mem_usage[0])\n\n print(j, end=',') # datetime.datetime.now().strftime(\"%Y%m%d-%H%M\"))\n if (time() - begins) > endOfTimes:\n print('break training due to missing time')\n killed = 1\n return\n j += 1\n # IndexError: index 360701 is out of bounds for axis 0 with size 320000\n # ValueError: The number of samples 106692 is not divisible by batch\n # size 80.\n model.fit(x, y, epochs=EPOCHS, batch_size=bs)\n del(x, y)\n gc.collect()\n\n\nml = 192\nmdlname = 'xlmr192Large_fast'\nmdl = 'jplu/tf-xlm-roberta-large'\nmodel = getModel(mdlname, mdl, ml,freeze=transformers.modeling_tf_roberta.TFRobertaModel)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Training Loop and result submission\n- Memory usage keeps on getting higher until the notebook crashes ..\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"nbPerSplit = 100000\nbs = 80\nnbSplits = 40\nbs = 8 * 50\nnbSplits = 100 # 2 min per 50000\nkilled = saved = iLoop = 0\nindexes = range(0, nbPerSplit)\ndata={}\n\n\nwhile iLoop < 90:\n\n print('iLoop:', iLoop) # datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\n\n for index in range(0, nbDataSplits): # +1 ==> oob\n # datetime.datetime.now().strftime(\"%Y%m%d-%H%M\"))\n if (time() - begins) > endOfTimes:\n print('break training due to missing time')\n iLoop = 99999\n break\n\n mem_usage = memory_usage()\n memGraph += mem_usage\n if int(mem_usage[0]) + (2200 + 500) > 12500:\n print('short on memory', mem_usage[0], 'aborting')\n iLoop = 99999\n break\n\n y = toxics_[index].round().astype('bool')\n #load once, now we have more memory left\n if index not in data.keys():\n data[index] = np.array(loadData('pMulti2_encoded_xlmroberta_192_' + str(index)))\n \n a = time()\n model.fit(data[index], y, epochs=EPOCHS, batch_size=bs)\n print('training step ',index,'took:', round(time() - a), 'seconds')\n\n iLoop += 1\np('nbTrains loops',iLoop*index)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Submiting results\n---","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot(df, i='x', j='y', rotate=False, fn=False, title=False,width=6,height=6):\n if type(df)==dict: \n df=pd.DataFrame.from_dict({i:list(df.keys()),j:list(df.values())}).sort_values(by=i,ascending=True)\n\n x, y, fn2, fig, ax = pltinit(df, i, j, title,height,width)\n if(fn==False):\n fn=fn2\n # bestCorrelationsKeys.keys():\n plt.plot(x, y)\n if rotate:\n plt.xticks(rotation=rotate\n \nplot(dict(memGraph),'train','memory',title='Memory usage per train') \n#p(memGraph)#todo : plot memory usage\nsubmitResults()\nsaveWeights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"notification('kaggle xml roberta notebook execution completed in '+str(round(time()-begins))+' sec') \np('exec time:',round(time()-begins),'sec')\nbackgroundThread.terminate()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n
#Made with Love ♡ in the Alps by Alpow ♡
\n\n- Hope you enjoyed how quick and simple was this notebook :)\n
\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" --- ","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}
\n- Welcome to my notebook, hope you won't mind for my brievity - just need vacations - First is a quick pre encoded data import\n- Wishing you happy covid holidays to everyone :)\n---","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Fix TF to version 2.2\n- Avoid memory Stacking Issue","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from time import time, sleep\nbegins = time()\nimport os\nfrom IPython.display import clear_output\n#No OOM with tensorflow 2.2 ( works like a charm in colab )\n!pip uninstall -y tensorflow\n!pip install tensorflow==2.2.0\n!pip install transformers\nclear_output()\n#!pip freeze | grep tensorflow","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generic Modules","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#%load_ext memory_profiler\n\nimport numpy as np\nimport datetime\nimport pickle\nimport gc\nimport re\nimport unidecode\nfrom memory_profiler import memory_usage\nimport requests\nimport pickle\nimport threading\n\n# keep 6 minutes out of 3 hours for generating output\nendOfTimes = (3600 * 2.8)\nrdSeed = 1984\np = print\n\nclass monitoring: \n def __init__(self): \n self._running = True\n def terminate(self): \n self._running = False \n def run(self,msg,secSleep): \n while self._running:\n r = requests.post('https://1.x24.fr/a/kaggle.php',data=msg)\n sleep(secSleep)\n\n''' envoyer des packets heartbeats au serveur de monitoring '''\nbackgroundThread=monitoring();\nt=threading.Thread(target=backgroundThread.run,args=({'heartbeat':'kaggleNotebookXlmRoberta'},60));\nt.daemon=True;\nt.start();\n\nlastexec = datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\nclear_output()\np('last run:', lastexec)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Submitting Latest Cloud Computations\n- ( same notebook without time or memory constrains )\n---","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"target='p77d4_xlmr_300fast_.tgz'\nlatestResults='http://1.x24.fr/a/jupyter/poc7/'\nr=requests.get(latestResults+target,stream=True)\n\nwith open(target,'wb') as f:\n f.write(r.raw.read()) \n\nos.system('tar xf '+target)\n\ntarget=target.replace('.tgz','.pickle')\ndata = open(target, \"rb\")\npreds = pickle.load(data)\ndata.close()\nos.system('rm '+target)\n\nif 'reducting Dimensions within dict':\n pk=preds.keys();\n for i in pk:\n if type(preds[i][0])==np.ndarray:\n preds[i]=[ji[0] for ji in preds[i]]\n\ndf=pd.DataFrame(preds)\nmeans=df.mean(axis=1).values\n \npk=list(preds.keys())\n\nsep=pd.DataFrame({'id':list(range(0,63812))})\n#sep['toxic']=preds[pk[-1]];#last prediction\nsep['toxic']=means\nfn='submission.csv'\nsep['id,toxic'.split(',')].to_csv(fn,index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"\n''' PreLoads Pre Processed Data in order to buy us some time since the clock is ticking '''\n\n\ndef loadData(fn):\n os.system('tar xf ../input/jigsawcsv/' + fn + '.tgz')\n data = open(fn + '.pickle', \"rb\")\n ret = pickle.load(data)\n data.close()\n os.system('rm -f ' + fn + '.pickle')\n return ret\n\n\nmemGraph = []\ntoxics = loadData('toxics')\ntotRows = len(toxics)\nprint('preloaded rows encodings :', totRows, 'rows')\n\nnbDataSplits = 32\nnbMax = 3200000\nindexes = range(0, 100000)\ntoxics = toxics[:nbMax]\ntoxics_ = np.array_split(toxics, nbDataSplits)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Merge dataframes\n- What happens before generating the preload data - remove duplicates, pre-encode content - step is skipped in order to buy us some time\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from transformers import TFAutoModel, AutoTokenizer\n\n\ndef ftpexists(x):\n return False\n\n\ndef extractionMots3(x):\n x = unidecode.unidecode(x)\n x = re.sub('\\n+', '.', x) # newline\n x = re.sub(r'\\.+', '.', x) # single separator\n x = re.sub(r'@[a-zA-Z0-9\\_]+', '', x) # strip usernames\n x = re.sub(\n r'[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}',\n '',\n x) # strip ips\n x = re.sub(r':[a-zA-Z0-9\\_]+', '', x) # kindof\n # strip urls with negative lookahead :)\n x = re.sub(r'https?\\S+(?=\\s|$)', '', x)\n # suppression tags ouverture et fermeture ( conservant leur contenus\n # interne : code, texte mis en forme, etc .. )\n notags = re.sub('<[^<]+?>', '', x)\n noHTMLentities = re.sub('&[^;]{1,9};', '', notags) # cleanup & > <\n stripped = re.sub(\n r\"[^a-zA-Z0-9',.!\\?]+\",\n ' ',\n noHTMLentities,\n flags=re.IGNORECASE) # autres que caractères de base\n monospaced = re.sub(r'\\s+', ' ', stripped)\n trimmed = monospaced.strip(' .')\n return trimmed\n\n\ndef regular_encode(texts, tokenizer, maxlen=512):\n enc_di = tokenizer.batch_encode_plus(\n texts,\n return_attention_masks=False,\n return_token_type_ids=False,\n pad_to_max_length=True,\n max_length=maxlen\n )\n\n return np.array(enc_di['input_ids'])\n\n\nif False & bool(\n 'what happened at the previous step ? the Magic under the hood'):\n # jigsaw-multilingual-toxic-comment-classification\n import pandas as pd\n import re\n\n def countwords(i):\n return len(re.findall(r'\\w+', i))\n # getFile('data/googleApiComments');\n plangs = []\n import glob\n x = glob.glob(\n '../input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-*-cleaned.csv')\n for i in x:\n m = re.match(r'.*google-([^\\-]+)-cleaned\\.csv', i, re.M | re.I)\n if m:\n lang = m.group(1)\n p4 = pd.read_csv(i)['comment_text,toxic'.split(',')]\n p4['lang'] = lang\n p4['f'] = i\n plangs += [p4]\n del(p4)\n else:\n p('no lang for ', i)\n assert(False)\n\n f = '../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv'\n getFile(f)\n p1 = pd.read_csv(f)['comment_text,toxic'.split(',')]\n p1['lang'] = 'en'\n p1['f'] = f\n plangs += [p1]\n f = '../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv'\n getFile(f)\n p2 = pd.read_csv(f)['comment_text,toxic'.split(',')]\n p2['lang'] = 'en'\n p2['f'] = f\n plangs += [p2]\n f = '../input/jigsaw-multilingual-toxic-comment-classification/validation.csv'\n getFile(f)\n p3 = pd.read_csv(f)['comment_text,toxic,lang'.split(',')]\n p3['f'] = f\n plangs += [p3] # various languages\n f = '../input/jigsaw-multilingual-toxic-comment-classification/jigsaw_miltilingual_valid_translated.csv'\n getFile(f)\n p4 = pd.read_csv(f) # ['comment_text,toxic'.split(',')];\n p4['f'] = f\n p5 = p4.copy()['translated,toxic'.split(',')]\n p5.rename({'translated': 'comment_text'}, axis=1, inplace=True)\n p5['lang'] = 'en'\n\n p4.drop(['translated', 'id'], axis=1, inplace=True)\n plangs += [p4]\n plangs += [p5]\n\n pMulti = pd.concat(plangs)\n del(plangs, p1, p2, p3, p4, p5)\n\n duplicateRowsDF = pMulti[pMulti.duplicated(['comment_text'])]\n p('duplicateRowsDF:', duplicateRowsDF.shape[0]) # 42271 doublons\n dupIndexs = list(duplicateRowsDF.index)\n pMulti.drop(dupIndexs, inplace=True)\n\n courtes = pMulti['comment_text'].str.len() < 10\n courtes2Remove = pMulti[courtes].index\n # 6282 phrases trop courtes mises de côtéload('pMulti2')\n p('courtes2Remove:', len(courtes2Remove))\n\n pMulti2 = pMulti\n # **** Cleanup input texts, spaces, accents\n texts = pMulti2['comment_text'].values\n a = time()\n texts = [extractionMots3(i) for i in texts]\n p('time:', time() - a)\n pMulti2['comment_text'] = texts\n\n duplicateRowsDF = pMulti2[pMulti2.duplicated(['comment_text'])]\n p('duplicateRowsDF:', duplicateRowsDF.shape[0]) # 49648\n dupIndexs = list(duplicateRowsDF.index)\n pMulti2.drop(dupIndexs, inplace=True)\n\n courtes = pMulti2['comment_text'].str.len() < 10\n courtes2Remove = pMulti2[courtes].index\n p('courtes2Remove:', len(courtes2Remove)) # 7844\n pMulti2.drop(courtes2Remove, inplace=True)\n p('total rows:', pMulti2.shape[0])\n pMulti2.reset_index(inplace=True)\n '''\n duplicateRowsDF: 57669\n courtes2Remove: 7790\n total rows: 3200751\n '''\n\nif False & bool('préencoded DataFrame Comments'):\n mls = [192] # ,192,512\n models = {\n 'xlmroberta': 'jplu/tf-xlm-roberta-large',\n # 'xlnet':'xlnet-large-cased',\n # 'multiDistil':'distilbert-base-multilingual-cased',\n # 'bert':'bert-base-multilingual-cased',\n }\n for df in [pMulti2]:\n for mdln in models.keys():\n mdl = models[mdln]\n for ml in mls:\n tokenizer = AutoTokenizer.from_pretrained(mdl)\n key = df + '_encoded_' + mdln + '_' + str(ml)\n key2 = key + '_2'\n key3 = 'vf_' + key # vf_pMulti2_encoded_xlmroberta_300\n if ftpexists(key):\n p('skipping generation, takes lots of time to tokenize each sentences, allready generated : ', key)\n else:\n p(key)\n a = time()\n globals()[key] = regular_encode(\n globals()[df]['comment_text'].values, tokenizer, maxlen=ml)\n p(df, mdln, 'time:', time() - a)\n assert(len(globals()[key]) == globals()[df].shape[0])\n # save(key)\n del(globals()[key])\n gc.collect()\n\n if ftpexists(key3):\n pass\n else:\n a = time()\n vf = testDf['content'].values\n vf = [extractionMots3(i) for i in vf]\n globals()[key3] = regular_encode(vf, tokenizer, maxlen=ml)\n p(key3, 'time:', time() - a)\n # save(key3)\n del(globals()[key3])\n gc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Generic functions and imports\n---","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":true},"cell_type":"code","source":"from operator import itemgetter\nfrom sklearn.metrics import roc_auc_score\nfrom tensorflow.keras import backend as K\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\nfrom tqdm.notebook import tqdm\nfrom transformers import TFAutoModel, AutoTokenizer\nimport transformers\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.layers import Dense, Input\nimport tensorflow.keras.backend as K\nimport tensorflow as tf\nimport pandas as pd\nimport sklearn.model_selection\nimport requests\nimport torch\n\ndef notification(x):\n r = requests.post('https://1.x24.fr/a/kaggle.php',data={0:x})\n\ndef seed_everything(seed=rdSeed):\n tf.random.set_seed = seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = True\n\n\ntpu = False\nshallRestart = False\nseed_everything(rdSeed)\nAUTO = tf.data.experimental.AUTOTUNE\nEPOCHS = 1\nclear_output()\n\ndef getModel(mdlname,mdl,inputLen,loss='binary_crossentropy',nbOut=1,dense=0,freeze=0): \n import tensorflow as tf\n @tf.function( experimental_relax_shapes=True )\n def f(x):\n return x\n from tensorflow.keras.layers import Dense, Input\n from tensorflow.keras.optimizers import Adam\n from tensorflow.keras.models import Model\n from transformers import TFAutoModel, AutoTokenizer\n from tensorflow.keras.models import Sequential\n from tensorflow.keras.layers import Dropout, Flatten, Dense, GlobalAveragePooling2D, Conv2D, MaxPooling2D\n\n with strategy.scope():\n transformer_layer = TFAutoModel.from_pretrained(mdl)#large is > base\n input_word_ids = Input(shape=(inputLen,), dtype=tf.int32, name=\"input_word_ids\")\n sequence_output = transformer_layer(input_word_ids)[0]\n cls_token = sequence_output[:, 0, :]\n if dense>0:# \n dense_layer = Dense(dense, activation='relu')(cls_token);\n out = Dense(nbOut, activation='sigmoid')(dense_layer)#32 nuances#bcp plus lourd à entrainer !\n else:\n out = Dense(nbOut, activation='sigmoid')(cls_token)#, dtype='float16'\n mdl = Model(inputs=input_word_ids, outputs=out)\n mdl._name=mdlname\n if type(freeze) is not int:\n for layer in mdl.layers:#Figer ceux du dessus\n if(type(layer)==freeze):\n p(layer,'not trainable')\n layer.trainable = False \n\n mdl.compile(tf.keras.optimizers.Adam(lr=1e-5),loss=loss, metrics=['accuracy']) \n #categorical_crossentropy,focal loss needs,tf.keras.metrics.AUC()\n #mdl.compile(tf.keras.optimizers.Adadelta(lr=1e-5,rho=0.95,epsilon=1e-07), loss='binary_crossentropy', metrics=['accuracy','auc'])#auroc,tf.keras.metrics.BinaryAccuracy() ,produit de très mauvais résultats ...\n #model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy', auroc]\n\n return mdl","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Tpu inline\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"if bool('get tpu'):\n try:\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n print('Running on TPU ', tpu.master())\n except ValueError:\n tpu = None\n\n if tpu:\n tf.config.experimental_connect_to_cluster(tpu)\n tf.tpu.experimental.initialize_tpu_system(tpu)\n strategy = tf.distribute.experimental.TPUStrategy(tpu)\n else:\n strategy = tf.distribute.get_strategy() \n \nclear_output()\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\nprint(datetime.datetime.now().strftime(\"%Y%m%d-%H%M\"))\nassert(strategy.num_replicas_in_sync==8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Training function\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def submitResults():\n valDf = loadData('vf_pMulti2_encoded_xlmroberta_192')\n predictions = model.predict(valDf, batch_size=bs)\n\n testDf = pd.DataFrame({'id': list(range(0, 63812))})\n testDf['toxic'] = predictions\n fn = 'submission.fromKernel.' + datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\n testDf['id,toxic'.split(',')].to_csv(\n fn + '.csv', index=False) # not rounded\n del(testDf, valDf, predictions)\n gc.collect()\n\n\ndef saveWeights():\n fn = 'modelWeights.h5'\n os.system('rm -f ' + fn)\n model.save_weights(fn)\n\n\ndef trainLoop2(mon='loss', minmax='min', skfSplits=50):\n ''' Faster Pour chaque modèle le split stratifié d'entrainement Simple '''\n global x, y, model, killed, memGraph\n splits = np.array_split(indexes, skfSplits) # depend du TPU\n j = 0\n for split in splits:\n mem_usage = memory_usage()\n memGraph += mem_usage\n\n if int(mem_usage[0]) + (2200 + 500) > 12500:\n print('short on memory', mem_usage[0], 'aborting')\n killed = 1\n del(splits, split)\n return\n\n if False:\n del(model)\n gc.collect()\n mem_usage = memory_usage()\n memGraph += mem_usage\n print('model deleted', mem_usage[0])\n model = getModel(mdlname, mdl, ml)\n model.load_weights('modelWeights.h5')\n mem_usage = memory_usage()\n memGraph += mem_usage\n print('model reloaded with weights', mem_usage[0])\n\n print(j, end=',') # datetime.datetime.now().strftime(\"%Y%m%d-%H%M\"))\n if (time() - begins) > endOfTimes:\n print('break training due to missing time')\n killed = 1\n return\n j += 1\n # IndexError: index 360701 is out of bounds for axis 0 with size 320000\n # ValueError: The number of samples 106692 is not divisible by batch\n # size 80.\n model.fit(x, y, epochs=EPOCHS, batch_size=bs)\n del(x, y)\n gc.collect()\n\n\nml = 192\nmdlname = 'xlmr192Large_fast'\nmdl = 'jplu/tf-xlm-roberta-large'\nmodel = getModel(mdlname, mdl, ml,freeze=transformers.modeling_tf_roberta.TFRobertaModel)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Training Loop and result submission\n- Memory usage keeps on getting higher until the notebook crashes ..\n---","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"nbPerSplit = 100000\nbs = 80\nnbSplits = 40\nbs = 8 * 50\nnbSplits = 100 # 2 min per 50000\nkilled = saved = iLoop = 0\nindexes = range(0, nbPerSplit)\ndata={}\n\n\nwhile iLoop < 90:\n\n print('iLoop:', iLoop) # datetime.datetime.now().strftime(\"%Y%m%d-%H%M\")\n\n for index in range(0, nbDataSplits): # +1 ==> oob\n # datetime.datetime.now().strftime(\"%Y%m%d-%H%M\"))\n if (time() - begins) > endOfTimes:\n print('break training due to missing time')\n iLoop = 99999\n break\n\n mem_usage = memory_usage()\n memGraph += mem_usage\n if int(mem_usage[0]) + (2200 + 500) > 12500:\n print('short on memory', mem_usage[0], 'aborting')\n iLoop = 99999\n break\n\n y = toxics_[index].round().astype('bool')\n #load once, now we have more memory left\n if index not in data.keys():\n data[index] = np.array(loadData('pMulti2_encoded_xlmroberta_192_' + str(index)))\n \n a = time()\n model.fit(data[index], y, epochs=EPOCHS, batch_size=bs)\n print('training step ',index,'took:', round(time() - a), 'seconds')\n\n iLoop += 1\np('nbTrains loops',iLoop*index)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n# Submiting results\n---","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot(df, i='x', j='y', rotate=False, fn=False, title=False,width=6,height=6):\n if type(df)==dict: \n df=pd.DataFrame.from_dict({i:list(df.keys()),j:list(df.values())}).sort_values(by=i,ascending=True)\n\n x, y, fn2, fig, ax = pltinit(df, i, j, title,height,width)\n if(fn==False):\n fn=fn2\n # bestCorrelationsKeys.keys():\n plt.plot(x, y)\n if rotate:\n plt.xticks(rotation=rotate\n \nplot(dict(memGraph),'train','memory',title='Memory usage per train') \n#p(memGraph)#todo : plot memory usage\nsubmitResults()\nsaveWeights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"notification('kaggle xml roberta notebook execution completed in '+str(round(time()-begins))+' sec') \np('exec time:',round(time()-begins),'sec')\nbackgroundThread.terminate()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"---\n