{"cells":[{"metadata":{},"cell_type":"markdown","source":"---\nFull Notebook is here : https://www.kaggle.com/benjaminfontaine/xlm-roberta-toxicity-predictions\n\n---","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import requests,pickle,os,pandas as pd,numpy as np\nfrom time import time\nbegins=time()\n\n#target='p77d3_xlmr_300fast_.tgz'#older version more runs ..\ntarget='p77d4_xlmr_300fast_.tgz'\ntarget='P8b_DistillRoberta_192_Fast_.pickle'\ntarget='p77d4_xlmr192Fast_.tgz'\ntarget='p8_p2xlmr192Fast_All8.tgz';#0.8275 -- Left, pas terrible ( moins entrainé pour l'instant .. )\ntarget='p8_p2xlmr192Fast_NotEn_17.tgz'#0.854, puis 0.854 toujours pas terrible -- Right,last one -- supposed be the best -- needs more training\ntarget='p8_p2xlmr192Fast_NotEn7.tgz'#0.9178, puis 0.9187 -- Center excluding en train, has more trainings -- works better\ntarget='ptotal.tgz'#0.9178, puis 0.9187 -- Center excluding en train, has more trainings -- works better\n\n\navg=1\nlatestResults='http://1.x24.fr/a/jupyter/poc7/'#\nr=requests.get(latestResults+target,stream=True)\n\nwith open(target,'wb') as f:\n    f.write(r.raw.read())       \n\nif target.endswith('.tgz'):\n    os.system('tar xf '+target+';rm -f '+target)\n\ntarget=target.replace('.tgz','.pickle')\ndata = open(target, \"rb\")\npreds = pickle.load(data)\ndata.close()\nos.system('rm '+target)\n\nsep=pd.DataFrame({})#\n\nif'Direct':\n    sep['id']=list(preds.keys())\n    sep['toxic']=list(preds.values())\nsep=sep.sort_values(by='id')\nfn='submission.csv'\nsep['id,toxic'.split(',')].to_csv(fn,index=False)\n#display(sep)\nprint('exec time:',round(time()-begins),'sec')\nassert(False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%script False\nsep=pd.DataFrame({'id':list(range(0,63812))})\n\nif(type(preds)==list):\n  j=0;res={}\n  for i in preds:\n    p(i)\n    res[j]=i\n    j+=1\n  preds=res\n\nif True & bool('dict'):\n  pk=preds.keys();\n  for i in pk:\n    if type(preds[i][0])==np.ndarray:\n      preds[i]=[ji[0] for ji in preds[i]]\n\nprint('number of predictions',len(pk))    \n    \ndf=pd.DataFrame(preds)\nmeans=df.mean(axis=1).values\n    \npk=list(preds.keys())\n\nif avg:\n    sep['toxic']=means\nelse:\n    sep['toxic']=preds[pk[-1]];#last prediction : scores less than the average of submissions\n\nfn='submission.csv'\nsep['id,toxic'.split(',')].to_csv(fn,index=False)\nprint('exec time:',round(time()-begins),'sec')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}