{"cells":[{"metadata":{},"cell_type":"markdown","source":"## References\n* https://www.kaggle.com/kalashnimov/distilbert-baseline/notebook\n* https://www.kaggle.com/sophiechampagne/jigsaw-baseline-models-v1/data"},{"metadata":{},"cell_type":"markdown","source":"## Load Libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport re\nimport matplotlib.pyplot as plt\nimport glob","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Params"},{"metadata":{"trusted":true},"cell_type":"code","source":"needed_cols = ['toxic', 'comment_text']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Simple approach"},{"metadata":{"trusted":true},"cell_type":"code","source":"import string\nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV, GridSearchCV, StratifiedKFold\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.pipeline import Pipeline\nfrom gensim import utils\nimport gensim.parsing.preprocessing as gsp\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading train data\n\ntranslated_train_files = glob.glob('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-*-cleaned.csv')\ntranslated_train_dfs = []\nfor filename in translated_train_files:\n    df = pd.read_csv(filename, usecols=needed_cols)\n    lang = re.findall('train-google-(.*)-cleaned.csv', filename)[0]\n    df['lang'] = lang\n    translated_train_dfs.append(df)\n\ntrain_en = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=needed_cols)\ntrain_en['lang'] = 'en'\n\ntranslated_train_dfs.append(train_en)\n    \n#train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n#train2.toxic = train2.toxic.round().astype(int)\n#train2['lang'] = 'en'\n\ntrain_df = pd.concat(translated_train_dfs).sample(n=600000).reset_index(drop=True)\n# len 1563650\n# train_df = pd.concat(translated_train_dfs).reset_index(drop=True)\n\n\ndel df, translated_train_dfs, train_en\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['toxic'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')\ndef tokenize(s): \n    return re_tok.sub(r' \\1 ', s).split()\n \nfilters = [\n           gsp.strip_tags, #remove tags \n           gsp.strip_punctuation, #remove punctuation\n           gsp.strip_multiple_whitespaces, #standarized the spaces \n           gsp.strip_numeric,\n           gsp.remove_stopwords, #stop words  \n           gsp.strip_short, \n           gsp.stem_text #stemming \n          ]\n\ndef clean_text(s):\n    s = str(s).lower() #lower case for all words\n    s = utils.to_unicode(s)\n    for f in filters:\n        s = f(s)\n    return s","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#clean the text first \ntrain_df['comment_text'].fillna(\"unknown\", inplace=True)\ntrain_df[\"comment_text\"] = train_df[\"comment_text\"].apply(clean_text)\n\n#split the train and test from the whole table \n# x_train, x_test, y_train, y_test = train_test_split(train_df[\"comment_text\"], train_df['toxic'], test_size=0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#vectorization of the model \nvec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,\n               strip_accents='unicode', use_idf=1,\n               smooth_idf=1, sublinear_tf=1)\n\npipeline = Pipeline([\n    ('tfidf', vec),\n    ('logreg', LogisticRegression(penalty='elasticnet')),\n])\n\nparameters = {\n    'tfidf__max_features': [None, 1000, 5000, 50000],\n    'tfidf__ngram_range': [(1, 1), (1, 2)],  # unigrams or unigrams + bigrams\n    'logreg__penalty' : ['l1', 'l2'],\n    'logreg__C' : np.logspace(-4, 4, 20),\n    'logreg__solver' : ['liblinear'],\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\nrandom_search = RandomizedSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, \n                                   scoring=['roc_auc'],\n                                   cv=cv, n_iter=10, refit='roc_auc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nrandom_search.fit(train_df[\"comment_text\"], train_df['toxic']);\n\ndel train_df; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(random_search.cv_results_).sort_values('mean_test_roc_auc', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nval_df['comment_text'].fillna(\"unknown\", inplace=True)\nval_df[\"comment_text\"] = val_df[\"comment_text\"].apply(clean_text)\n\n#score the validation dataset\ny_valid = val_df['toxic']\ny_pred_valid = random_search.best_estimator_.predict_proba(val_df[\"comment_text\"])\n# print('Testing accuracy %s' % accuracy_score(y_valid, y_pred_valid))\n# print('Testing F1 score: {}'.format(f1_score(y_valid, y_pred_valid, average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_valid, y_pred_valid[:, 1]))\n\ndel val_df, y_valid, y_pred_valid; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_subm_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntest_subm_df[\"comment_text\"] = test_subm_df[\"content\"].apply(clean_text)\n\n#score the submission file \ny_pred_submission = random_search.best_estimator_.predict_proba(test_subm_df[\"comment_text\"])\n#load the sample submission file \nsample_sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n\nsubmid = pd.DataFrame({'id': sample_sub[\"id\"]})\nsubmission = pd.concat([submid, pd.DataFrame(y_pred_submission[:, 1],columns=['toxic'])], axis=1)\ndisplay(submission.head())\nsubmission['toxic'].hist(bins=100, log=False, alpha=1);\nsubmission.to_csv('submission_logreg.csv', index=False)\n\ndel test_subm_df, y_pred_submission, sample_sub, submid, submission; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Distil Bert"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U transformers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Dropout, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\nfrom tqdm.notebook import tqdm\nimport tokenizers\nfrom tokenizers import BertWordPieceTokenizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(text):\n    text = str(text)\n    text = re.sub(r'[0-9\"]', '', text) # number\n    text = re.sub(r'#[\\S]+\\b', '', text) # hash\n    text = re.sub(r'@[\\S]+\\b', '', text) # mention\n    text = re.sub(r'https?\\S+', '', text) # link\n    text = re.sub(r'\\s+', ' ', text) # multiple white spaces\n#     text = re.sub(r'\\W+', ' ', text) # non-alphanumeric\n    return text.strip()\n\ndef text_process(text):\n    ws = text.split(' ')\n    if(len(ws)>160):\n        text = ' '.join(ws[:160]) + ' ' + ' '.join(ws[-32:])\n    return text\n\ndef fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n    print('encoding with', tokenizer)\n    \n    # for transformers 3.5\n    if isinstance(tokenizer, transformers.tokenization_distilbert.DistilBertTokenizer) or \\\n        isinstance(tokenizer, transformers.DistilBertTokenizerFast):\n    #     tokenizer.enable_truncation(max_length=maxlen)\n    #     tokenizer.enable_padding(max_length=maxlen)\n        all_ids = []\n\n        for i in tqdm(range(0, len(texts), chunk_size)):\n            text_chunk = texts[i:i+chunk_size].tolist()\n    #         encs = tokenizer.encode_batch(text_chunk)\n            encs = tokenizer(text_chunk, padding='max_length', truncation=True, max_length=maxlen)\n    #         all_ids.extend([enc.ids for enc in encs])\n            all_ids.extend(encs['input_ids']) \n        \n    elif isinstance(fast_tokenizer, tokenizers.implementations.bert_wordpiece.BertWordPieceTokenizer): \n        tokenizer.enable_truncation(max_length=maxlen)\n        tokenizer.enable_padding(max_length=maxlen)\n        all_ids = []\n\n        for i in tqdm(range(0, len(texts), chunk_size)):\n            text_chunk = texts[i:i+chunk_size].tolist()\n            encs = tokenizer.encode_batch(text_chunk)\n            all_ids.extend([enc.ids for enc in encs])\n\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = transformers.DistilBertTokenizerFast.from_pretrained('distilbert-base-multilingual-cased')\n\nsave_path = '/kaggle/working/distilbert_base_cased/'\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\ntokenizer.save_pretrained(save_path)\n\n# \"faster as the tokenizers from transformers because they are implemented in Rust.\"\n# fast_tokenizer = BertWordPieceTokenizer('distilbert_base_cased/vocab.txt', lowercase=False)\nfast_tokenizer = tokenizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU config"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Configuration\nAUTO = tf.data.experimental.AUTOTUNE\nSHUFFLE = 2048\nEPOCHS1 = 20\nEPOCHS2 = 4\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192\nVERBOSE = 2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read data"},{"metadata":{},"cell_type":"markdown","source":"may be switched from original to multilingual"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \n\n# # Label distributions are pretty the same in different train sets, so let's make full use of both.  \n# # If balanced subsample is needed, it may be done with the func below\n\n\n# train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", \n#                      usecols=needed_cols)\n# train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\",\n#                      usecols=needed_cols)\n# # convert to binary\n# train2.toxic = train2.toxic.round().astype(int)\n# display(train1['toxic'].value_counts(normalize=True))\n# display(train2['toxic'].value_counts(normalize=True))\n\n# def downsample(df, negative_ratio=1):\n#     \"\"\"Subsample the train dataframe to 50%-50%\"\"\"\n#     ds_df= pd.concat([\n#         df.query('toxic==1'),\n#         df.query('toxic==0').sample(sum(df['toxic'])*negative_ratio)\n#     ])\n    \n#     return ds_df\n\n# train_df = pd.concat([\n#     train1[needed_cols],\n#     train2[needed_cols]\n# ])\n# train_df['toxic'] = train_df['toxic']\n\n# train_df = downsample(train_df, negative_ratio=3)\n\n# display(train_df.head(5), train_df.shape)\n\n# y_train = train_df['toxic'].values\n\n# del train1, train2; gc.collect()\n\n\n\n\n# Loading train data\n\ntranslated_train_files = glob.glob('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-*-cleaned.csv')\ntranslated_train_dfs = []\nfor filename in translated_train_files:\n    df = pd.read_csv(filename, usecols=needed_cols)\n    lang = re.findall('train-google-(.*)-cleaned.csv', filename)[0]\n    df['lang'] = lang\n    translated_train_dfs.append(df)\n\ntrain_en = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=needed_cols)\ntrain_en['lang'] = 'en'\n\ntranslated_train_dfs.append(train_en)\n    \n#train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n#train2.toxic = train2.toxic.round().astype(int)\n#train2['lang'] = 'en'\n\ntrain_df = pd.concat(translated_train_dfs).sample(n=600000).reset_index(drop=True)\n# train_df = pd.concat(translated_train_dfs).reset_index(drop=True)\n\n\ny_train = train_df['toxic'].values\n\n\ndel df, translated_train_dfs, train_en\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def len_sent(data):\n    return len(data.split())\ntrain_df[\"num_words_comment_text\"] = train_df[\"comment_text\"].apply(lambda x : len_sent(x))\nsns.kdeplot(train_df[train_df[\"toxic\"] == 0][\"num_words_comment_text\"].values, shade = True, color = \"red\", label='non_toxity')\nsns.kdeplot(train_df[train_df[\"toxic\"] == 1][\"num_words_comment_text\"].values, shade = True, color = \"blue\", label='toxity')\n\ndel train_df['toxic']; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['comment_text'] = train_df['comment_text'].apply(lambda x: clean_text(x))\ntrain_df['comment_text'] = train_df['comment_text'].apply(lambda x: text_process(x))\nx_train = fast_encode(train_df['comment_text'].astype(str), fast_tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build dataset objects"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(SHUFFLE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\ndel x_train; gc.collect()\n\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nvalid['comment_text'] = valid.apply(lambda x: clean_text(x['comment_text']), axis=1)\nvalid['comment_text'] = valid['comment_text'].apply(lambda x: text_process(x))\nx_valid = fast_encode(valid['comment_text'].astype(str), fast_tokenizer, maxlen=MAX_LEN)\ny_valid = valid['toxic'].values\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ndel x_valid; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"lrs = ReduceLROnPlateau(monitor='val_auc', mode ='max', factor = 0.7, min_lr= 1e-7, verbose = 1, patience = 2)\nes1 = EarlyStopping(monitor='val_auc', mode='max', verbose = 1, patience = 5, restore_best_weights=True)\nes2 = EarlyStopping(monitor='auc', mode='max', verbose = 1, patience = 1, restore_best_weights=True)\ncallbacks_list1 = [lrs,es1]\ncallbacks_list2 = [lrs,es2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    x = tf.keras.layers.Dropout(0.4)(cls_token)\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC(name='auc'), 'accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load model into TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Run model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# not train on order to save memory\nn_steps = len(y_train) // (BATCH_SIZE*8)\n\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS1,\n    callbacks=callbacks_list1,\n    verbose=VERBOSE\n)\n\ndel train_dataset; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history_df = pd.DataFrame.from_dict(train_history.history)\ntrain_history_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(10, 5))\nplt.plot(train_history_df['val_auc'], label='val')\nplt.plot(train_history_df['auc'], label='train')\nplt.legend(fontsize=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = len(y_valid) // (BATCH_SIZE)\n\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS2,\n    callbacks=callbacks_list2,\n    verbose=VERBOSE\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history2_df = pd.DataFrame.from_dict(train_history_2.history)\ntrain_history2_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntest['content'] = test.apply(lambda x: clean_text(x['content']), axis=1)\ntest['content'] = test['content'].apply(lambda x: text_process(x))\nx_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)\n\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = model.predict(test_dataset, verbose=1)\nsub['toxic'].hist(bins=100, log=False, alpha=1)\nsub.to_csv('submission_distilbert.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Blending"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_logreg = pd.read_csv('submission_logreg.csv')\ndisplay(sub_logreg.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_distilbert = pd.read_csv('submission_distilbert.csv')\ndisplay(sub_distilbert.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = 1\n\nblend_sub = sub_logreg.copy()\n\nblend_sub['toxic'] = \\\n    sub_logreg['toxic']**p * 0.25 + \\\n    sub_distilbert['toxic']**p * 0.75","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15, 6))\nplt.hist(sub_distilbert['toxic'], bins=100, log=False, alpha=0.3, label='distilbert');\nplt.hist((sub_distilbert['toxic']**p), bins=100, log=False, alpha=0.3, label='distilbert power_'+str(p));\nplt.hist(sub_logreg['toxic'], bins=100, log=False, alpha=0.3, label='logreg');\nplt.hist((sub_logreg['toxic']**p), bins=100, log=False, alpha=0.3, label='logreg power_'+str(p));\nplt.hist(blend_sub['toxic'], bins=100, log=False, alpha=0.3, label='blend');\nplt.legend(fontsize=15);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"blend_sub.head()\nblend_sub.to_csv('submission.csv', index=False)\n# sub_distilbert.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}