{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport time\nfrom IPython.core.display import display, HTML\nfrom tqdm.notebook import tqdm\n\nimport tensorflow as tf\n\nfrom tensorflow.keras.layers import Dense, Input,Embedding, GlobalAveragePooling1D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\nfrom tensorflow.python.keras.preprocessing.text import Tokenizer\nfrom tensorflow.python.keras.preprocessing.sequence import pad_sequences\n\nfrom sklearn.model_selection import train_test_split\n\nfrom transformers import AutoTokenizer\nimport pickle\nfrom keras.callbacks import ModelCheckpoint\nimport os\nimport pandas as pd\nfrom kaggle_datasets import KaggleDatasets\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# With 80 percent data dropped accuracy was 0.4785\n# With 70 percent data dropped accuracy was 0.4560\n# With optimizer RMSProp and 10 EPOCH accuracy accuracy was 0.4182. So what does this mean? There was a bug. We were reinitializing tokenizer for test!!!\n# With 10 percent dropped and 10 EPOCH, toxic as label instead of int \n# Also check what happens with increase in word size, learning rate settings, steps, best model, toxic as int","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_strategy():\n    # Detect hardware, return appropriate distribution strategy\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    print(\"REPLICAS: \", strategy.num_replicas_in_sync)\n    return strategy\n\nstrategy = get_strategy()\n\nEPOCHS = 1\nBATCH_SIZE = 128 * strategy.num_replicas_in_sync\nVOCAB_SIZE = 100000\nEMBEDDING_DIM=128\nMAX_LEN=256\n\nSMALL_DATA = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_file_path():\n    os.listdir(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\")\n    base_path = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\"\n    train_file = base_path + \"jigsaw-toxic-comment-train.csv\"\n    bias_file = base_path + \"jigsaw-unintended-bias-train.csv\"\n    validation_file = base_path + \"validation.csv\"\n    test_file = base_path + \"test.csv\"\n    sub_file = base_path + \"sample_submission.csv\"\n    return train_file, bias_file, validation_file, test_file, sub_file","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data_using_pandas(train_file, bias_file, validation_file):\n    train_data = pd.read_csv(train_file)\n    bias_data = pd.read_csv(bias_file)\n    validation_data = pd.read_csv(validation_file)\n    \n    train_data[\"lang\"] = \"en\"\n    bias_data[\"lang\"] = \"en\"\n    train_data.fillna(-1, inplace=True)\n    \n    \n    display(HTML(train_data.head(1).to_html()))\n    display(HTML(bias_data.head(1).to_html()))\n    display(HTML(validation_data.head(1).to_html()))\n    if SMALL_DATA:\n        return train_data.head(10000), bias_data.head(10000), validation_data.head(10000)\n    else:\n        return train_data, bias_data, validation_data\n\ndef describe_data(train_data, bias_data, validation_data):\n    display(HTML(train_data.describe().T.to_html()))\n    display(HTML(bias_data.describe().T.to_html()))\n    display(HTML(validation_data.describe().T.to_html()))\n    \n    \ndef get_statistics(train_data, bias_data, validation_data):\n    print(\"Train Data:------------------------\\n\", train_data.shape[0])\n    print(\"Bias Data:------------------------\\n\", bias_data.shape[0])\n    print(\"Valdation Data:--------------------\\n\", validation_data[\"lang\"].value_counts())\n   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tokenized_data(data):\n    arr_data = []\n    for i in tqdm(range(0, data.shape[0], CHUNK_SIZE )):\n        slice_data = data.iloc[i:i+CHUNK_SIZE].copy()\n        arr_data.append(tokenize_data(slice_data))\n        print(\"Process from{} to {}. Now we have {} records\".format(i, i+CHUNK_SIZE, len(arr_data)))\n    data = pd.concat(arr_data)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = None\n\ndef tokenize_data(data):\n    global tokenizer\n    data[\"sequences\"] = tokenizer.texts_to_sequences(data[\"comment_text\"].values)\n    data[\"embedding_input\"] = pad_sequences(data[\"sequences\"].values, padding=\"post\", maxlen=MAX_LEN, truncating=\"post\").tolist()\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model():\n    model = tf.keras.Sequential([\n        Embedding(input_dim=VOCAB_SIZE, output_dim=EMBEDDING_DIM, input_length=MAX_LEN, name=\"embedding\"),\n        GlobalAveragePooling1D(),\n        Dense(24, activation=\"relu\"),\n        Dense(1, activation=\"sigmoid\")\n        \n    ])\n    \n    model.compile(optimizer=\"adam\", loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fix_data(data, label):\n    if 1==1:\n        s = tf.strings.strip(data[\"embedding_input\"])\n        s = tf.strings.substr(\n            s, 1, tf.strings.length(s) - 2)  # Remove parentheses around list\n        s = tf.strings.split(s, ',')\n        s = tf.strings.to_number(s, tf.int32)\n        s = tf.reshape(s, [MAX_LEN])  # Force shape here needed for XLA compilation (TPU)\n    \n    data[\"embedding_input\"] = s\n    return data, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bln_skip = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PATH = KaggleDatasets().get_gcs_path(\"toxic-ds\") # you can list the bucket with \"!gsutil ls $GCS_PATH\"\n    \ntrain_dataset_file  = GCS_PATH + \"/train_dataset.csv\"\nvalidation_dataset_file  = GCS_PATH + \"/validation_dataset.csv\"\n#test_dataset_file = GCS_PATH + \"/x_test.pkl\"\n!gsutil ls $GCS_PATH\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_file, bias_file, validation_file, test_file, sub_file = get_file_path()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_data, bias_data, validation_data = get_data_using_pandas(train_file, bias_file, validation_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not bln_skip:\n    describe_data(train_data, bias_data, validation_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not bln_skip:\n    get_statistics(train_data, bias_data, validation_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not bln_skip:\n    train_data = pd.concat([train_data, bias_data, validation_data])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not bln_skip:\n    del bias_data\n    del validation_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bln_skip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not bln_skip:\n    # Fit tokenizer\n    tokenizer = Tokenizer(num_words=VOCAB_SIZE, oov_token=\"<OOV>\")\n    tokenizer.fit_on_texts(train_data[\"comment_text\"].values.tolist())\n    CHUNK_SIZE = 50000\n\n\n    train_data[\"label\"] = train_data[\"toxic\"].map(lambda x: 1 if x >=0.5 else 0)\n    train_data[\"lang_label\"] = train_data[\"label\"].astype(str) + train_data[\"lang\"]\n\n    train_data[\"len\"] = train_data[\"comment_text\"].map(lambda x: len(x.split(\" \")))\n    train_data[\"len\"] = train_data[\"len\"].map(lambda x: x//10 if x < 500 else 50)\n    en_notoxic_data = train_data[(train_data[\"lang\"]==\"en\") & (train_data[\"label\"]==0)]\n    drop_data, _ = train_test_split(en_notoxic_data, train_size=0.1, stratify=en_notoxic_data[\"len\"])\n    train_data.drop(drop_data.index.values, inplace=True)\n\n\n    train_nlp_data, validation_nlp_data = train_test_split(train_data, train_size=0.8, stratify=train_data[\"lang_label\"], shuffle=True)\n   \n\n    tokenized_train_data = get_tokenized_data(train_nlp_data)\n    tokenized_validation_data = get_tokenized_data(validation_nlp_data)\n\n\n    tokenized_train_data[[\"embedding_input\",\"toxic\"]].to_csv(\"./train_dataset.csv\", index=False)\n    tokenized_validation_data[[\"embedding_input\",\"toxic\"]].to_csv(\"./validation_dataset.csv\", index=False)\n    \n   \n    del train_data, train_nlp_data, validation_nlp_data\n    \n    test_data = pd.read_csv(test_file)\n\n    test_data = test_data.rename(columns={\"content\":\"comment_text\"})\n    print(test_data.head(1))\n    tokenize_data(test_data)\n    x_test = test_data[\"embedding_input\"].values.tolist()\n    pickle.dump(x_test, open(\"x_test.pkl\", \"wb\"))\n    \nelse:\n    GCS_PATH = KaggleDatasets().get_gcs_path(\"toxic-ds\") # you can list the bucket with \"!gsutil ls $GCS_PATH\"\n    \n    train_dataset_file  = GCS_PATH + \"/train_dataset.csv\"\n    validation_dataset_file  = GCS_PATH + \"/validation_dataset.csv\"\n    test_dataset_file = \"../input/toxic-ds\" + \"/x_test.pkl\"\n    with open(test_dataset_file, 'rb') as pickle_file:\n        x_test = pickle.load(pickle_file)\n\n    tokenized_train_data = pd.read_csv(train_dataset_file)\n    tokenized_validation_data = pd.read_csv(validation_dataset_file)\n\n    train_size = tokenized_train_data.shape[0]\n    print(\"Train data size:\", train_size)\n\n    valid_size = tokenized_validation_data.shape[0]\n    print(\"Valid data size:\", valid_size)\n    del tokenized_train_data, tokenized_validation_data\n\n    train_dataset = tf.data.experimental.make_csv_dataset(train_dataset_file, label_name=\"toxic\", batch_size=1)\n    validation_dataset = tf.data.experimental.make_csv_dataset(validation_dataset_file, label_name=\"toxic\", batch_size=1)\n\n    train_dataset = train_dataset.unbatch()\n    train_dataset = train_dataset.map(lambda data, label: fix_data(data, label), num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(BATCH_SIZE).prefetch(tf.data.experimental.AUTOTUNE)\n\n    validation_dataset = validation_dataset.unbatch()\n    validation_dataset = validation_dataset.map(lambda data, label: fix_data(data, label), num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(BATCH_SIZE).prefetch(tf.data.experimental.AUTOTUNE)\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if bln_skip:\n    for data,label in train_dataset.take(1):\n        print(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if bln_skip:\n    from tensorflow.keras.models import model_from_json\n    if 1==1: #not bln_skip:\n        with strategy.scope():\n            model = build_model()\n    else:\n        prefix = \"../input/from-first-principles-toxic-or-not/\"\n        json_file = open(prefix + 'model.json', 'r')\n        loaded_model_json = json_file.read()\n        json_file.close()\n        model = model_from_json(loaded_model_json)\n        # load weights into new model\n        model.load_weights(prefix + \"model.h5\")\n    rmsprop = RMSprop()\n    model.compile(optimizer=rmsprop, loss='binary_crossentropy', metrics=['accuracy'])\n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if 1==2:\n    del x_train\n    del y_train\n    del x_valid\n    del y_valid\n    del x_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if  bln_skip:\n    EPOCHS = 10\n    filepath=\"toxic_model.hdf5\"\n    checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')\n    earlystopping = EarlyStopping(monitor='val_loss', patience=30, mode='min', min_delta=0.0001),\n    callbacks_list = [checkpoint, earlystopping]\n    if 1==1: #not bln_skip:\n        n_steps = train_size // BATCH_SIZE\n        print(\"Steps:\", n_steps)\n        train_history = model.fit(\n            train_dataset,\n            steps_per_epoch=n_steps,\n            validation_data=validation_dataset,\n            validation_steps=valid_size//BATCH_SIZE,\n            epochs=EPOCHS\n        )\n\nif 1==2:\n    model_json = model.to_json()\n    with open(\"./model.json\", \"w\") as json_file:\n        json_file.write(model_json)\n    # serialize weights to HDF5\n    model.save_weights(\"./model.h5\")\n    print(\"Saved model to disk\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if bln_skip:\n    test_dataset = (\n        tf.data.Dataset\n        .from_tensor_slices(x_test)\n        .batch(8)\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if bln_skip:\n    sub = pd.read_csv(sub_file)\n    sub['toxic'] = model.predict(test_dataset, verbose=1)\n    sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from kaggle_datasets import KaggleDataset\n#GCS_PATH = KaggleDatasets().get_gcs_path() # you can list the bucket with \"!gsutil ls $GCS_PATH\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}