{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is default import statements code cell, that initializes when creating a new notebook."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nimport gc\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.utils import to_categorical\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Embedding\nfrom keras.layers import Input\nfrom keras.layers import Conv1D\nfrom keras.layers import MaxPooling1D\nfrom keras.layers import Flatten\nfrom keras.layers import Dropout\nfrom keras.layers import Dense\nfrom keras.optimizers import RMSprop\nfrom keras.models import Model\nfrom keras.models import load_model\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/tensorflow2-question-answering/sample_submission.csv\")\nsample_submission.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_file_path='/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl'\ntest_file_path='/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl'\n\n#written by ragnar\ndef read_data(path, sample = True, chunksize = 30000):\n    if sample == True:\n        df = []\n        with open(path, 'r') as reader:\n            for i in range(chunksize):\n                df.append(json.loads(reader.readline()))\n        df = pd.DataFrame(df)\n        print('Our sampled dataset have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n    else:\n        df = pd.read_json(path, orient = 'records', lines = True)\n        print('Our dataset have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n        gc.collect()\n    return df\n\ntrain = read_data(train_file_path, sample = True)\ntest = read_data(test_file_path, sample = False)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_missing_data(df):\n    flag=df.isna().sum().any()\n    if flag==True:\n        total = df.isnull().sum()\n        percent = (df.isnull().sum())/(df.isnull().count()*100)\n        output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n        data_type = []\n        # written by MJ Bahmani\n        for col in df.columns:\n            dtype = str(train[col].dtype)\n            data_type.append(dtype)\n        output['Types'] = data_type\n        return(np.transpose(output))\n    else:\n        return(False)\n    \ncheck_missing_data(train)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We don't have any missing values in training data."},{"metadata":{"trusted":true},"cell_type":"code","source":"check_missing_data(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Similarly, we don't have any missing values in test data."},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.tokenize import sent_tokenize, word_tokenize\nsample_text=train.document_text[0]\nphrases = sent_tokenize(sample_text)\nwords = word_tokenize(sample_text)\nprint(phrases)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type(words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.tokenize import sent_tokenize, word_tokenize\nfrom nltk.corpus import stopwords\n\nstopWords = set(stopwords.words('english'))\nwords = word_tokenize(sample_text)\nwordsFiltered = []\n \nfor w in words:\n    if w not in stopWords:\n        wordsFiltered.append(w)\n \nprint(wordsFiltered)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud as wc\nfrom nltk.corpus import stopwords \nimport matplotlib.pyplot as plt\nfrom nltk.corpus import stopwords\n\neng_stopwords = set(stopwords.words(\"english\"))\n\ndef generate_wordcloud(text): \n    wordcloud = wc(relative_scaling = 1.0,stopwords = eng_stopwords).generate(text)\n    fig,ax = plt.subplots(1,1,figsize=(10,10))\n    ax.imshow(wordcloud, interpolation='bilinear')\n    ax.axis(\"off\")\n    ax.margins(x=0, y=0)\n    plt.show()\n    \ngenerate_wordcloud(train.document_text[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_NUM_WORDS = 10000\nTEXT_COLUMN = 'question_text'\n\n# Create a text tokenizer.\ntokenizer = Tokenizer(num_words=MAX_NUM_WORDS)\ntokenizer.fit_on_texts(train[TEXT_COLUMN])\n\n# All comments must be truncated or padded to be the same length.\nMAX_SEQUENCE_LENGTH = 250\ndef pad_text(texts, tokenizer):\n    return pad_sequences(tokenizer.texts_to_sequences(texts), maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDINGS_PATH = '../input/glove-global-vectors-for-word-representation/glove.6B.100d.txt'\nEMBEDDINGS_DIMENSION = 100\nDROPOUT_RATE = 0.3\nLEARNING_RATE = 0.00005\nNUM_EPOCHS = 10\nBATCH_SIZE = 128\n\ndef train_model(train_df, test, tokenizer):\n    # Prepare data\n    train_text = pad_text(train_df[TEXT_COLUMN], tokenizer)\n   # train_labels = to_categorical(train_df[TOXICITY_COLUMN])\n  #  validate_text = pad_text(validate_df[TEXT_COLUMN], tokenizer)\n   # validate_labels = to_categorical(validate_df[TOXICITY_COLUMN])\n\n    # Load embeddings\n    print('loading embeddings')\n    embeddings_index = {}\n    with open(EMBEDDINGS_PATH) as f:\n        for line in f:\n            values = line.split()\n            word = values[0]\n            coefs = np.asarray(values[1:], dtype='float32')\n            embeddings_index[word] = coefs\n\n    embedding_matrix = np.zeros((len(tokenizer.word_index) + 1,\n                                 EMBEDDINGS_DIMENSION))\n    num_words_in_embedding = 0\n    for word, i in tokenizer.word_index.items():\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            num_words_in_embedding += 1\n            # words not found in embedding index will be all-zeros.\n            embedding_matrix[i] = embedding_vector\n\n    # Create model layers.\n    def get_convolutional_neural_net_layers():\n        \"\"\"Returns (input_layer, output_layer)\"\"\"\n        sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n        embedding_layer = Embedding(len(tokenizer.word_index) + 1,\n                                    EMBEDDINGS_DIMENSION,\n                                    weights=[embedding_matrix],\n                                    input_length=MAX_SEQUENCE_LENGTH,\n                                    trainable=False)\n        x = embedding_layer(sequence_input)\n        x = Conv1D(128, 2, activation='relu', padding='same')(x)\n        x = MaxPooling1D(5, padding='same')(x)\n        x = Conv1D(128, 3, activation='relu', padding='same')(x)\n        x = MaxPooling1D(5, padding='same')(x)\n        x = Conv1D(128, 4, activation='relu', padding='same')(x)\n        x = MaxPooling1D(40, padding='same')(x)\n        x = Flatten()(x)\n        x = Dropout(DROPOUT_RATE)(x)\n        x = Dense(128, activation='relu')(x)\n        preds = Dense(2, activation='softmax')(x)\n        return sequence_input, preds\n\n    # Compile model.\n    print('compiling model')\n    input_layer, output_layer = get_convolutional_neural_net_layers()\n    model = Model(input_layer, output_layer)\n    model.compile(loss='categorical_crossentropy',\n                  optimizer=RMSprop(lr=LEARNING_RATE),\n                  metrics=['acc'])\n\n    # Train model.\n    print('training model')\n    model.fit(train_text,\n              train_labels,\n              batch_size=BATCH_SIZE,\n              epochs=NUM_EPOCHS,\n              validation_data=(validate_text, validate_labels),\n              verbose=2)\n\n    return model\n\nmodel = train_model(train, test, tokenizer)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}