{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nfrom collections import Counter\nimport numpy as np\nimport time\nimport operator\nimport re\nimport gc\n\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import Model, load_model\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras import optimizers\nimport tensorflow as tf\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn import metrics\n\nimport nltk\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.corpus import wordnet\nfrom nltk.tokenize import WordPunctTokenizer\nfrom nltk.corpus import stopwords\nfrom string import punctuation\n\nnltk.download('stopwords')","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:50.008934Z","iopub.execute_input":"2021-12-16T19:50:50.009266Z","iopub.status.idle":"2021-12-16T19:50:50.022224Z","shell.execute_reply.started":"2021-12-16T19:50:50.009229Z","shell.execute_reply":"2021-12-16T19:50:50.021194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"TRAIN_DATA_PATH = '/kaggle/input/quora-insincere-questions-classification/train.csv'\nTEST_DATA_PATH = '/kaggle/input/quora-insincere-questions-classification/test.csv'","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:50.024634Z","iopub.execute_input":"2021-12-16T19:50:50.025259Z","iopub.status.idle":"2021-12-16T19:50:50.03875Z","shell.execute_reply.started":"2021-12-16T19:50:50.02521Z","shell.execute_reply":"2021-12-16T19:50:50.037828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(TRAIN_DATA_PATH)\ntest_data = pd.read_csv(TEST_DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:50.040127Z","iopub.execute_input":"2021-12-16T19:50:50.040343Z","iopub.status.idle":"2021-12-16T19:50:54.044931Z","shell.execute_reply.started":"2021-12-16T19:50:50.040315Z","shell.execute_reply":"2021-12-16T19:50:54.043862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:54.046468Z","iopub.execute_input":"2021-12-16T19:50:54.046932Z","iopub.status.idle":"2021-12-16T19:50:54.056012Z","shell.execute_reply.started":"2021-12-16T19:50:54.046882Z","shell.execute_reply":"2021-12-16T19:50:54.05549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:54.05774Z","iopub.execute_input":"2021-12-16T19:50:54.05805Z","iopub.status.idle":"2021-12-16T19:50:54.075321Z","shell.execute_reply.started":"2021-12-16T19:50:54.058023Z","shell.execute_reply":"2021-12-16T19:50:54.074549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Train data shape: {train_data.shape}')\nprint(f'Test data shape: {test_data.shape}')","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:54.076653Z","iopub.execute_input":"2021-12-16T19:50:54.077411Z","iopub.status.idle":"2021-12-16T19:50:54.087381Z","shell.execute_reply.started":"2021-12-16T19:50:54.077365Z","shell.execute_reply":"2021-12-16T19:50:54.086586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_counts = train_data['target'].value_counts()\nvalue_counts_percentage = train_data['target'].value_counts(normalize=True).mul(100).round(2).astype(str) + '%'\npd.concat([value_counts, value_counts_percentage], axis=1, keys=['Counts', 'Percentage'])","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:54.088569Z","iopub.execute_input":"2021-12-16T19:50:54.089448Z","iopub.status.idle":"2021-12-16T19:50:54.121275Z","shell.execute_reply.started":"2021-12-16T19:50:54.089406Z","shell.execute_reply":"2021-12-16T19:50:54.120323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Классы сильно несбалансированы. Доля провокационных вопросов в датасете составляет 6.19% от общего числа вопросов.","metadata":{}},{"cell_type":"markdown","source":"# Prerocessing","metadata":{}},{"cell_type":"code","source":"train_data['question_text'][35]","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:54.122407Z","iopub.execute_input":"2021-12-16T19:50:54.122806Z","iopub.status.idle":"2021-12-16T19:50:54.129092Z","shell.execute_reply.started":"2021-12-16T19:50:54.122776Z","shell.execute_reply":"2021-12-16T19:50:54.128161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Препроцессим текст: приводим все слова к нижнему регистру, удаляем пунктуацию и стопворды (слова, не несущие особого смысла).","metadata":{}},{"cell_type":"code","source":"STOPWORDS = nltk.corpus.stopwords.words('english')\n\ntokenizer = WordPunctTokenizer()\n\nlemmatizer = WordNetLemmatizer()\nwordnet_map = {\"N\":wordnet.NOUN, \"V\":wordnet.VERB, \"J\":wordnet.ADJ, \"R\":wordnet.ADV}\n\ndef lemmatize_words(text):\n    pos_tagged_text = nltk.pos_tag(text)\n    return [lemmatizer.lemmatize(word, wordnet_map.get(pos[0], wordnet.NOUN)) for word, pos in pos_tagged_text]\n\ndef remove_punctuation(text):\n    return \"\".join([i for i in text if i not in punctuation])\n\ndef remove_stopwords(text):\n    return \" \".join([word for word in str(text).split() if word not in STOPWORDS])\n\n\ndef preprocess_data(data):\n    # Lower Casing\n    data['preprocessed_text'] = data['question_text'].apply(lambda x: x.lower())\n\n    # Remove Punctuation\n    data['preprocessed_text'] = data['preprocessed_text'].apply(lambda x: remove_punctuation(x))\n\n    # Remove Stopwords\n    data['preprocessed_text'] = data['preprocessed_text'].apply(lambda x: remove_stopwords(x))\n\n    # Tokenization\n    # data['preprocessed_text'] = data['preprocessed_text'].apply(lambda x: tokenizer.tokenize(x))\n\n    # Lemmitization\n    # data['preprocessed_text'] = data['preprocessed_text'].apply(lambda x: lemmatize_words(x))","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:54.130145Z","iopub.execute_input":"2021-12-16T19:50:54.130674Z","iopub.status.idle":"2021-12-16T19:50:56.24447Z","shell.execute_reply.started":"2021-12-16T19:50:54.130629Z","shell.execute_reply":"2021-12-16T19:50:56.243569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\npreprocess_data(train_data)\npreprocess_data(test_data)\n\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:50:56.245501Z","iopub.execute_input":"2021-12-16T19:50:56.24572Z","iopub.status.idle":"2021-12-16T19:51:51.177024Z","shell.execute_reply.started":"2021-12-16T19:50:56.245695Z","shell.execute_reply":"2021-12-16T19:51:51.176187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef vocab_to_integer(vocab):\n    return {word: ii for ii, word in enumerate(vocab, 1)}","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:51:51.178491Z","iopub.execute_input":"2021-12-16T19:51:51.178921Z","iopub.status.idle":"2021-12-16T19:51:51.185718Z","shell.execute_reply.started":"2021-12-16T19:51:51.178874Z","shell.execute_reply":"2021-12-16T19:51:51.185037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Строим словарь из обработанных данных и маппим слова на соответсвтующие индексы.","metadata":{}},{"cell_type":"code","source":"%%time\nall_questions = pd.concat([train_data['preprocessed_text'], test_data['preprocessed_text']])\nfinal_vocab = build_vocab(all_questions)\nword_to_idx = vocab_to_integer(final_vocab)","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:51:51.186652Z","iopub.execute_input":"2021-12-16T19:51:51.187175Z","iopub.status.idle":"2021-12-16T19:51:59.324883Z","shell.execute_reply.started":"2021-12-16T19:51:51.18714Z","shell.execute_reply":"2021-12-16T19:51:59.323865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Строим словарь из исходных вопросов.","metadata":{}},{"cell_type":"code","source":"vocab_original = build_vocab(pd.concat([train_data['question_text'], test_data['question_text']]))","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:51:59.327641Z","iopub.execute_input":"2021-12-16T19:51:59.327879Z","iopub.status.idle":"2021-12-16T19:52:11.845206Z","shell.execute_reply.started":"2021-12-16T19:51:59.32785Z","shell.execute_reply":"2021-12-16T19:52:11.844356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Задаем параметры некоторые гиперпараметры для модели + константы","metadata":{}},{"cell_type":"code","source":"hparam = {}\nhparam['VOCAB_SIZE'] = len(final_vocab) + 1\nhparam['PAD_LENGTH'] = 77\nhparam['MINIBATCH_SIZE'] = 512\nhparam['LEARNING_RATE'] = 1e-3\nhparam['EPOCHS'] = 4\nhparam['LSTM_HIDDEN_SIZE'] = 128\nhparam['WORD_EMB_DIM'] = 0\nhparam['KFOLDS'] = 3","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:52:11.846586Z","iopub.execute_input":"2021-12-16T19:52:11.847454Z","iopub.status.idle":"2021-12-16T19:52:11.853974Z","shell.execute_reply.started":"2021-12-16T19:52:11.847406Z","shell.execute_reply":"2021-12-16T19:52:11.853183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Функция для загрузки готовых эмбеддингов.","metadata":{}},{"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word, *arr):\n        return word, np.asarray(arr, dtype='float32')\n    \n    if file.split('/')[-1] == 'wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o) > 100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:52:11.856458Z","iopub.execute_input":"2021-12-16T19:52:11.856742Z","iopub.status.idle":"2021-12-16T19:52:11.869687Z","shell.execute_reply.started":"2021-12-16T19:52:11.856701Z","shell.execute_reply":"2021-12-16T19:52:11.869109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ниже описана функция определения слов, которых нет в готовых эмбеддингах.","metadata":{}},{"cell_type":"code","source":"def check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n    print('{} known words, {} unique'.format(nb_known_words, len(known_words)))\n    print('{} unknown words, {} unique'.format(nb_unknown_words, len(unknown_words)))\n    print('Found embeddings for {:.3%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.3%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n    return unknown_words","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:52:11.871007Z","iopub.execute_input":"2021-12-16T19:52:11.871397Z","iopub.status.idle":"2021-12-16T19:52:11.881238Z","shell.execute_reply.started":"2021-12-16T19:52:11.871368Z","shell.execute_reply":"2021-12-16T19:52:11.880674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_emb_matrix(nb_words, embed_size):\n    # Создаем исходную матрицу эмбеддингов (слова, у которых нет эмбеддингов, будут представлены в виде нулевого вектора)\n    return np.zeros((nb_words, embed_size), dtype=np.float32)\n\ndef fill_emb_matrix(word_idx, emb_matrix, emb_index):\n    for word, i in word_idx:\n        emb_vector = emb_index.get(word)\n        if emb_vector is not None:\n            emb_matrix[i] = emb_vector\n    return emb_matrix\n\ndef add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:52:11.882291Z","iopub.execute_input":"2021-12-16T19:52:11.882756Z","iopub.status.idle":"2021-12-16T19:52:11.893738Z","shell.execute_reply.started":"2021-12-16T19:52:11.882709Z","shell.execute_reply":"2021-12-16T19:52:11.892961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Извлекаем готовые эмбеддинги из архива","metadata":{}},{"cell_type":"code","source":"import zipfile\n\nz= zipfile.ZipFile('../input/quora-insincere-questions-classification/embeddings.zip')\nz.extractall()","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:52:11.894992Z","iopub.execute_input":"2021-12-16T19:52:11.895211Z","iopub.status.idle":"2021-12-16T19:55:49.09272Z","shell.execute_reply.started":"2021-12-16T19:52:11.895185Z","shell.execute_reply":"2021-12-16T19:55:49.091275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_glove = './glove.840B.300d/glove.840B.300d.txt'\n_paragram =  './paragram_300_sl999/paragram_300_sl999.txt'\n_wiki_news = './wiki-news-300d-1M/wiki-news-300d-1M.vec'\n_google_news = './GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n\nembeddings = [{'name': 'glove', 'path': _glove},\n              {'name': 'paragram', 'path': _paragram},\n              {'name': 'fasttext', 'path': _wiki_news}]","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:55:49.096102Z","iopub.execute_input":"2021-12-16T19:55:49.096532Z","iopub.status.idle":"2021-12-16T19:55:49.104561Z","shell.execute_reply.started":"2021-12-16T19:55:49.096452Z","shell.execute_reply":"2021-12-16T19:55:49.103593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nconc_embedding = None\nword_index = word_to_idx\nnb_words = min(hparam['VOCAB_SIZE'], len(word_index) + 1)\nhparam['VOCAB_SIZE'] = nb_words\nprint(hparam['VOCAB_SIZE'], len(word_index) + 1)\nprint(f\"Got a vocab size of {nb_words} number of words\")\n\nfor embedding in embeddings:\n    emb_name = embedding['name']\n    emb_path = embedding['path']\n    print(\"Running procedure on {}\".format(emb_name))\n    \n    # Загружаем эмбеддинги\n    print(\"Loading {}\".format(emb_name))\n    emb_index = load_embed(emb_path)\n    \n    # Добавляем слова в нижнем регистре\n    print(\"Adding lowercase to {}\".format(emb_name))\n    add_lower(emb_index, vocab_original)\n    \n    \n    _ = check_coverage(final_vocab, emb_index)\n    \n    emb_size = 300\n    hparam['WORD_EMB_DIM'] += emb_size\n    \n    # Конвертируем в формат word2vec\n    emb_matrix = create_emb_matrix(nb_words, emb_size)\n    print(emb_matrix.size)\n    print(emb_matrix.shape)\n    emb_matrix = fill_emb_matrix(word_index.items(), emb_matrix, emb_index)\n    \n    # Конкатенируем новые эмбеддинги с предыдущими\n    if conc_embedding is not None:\n        conc_embedding = np.concatenate((conc_embedding, emb_matrix), axis=1)\n        print(\"Concatenated! New shape: {}\".format(conc_embedding.shape))\n    else:\n        conc_embedding = emb_matrix\n    print(\"=================================================\")\n    \n    del emb_matrix, emb_index, emb_name, emb_path, emb_size\n    import gc; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-12-16T19:55:49.106415Z","iopub.execute_input":"2021-12-16T19:55:49.106832Z","iopub.status.idle":"2021-12-16T20:06:31.665446Z","shell.execute_reply.started":"2021-12-16T19:55:49.106784Z","shell.execute_reply":"2021-12-16T20:06:31.663826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def embed_word_to_int(X, vocab_to_int):\n    embedded_X = []\n    for q in X:\n        tmp_X = []\n        for w in q.split():\n            tmp_X.append(vocab_to_int[w])\n        embedded_X.append(tmp_X)\n    return embedded_X\n\n# Ставим в соответствие каждому слову уникальное целое число\nX_train = embed_word_to_int(train_data['preprocessed_text'].values, word_to_idx)\nX_test = embed_word_to_int(test_data['preprocessed_text'].values, word_to_idx)\n\npad_length = hparam['PAD_LENGTH']\n\n# Приводим данные к единому размеру\nX_train_pad = pad_sequences(X_train, maxlen=pad_length, padding='pre', truncating='pre')\nX_test_pad = pad_sequences(X_test, maxlen=pad_length, padding='pre', truncating='pre')\n\nprint(train_data['preprocessed_text'][25])\nprint(X_train[25])\nprint(X_train_pad[25])","metadata":{"execution":{"iopub.status.busy":"2021-12-16T20:06:31.667123Z","iopub.execute_input":"2021-12-16T20:06:31.667349Z","iopub.status.idle":"2021-12-16T20:06:50.032717Z","shell.execute_reply.started":"2021-12-16T20:06:31.667314Z","shell.execute_reply":"2021-12-16T20:06:50.031728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Для оценки модели используется метод кросс-валидации, метрика качества - f-мера","metadata":{}},{"cell_type":"code","source":"def train_val_pred(dataset, hparam, embedding_matrix):\n    \n    # Достаем данные\n    X_train = dataset['X_train']\n    y_train = dataset['y_train']\n    X_val = dataset['X_val']\n    y_val = dataset['y_val']\n    X_test = dataset['X_test']\n\n    # Достаем гиперпараметры\n    VOCAB_SIZE = hparam['VOCAB_SIZE']\n    PAD_LENGTH = hparam['PAD_LENGTH']\n    MINIBATCH_SIZE = hparam['MINIBATCH_SIZE']\n    LEARNING_RATE = hparam['LEARNING_RATE']\n    EPOCHS = hparam['EPOCHS']\n    LSTM_HIDDEN_SIZE = hparam['LSTM_HIDDEN_SIZE']\n    WORD_EMB_DIM = hparam['WORD_EMB_DIM']\n    \n    # Создаем модель (2-x BiLSTM)\n    inp = Input(shape=(PAD_LENGTH,))\n    x = Embedding(VOCAB_SIZE, WORD_EMB_DIM, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.3)(x)\n    x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)\n    x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)\n    max_pool1 = GlobalMaxPooling1D()(x1)\n    max_pool2 = GlobalMaxPooling1D()(x2)\n    conc = Concatenate()([max_pool1, max_pool2])\n    predictions = Dense(1, activation='sigmoid')(conc)\n    model = Model(inputs=inp, outputs=predictions)\n    adam = tf.keras.optimizers.Adam(lr=LEARNING_RATE)\n    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n    model.summary()\n    \n    # Обучаем модель\n    model.fit(X_train, y_train, epochs=EPOCHS, batch_size=MINIBATCH_SIZE, \n          validation_data = (X_val, y_val))\n    \n    \n    val_preds = model.predict(X_val, batch_size=MINIBATCH_SIZE, verbose=1)\n    best_f1 = -1\n    best_thresh = -1\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        f1 = metrics.f1_score(y_val, (val_preds > thresh).astype(int))\n        if f1 > best_f1:\n            best_f1 = f1\n            best_thresh = thresh\n    print(\"Best f1 score = {} at thresh {}\".format(best_f1, best_thresh))\n    \n    # Предсказания на тестовой выборке\n    test_preds = model.predict(X_test)\n    \n    del embedding_matrix, model, inp, x, adam\n    import gc; gc.collect()\n    \n    return test_preds, val_preds, best_thresh, best_f1","metadata":{"execution":{"iopub.status.busy":"2021-12-16T20:06:50.03388Z","iopub.status.idle":"2021-12-16T20:06:50.034643Z","shell.execute_reply.started":"2021-12-16T20:06:50.034409Z","shell.execute_reply":"2021-12-16T20:06:50.034435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Разбиваем обучающий датасет на несколько маленьких кусков для кросс-валидации.","metadata":{}},{"cell_type":"code","source":"kfold = StratifiedKFold(n_splits=hparam['KFOLDS'], shuffle=True, random_state=2019)","metadata":{"execution":{"iopub.status.busy":"2021-12-16T20:06:50.035665Z","iopub.status.idle":"2021-12-16T20:06:50.036955Z","shell.execute_reply.started":"2021-12-16T20:06:50.03666Z","shell.execute_reply":"2021-12-16T20:06:50.036692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X_train_pad\ny = train_data['target'].values\n\nresults = []\n\nfor fold, (train_index, val_index) in enumerate(kfold.split(X, y)):\n    X_train, X_val = X[train_index], X[val_index]\n    y_train, y_val = y[train_index], y[val_index]\n    \n    print(f\"Training on {len(X_train)} and validating on {len(X_val)} number of words\")\n    \n    dataset = {'X_train': X_train, 'y_train': y_train,\n          'X_val': X_val, 'y_val': y_val,\n          'X_test': X_test_pad}\n    \n    test_preds, val_preds, thresh, f1 = train_val_pred(dataset, hparam, conc_embedding)\n    \n    print(\"len(test_preds) = {}, len(val_preds) = {}, thresh = {} at f1 = {}\".format(len(test_preds), \n                                                                                     len(val_preds), \n                                                                                     thresh, \n                                                                                     f1))\n    new_result = {'name': 'fold-' + str(fold), \n                  'test_preds': test_preds, \n                  'val_preds': val_preds, \n                  'thresh': thresh, \n                  'f1': f1}\n    results.append(new_result)\n    \n    import gc; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-12-16T20:06:50.038385Z","iopub.status.idle":"2021-12-16T20:06:50.039068Z","shell.execute_reply.started":"2021-12-16T20:06:50.038803Z","shell.execute_reply":"2021-12-16T20:06:50.03883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Выводим список f-мер и трешхолды для соответствующих кусков датасета.","metadata":{}},{"cell_type":"code","source":"print(\"Got {} number of results!\".format(len(results)))\navg_thresh = 0\nfor result in results:\n    print(\"{} gave f1 score {} with thresh {}\".format(result['name'], result['f1'], result['thresh']))\n    avg_thresh += result['thresh']\n\navg_thresh = avg_thresh / len(results)\nprint(\"Got an average threshold at {}\".format(avg_thresh))","metadata":{"execution":{"iopub.status.busy":"2021-12-16T20:06:50.040364Z","iopub.status.idle":"2021-12-16T20:06:50.040864Z","shell.execute_reply.started":"2021-12-16T20:06:50.040616Z","shell.execute_reply":"2021-12-16T20:06:50.040642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Avg treshold {}\".format(avg_thresh))\n\nfactor = 1.0 / len(results)\npred_test_y = results[0]['test_preds'] * factor\n\nprint(\"Using factor: \", factor)\n\nfor i in range(1, len(results)):\n    pred_test_y += factor * results[i]['test_preds']\n    \n\npred_test_y_res = (pred_test_y > avg_thresh).astype(int)\n\nresults_dict = {'qid':test_data['qid'].values, 'prediction':[]}\n\nfor prediction in pred_test_y_res:\n    results_dict['prediction'].append(prediction[0])\n    \nprint(results_dict['qid'][:15])\nprint(results_dict['prediction'][:15])\n    \ndf = pd.DataFrame(data=results_dict)\ndf.to_csv('submission.csv', index=False)\nprint(\"Saved\")","metadata":{"execution":{"iopub.status.busy":"2021-12-16T20:06:50.042377Z","iopub.status.idle":"2021-12-16T20:06:50.04287Z","shell.execute_reply.started":"2021-12-16T20:06:50.042605Z","shell.execute_reply":"2021-12-16T20:06:50.042629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}