{"cells":[{"metadata":{"_uuid":"8d984a97b8b1f9bf9b89300bca44ea5ba74fdbbe"},"cell_type":"markdown","source":"Here we take [the top scoring model from the Mercari competition](https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s) and attempt to apply it to judging insincere Quora questions. Will the state of the art there work here?"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print('Starting')\nimport os; os.environ['OMP_NUM_THREADS'] = '1'\nfrom contextlib import contextmanager\nfrom functools import partial\nfrom operator import itemgetter\nfrom multiprocessing.pool import ThreadPool\nimport time\nfrom typing import List, Dict\n\nimport keras as ks\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer as Tfidf\nfrom sklearn.pipeline import make_pipeline, make_union, Pipeline\nfrom sklearn.preprocessing import FunctionTransformer, StandardScaler\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.model_selection import KFold\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n\ndef on_field(f: str, *vec) -> Pipeline:\n    return make_pipeline(FunctionTransformer(itemgetter(f), validate=False), *vec)\n\ndef to_records(df: pd.DataFrame) -> List[Dict]:\n    return df.to_dict(orient='records')\n\ndef fit_predict(xs, y_train) -> np.ndarray:\n    X_train, X_valid, X_test = xs\n    config = tf.ConfigProto(\n        intra_op_parallelism_threads=1, use_per_session_threads=1, inter_op_parallelism_threads=1)\n    with tf.Session(graph=tf.Graph(), config=config) as sess, timer('fit_predict'):\n        ks.backend.set_session(sess)\n        model_in = ks.Input(shape=(X_train.shape[1],), dtype='float32', sparse=True)\n        out = ks.layers.Dense(192, activation='relu')(model_in)\n        out = ks.layers.Dense(64, activation='relu')(out)\n        out = ks.layers.Dense(64, activation='relu')(out)\n        out = ks.layers.Dense(1, activation=\"sigmoid\")(out)\n        model = ks.Model(model_in, out)\n        model.compile(loss='binary_crossentropy', optimizer=ks.optimizers.Adam(lr=3e-3), metrics=['accuracy'])\n        for i in range(3):\n            with timer(f'epoch {i + 1}'):\n                model.fit(x=X_train, y=y_train, batch_size=2**(11 + i), epochs=1, verbose=0)\n        with timer('predict'):\n            return model.predict(X_valid)[:, 0], model.predict(X_test)[:, 0]\n\nvectorizer = make_union(\n    on_field('question_text', Tfidf(max_features=100000, token_pattern='\\w+', ngram_range=(1, 2))),\n    n_jobs=4)\nwith timer('process train'):\n    train = pd.read_csv(\"../input/train.csv\")\n    cv = KFold(n_splits=20, shuffle=True, random_state=42)\n    train_ids, valid_ids = next(cv.split(train))\n    train, valid = train.iloc[train_ids], train.iloc[valid_ids]\n    y_train = train['target']\n    X_train = vectorizer.fit_transform(train).astype(np.float32)\n    print(f'X_train: {X_train.shape} of {X_train.dtype}')\n    del train\nwith timer('process valid'):\n    X_valid = vectorizer.transform(valid).astype(np.float32)\n    print(f'X_valid: {X_valid.shape} of {X_valid.dtype}')\nwith timer('process test'):\n    test = pd.read_csv(\"../input/test.csv\")\n    X_test = vectorizer.transform(test).astype(np.float32)\n    print(f'X_test: {X_test.shape} of {X_test.dtype}')\nwith ThreadPool(processes=4) as pool:\n    Xb_train, Xb_valid, Xb_test = [x.astype(np.bool).astype(np.float32) for x in [X_train, X_valid, X_test]]\n    xs = [[Xb_train, Xb_valid, Xb_test], [X_train, X_valid, X_test]] * 2\n    preds = pool.map(partial(fit_predict, y_train=y_train), xs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6a9680f328118ca5734a606e214bd86c955e5b6"},"cell_type":"code","source":"val_preds = [p[0] for p in preds]\nval_pred_average = np.mean(val_preds, axis=0)\ntest_preds = [p[1] for p in preds]\ntest_pred_average = np.mean(test_preds, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"37825b5a8946ce4c48576010b93064f605d8abf1"},"cell_type":"markdown","source":"Now let's find the best threshold for each model and the average."},{"metadata":{"trusted":true,"_uuid":"04325526a1a613910b2a9e145105a676b3caa83f"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ny_val = valid['target']\nfor i, pred in enumerate(val_preds + [val_pred_average]):\n    print('-')\n    if i == 4:\n        print('Ensemble')\n    else:\n        print('Model {}'.format(i))\n    pred = np.array(pred)\n    best_threshold = 0.01\n    best_score = 0.0\n    for threshold in range(1, 100):\n        threshold = threshold / 100\n        score = f1_score(y_val, pred > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print(\"Score at threshold=0.5 is {}\".format(f1_score(y_val, pred > 0.5)))\n    print(\"Optimal threshold is {} with a score of {}\".format(best_threshold, best_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd5e87e488c8839cc56725a4581ef566fdd83dad"},"cell_type":"code","source":"y_te = (np.array(test_pred_average) > best_threshold).astype(np.int)\nsubmit_df = pd.DataFrame({\"qid\": test[\"qid\"], \"prediction\": y_te})\nsubmit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b63a1ff2408aaf913ea3e001b018951eb9bb5fd"},"cell_type":"code","source":"submit_df['prediction'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e58077d8812f879fc250f0f7752778b1197f473b"},"cell_type":"code","source":"submit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}