{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install \"tensorflow>=2\"\n!pip install \"tensorflow_hub>=0.7\"\n!pip install bert-for-tf2\n!pip install sentencepiece","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\nprint(\"TF version: \", tf.__version__)\nprint(\"Hub version: \", hub.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow_hub as hub\nimport tensorflow as tf\nimport bert\nFullTokenizer = bert.bert_tokenization.FullTokenizer\nfrom tensorflow.keras.models import Model       # Keras is the new high level API for TensorFlow\nimport math\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_seq_length = 512  # Your choice here.\ninput_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n                                       name=\"input_word_ids\")\ninput_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n                                   name=\"input_mask\")\nsegment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n                                    name=\"segment_ids\")\nbert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n                            trainable=True)\npooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n\nmodel = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=[pooled_output, sequence_output])\n\nvocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\ndo_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\ntokenizer = FullTokenizer(vocab_file, do_lower_case)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# See BERT paper: https://arxiv.org/pdf/1810.04805.pdf\n# And BERT implementation convert_single_example() at https://github.com/google-research/bert/blob/master/run_classifier.py\n\ndef get_masks(tokens, max_seq_length):\n    \"\"\"Mask for padding\"\"\"\n    if len(tokens)>max_seq_length:\n        raise IndexError(\"Token length more than max seq length!\")\n    return [1]*len(tokens) + [0] * (max_seq_length - len(tokens))\n\n\ndef get_segments(tokens, max_seq_length):\n    \"\"\"Segments: 0 for the first sequence, 1 for the second\"\"\"\n    if len(tokens)>max_seq_length:\n        raise IndexError(\"Token length more than max seq length!\")\n    segments = []\n    current_segment_id = 0\n    for token in tokens:\n        segments.append(current_segment_id)\n        if token == \"[SEP]\":\n            current_segment_id = 1\n    return segments + [0] * (max_seq_length - len(tokens))\n\n\ndef get_ids(tokens, tokenizer, max_seq_length):\n    \"\"\"Token ids from Tokenizer vocab\"\"\"\n    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n    input_ids = token_ids + [0] * (max_seq_length-len(token_ids))\n    return input_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tokenize_sentence(sentence):\n    stokens = tokenizer.tokenize(sentence)\n    stokens = [\"[CLS]\"] + stokens + [\"[SEP]\"]\n    \n    input_ids = get_ids(stokens, tokenizer, max_seq_length)\n    input_masks = get_masks(stokens, max_seq_length)\n    input_segments = get_segments(stokens, max_seq_length)\n    \n    return input_ids, input_masks, input_segments\n\ndef compare_sentences(sentence_1, sentence_2, distance_metric):\n    input_ids_1, input_masks_1, input_segments_1 = tokenize_sentence(sentence_1)\n    input_ids_2, input_masks_2, input_segments_2 = tokenize_sentence(sentence_2)\n    \n    pool_embs_1, all_embs_1 = model.predict([[input_ids_1],[input_masks_1],[input_segments_1]])\n    pool_embs_2, all_embs_2 = model.predict([[input_ids_2],[input_masks_2],[input_segments_2]])\n#     print(pool_embs_1, all_embs_1)\n#     print(pool_embs_2, all_embs_2)\n    return distance_metric(pool_embs_1[0], pool_embs_2[0])\n    \ndef square_rooted(x):\n    return math.sqrt(sum([a*a for a in x]))\n\ndef cosine_similarity(x,y):\n    numerator = sum(a*b for a,b in zip(x,y))\n    denominator = square_rooted(x)*square_rooted(y)\n    return numerator/float(denominator)\n\ndef dummy_metric(x,y):\n    return 42","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Natural Language"},{"metadata":{"trusted":true},"cell_type":"code","source":"s = []\ns.append('How are you doing?')\ns.append('How much are we feeling?')\ns.append('What are you doing?')\ns.append('What`s up?')\ns.append('Are you doing?')\ncentral = s[0]\nprint(\"Central phrase: '{}'\".format(central))\nfor sentence in s:\n    print(\"Distance to '{}' = {}\".format(sentence, round(compare_sentences(central, sentence, cosine_similarity), 3)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Source Code \"words\""},{"metadata":{"trusted":true},"cell_type":"code","source":"w = []\nw.append('data')\nw.append('Data')\nw.append('Data Set')\nw.append('datadata')\nw.append('dataset')\nw.append('DataFrame')\nw.append('dataframe')\nw.append('df')\nw.append('pd.DataFrame')\ncentral = w[0]\nprint(\"Central phrase: '{}'\".format(central))\nfor word in w:\n    print(\"Distance to '{}' = {}\".format(word, round(compare_sentences(central, word, cosine_similarity), 3)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Source Code Chunks"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c = []\ncnot = []\nc.append(\"\"\"\ndef apply_window(image, center, width):\n    image = image.copy()\n\n    min_value = center - width // 2\n    max_value = center + width // 2\n\n    image[image < min_value] = min_value\n    image[image > max_value] = max_value\n\n    return image\n\"\"\")\nc.append(\"\"\"\ndef image_windowed(image, custom_center=50, custom_width=130, out_side_val=False):\n    '''\n    Important thing to note in this function: The image migth be changed in place!\n    '''\n    # see: https://www.kaggle.com/allunia/rsna-ih-detection-eda-baseline\n    min_value = custom_center - (custom_width/2)\n    max_value = custom_center + (custom_width/2)\n    \n    # Including another value for values way outside the range, to (hopefully) make segmentation processes easier. \n    out_value_min = custom_center - custom_width\n    out_value_max = custom_center + custom_width\n    \n    if out_side_val:\n        image[np.logical_and(image < min_value, image > out_value_min)] = min_value\n        image[np.logical_and(image > max_value, image < out_value_max)] = max_value\n        image[image < out_value_min] = out_value_min\n        image[image > out_value_max] = out_value_max\n    \n    else:\n        image[image < min_value] = min_value\n        image[image > max_value] = max_value\n    \n    return image\n\"\"\")\nc.append(\"\"\"\ndef image_crop(image):\n    # Based on this stack overflow post: https://stackoverflow.com/questions/26310873/how-do-i-crop-an-image-on-a-white-background-with-python\n    mask = image == 0\n\n    # Find the bounding box of those pixels\n    coords = np.array(np.nonzero(~mask))\n    top_left = np.min(coords, axis=1)\n    bottom_right = np.max(coords, axis=1)\n\n    out = image[top_left[0]:bottom_right[0],\n                top_left[1]:bottom_right[1]]\n    \n    return out\n\"\"\")\ncnot.append(\"\"\"\ndef normalize_minmax(img):\n    mi, ma = img.min(), img.max()\n    return (img - mi) / (ma - mi)\"\"\")\ncnot.append(\"\"\"def normalize(img, means, stds, tensor=False):\n    return (img - means)/stds\"\"\")\ncnot.append(\"\"\"X_train = X_train / 255.0\ntest = test / 255.0\nmean_px = X_train.mean().astype(np.float32)\nstd_px = X_train.std().astype(np.float32)\"\"\")\ncnot.append(\"\"\"def standardize(x): \n    return (x-mean_px)/std_px\"\"\")\ncnot.append(\"\"\"def rle_decode(mask_rle, shape=(768, 768)):\n    s = mask_rle.split()\n    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n    starts -= 1\n    ends = starts + lengths\n    img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n    for lo, hi in zip(starts, ends):\n        img[lo:hi] = 1\n    return img.reshape(shape).T  # Needed to align to RLE direction\"\"\")\ncnot.append(\"\"\"def mask2rle(img):\n    pixels = img.T.flatten()\n    pixels = np.concatenate([[0], pixels, [0]])\n    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n    runs[1::2] -= runs[::2]\n    return ' '.join(str(x) for x in runs)\"\"\")\ncnot.append(\"\"\"def deskew(img):\n    m = cv2.moments(img)\n    if abs(m['mu02']) < 1e-2:\n        return img.copy()\n    skew = m['mu11']/m['mu02']\n    M = np.float32([[1, skew, -0.5*SZ*skew], [0, 1, 0]])\n    img = cv2.warpAffine(img,M,(SZ, SZ),flags=affine_flags)\n    return img\"\"\")\ncnot.append(\"\"\"image = imageio.imread('../input/image1.jpg')\nimage_rotated = rotate.augment_images([image])\nimage_noise = gaussian_noise.augment_images([image])\nimage_crop = crop.augment_images([image])\nimage_hue = hue.augment_images([image])\nimage_trans = elastic_trans.augment_images([image])\nimage_coarse = coarse_drop.augment_images([image])\"\"\")\n\ncentral = c[0]\nresults_c = []\nfor chunk in c:\n    results_c.append(compare_sentences(central, chunk, cosine_similarity))\n\nresults_cnot = []\nfor chunk in cnot:\n    results_cnot.append(compare_sentences(central, chunk, cosine_similarity))\nprint(np.mean(results_c), np.mean(results_cnot))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## NL2ML Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nl2ml = pd.read_csv('../input/nl2ml-images/nl2ml_images.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_names = nl2ml['Preprocessing class (for doc about methods)'].value_counts().head(5).keys().tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"central = nl2ml[(nl2ml['Preprocessing class (for doc about methods)'] == cat_names[0])]['Code'].reset_index(drop=True)[0]\nresults = {\"central\":cat_names[0]}\n\nfor cat in cat_names:\n    rows = nl2ml[(nl2ml['Preprocessing class (for doc about methods)'] == cat)].reset_index(drop=True)\n    chunks = rows['Code']\n    results_c = []\n    for chunk in chunks:\n        if len(chunk) <= 512:\n            results_c.append(compare_sentences(central, chunk, cosine_similarity))\n        else: continue\n    results.update({cat:round(np.mean(results_c), 3)})\n    del results_c","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}