{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**IMPORT DES MODULES**","metadata":{}},{"cell_type":"code","source":"! pip install numpy==1.16.1\n! pip install nltk==3.4\n! pip install Pillow==5.4.1","metadata":{"execution":{"iopub.status.busy":"2021-12-28T19:15:51.748391Z","iopub.execute_input":"2021-12-28T19:15:51.748774Z","iopub.status.idle":"2021-12-28T19:16:32.758084Z","shell.execute_reply.started":"2021-12-28T19:15:51.748681Z","shell.execute_reply":"2021-12-28T19:16:32.757047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nfrom keras.models import load_model\n\nimport pandas as pd\nimport numpy as np\n\nfrom PIL import Image\nfrom urllib.parse import unquote\n\nfrom sklearn.metrics.pairwise import linear_kernel\nimport requests\n\nimport os\nimport io\nfrom PIL import Image\n","metadata":{"id":"n9IK-1T5DqmC","execution":{"iopub.status.busy":"2021-12-28T19:16:32.760251Z","iopub.execute_input":"2021-12-28T19:16:32.760583Z","iopub.status.idle":"2021-12-28T19:16:39.74344Z","shell.execute_reply.started":"2021-12-28T19:16:32.760538Z","shell.execute_reply":"2021-12-28T19:16:39.742774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**DOWNLOAD TEST AND PROPOSAL**","metadata":{"id":"_o1ApTmjETTS"}},{"cell_type":"code","source":"! wget https://www.dropbox.com/s/wtffsnuwbs8mk44/test_caption_list.csv\n! wget https://www.dropbox.com/s/epmt2ja9tfhba33/test.tsv","metadata":{"id":"6R_4H3_JESva","outputId":"dd522f31-2635-4dd8-e208-e50c1431d3f3","execution":{"iopub.status.busy":"2021-12-28T19:16:39.745777Z","iopub.execute_input":"2021-12-28T19:16:39.746097Z","iopub.status.idle":"2021-12-28T19:16:45.426364Z","shell.execute_reply.started":"2021-12-28T19:16:39.746046Z","shell.execute_reply":"2021-12-28T19:16:45.425226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('test.tsv',sep='\\t')","metadata":{"id":"s_ri0ytcHmaT","execution":{"iopub.status.busy":"2021-12-28T19:16:45.429172Z","iopub.execute_input":"2021-12-28T19:16:45.429418Z","iopub.status.idle":"2021-12-28T19:16:45.576243Z","shell.execute_reply.started":"2021-12-28T19:16:45.429388Z","shell.execute_reply":"2021-12-28T19:16:45.57565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_caption_list = pd.read_csv('test_caption_list.csv')","metadata":{"id":"qFbVFsx0EnbX","execution":{"iopub.status.busy":"2021-12-28T19:16:45.577293Z","iopub.execute_input":"2021-12-28T19:16:45.578076Z","iopub.status.idle":"2021-12-28T19:16:45.758947Z","shell.execute_reply.started":"2021-12-28T19:16:45.578035Z","shell.execute_reply":"2021-12-28T19:16:45.757993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Caption = test_caption_list.caption_title_and_reference_description.values.tolist()","metadata":{"id":"kUIc4m-UISPC","execution":{"iopub.status.busy":"2021-12-28T19:16:45.760278Z","iopub.execute_input":"2021-12-28T19:16:45.760604Z","iopub.status.idle":"2021-12-28T19:16:45.772739Z","shell.execute_reply.started":"2021-12-28T19:16:45.760563Z","shell.execute_reply":"2021-12-28T19:16:45.771692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\n\n\ndef getWordFreq(corpus):\n    result = {}\n    for data in corpus:\n        for word in data.split(\" \"):\n            if word in result:\n                result[word] += 1 #adding result in the dictionary\n            else:\n                result[word] = 1\n\n    return result #returning items\n\n\nfreq = getWordFreq(Caption)","metadata":{"id":"n46f1LHcIKRn","execution":{"iopub.status.busy":"2021-12-28T19:16:45.774206Z","iopub.execute_input":"2021-12-28T19:16:45.774492Z","iopub.status.idle":"2021-12-28T19:16:46.2733Z","shell.execute_reply.started":"2021-12-28T19:16:45.774443Z","shell.execute_reply":"2021-12-28T19:16:46.272378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.iloc[:10,:]","metadata":{"id":"XKrSBA6ICcFn","execution":{"iopub.status.busy":"2021-12-28T19:16:46.543154Z","iopub.execute_input":"2021-12-28T19:16:46.543544Z","iopub.status.idle":"2021-12-28T19:16:46.549474Z","shell.execute_reply.started":"2021-12-28T19:16:46.543494Z","shell.execute_reply":"2021-12-28T19:16:46.548868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SYSTEME WORD EMBEDDING AVEC COSIM**","metadata":{"id":"d_IeDdQSzj_1"}},{"cell_type":"code","source":"import nltk\nnltk.download('punkt')\nnltk.download('stopwords')","metadata":{"id":"pY6Kt60oypvM","outputId":"187b103e-ac93-4900-c057-8896485f1094","execution":{"iopub.status.busy":"2021-12-28T19:16:46.55237Z","iopub.execute_input":"2021-12-28T19:16:46.552654Z","iopub.status.idle":"2021-12-28T19:16:47.151677Z","shell.execute_reply.started":"2021-12-28T19:16:46.552598Z","shell.execute_reply":"2021-12-28T19:16:47.150814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport scipy\nimport itertools \nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\n\ndef tokenize_cst(sent):\n  stop_words = set(stopwords.words('english'))\n \n  word_tokens = word_tokenize(sent)\n \n  filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]\n \n  filtered_sentence = []\n  \n  for w in word_tokens:\n    if w not in stop_words:\n        filtered_sentence.append(w)\n\n  return filtered_sentence    ","metadata":{"id":"uFc3mYc6tIoq","execution":{"iopub.status.busy":"2021-12-28T19:16:47.153432Z","iopub.execute_input":"2021-12-28T19:16:47.154027Z","iopub.status.idle":"2021-12-28T19:16:47.163094Z","shell.execute_reply.started":"2021-12-28T19:16:47.153982Z","shell.execute_reply":"2021-12-28T19:16:47.16156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Caption = test_caption_list.caption_title_and_reference_description.values.tolist()","metadata":{"id":"aCV6uqUgH_3s","execution":{"iopub.status.busy":"2021-12-28T19:16:47.16433Z","iopub.execute_input":"2021-12-28T19:16:47.164967Z","iopub.status.idle":"2021-12-28T19:16:47.209067Z","shell.execute_reply.started":"2021-12-28T19:16:47.164936Z","shell.execute_reply":"2021-12-28T19:16:47.208428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity","metadata":{"id":"qwmyAgyuKUfD","execution":{"iopub.status.busy":"2021-12-28T19:16:47.210943Z","iopub.execute_input":"2021-12-28T19:16:47.211535Z","iopub.status.idle":"2021-12-28T19:16:47.221261Z","shell.execute_reply.started":"2021-12-28T19:16:47.211497Z","shell.execute_reply":"2021-12-28T19:16:47.220444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nimport urllib.parse\ndef url_target_process(s):\n    when = \"\"\n    who = \"\"\n    where = \"\"\n    lcl_url = s\n    s = urllib.parse.unquote(s) \n    s = s.rsplit('/',1)[1]\n    if len(s.split('.')) > 2:\n      s = \" \".join(s.split('.')[:-1])\n    else:\n      s = s.split('.')[0]\n\n    s = unquote(s)\n    s = s.replace('_',' ')\n    s = s.replace('.',' ')\n    #doc = nlp(s)\n    list_tk = []\n\n    try:\n      if len(list_tk)> 1:\n        s = \" \".join(list_tk)\n        return s\n      else:\n        return s\n    except Exception as e:\n      print(e)\n      return s ","metadata":{"id":"0Owlf3PwH5Gy","execution":{"iopub.status.busy":"2021-12-28T19:16:47.222783Z","iopub.execute_input":"2021-12-28T19:16:47.223782Z","iopub.status.idle":"2021-12-28T19:16:51.42079Z","shell.execute_reply.started":"2021-12-28T19:16:47.223734Z","shell.execute_reply":"2021-12-28T19:16:51.419961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nimport urllib.parse\ndef vector_process(s):\n    lcl_url = s\n    s = urllib.parse.unquote(s) \n    s = s.rsplit('/',1)[1]\n    if len(s.split('.')) > 2:\n      s = \" \".join(s.split('.')[:-1])\n    else:\n      s = s.split('.')[0]\n    \n    s = unquote(s)\n    s = s.replace('_',' ')\n    s = s.replace('.',' ')\n       ","metadata":{"id":"iHgvIiOhdLXP","execution":{"iopub.status.busy":"2021-12-28T19:16:51.421958Z","iopub.execute_input":"2021-12-28T19:16:51.422166Z","iopub.status.idle":"2021-12-28T19:16:51.428992Z","shell.execute_reply.started":"2021-12-28T19:16:51.42214Z","shell.execute_reply":"2021-12-28T19:16:51.428451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(Caption))","metadata":{"id":"_X33d1yTZqkg","outputId":"85824052-9492-488e-9c24-3ed8dbae623c","execution":{"iopub.status.busy":"2021-12-28T19:16:51.429964Z","iopub.execute_input":"2021-12-28T19:16:51.430528Z","iopub.status.idle":"2021-12-28T19:16:51.441712Z","shell.execute_reply.started":"2021-12-28T19:16:51.430497Z","shell.execute_reply":"2021-12-28T19:16:51.440702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.iloc[0:3]","metadata":{"id":"pXvst5b5DpjM","execution":{"iopub.status.busy":"2021-12-28T19:16:51.442942Z","iopub.execute_input":"2021-12-28T19:16:51.443184Z","iopub.status.idle":"2021-12-28T19:16:51.452252Z","shell.execute_reply.started":"2021-12-28T19:16:51.443155Z","shell.execute_reply":"2021-12-28T19:16:51.451385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['prediction'] = test['image_url'].apply(url_target_process)","metadata":{"id":"DPhf9U0-I91q","execution":{"iopub.status.busy":"2021-12-28T19:16:51.45358Z","iopub.execute_input":"2021-12-28T19:16:51.454027Z","iopub.status.idle":"2021-12-28T19:16:51.467775Z","shell.execute_reply.started":"2021-12-28T19:16:51.453995Z","shell.execute_reply":"2021-12-28T19:16:51.466859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**TF_IDF**","metadata":{"id":"lZQJ0xLKP5ss"}},{"cell_type":"markdown","source":"**IMPORT**","metadata":{}},{"cell_type":"code","source":"import gensim\nfrom gensim import corpora\nfrom pprint import pprint\n#from gensim.utils import simple_preprocess, lemmatize\nfrom gensim import models\nimport numpy as np\nfrom sklearn.datasets import fetch_20newsgroups\nfrom gensim import corpora\nfrom gensim.parsing import strip_tags, strip_numeric, \\\n    strip_multiple_whitespaces, stem_text, strip_punctuation, \\\n    remove_stopwords, preprocess_string\nimport pprint\nimport re","metadata":{"id":"JavFpCgx6fMY","execution":{"iopub.status.busy":"2021-12-28T19:16:51.469022Z","iopub.execute_input":"2021-12-28T19:16:51.469462Z","iopub.status.idle":"2021-12-28T19:16:51.606327Z","shell.execute_reply.started":"2021-12-28T19:16:51.46943Z","shell.execute_reply":"2021-12-28T19:16:51.605396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STRING CLEANING PIPELINE**","metadata":{}},{"cell_type":"code","source":"# preprocess using gensim.parsing\n# ref: https://www.kaggle.com/venkatkrishnan/gensim-text-mining-techniques\ntransform_to_lower = lambda s: s.lower()\n\nremove_single_char = lambda s: re.sub(r'\\s+\\w{1}\\s+', '', s)\n\n# Filters to be executed in pipeline\nCLEAN_FILTERS = [strip_tags,\n                strip_numeric,\n                strip_punctuation, \n                strip_multiple_whitespaces, \n                transform_to_lower,\n                remove_stopwords,\n                remove_single_char]\n\n# Method does the filtering of all the unrelevant text elements\ndef cleaning_pipe(document):\n    # Invoking gensim.parsing.preprocess_string method with set of filters\n    processed_words = preprocess_string(document, CLEAN_FILTERS)\n    \n    return processed_words\nprint(cleaning_pipe(Caption[0]))","metadata":{"id":"SmVnt38W6Ucp","outputId":"cdab4058-f628-411f-f0f8-93fdf1cfaf48","execution":{"iopub.status.busy":"2021-12-28T19:16:51.607585Z","iopub.execute_input":"2021-12-28T19:16:51.607991Z","iopub.status.idle":"2021-12-28T19:16:51.616339Z","shell.execute_reply.started":"2021-12-28T19:16:51.607949Z","shell.execute_reply":"2021-12-28T19:16:51.615566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CREATE CORPUS DICTIONNARY**","metadata":{}},{"cell_type":"code","source":"def create_dictionary(docs):\n    'create dictionary of words in preprocessed corpus'\n    pdocs = [cleaning_pipe(doc) for doc in docs]\n    dictionary = corpora.Dictionary(pdocs)\n    dictionary.save('newsgroup.dict')\n    return dictionary,pdocs","metadata":{"id":"UKYAeen7_DvM","execution":{"iopub.status.busy":"2021-12-28T19:16:51.617352Z","iopub.execute_input":"2021-12-28T19:16:51.617929Z","iopub.status.idle":"2021-12-28T19:16:51.631182Z","shell.execute_reply.started":"2021-12-28T19:16:51.617893Z","shell.execute_reply":"2021-12-28T19:16:51.630094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dictionary, pdocs = create_dictionary(Caption)","metadata":{"id":"7B7Sl0j3_G-s","execution":{"iopub.status.busy":"2021-12-28T19:16:51.632691Z","iopub.execute_input":"2021-12-28T19:16:51.633022Z","iopub.status.idle":"2021-12-28T19:16:59.120881Z","shell.execute_reply.started":"2021-12-28T19:16:51.632988Z","shell.execute_reply":"2021-12-28T19:16:59.12005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bow_corpus = [dictionary.doc2bow(text) for text in pdocs]","metadata":{"id":"Wvksv7en_L6C","execution":{"iopub.status.busy":"2021-12-28T19:16:59.121946Z","iopub.execute_input":"2021-12-28T19:16:59.122162Z","iopub.status.idle":"2021-12-28T19:17:00.295747Z","shell.execute_reply.started":"2021-12-28T19:16:59.122136Z","shell.execute_reply":"2021-12-28T19:17:00.294927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **TRAINING TF-IDF**","metadata":{}},{"cell_type":"code","source":"from gensim import models\n# train the model\ntfidf = models.TfidfModel(bow_corpus)","metadata":{"id":"tXLRXu0e_U0Y","execution":{"iopub.status.busy":"2021-12-28T19:17:00.297323Z","iopub.execute_input":"2021-12-28T19:17:00.297738Z","iopub.status.idle":"2021-12-28T19:17:01.44069Z","shell.execute_reply.started":"2021-12-28T19:17:00.297693Z","shell.execute_reply":"2021-12-28T19:17:01.439804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# index the tfidf vector of corpus as sparse matrix\nfrom gensim import similarities\nindex = similarities.SparseMatrixSimilarity(tfidf[bow_corpus], num_features=len(dictionary))","metadata":{"id":"v1ze1U6o6y5S","execution":{"iopub.status.busy":"2021-12-28T19:17:01.442013Z","iopub.execute_input":"2021-12-28T19:17:01.442381Z","iopub.status.idle":"2021-12-28T19:17:10.185556Z","shell.execute_reply.started":"2021-12-28T19:17:01.442336Z","shell.execute_reply":"2021-12-28T19:17:10.184671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **INFERENCE FUNCTION**","metadata":{}},{"cell_type":"code","source":"def get_closest_n(query, n):\n    '''get the top matching docs as per cosine similarity\n    between tfidf vector of query and all docs'''\n    query_document = cleaning_pipe(query)\n    query_bow = dictionary.doc2bow(query_document)\n    sims = index[tfidf[query_bow]]\n    top_idx = sims.argsort()[-1*n:][::-1]\n    return [Caption[i] for i in top_idx]","metadata":{"id":"vU46Qc-J77xe","execution":{"iopub.status.busy":"2021-12-28T19:17:10.186678Z","iopub.execute_input":"2021-12-28T19:17:10.186898Z","iopub.status.idle":"2021-12-28T19:17:10.192766Z","shell.execute_reply.started":"2021-12-28T19:17:10.186871Z","shell.execute_reply":"2021-12-28T19:17:10.192111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for d in get_closest_n(\"Map of New York highlighting Sullivan County\",5):\n    print(d)","metadata":{"id":"LGO0JlOY_kYo","outputId":"a06f59cb-e70d-477c-ce76-48a10e0acf6b","execution":{"iopub.status.busy":"2021-12-28T19:17:10.193926Z","iopub.execute_input":"2021-12-28T19:17:10.194271Z","iopub.status.idle":"2021-12-28T19:17:10.218815Z","shell.execute_reply.started":"2021-12-28T19:17:10.194232Z","shell.execute_reply":"2021-12-28T19:17:10.218146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **PREDICT CAPTION**","metadata":{}},{"cell_type":"code","source":"def predict_caption_tf_idf(m):\n    res_score = []\n    for d in get_closest_n(m,5):\n      #if cos_sim(d, m) > 0.2: \n      res_score.append(d)\n    #if len(res_score) == 0:\n    #    res_score = process.extract(m, Caption, scorer=fuzz.token_set_ratio, processor=None, limit=5)\n    return res_score","metadata":{"id":"qASahncusQ2J","execution":{"iopub.status.busy":"2021-12-28T19:21:09.105494Z","iopub.execute_input":"2021-12-28T19:21:09.106011Z","iopub.status.idle":"2021-12-28T19:21:09.111176Z","shell.execute_reply.started":"2021-12-28T19:21:09.105978Z","shell.execute_reply":"2021-12-28T19:21:09.109819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"id":"BHauQg_7ELPY","execution":{"iopub.status.busy":"2021-12-28T19:21:09.113179Z","iopub.execute_input":"2021-12-28T19:21:09.11343Z","iopub.status.idle":"2021-12-28T19:21:09.132379Z","shell.execute_reply.started":"2021-12-28T19:21:09.113402Z","shell.execute_reply":"2021-12-28T19:21:09.131182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()","metadata":{"id":"gccqzA1eEkCn","execution":{"iopub.status.busy":"2021-12-28T19:21:28.074787Z","iopub.execute_input":"2021-12-28T19:21:28.075516Z","iopub.status.idle":"2021-12-28T19:21:28.081281Z","shell.execute_reply.started":"2021-12-28T19:21:28.075463Z","shell.execute_reply":"2021-12-28T19:21:28.080359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **APPLY ON DATASET**","metadata":{}},{"cell_type":"code","source":"test['caption_title_and_reference_description'] = test['prediction'].progress_apply(predict_caption_tf_idf)","metadata":{"id":"cxOayzbbIQN6","outputId":"456149cf-5219-4a90-98d6-f73826140c6e","execution":{"iopub.status.busy":"2021-12-28T19:21:28.083155Z","iopub.execute_input":"2021-12-28T19:21:28.083713Z","iopub.status.idle":"2021-12-28T19:21:28.129131Z","shell.execute_reply.started":"2021-12-28T19:21:28.083676Z","shell.execute_reply":"2021-12-28T19:21:28.128305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(100)","metadata":{"id":"AygRf6ZJYkan","execution":{"iopub.status.busy":"2021-12-28T19:21:28.130581Z","iopub.execute_input":"2021-12-28T19:21:28.131086Z","iopub.status.idle":"2021-12-28T19:21:28.150091Z","shell.execute_reply.started":"2021-12-28T19:21:28.131043Z","shell.execute_reply":"2021-12-28T19:21:28.149214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = test[['id','caption_title_and_reference_description']]\nsub = sub.explode('caption_title_and_reference_description')\nsub.head(1000)","metadata":{"id":"EaVy8zeRIVpA","outputId":"167510f2-4784-46f0-b241-3ab35e9f9055","execution":{"iopub.status.busy":"2021-12-28T19:21:28.15209Z","iopub.execute_input":"2021-12-28T19:21:28.152491Z","iopub.status.idle":"2021-12-28T19:21:28.184814Z","shell.execute_reply.started":"2021-12-28T19:21:28.152448Z","shell.execute_reply":"2021-12-28T19:21:28.184125Z"},"trusted":true},"execution_count":null,"outputs":[]}]}