{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-03T09:40:38.476472Z","iopub.execute_input":"2021-10-03T09:40:38.47704Z","iopub.status.idle":"2021-10-03T09:40:38.591267Z","shell.execute_reply.started":"2021-10-03T09:40:38.476942Z","shell.execute_reply":"2021-10-03T09:40:38.59057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rapidfuzz -qq","metadata":{"execution":{"iopub.status.busy":"2021-10-03T09:40:38.594002Z","iopub.execute_input":"2021-10-03T09:40:38.594192Z","iopub.status.idle":"2021-10-03T09:40:48.385842Z","shell.execute_reply.started":"2021-10-03T09:40:38.59417Z","shell.execute_reply":"2021-10-03T09:40:48.3849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport base64 \nfrom PIL import Image\nimport io\nfrom rapidfuzz import process, fuzz\nfrom tqdm import tqdm\nimport urllib","metadata":{"execution":{"iopub.status.busy":"2021-10-03T09:40:48.387642Z","iopub.execute_input":"2021-10-03T09:40:48.387937Z","iopub.status.idle":"2021-10-03T09:40:48.403228Z","shell.execute_reply.started":"2021-10-03T09:40:48.387901Z","shell.execute_reply":"2021-10-03T09:40:48.402447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Languages\nHistograms of languages in each train tsv file\n> \nTop 10 languages:\n1. en\n2. de\n3. fr\n4. en\n5. ru\n6. it\n7. nl\n8. pl\n9. ja\n10. uk","metadata":{}},{"cell_type":"code","source":"'''\nfor i in range(5):\n    filename = f'/kaggle/input/wikipedia-image-caption/train-0000{i}-of-00005.tsv'\n    df = pd.read_csv(filename, sep='\\t', usecols=['language'])\n    plt.figure(figsize=[20,3])\n    df[\"language\"].value_counts().plot(kind=\"bar\")\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preditions only from image URLs\n- extract texts from the images urls\n- find captions that matche the extracted texts.\n\n## Updates (Sep 23)\n- use `urllib.parse.unquote` to decode percent-encoded sequences into Unicode characters\n- use `fuzz.token_set_ratio` as a scorer","metadata":{}},{"cell_type":"code","source":"'''\ndf_test = pd.read_csv('/kaggle/input/wikipedia-image-caption/test.tsv', sep='\\t')\ndf_test['txt_from_url'] = df_test['image_url'].apply(lambda x: x.split('/')[-1][:-4].replace('_', ' '))\ndf_captions = pd.read_csv('../input/wikipedia-image-caption/test_caption_list.csv')\ncaptions = df_captions['caption_title_and_reference_description'].tolist()\n\npreds_list = []\nfor ID, txt in tqdm(df_test[['id', 'txt_from_url']].values):\n    result = process.extract(txt, captions, scorer=fuzz.ratio, processor=N, limit=50)\n    for line in result:\n        preds_list.append([ID, line[0]])\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/wikipedia-image-caption/test.tsv', sep='\\t')\ndf_test['txt_from_url'] = df_test['image_url'].apply(lambda x: urllib.parse.unquote(x).split('/')[-1][:-4].replace('_', ' '))\ndf_captions = pd.read_csv('../input/wikipedia-image-caption/test_caption_list.csv')\ncaptions = df_captions['caption_title_and_reference_description'].tolist()","metadata":{"execution":{"iopub.status.busy":"2021-10-03T09:41:23.653414Z","iopub.execute_input":"2021-10-03T09:41:23.654072Z","iopub.status.idle":"2021-10-03T09:41:24.858657Z","shell.execute_reply.started":"2021-10-03T09:41:23.654023Z","shell.execute_reply":"2021-10-03T09:41:24.857869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_list = []\nfor ID, txt in tqdm(df_test[['id', 'txt_from_url']].values):\n    result = process.extract(txt, captions, scorer=fuzz.token_set_ratio, processor=None, limit=250)\n    for line in result:\n        preds_list.append([ID, line[0]])","metadata":{"execution":{"iopub.status.busy":"2021-10-03T09:42:09.669512Z","iopub.execute_input":"2021-10-03T09:42:09.669812Z","iopub.status.idle":"2021-10-03T09:42:16.962133Z","shell.execute_reply.started":"2021-10-03T09:42:09.669762Z","shell.execute_reply":"2021-10-03T09:42:16.960992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.DataFrame(preds_list, columns=['id', 'caption_title_and_reference_description'])\nsub.to_csv('submission.csv', index=None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}