{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Competition\n\nIn this competition, you’ll build a model that automatically retrieves the text closest to an image. Specifically, you'll train your model to associate given images with article titles or complex captions, in multiple languages. The best models will account for the semantic granularity of Wikipedia images.\n\nThe objective is to predict the target *'caption_title_and_reference_description'* given information about an images\n# Evaluation\n\nSubmissions will be evaluated using NDCG@5 (Normalized Discounted Cumulative Gain).","metadata":{}},{"cell_type":"markdown","source":"This Notebook uses external dataset [feather format](https://www.kaggle.com/msafi04/train-tsv-file-to-feather-files) created from the train tsv files","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport dask\nimport dask.dataframe as dd\n\nimport cv2\nfrom PIL import Image\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom plotly.offline import iplot\n\nimport cufflinks as cf\ncf.go_offline()\ncf.set_config_file(offline = False, world_readable = True)\n\nplt.rcParams[\"figure.figsize\"] = (12, 8)\nplt.rcParams['axes.titlesize'] = 16\nplt.style.use('seaborn-whitegrid')\nsns.set_palette('Set2')\n\nimport glob as glob\nimport gc\n\nimport requests\nimport urllib\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom time import time, strftime, gmtime\nstart = time()\nimport datetime\nprint(str(datetime.datetime.now()))\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-16T04:17:34.317298Z","iopub.execute_input":"2021-09-16T04:17:34.317657Z","iopub.status.idle":"2021-09-16T04:17:38.313703Z","shell.execute_reply.started":"2021-09-16T04:17:34.317568Z","shell.execute_reply":"2021-09-16T04:17:38.312527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use dask to read the dataframes, its fast!","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"def tsv_to_feather(tsv_files):\n    for i, tsv in enumerate(tsv_files):\n        print(f\"Processingc file: {i + 1} ...\")\n        df = dd.read_csv(tsv, sep = '\\t', quoting = 3, escapechar = '\\n', \n                         on_bad_lines = 'skip', dtype = 'string')\n        df = df.dropna()\n        df = df.compute() #Convert to pandas df\n        df = df.reset_index(drop = True)\n        #Save as feature to save disk space and read faster\n        df.to_feather(tsv.split('/')[-1].split('.')[0])\n        print(f\"Tsv file {tsv.split('/')[-1].split('.')[0]} stored as feather file\")\n\n        del df\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:38.316456Z","iopub.execute_input":"2021-09-16T04:17:38.317231Z","iopub.status.idle":"2021-09-16T04:17:38.326115Z","shell.execute_reply.started":"2021-09-16T04:17:38.317177Z","shell.execute_reply":"2021-09-16T04:17:38.325182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_tsvs = glob.glob('/kaggle/input/wikipedia-image-caption/train*.tsv')\n#tsv_to_feather(train_tsvs)","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:38.327432Z","iopub.execute_input":"2021-09-16T04:17:38.327647Z","iopub.status.idle":"2021-09-16T04:17:38.342662Z","shell.execute_reply.started":"2021-09-16T04:17:38.327624Z","shell.execute_reply":"2021-09-16T04:17:38.342058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_external = '/kaggle/input/train-tsv-file-to-feather-files/'","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:38.344277Z","iopub.execute_input":"2021-09-16T04:17:38.345008Z","iopub.status.idle":"2021-09-16T04:17:38.350936Z","shell.execute_reply.started":"2021-09-16T04:17:38.344973Z","shell.execute_reply":"2021-09-16T04:17:38.350271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/wikipedia-image-caption/test.tsv', sep = '\\t')\nprint(test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:38.351872Z","iopub.execute_input":"2021-09-16T04:17:38.352752Z","iopub.status.idle":"2021-09-16T04:17:38.638399Z","shell.execute_reply.started":"2021-09-16T04:17:38.352723Z","shell.execute_reply":"2021-09-16T04:17:38.637461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/wikipedia-image-caption/sample_submission.csv')\nprint(sub.shape)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:38.639834Z","iopub.execute_input":"2021-09-16T04:17:38.640155Z","iopub.status.idle":"2021-09-16T04:17:39.247872Z","shell.execute_reply.started":"2021-09-16T04:17:38.640117Z","shell.execute_reply":"2021-09-16T04:17:39.246961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_feathers = glob.glob(train_external + 'train*')\nprint(train_feathers)","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:39.249030Z","iopub.execute_input":"2021-09-16T04:17:39.249244Z","iopub.status.idle":"2021-09-16T04:17:39.254061Z","shell.execute_reply.started":"2021-09-16T04:17:39.249221Z","shell.execute_reply":"2021-09-16T04:17:39.253442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Read all the feather files and make it into one dataframe","metadata":{}},{"cell_type":"code","source":"train_df = pd.DataFrame()\nfor file in train_feathers:\n    df = pd.read_feather(file)\n    train_df = pd.concat([train_df, df])\nprint(f\"Before removing duplicate rows: {train_df.shape}\")\ntrain_df = train_df.drop_duplicates() #Drop duplicate rows if any\nprint(f\"After removing duplicate rows: {train_df.shape}\")\ntrain_df = train_df.sample(frac = 1).reset_index(drop = True)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:39.255210Z","iopub.execute_input":"2021-09-16T04:17:39.255456Z","iopub.status.idle":"2021-09-16T04:17:40.390783Z","shell.execute_reply.started":"2021-09-16T04:17:39.255428Z","shell.execute_reply":"2021-09-16T04:17:40.389962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"__Missing Values__","metadata":{"execution":{"iopub.status.busy":"2021-09-15T06:32:11.749555Z","iopub.execute_input":"2021-09-15T06:32:11.751083Z","iopub.status.idle":"2021-09-15T06:32:11.757014Z","shell.execute_reply.started":"2021-09-15T06:32:11.751035Z","shell.execute_reply":"2021-09-15T06:32:11.755604Z"}}},{"cell_type":"code","source":"train_df.isna().any()","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:40.392240Z","iopub.execute_input":"2021-09-16T04:17:40.392580Z","iopub.status.idle":"2021-09-16T04:17:40.436430Z","shell.execute_reply.started":"2021-09-16T04:17:40.392540Z","shell.execute_reply":"2021-09-16T04:17:40.435472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['language'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:40.437679Z","iopub.execute_input":"2021-09-16T04:17:40.437932Z","iopub.status.idle":"2021-09-16T04:17:40.447295Z","shell.execute_reply.started":"2021-09-16T04:17:40.437897Z","shell.execute_reply":"2021-09-16T04:17:40.446297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['language'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:40.450319Z","iopub.execute_input":"2021-09-16T04:17:40.450557Z","iopub.status.idle":"2021-09-16T04:17:40.467513Z","shell.execute_reply.started":"2021-09-16T04:17:40.450527Z","shell.execute_reply":"2021-09-16T04:17:40.466386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Image and Caption Visualization","metadata":{}},{"cell_type":"code","source":"def url_to_images(df: pd.DataFrame, num_images: int, flag: str) -> tuple:\n    images_to_display = []\n    #Get random 'num_images' url from df\n    if flag == 'train':\n        sample_df = df[['image_url', 'caption_title_and_reference_description']].sample(num_images)\n    else:\n        sample_df = df[['image_url']].sample(num_images)\n    URLS = sample_df['image_url'].tolist()\n    for img_url in URLS:\n        try:\n            with urllib.request.urlopen(img_url) as url:\n                with open('./temp.jpg', 'wb') as f:\n                    f.write(url.read())\n            img = cv2.imread('./temp.jpg')\n            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n            images_to_display.append(img)\n        except:\n            continue\n    if flag == 'train':\n        return images_to_display, sample_df['caption_title_and_reference_description'].tolist()\n    else:\n        return images_to_display","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-09-16T04:17:40.468953Z","iopub.execute_input":"2021-09-16T04:17:40.469202Z","iopub.status.idle":"2021-09-16T04:17:40.480461Z","shell.execute_reply.started":"2021-09-16T04:17:40.469176Z","shell.execute_reply":"2021-09-16T04:17:40.479550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from textwrap import wrap\n\ndef display_images(df: pd.DataFrame, rows: int, cols: int, flag: str = 'train') -> None:\n    if flag == 'train':\n        images, captions = url_to_images(df, num_images = rows * cols, flag = flag)\n    else:\n        images = url_to_images(df, num_images = rows * cols, flag = flag)\n    \n    fig, ax = plt.subplots(rows, cols, figsize = (20, 12))\n    ax = ax.flatten()\n    for p in range(rows * cols):\n        try:\n            ax[p].imshow(images[p])\n            ax[p].grid(False)\n            ax[p].axis('off')\n        except:\n            continue\n        if flag == 'train':\n            ax[p].set_title('\\n'.join(wrap(captions[p], 30)))\n    fig.tight_layout()\n    plt.show()\n    return None","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:40.481863Z","iopub.execute_input":"2021-09-16T04:17:40.482101Z","iopub.status.idle":"2021-09-16T04:17:40.494103Z","shell.execute_reply.started":"2021-09-16T04:17:40.482077Z","shell.execute_reply":"2021-09-16T04:17:40.493273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_images(train_df, 3, 3, 'train')","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:17:40.495295Z","iopub.execute_input":"2021-09-16T04:17:40.495716Z","iopub.status.idle":"2021-09-16T04:18:01.581044Z","shell.execute_reply.started":"2021-09-16T04:17:40.495686Z","shell.execute_reply":"2021-09-16T04:18:01.576578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_images(train_df, 3, 3, 'train')","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:18:01.582732Z","iopub.execute_input":"2021-09-16T04:18:01.583129Z","iopub.status.idle":"2021-09-16T04:18:20.923854Z","shell.execute_reply.started":"2021-09-16T04:18:01.583088Z","shell.execute_reply":"2021-09-16T04:18:20.920027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Display Test Images","metadata":{}},{"cell_type":"markdown","source":"- Looks like there are GIF, SVG, TIF, PNG formats as well in addition to JPG","metadata":{}},{"cell_type":"code","source":"display_images(test_df, 3, 3, 'test')","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:18:20.924980Z","iopub.execute_input":"2021-09-16T04:18:20.925774Z","iopub.status.idle":"2021-09-16T04:18:44.163393Z","shell.execute_reply.started":"2021-09-16T04:18:20.925727Z","shell.execute_reply":"2021-09-16T04:18:44.162666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_format(x):\n    if (x.lower() == 'jpg') or (x.lower() == 'jpeg'):\n        return 'jpg'\n    elif (x.lower() == 'png'):\n        return 'png'\n    elif (x.lower() == 'tif') or (x.lower() == 'tiff'):\n        return 'tif'\n    elif (x.lower() == 'svg'):\n        return 'svg'\n    elif (x.lower() == 'gif'):\n        return 'gif'","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:28:07.472440Z","iopub.execute_input":"2021-09-16T04:28:07.472899Z","iopub.status.idle":"2021-09-16T04:28:07.481229Z","shell.execute_reply.started":"2021-09-16T04:28:07.472855Z","shell.execute_reply":"2021-09-16T04:28:07.480029Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['format'] = test_df['image_url'].apply(lambda x: x.split('.')[-1])\ntest_df['format'] = test_df['format'].apply(clean_format)\nax = sns.countplot(test_df['format'])\nfor p in ax.patches:\n    ax.annotate(str(p.get_height()), (p.get_x() * 1.005, p.get_height() * 1.005))\nplt.title('Test Image Formats Counts');","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:30:23.946621Z","iopub.execute_input":"2021-09-16T04:30:23.946912Z","iopub.status.idle":"2021-09-16T04:30:24.295849Z","shell.execute_reply.started":"2021-09-16T04:30:23.946874Z","shell.execute_reply":"2021-09-16T04:30:24.294976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target Text","metadata":{}},{"cell_type":"code","source":"train_df['target_len'] = train_df['caption_title_and_reference_description'].apply(lambda x: len(x.split()))\ntrain_df['target_len'].hist();","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:18:44.164469Z","iopub.execute_input":"2021-09-16T04:18:44.164889Z","iopub.status.idle":"2021-09-16T04:18:44.388647Z","shell.execute_reply.started":"2021-09-16T04:18:44.164846Z","shell.execute_reply":"2021-09-16T04:18:44.387796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Max no. of words in target: {train_df['target_len'].max()}\")\nprint(f\"Min no. of words in target: {train_df['target_len'].min()}\")\nprint(f\"Avg. no. of words in target: {train_df['target_len'].mean()}\")","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:18:44.389877Z","iopub.execute_input":"2021-09-16T04:18:44.390110Z","iopub.status.idle":"2021-09-16T04:18:44.396339Z","shell.execute_reply.started":"2021-09-16T04:18:44.390084Z","shell.execute_reply":"2021-09-16T04:18:44.395381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is a sample with no target caption, remove this from train dataset","metadata":{}},{"cell_type":"code","source":"train_df = train_df[train_df['target_len'] != 0]\nprint(f\"Max no. of words in target: {train_df['target_len'].max()}\")\nprint(f\"Min no. of words in target: {train_df['target_len'].min()}\")\nprint(f\"Avg. no. of words in target: {train_df['target_len'].mean()}\")\ntrain_df.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-16T04:18:44.398166Z","iopub.execute_input":"2021-09-16T04:18:44.398487Z","iopub.status.idle":"2021-09-16T04:18:44.434964Z","shell.execute_reply.started":"2021-09-16T04:18:44.398449Z","shell.execute_reply":"2021-09-16T04:18:44.433860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WIP\nMore to come...","metadata":{}}]}