{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "05a4dccb-a7eb-bd06-c172-d2512b411cab"
      },
      "source": [
        "# Summary\n",
        "\n",
        "I compute hashsum (MD5 and SHA1) of 7z files and missing data jpeg files on my local NAS, Colfax Cluster and kaggle's kernel.  \n",
        "\n",
        "All of hashsum are coincident (Kaggle's kernel doesn't have 7z files, only have extracted jpeg files.).  \n",
        "Their dataset can be regarded as downloaded and extracted independently.  \n",
        "\n",
        "So, I think their hashsum are correct, and their images are certainly broken.   \n",
        "I'm worried about how to handle their files...\n",
        "\n",
        "Note:  [Missing Data jpeg files](https://www.kaggle.com/c/intel-mobileodt-cervical-cancer-screening/discussion/30337)\n",
        "\n",
        "* 0 byte files  \n",
        "additional/Type_2/2845.jpg  \n",
        "additional/Type_2/5892.jpg  \n",
        "additional/Type_2/5893.jpg  \n",
        "\n",
        "* Premature end of JPEG file  \n",
        "train/Type_1/1339.jpg (missing about 45% data)  \n",
        "additional/Type_1/3068.jpg (missing about 22% data)  \n",
        "additional/Type_2/7.jpg (missing about 25% data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "00ce58bf-7bd7-aeae-e443-c29a0f616a8e"
      },
      "outputs": [],
      "source": [
        "import pandas\n",
        "\n",
        "\n",
        "list_compute    = ['My NAS', 'Colfax Cluster', \"Kaggle's kernel\"]\n",
        "list_files      = 'md5sum(train.7z)', 'md5sum(test.7z)', 'md5sum(additional.7z)', 'md5sum(additional/Type_2/2845.jpg)', 'md5sum(additional/Type_2/5892.jpg)', 'md5sum(additional/Type_2/5893.jpg)', 'md5sum(train/Type_1/1339.jpg)', 'md5sum(additional/Type_1/3068.jpg)', 'md5sum(additional/Type_2/7.jpg)', 'sha1sum(train.7z)', 'sha1sum(test.7z)', 'sha1sum(additional.7z)', 'sha1sum(additional/Type_2/2845.jpg)', 'sha1sum(additional/Type_2/5892.jpg)', 'sha1sum(additional/Type_2/5893.jpg)', 'sha1sum(train/Type_1/1339.jpg)', 'sha1sum(additional/Type_1/3068.jpg)', 'sha1sum(additional/Type_2/7.jpg)'\n",
        "hash_on_my_nas  = ['4b3fd8c73b6ac21b1e106d5efd8713cd', 'ae9efe79d4efa87c0bf923d86fa5c10f', '3e914c7fe2dcc8a10584b9b7413acec4', 'd41d8cd98f00b204e9800998ecf8427e', 'd41d8cd98f00b204e9800998ecf8427e', 'd41d8cd98f00b204e9800998ecf8427e', 'cc2c4af9200f0e03ccf00f647ceb2adc', '0b8cfe3f0d6256532dade1c1216024d8', '715ae5f8509d6d5eddd98364663ad656', '140233b2fd2b42a68c80d8cf2719392d9ff39d1c', 'a4f1fca9acd5a1a01d6ae4c5fd6c7ca2c2f85f6d', 'cf0066cf2fccf5086c4a540083961b2bf47d5479', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', '239cf581a491df7c9890103efc40a85b6daf82c1', 'd375c65833730d51c3748d64aecee096ff2bdbd5', '75c1d81de299d477e9aa88a41474bc74a3ac447e']\n",
        "hash_on_colfax  = ['4b3fd8c73b6ac21b1e106d5efd8713cd', 'ae9efe79d4efa87c0bf923d86fa5c10f', '3e914c7fe2dcc8a10584b9b7413acec4', 'd41d8cd98f00b204e9800998ecf8427e', 'd41d8cd98f00b204e9800998ecf8427e', 'd41d8cd98f00b204e9800998ecf8427e', 'cc2c4af9200f0e03ccf00f647ceb2adc', '0b8cfe3f0d6256532dade1c1216024d8', '715ae5f8509d6d5eddd98364663ad656', '140233b2fd2b42a68c80d8cf2719392d9ff39d1c', 'a4f1fca9acd5a1a01d6ae4c5fd6c7ca2c2f85f6d', 'cf0066cf2fccf5086c4a540083961b2bf47d5479', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', '239cf581a491df7c9890103efc40a85b6daf82c1', 'd375c65833730d51c3748d64aecee096ff2bdbd5', '75c1d81de299d477e9aa88a41474bc74a3ac447e']\n",
        "hash_on_kaggle  = ['Nan', 'Nan', 'Nan', 'd41d8cd98f00b204e9800998ecf8427e', 'd41d8cd98f00b204e9800998ecf8427e', 'd41d8cd98f00b204e9800998ecf8427e', 'cc2c4af9200f0e03ccf00f647ceb2adc', '0b8cfe3f0d6256532dade1c1216024d8', '715ae5f8509d6d5eddd98364663ad656', 'Nan', 'Nan', 'Nan', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'da39a3ee5e6b4b0d3255bfef95601890afd80709', '239cf581a491df7c9890103efc40a85b6daf82c1', 'd375c65833730d51c3748d64aecee096ff2bdbd5', '75c1d81de299d477e9aa88a41474bc74a3ac447e']\n",
        "\n",
        "pandas.DataFrame([hash_on_my_nas, hash_on_colfax, hash_on_kaggle], index=list_compute, columns=list_files).transpose()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "271f0f47-83fd-3343-adc6-5fb4a2a5da09"
      },
      "source": [
        "# Check the MD5 and SHA1 hashsum\n",
        "\n",
        "I have used this script to compute hashsum on each environment."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3755ee97-f818-d5ec-797c-f9c7de0795c0"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "import os\n",
        "import platform\n",
        "import subprocess\n",
        "\n",
        "\n",
        "def res_cmd(cmd):\n",
        "  return str(subprocess.Popen(cmd, stdout=subprocess.PIPE,shell=True).communicate()[0])\n",
        "\n",
        "def str_md5sum(abspath_dataset_dir, abspath_file):\n",
        "    str_res = res_cmd('md5sum \"%s\"' % os.path.join(abspath_dataset_dir, abspath_file))\n",
        "    return str_res.split(\"'\")[1].replace('\\\\n', '')\n",
        "\n",
        "def str_sha1sum(abspath_dataset_dir, abspath_file):\n",
        "    str_res = res_cmd('sha1sum \"%s\"' % os.path.join(abspath_dataset_dir, abspath_file))\n",
        "    return str_res.split(\"'\")[1].replace('\\\\n', '')\n",
        "\n",
        "\n",
        "if 'c001' in platform.node():\n",
        "    # platform.node() => 'c001' or like 'c001-n030' on Colfax\n",
        "    abspath_dataset_dir = '/data/kaggle/'\n",
        "elif '.local' in platform.node():\n",
        "    # platform.node() => '*.local' on my local MacBook Air\n",
        "    abspath_dataset_dir = '/Volumes/TRANSCEND_G/intel-mobileodt-cervical-cancer-screening/'\n",
        "else:\n",
        "    # For kaggle's kernels environment (docker container?)\n",
        "    abspath_dataset_dir = '/kaggle/input/'\n",
        "\n",
        "\n",
        "print(str_md5sum(abspath_dataset_dir, 'train.7z'))                   # 4b3fd8c73b6ac21b1e106d5efd8713cd\n",
        "print(str_md5sum(abspath_dataset_dir, 'test.7z'))                    # ae9efe79d4efa87c0bf923d86fa5c10f\n",
        "print(str_md5sum(abspath_dataset_dir, 'additional.7z'))              # 3e914c7fe2dcc8a10584b9b7413acec4\n",
        "print(str_md5sum(abspath_dataset_dir, 'additional/Type_2/2845.jpg')) # d41d8cd98f00b204e9800998ecf8427e\n",
        "print(str_md5sum(abspath_dataset_dir, 'additional/Type_2/5892.jpg')) # d41d8cd98f00b204e9800998ecf8427e\n",
        "print(str_md5sum(abspath_dataset_dir, 'additional/Type_2/5893.jpg')) # d41d8cd98f00b204e9800998ecf8427e\n",
        "print(str_md5sum(abspath_dataset_dir, 'train/Type_1/1339.jpg'))      # cc2c4af9200f0e03ccf00f647ceb2adc\n",
        "print(str_md5sum(abspath_dataset_dir, 'additional/Type_1/3068.jpg')) # 0b8cfe3f0d6256532dade1c1216024d8\n",
        "print(str_md5sum(abspath_dataset_dir, 'additional/Type_2/7.jpg'))    # 715ae5f8509d6d5eddd98364663ad656\n",
        "\n",
        "print(str_sha1sum(abspath_dataset_dir, 'train.7z'))                   # 140233b2fd2b42a68c80d8cf2719392d9ff39d1c\n",
        "print(str_sha1sum(abspath_dataset_dir, 'test.7z'))                    # a4f1fca9acd5a1a01d6ae4c5fd6c7ca2c2f85f6d\n",
        "print(str_sha1sum(abspath_dataset_dir, 'additional.7z'))              # cf0066cf2fccf5086c4a540083961b2bf47d5479\n",
        "print(str_sha1sum(abspath_dataset_dir, 'additional/Type_2/2845.jpg')) # da39a3ee5e6b4b0d3255bfef95601890afd80709\n",
        "print(str_sha1sum(abspath_dataset_dir, 'additional/Type_2/5892.jpg')) # da39a3ee5e6b4b0d3255bfef95601890afd80709\n",
        "print(str_sha1sum(abspath_dataset_dir, 'additional/Type_2/5893.jpg')) # da39a3ee5e6b4b0d3255bfef95601890afd80709\n",
        "print(str_sha1sum(abspath_dataset_dir, 'train/Type_1/1339.jpg'))      # 239cf581a491df7c9890103efc40a85b6daf82c1\n",
        "print(str_sha1sum(abspath_dataset_dir, 'additional/Type_1/3068.jpg')) # d375c65833730d51c3748d64aecee096ff2bdbd5\n",
        "print(str_sha1sum(abspath_dataset_dir, 'additional/Type_2/7.jpg'))    # 75c1d81de299d477e9aa88a41474bc74a3ac447e"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}