{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"%%bash\n\npip install --upgrade pip\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ig_resnext101_32x8d-224 ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%%bash\ntime wget https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/seresnext50_32x4d_racm-a304a460.pth\ntime wget https://github.com/rwightman/pytorch-pretrained-gluonresnet/releases/download/v0.1/gluon_seresnext101_32x4d-cf52900d.pth\ntime wget https://storage.googleapis.com/tensorflow/keras-applications/nasnet/NASNet-large-no-top.h5\ntime wget https://storage.googleapis.com/tensorflow/keras-applications/inception_resnet_v2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb7_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb6_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb5_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb4_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb3_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb2_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb1_notop.h5\ntime wget https://storage.googleapis.com/keras-applications/efficientnetb0_notop.h5\ntime wget https://storage.googleapis.com/tensorflow/keras-applications/mobilenet_v2/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_1.0_224_no_top.h5\ntime git clone https://github.com/titu1994/keras-normalized-optimizers.git\ntime git clone https://github.com/ypeleg/nfnets-keras.git\ntime git clone https://github.com/sayakpaul/Adaptive-Gradient-Clipping.git\ntime git clone https://github.com/rwightman/pytorch-image-models.git\n\nls -hltar\npwd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torchvision","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import torch\n# from .resnext_wsl import resnext101_32x48d_wsl\n\n# model=resnext101_32x48d_wsl(progress=True) # example with the ResNeXt-101 32x48d \n\n# pretrained_dict=torch.load('ResNeXt101_32x48d.pth',map_location='cpu')['model']\n\n# model_dict = model.state_dict()\n# for k in model_dict.keys():\n#     if(('module.'+k) in pretrained_dict.keys()):\n#         model_dict[k]=pretrained_dict.get(('module.'+k))\n# model.load_state_dict(model_dict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import timm\n# model = timm.create_model(\n#     model_name,\n#     num_classes=1000,\n#     in_chans=3,\n#     pretrained=pretrained,\n#     checkpoint_path=checkpoint_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if False:\n    import requests\n    package = requests.get(\"https://pypi.python.org/pypi/timm/json\").json()\n    versions = sorted(package[\"releases\"].keys(), reverse=True)\n    max_ver = versions[0]\n    files = [d['url'] for d in package[\"releases\"][max_ver] if '.whl' in d['url'].lower()]\n    files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget https://files.pythonhosted.org/packages/22/c6/ba02d533cec7329323c7d7a317ab49f673846ecef202d4cc40988b6b7786/timm-0.3.4-py3-none-any.whl\n# !pip install https://files.pythonhosted.org/packages/22/c6/ba02d533cec7329323c7d7a317ab49f673846ecef202d4cc40988b6b7786/timm-0.3.4-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget https://s3-us-west-2.amazonaws.com/imagenetv2public/imagenetv2-matched-frequency.tar.gz ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -hltar  .\nexit()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!find imagenetv2-matched-frequency-format-val | wc -l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\ntorch.backends.cudnn.benchmark = True\n\nimport timm\nfrom timm.data import *\nfrom timm.utils import *\n\nimport pandas as pd\nimport numpy as np\nimport pynvml\nfrom collections import OrderedDict\nimport logging\nimport time\n\ndef log_gpu_memory():\n    handle = pynvml.nvmlDeviceGetHandleByIndex(0)\n    info = pynvml.nvmlDeviceGetMemoryInfo(handle)\n    info.free = round(info.free / 1024**2)\n    info.used = round(info.used / 1024**2)\n    logging.info('GPU memory free: {}, memory used: {}'.format(info.free, info.used))\n    return info.used\n\ndef get_gpu_memory_total():\n    handle = pynvml.nvmlDeviceGetHandleByIndex(0)\n    info = pynvml.nvmlDeviceGetMemoryInfo(handle)\n    info.total = round(info.total / 1024**2)\n    return info.total\n\nsetup_default_logging()\n    \nprint('PyTorch version:', torch.__version__)\nif torch.cuda.is_available():\n    print('CUDA available')\n    device='cuda'\nelse:\n    print('CUDA is not available')\n    device='cpu'\n\nBATCH_SIZE = 128\nif device == 'cuda':\n    pynvml.nvmlInit()\n    log_gpu_memory()\n    total_gpu_mem = get_gpu_memory_total()\n    HAS_T4 = False\n    if total_gpu_mem > 12300:\n        HAS_T4 = True\n        logging.info('Running on a T4 GPU or other with > 12GB memory, setting batch size to {}'.format(BATCH_SIZE))\n    else:\n        BATCH_SIZE = 64\n        logging.info('Running on a K80 GPU or other with < 12GB memory, batch size set to {}'.format(BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n# if not os.path.exists('./imagenetv2-matched-frequency-format-val'):\n#     !curl -s https://s3-us-west-2.amazonaws.com/imagenetv2public/imagenetv2-matched-frequency.tar.gz | tar x\n# dataset = Dataset('./imagenetv2-matched-frequency-format-val/')\n# assert len(dataset) == 10000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torchvision.utils import make_grid\nimport torchvision.transforms as transforms\nimport matplotlib.pyplot as plt\n\ndef show_img(ax, img):\n    npimg = img.numpy()\n    ax.imshow(np.transpose(npimg, (1,2,0)), interpolation='bicubic')\n\nfig = plt.figure(figsize=(8, 16), dpi=100)\nax = fig.add_subplot('111')\nnum_images = 4*8\nimages = []\ndataset.transform = transforms.Compose([\n    transforms.Resize(320),\n    transforms.CenterCrop(320),\n    transforms.ToTensor()])\nfor i in np.random.permutation(np.arange(len(dataset)))[:num_images]:\n    images.append(dataset[i][0])\n\ngrid_img = make_grid(images, nrow=4, padding=10, normalize=True, scale_each=True)\nshow_img(ax, grid_img)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# a basic validation routine and runner that configures each model and loader\nfrom timm.models import TestTimePoolHead\n\ndef validate(model, loader, criterion=None, device='cuda'):\n    # metrics\n    batch_time = timm.utils.AverageMeter()\n    losses = AverageMeter()\n    top1 = AverageMeter()\n    top5 = AverageMeter()\n    \n    # for collecting per sample prediction/loss details\n    losses_val = []\n    top5_idx = []\n    top5_val = []\n    \n    end = time.time()\n    with torch.no_grad():\n        for i, (input, target) in enumerate(loader):\n            target = target.to(device)\n            input = input.to(device)\n            output = model(input)\n            \n            if criterion is not None:\n                loss = criterion(output, target)\n                if not loss.size():\n                    losses.update(loss.item(), input.size(0))\n                else:\n                    # only bother collecting top5 we're also collecting per-example loss\n                    output = output.softmax(1)\n                    top5v, top5i = output.topk(5, 1, True, True)\n                    top5_val.append(top5v.cpu().numpy())\n                    top5_idx.append(top5i.cpu().numpy())\n                    losses_val.append(loss.cpu().numpy())\n                    losses.update(loss.mean().item(), input.size(0))\n                \n            prec1, prec5 = timm.utils.accuracy(output, target, topk=(1, 5))\n            top1.update(prec1.item(), input.size(0))\n            top5.update(prec5.item(), input.size(0))\n\n            batch_time.update(time.time() - end)\n            end = time.time()\n\n            if i % 20 == 0:\n                print('Test: [{0}/{1}]\\t'\n                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f}, {rate_avg:.3f}/s) \\t'\n                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\\t'\n                      'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format(\n                    i, len(loader), batch_time=batch_time,\n                    rate_avg=input.size(0) / batch_time.avg,\n                    top1=top1, top5=top5))\n\n    results = OrderedDict(\n        top1=top1.avg, top1_err=100 - top1.avg,\n        top5=top5.avg, top5_err=100 - top5.avg,\n    )\n    if criterion is not None:\n        results['loss'] = losses.avg\n    if len(top5_idx):\n        results['top5_val'] = np.concatenate(top5_val, axis=0)\n        results['top5_idx'] = np.concatenate(top5_idx, axis=0)\n    if len(losses_val):\n        results['losses_val'] = np.concatenate(losses_val, axis=0)\n    print(' * Prec@1 {:.3f} ({:.3f}) Prec@5 {:.3f} ({:.3f})'.format(\n       results['top1'], results['top1_err'], results['top5'], results['top5_err']))\n    return results\n\n\ndef runner(model_args, dataset, device='cuda', collect_loss=False):\n    model_name = model_args['model']\n    model = timm.create_model(model_name, pretrained=True)\n    ttp = False\n    if 'ttp' in model_args and model_args['ttp']:\n        ttp = True\n        logging.info('Applying test time pooling to model')\n        model = TestTimePoolHead(model, original_pool=model.default_cfg['pool_size'])\n    model = model.to(device)\n    model.eval()\n    if HAS_T4:\n        model = model.half()\n\n    data_config = timm.data.resolve_data_config(model_args, model=model, verbose=True)\n        \n    loader = timm.data.create_loader(\n        dataset,\n        input_size=data_config['input_size'],\n        batch_size=BATCH_SIZE,\n        use_prefetcher=True,\n        interpolation='bicubic',\n        mean=data_config['mean'],\n        std=data_config['std'],\n        fp16=HAS_T4,\n        crop_pct=1.0 if ttp else data_config['crop_pct'],\n        num_workers=2)\n\n    criterion = None\n    if collect_loss:\n        criterion = torch.nn.CrossEntropyLoss(reduction='none').to(device)\n    results = validate(model, loader, criterion, device)\n    \n    # cleanup checkpoint cache to avoid running out of disk space\n    shutil.rmtree(os.path.join(os.environ['HOME'], '.cache', 'torch', 'checkpoints'), True)\n    \n    # add some non-metric values for charting / comparisons\n    results['model'] = model_name\n    results['img_size'] = data_config['input_size'][-1]\n\n    # create key to identify model in charts\n    key = [model_name, str(data_config['input_size'][-1])]\n    if ttp:\n        key += ['ttp']\n    key = '-'.join(key)\n    return key, results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmodel_name = 'tf_efficientnet_b7_ns'\nmodel = timm.create_model(model_name, pretrained=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = [\n    dict(model='mobilenetv3_100'),\n    dict(model='dpn68b'),\n    dict(model='gluon_resnet50_v1d'),\n    dict(model='efficientnet_b2'),\n    dict(model='gluon_seresnext50_32x4d'),\n    dict(model='dpn92'),\n    dict(model='gluon_seresnext101_32x4d'),\n    dict(model='inception_resnet_v2'),\n    dict(model='pnasnet5large'),\n    dict(model='tf_efficientnet_b5'),\n    dict(model='ig_resnext101_32x8d'),\n    dict(model='ig_resnext101_32x16d'),\n    dict(model='ig_resnext101_32x32d'),\n    dict(model='ig_resnext101_32x48d'),\n]\n\nresults = OrderedDict()\nfor ma in models:\n    mk, mr = runner(ma, dataset, device)\n    results[mk] = mr\n\nresults_df = pd.DataFrame.from_dict(results, orient='index')\nresults_df.to_csv('./cached-results.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install timm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_model = timm.create_model(model_name='tf_efficientnet_b7_ns', pretrained=True, num_classes=1000, in_chans=3, checkpoint_path='', scriptable=None, exportable=None, no_jit=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nhttps://dl.fbaipublicfiles.com/semiweaksupervision/model_files/semi_supervised_resnext101_32x4-dc43570a.pth\n\"\"\";\nimport torch\nimport timm\ntorch_model = timm.create_model('ssl_resnext101_32x4d', pretrained=True, num_classes=5, global_pool='avg',)\no = model(torch.randn(2, 3, 224, 224))\no.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"timm.list_models()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision.models import vgg16 #https://download.pytorch.org/models/vgg16-397923af.pth\nfrom collections import namedtuple\n\nclass Vgg16(torch.nn.Module):\n    def __init__(self):\n        super(Vgg16, self).__init__()\n        features = list(t_model.features)\n        print(len(features),'features')\n        features = features\n        # features的第3，8，15，22层分别是: relu1_2,relu2_2,relu3_3,relu4_3\n        self.features = nn.ModuleList(features).eval() \n        \n    def forward(self, x):\n        results = []\n        for ii,model in enumerate(self.features):\n            x = model(x)\n            if ii in {30,31}:\n                results.append(x)\n        return results\n\nmodel = Vgg16()                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"o = t_model.forward(torch.randn(2, 3, 224, 224))\no.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list(t_model.modules())[-2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}