{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":21154,"databundleVersionId":1243559,"sourceType":"competition"}],"dockerImageVersionId":30839,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ================================\n# 基础库导入\n# ================================\nimport io\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import f1_score\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torch.cuda.amp import autocast, GradScaler  # 混合精度训练\n\nimport tensorflow as tf\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport time\n\n# ================================\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:09:26.204031Z","iopub.execute_input":"2025-02-02T02:09:26.204324Z","iopub.status.idle":"2025-02-02T02:09:43.897673Z","shell.execute_reply.started":"2025-02-02T02:09:26.204303Z","shell.execute_reply":"2025-02-02T02:09:43.896786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 配置参数\n# ================================\nBATCH_SIZE = 256       # P100显存充足，增大批次\nNUM_EPOCHS = 10        # 增加训练轮次\nNUM_WORKERS = 8        # 数据加载线程数（匹配CPU核心数）\nIMAGE_SIZE = 224       # 图像尺寸\nLR = 0.001             # 初始学习率\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\ntorch.backends.cudnn.benchmark = True  # 启用cudnn优化\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:09:53.154193Z","iopub.execute_input":"2025-02-02T02:09:53.154792Z","iopub.status.idle":"2025-02-02T02:09:53.159225Z","shell.execute_reply.started":"2025-02-02T02:09:53.154763Z","shell.execute_reply":"2025-02-02T02:09:53.158247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# ================================\n# 类别定义（必须完整保留）\n# ================================\nCLASSES = ['pink primrose',    'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',     'wild geranium',     'tiger lily',           'moon orchid',              'bird of paradise', 'monkshood',        'globe thistle',         # 00 - 09\n           'snapdragon',       \"colt's foot\",               'king protea',      'spear thistle', 'yellow iris',       'globe-flower',         'purple coneflower',        'peruvian lily',    'balloon flower',   'giant white arum lily', # 10 - 19\n           'fire lily',        'pincushion flower',         'fritillary',       'red ginger',    'grape hyacinth',    'corn poppy',           'prince of wales feathers', 'stemless gentian', 'artichoke',        'sweet william',         # 20 - 29\n           'carnation',        'garden phlox',              'love in the mist', 'cosmos',        'alpine sea holly',  'ruby-lipped cattleya', 'cape flower',              'great masterwort', 'siam tulip',       'lenten rose',           # 30 - 39\n           'barberton daisy',  'daffodil',                  'sword lily',       'poinsettia',    'bolero deep blue',  'wallflower',           'marigold',                 'buttercup',        'daisy',            'common dandelion',      # 40 - 49\n           'petunia',          'wild pansy',                'primula',          'sunflower',     'lilac hibiscus',    'bishop of llandaff',   'gaura',                    'geranium',         'orange dahlia',    'pink-yellow dahlia',    # 50 - 59\n           'cautleya spicata', 'japanese anemone',          'black-eyed susan', 'silverbush',    'californian poppy', 'osteospermum',         'spring crocus',            'iris',             'windflower',       'tree poppy',            # 60 - 69\n           'gazania',          'azalea',                    'water lily',       'rose',          'thorn apple',       'morning glory',        'passion flower',           'lotus',            'toad lily',        'anthurium',             # 70 - 79\n           'frangipani',       'clematis',                  'hibiscus',         'columbine',     'desert-rose',       'tree mallow',          'magnolia',                 'cyclamen ',        'watercress',       'canna lily',            # 80 - 89\n           'hippeastrum ',     'bee balm',                  'pink quill',       'foxglove',      'bougainvillea',     'camellia',             'mallow',                   'mexican petunia',  'bromelia',         'blanket flower',        # 90 - 99\n           'trumpet creeper',  'blackberry lily',           'common tulip',     'wild rose']                                                                                                                                               # 100 - 102\nprint('total number of classes: ', len(CLASSES))\n# ================================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:09.500168Z","iopub.execute_input":"2025-02-02T02:10:09.500444Z","iopub.status.idle":"2025-02-02T02:10:09.507038Z","shell.execute_reply.started":"2025-02-02T02:10:09.500423Z","shell.execute_reply":"2025-02-02T02:10:09.506101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 数据路径配置（根据实际修改）\n# ================================\nTF_TRAIN_PATH = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/train/*.tfrec'\nTF_VAL_PATH = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/val/*.tfrec'\nTF_TEST_PATH = '/kaggle/input/tpu-getting-started/tfrecords-jpeg-224x224/test/*.tfrec'\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:12.943131Z","iopub.execute_input":"2025-02-02T02:10:12.943412Z","iopub.status.idle":"2025-02-02T02:10:12.947163Z","shell.execute_reply.started":"2025-02-02T02:10:12.943392Z","shell.execute_reply":"2025-02-02T02:10:12.946159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# ================================\n# 数据加载核心函数（完整实现）\n# ================================\ndef tfrecords_to_dataframe(fp, test=False):\n    \"\"\"解析TFRecord文件为DataFrame\"\"\"\n    def parse(example_proto, test_flag):\n        feature_desc = {\n            'id': tf.io.FixedLenFeature([], tf.string),\n            'image': tf.io.FixedLenFeature([], tf.string)\n        }\n        if not test_flag:\n            feature_desc['class'] = tf.io.FixedLenFeature([], tf.int64)\n        return tf.io.parse_single_example(example_proto, feature_desc)\n    \n    df = {'id': [], 'img': []}\n    if not test:\n        df['lab'] = []\n    \n    dataset = tf.data.TFRecordDataset(glob.glob(fp))\n    parsed_dataset = dataset.map(lambda x: parse(x, test))\n    \n    for record in parsed_dataset:\n        df['id'].append(record['id'].numpy().decode('utf-8'))\n        df['img'].append(record['image'].numpy())\n        if not test:\n            df['lab'].append(record['class'].numpy())\n    \n    return pd.DataFrame(df)\n\n# ================================\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:16.762867Z","iopub.execute_input":"2025-02-02T02:10:16.763152Z","iopub.status.idle":"2025-02-02T02:10:16.769102Z","shell.execute_reply.started":"2025-02-02T02:10:16.763131Z","shell.execute_reply":"2025-02-02T02:10:16.768191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 数据集类（完整实现）\n# ================================\nclass FlowerDataset(Dataset):\n    \"\"\"通用花卉数据集类\"\"\"\n    def __init__(self, df, transform, is_test=False):\n        self.df = df\n        self.transform = transform\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        # 加载图像\n        img_bytes = self.df.iloc[idx]['img']\n        image = Image.open(io.BytesIO(img_bytes))\n        image = np.array(image)\n        \n        # 应用增强\n        augmented = self.transform(image=image)\n        image = augmented['image'].float() / 255.0  # 归一化\n        \n        # 返回数据\n        if self.is_test:\n            return image, self.df.iloc[idx]['id']\n        else:\n            return image, torch.tensor(self.df.iloc[idx]['lab'], dtype=torch.long)\n\n# ================================\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:20.402840Z","iopub.execute_input":"2025-02-02T02:10:20.403125Z","iopub.status.idle":"2025-02-02T02:10:20.408708Z","shell.execute_reply.started":"2025-02-02T02:10:20.403104Z","shell.execute_reply":"2025-02-02T02:10:20.407740Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 数据增强配置\n# ================================\ntrain_transform = A.Compose([\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.2),\n    A.RandomBrightnessContrast(p=0.3),\n    A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.2, rotate_limit=20, p=0.5),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])\n\nval_test_transform = A.Compose([\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])\n\n# ================================\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:23.360282Z","iopub.execute_input":"2025-02-02T02:10:23.360570Z","iopub.status.idle":"2025-02-02T02:10:23.368345Z","shell.execute_reply.started":"2025-02-02T02:10:23.360548Z","shell.execute_reply":"2025-02-02T02:10:23.367358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 加载所有数据集（完整流程）\n# ================================\nprint(\"Loading training data...\")\ntrain_df = tfrecords_to_dataframe(TF_TRAIN_PATH)\nprint(f\"Training data shape: {train_df.shape}\")\n\nprint(\"Loading validation data...\")\nval_df = tfrecords_to_dataframe(TF_VAL_PATH)\nprint(f\"Validation data shape: {val_df.shape}\")\n\nprint(\"Loading test data...\")\ntest_df = tfrecords_to_dataframe(TF_TEST_PATH, test=True)\nprint(f\"Test data shape: {test_df.shape}\")\n\n# ================================\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:26.361199Z","iopub.execute_input":"2025-02-02T02:10:26.361673Z","iopub.status.idle":"2025-02-02T02:10:41.162423Z","shell.execute_reply.started":"2025-02-02T02:10:26.361608Z","shell.execute_reply":"2025-02-02T02:10:41.161612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 创建数据集和数据加载器\n# ================================\ntrain_dataset = FlowerDataset(train_df, train_transform)\nval_dataset = FlowerDataset(val_df, val_test_transform)\ntest_dataset = FlowerDataset(test_df, val_test_transform, is_test=True)\n\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    num_workers=NUM_WORKERS,\n    pin_memory=True,\n    persistent_workers=True\n)\n\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    pin_memory=True,\n    persistent_workers=True\n)\n\ntest_loader = DataLoader(\n    test_dataset,\n    batch_size=BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    pin_memory=True\n)\n\n# ================================\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:45.322977Z","iopub.execute_input":"2025-02-02T02:10:45.323258Z","iopub.status.idle":"2025-02-02T02:10:45.329802Z","shell.execute_reply.started":"2025-02-02T02:10:45.323237Z","shell.execute_reply":"2025-02-02T02:10:45.328948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 模型定义（完整实现）\n# ================================\ndef create_model():\n    \"\"\"创建并配置EfficientNet模型\"\"\"\n    model = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.IMAGENET1K_V1)\n    num_features = model.classifier[1].in_features\n    model.classifier[1] = nn.Linear(num_features, len(CLASSES))\n    return model.to(DEVICE)\n\nmodel = create_model()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:53.406525Z","iopub.execute_input":"2025-02-02T02:10:53.406865Z","iopub.status.idle":"2025-02-02T02:10:53.961348Z","shell.execute_reply.started":"2025-02-02T02:10:53.406838Z","shell.execute_reply":"2025-02-02T02:10:53.960725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ================================\n# 训练工具配置\n# ================================\nscaler = GradScaler()  # 混合精度梯度缩放\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-5)\nscheduler = CosineAnnealingLR(optimizer, T_max=NUM_EPOCHS*len(train_loader), eta_min=1e-6)\n\n# ================================\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:10:59.503031Z","iopub.execute_input":"2025-02-02T02:10:59.503316Z","iopub.status.idle":"2025-02-02T02:10:59.509407Z","shell.execute_reply.started":"2025-02-02T02:10:59.503297Z","shell.execute_reply":"2025-02-02T02:10:59.508518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 训练验证函数（完整实现）\n# ================================\ndef train_epoch(model, loader, optimizer, scheduler, scaler):\n    \"\"\"混合精度训练单个epoch\"\"\"\n    model.train()\n    total_loss = 0.0\n    start_time = time.time()\n    \n    for inputs, targets in loader:\n        inputs = inputs.to(DEVICE, non_blocking=True)\n        targets = targets.to(DEVICE, non_blocking=True)\n        \n        optimizer.zero_grad(set_to_none=True)  # 更快的梯度清零\n        \n        with autocast():\n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n        \n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()  # 每个batch更新学习率\n        \n        total_loss += loss.item()\n    \n    return total_loss / len(loader), time.time() - start_time\n\ndef validate(model, loader):\n    \"\"\"验证集评估\"\"\"\n    model.eval()\n    total_loss = 0.0\n    all_preds = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for inputs, targets in loader:\n            inputs = inputs.to(DEVICE, non_blocking=True)\n            targets = targets.to(DEVICE, non_blocking=True)\n            \n            outputs = model(inputs)\n            loss = criterion(outputs, targets)\n            total_loss += loss.item()\n            \n            preds = torch.argmax(outputs, dim=1)\n            all_preds.extend(preds.cpu().numpy())\n            all_targets.extend(targets.cpu().numpy())\n    \n    return total_loss / len(loader), f1_score(all_targets, all_preds, average='weighted')\n\n# ================================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:11:10.798832Z","iopub.execute_input":"2025-02-02T02:11:10.799152Z","iopub.status.idle":"2025-02-02T02:11:10.806310Z","shell.execute_reply.started":"2025-02-02T02:11:10.799127Z","shell.execute_reply":"2025-02-02T02:11:10.805318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 主训练循环（完整日志）\n# ================================\nbest_f1 = 0.0\nhistory = {'train_loss': [], 'val_loss': [], 'val_f1': []}\n\nprint(\"\\nStarting training on device:\", DEVICE)\nprint(f\"Batch size: {BATCH_SIZE} | Workers: {NUM_WORKERS}\")\n\nfor epoch in range(NUM_EPOCHS):\n    # 训练\n    train_loss, epoch_time = train_epoch(model, train_loader, optimizer, scheduler, scaler)\n    history['train_loss'].append(train_loss)\n    \n    # 验证\n    val_loss, val_f1 = validate(model, val_loader)\n    history['val_loss'].append(val_loss)\n    history['val_f1'].append(val_f1)\n    \n    # 保存最佳模型\n    if val_f1 > best_f1:\n        best_f1 = val_f1\n        torch.save(model.state_dict(), f\"best_model_epoch{epoch}.pth\")\n        print(f\"🔥 New best model saved (F1: {val_f1:.4f})\")\n    \n    # 打印日志\n    print(f\"Epoch {epoch+1}/{NUM_EPOCHS} | \"\n          f\"Train Loss: {train_loss:.4f} | \"\n          f\"Val Loss: {val_loss:.4f} | \"\n          f\"Val F1: {val_f1:.4f} | \"\n          f\"Time: {epoch_time:.2f}s\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:11:23.522091Z","iopub.execute_input":"2025-02-02T02:11:23.522371Z","iopub.status.idle":"2025-02-02T02:18:53.239632Z","shell.execute_reply.started":"2025-02-02T02:11:23.522351Z","shell.execute_reply":"2025-02-02T02:18:53.238514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# ================================\n# 测试预测（完整实现）\n# ================================\n\n\nprint(\"\\nLoading best model for testing...\")\nmodel.load_state_dict(torch.load(f\"best_model_epoch{np.argmax(history['val_f1'])}.pth\"))\nmodel.eval()\n\nids = []\npreds = []\n\nwith torch.no_grad():\n    for inputs, img_ids in test_loader:\n        inputs = inputs.to(DEVICE, non_blocking=True)\n        \n        outputs = model(inputs)\n        batch_preds = torch.argmax(outputs, dim=1).cpu().numpy()\n        \n        ids.extend(img_ids)\n        preds.extend(batch_preds)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:20:19.782528Z","iopub.execute_input":"2025-02-02T02:20:19.782914Z","iopub.status.idle":"2025-02-02T02:20:30.869577Z","shell.execute_reply.started":"2025-02-02T02:20:19.782871Z","shell.execute_reply":"2025-02-02T02:20:30.868448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 生成提交文件\nsubmission = pd.DataFrame({'id': ids, 'label': preds})\nsubmission['label'] = submission['label'].apply(lambda x: CLASSES[x])  # 转换为类别名称\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"\\nSubmission file generated:\")\nprint(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-02T02:22:56.678596Z","iopub.execute_input":"2025-02-02T02:22:56.678946Z","iopub.status.idle":"2025-02-02T02:22:56.722885Z","shell.execute_reply.started":"2025-02-02T02:22:56.678919Z","shell.execute_reply":"2025-02-02T02:22:56.721984Z"}},"outputs":[],"execution_count":null}]}