{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"233a1de4-b1c6-4473-ad87-6cc9c7a17a67","cell_type":"markdown","source":"# Petals to the Metal - Flower Classification on TPU\nProjeto Final - Kaggle Competition\n\nEste notebook:\n1. Carrega os dados (TFRecords) direto do storage do Kaggle\n2. Faz uma analise exploratoria basica (distribuicao de classes, exemplos de imagens)\n3. Treina **3 arquiteturas diferentes** via transfer learning (EfficientNetB7, DenseNet201, Xception)\n4. Compara os resultados (accuracy / F1)\n5. Gera o arquivo `submission.csv`","metadata":{}},{"id":"0951042e-0734-47d3-9ae3-c067a4933d95","cell_type":"code","source":"import math, os, re, time\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import f1_score, classification_report, confusion_matrix\n\nprint(\"TF version:\", tf.__version__)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T20:55:10.975326Z","iopub.execute_input":"2026-07-02T20:55:10.975645Z","iopub.status.idle":"2026-07-02T20:55:16.494473Z","shell.execute_reply.started":"2026-07-02T20:55:10.975619Z","shell.execute_reply":"2026-07-02T20:55:16.493512Z"}},"outputs":[],"execution_count":null},{"id":"24791330-41f2-47ac-8d15-4740e02ec0a7","cell_type":"markdown","source":"## 1. Setup do TPU","metadata":{}},{"id":"799cba29-de9a-484c-95f4-8dd36e52dcc6","cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU:', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # sempre entra aqui porque não temos mais acesso ao TPU, apesar disso constar no tutorial da plataforma\n    strategy = tf.distribute.get_strategy()\n\nprint(\"Numero de replicas:\", strategy.num_replicas_in_sync)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T20:55:18.796551Z","iopub.execute_input":"2026-07-02T20:55:18.796847Z","iopub.status.idle":"2026-07-02T20:55:18.810607Z","shell.execute_reply.started":"2026-07-02T20:55:18.796822Z","shell.execute_reply":"2026-07-02T20:55:18.809745Z"}},"outputs":[],"execution_count":null},{"id":"f27e829e-56aa-4d06-84ac-f3c3187333ee","cell_type":"markdown","source":"## 2. Configuracao e caminhos dos dados","metadata":{}},{"id":"052652a1-892d-420f-9857-703ed3db94b2","cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')\n\nIMAGE_SIZE = [224,224]   # Escolhida para não estrapolar o limite de memória. Outras dimensoes possiveis sao: [192,192]/[331,331]/[512,512]\nEPOCHS = 5 # ja eh o suficiente para convergir nessa resolucao\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nGCS_PATH_SELECT = {\n    192: GCS_DS_PATH + '/tfrecords-jpeg-192x192',\n    224: GCS_DS_PATH + '/tfrecords-jpeg-224x224',\n    331: GCS_DS_PATH + '/tfrecords-jpeg-331x331',\n    512: GCS_DS_PATH + '/tfrecords-jpeg-512x512'\n}\nGCS_PATH = GCS_PATH_SELECT[IMAGE_SIZE[0]]\n\nTRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')\nVALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')\nTEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')\n\nCLASSES = ['pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',\n    'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood',\n    'globe thistle', 'snapdragon', \"colt's foot\", 'king protea', 'spear thistle',\n    'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily', 'balloon flower',\n    'giant white arum lily', 'fire lily', 'pincushion flower', 'fritillary', 'red ginger',\n    'grape hyacinth', 'corn poppy', 'prince of wales feathers', 'stemless gentian',\n    'artichoke', 'sweet william', 'carnation', 'garden phlox', 'love in the mist',\n    'cosmos', 'alpine sea holly', 'ruby-lipped cattleya', 'cape flower', 'great masterwort',\n    'siam tulip', 'lenten rose', 'barberton daisy', 'daffodil', 'sword lily', 'poinsettia',\n    'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy', 'common dandelion',\n    'petunia', 'wild pansy', 'primula', 'sunflower', 'lilac hibiscus', 'bishop of llandaff',\n    'gaura', 'geranium', 'orange dahlia', 'pink-yellow dahlia', 'cautleya spicata',\n    'japanese anemone', 'black-eyed susan', 'silverbush', 'californian poppy',\n    'osteospermum', 'spring crocus', 'iris', 'windflower', 'tree poppy', 'gazania',\n    'azalea', 'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower',\n    'lotus', 'toad lily', 'anthurium', 'frangipani', 'clematis', 'hibiscus', 'columbine',\n    'desert-rose', 'tree mallow', 'magnolia', 'cyclamen', 'watercress', 'canna lily',\n    'hippeastrum', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea', 'camellia',\n    'mallow', 'mexican petunia', 'bromelia', 'blanket flower', 'trumpet creeper',\n    'blackberry lily', 'common tulip', 'wild rose']\n\nprint(\"N classes:\", len(CLASSES))\nprint(\"Train files:\", len(TRAINING_FILENAMES), \"| Val files:\", len(VALIDATION_FILENAMES), \"| Test files:\", len(TEST_FILENAMES))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:06:50.223050Z","iopub.execute_input":"2026-07-02T21:06:50.223434Z","iopub.status.idle":"2026-07-02T21:06:50.654878Z","shell.execute_reply.started":"2026-07-02T21:06:50.223397Z","shell.execute_reply":"2026-07-02T21:06:50.653945Z"}},"outputs":[],"execution_count":null},{"id":"adc389fe-fa58-440f-a4fd-db8992e8c61f","cell_type":"markdown","source":"## 3. Funcoes de leitura dos TFRecords + Data Augmentation\n\n(Data augmentation conta como uma das tecnicas de pre-processamento do relatorio.)","metadata":{}},{"id":"c51a017d-1009-432e-bf36-ef2e0470e5a5","cell_type":"code","source":"def decode_image(image_data):\n    image = tf.image.decode_jpeg(image_data, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.reshape(image, [*IMAGE_SIZE, 3])\n    return image\n\ndef read_labeled_tfrecord(example):\n    LABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"class\": tf.io.FixedLenFeature([], tf.int64),\n    }\n    example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    label = tf.cast(example['class'], tf.int32)\n    return image, label\n\ndef read_unlabeled_tfrecord(example):\n    UNLABELED_TFREC_FORMAT = {\n        \"image\": tf.io.FixedLenFeature([], tf.string),\n        \"id\": tf.io.FixedLenFeature([], tf.string),\n    }\n    example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)\n    image = decode_image(example['image'])\n    idnum = example['id']\n    return image, idnum\n\ndef data_augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_brightness(image, 0.1)\n    image = tf.image.random_contrast(image, 0.8, 1.2)\n    return image, label\n\ndef load_dataset(filenames, labeled=True, ordered=False):\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False\n    dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)\n    dataset = dataset.with_options(ignore_order)\n    dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)\n    return dataset\n\ndef get_training_dataset():\n    dataset = load_dataset(TRAINING_FILENAMES, labeled=True)\n    dataset = dataset.map(data_augment, num_parallel_calls=AUTO)\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset(ordered=False):\n    dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset(ordered=True):\n    dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef count_data_items(filenames):\n    n = [int(re.compile(r\"-([0-9]*)\\.\").search(f).group(1)) for f in filenames]\n    return np.sum(n)\n\nNUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)\nNUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)\nNUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)\nSTEPS_PER_EPOCH = NUM_TRAINING_IMAGES // BATCH_SIZE\n\nprint(f\"Treino: {NUM_TRAINING_IMAGES} | Validacao: {NUM_VALIDATION_IMAGES} | Teste: {NUM_TEST_IMAGES}\")\nprint(f\"Steps por epoca: {STEPS_PER_EPOCH}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:06:53.196931Z","iopub.execute_input":"2026-07-02T21:06:53.197276Z","iopub.status.idle":"2026-07-02T21:06:53.215018Z","shell.execute_reply.started":"2026-07-02T21:06:53.197248Z","shell.execute_reply":"2026-07-02T21:06:53.214120Z"}},"outputs":[],"execution_count":null},{"id":"246d1830-54f7-4fd9-bb19-02c668ec4a81","cell_type":"markdown","source":"## 4. Analise Exploratoria dos Dados (EDA)","metadata":{}},{"id":"9283fd30-815d-4290-a2f5-f0049784b572","cell_type":"code","source":"# Distribuicao de classes no conjunto de treino\nlabel_ds = load_dataset(TRAINING_FILENAMES, labeled=True).map(lambda img, lbl: lbl)\nlabels = np.array(list(label_ds.as_numpy_iterator()))\n\ncounts = pd.Series(labels).value_counts().sort_index()\nplt.figure(figsize=(16,5))\nplt.bar(range(len(counts)), counts.values)\nplt.title(\"Distribuicao de imagens por classe (treino)\")\nplt.xlabel(\"Classe (indice)\")\nplt.ylabel(\"Numero de imagens\")\nplt.tight_layout()\nplt.savefig(\"class_distribution.png\", dpi=120)\nplt.show()\n\nprint(\"Classe com mais exemplos:\", CLASSES[counts.idxmax()], \"->\", counts.max())\nprint(\"Classe com menos exemplos:\", CLASSES[counts.idxmin()], \"->\", counts.min())\nprint(\"Desbalanceamento (max/min):\", round(counts.max()/counts.min(), 2))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:07:00.953649Z","iopub.execute_input":"2026-07-02T21:07:00.954013Z","iopub.status.idle":"2026-07-02T21:07:12.665496Z","shell.execute_reply.started":"2026-07-02T21:07:00.953964Z","shell.execute_reply":"2026-07-02T21:07:12.664365Z"}},"outputs":[],"execution_count":null},{"id":"9d37898e-887f-4acd-80a8-009b870e1ea6","cell_type":"markdown","source":"### 4.1 Estatistica descritiva - imagens por classe\n\nAqui aplicamos as medidas classicas de estatistica descritiva (media, mediana, moda, desvio padrao,\nquartis, assimetria e curtose) sobre a quantidade de imagens de cada uma das 104 classes.\nIsso mostra o quao desbalanceado esta o dataset.","metadata":{}},{"id":"fe3b2b10-ba12-4c3d-bcf2-b10a8eedef97","cell_type":"code","source":"from scipy import stats as scistats\n\ncounts_series = pd.Series(counts.values)\n\nmedia = counts_series.mean()\nmediana = counts_series.median()\nmoda_result = scistats.mode(counts.values)\nmoda = moda_result.mode[0]\nmoda_freq = moda_result.count[0]\ndesvio_padrao = counts_series.std()\nvariancia = counts_series.var()\nminimo = counts_series.min()\nmaximo = counts_series.max()\nq1 = counts_series.quantile(0.25)\nq3 = counts_series.quantile(0.75)\niqr = q3 - q1\nassimetria = counts_series.skew()\ncurtose = counts_series.kurtosis()\ncv = (desvio_padrao / media) * 100  # coeficiente de variacao\n\neda_class_stats = pd.DataFrame({\n    \"Estatistica\": [\"Media\", \"Mediana\", \"Moda\", \"Frequencia da moda\", \"Desvio Padrao\", \"Variancia\",\n                     \"Minimo\", \"Maximo\", \"Q1 (25%)\", \"Q3 (75%)\", \"IQR\", \"Assimetria (skewness)\",\n                     \"Curtose\", \"Coeficiente de Variacao (%)\"],\n    \"Valor\": [media, mediana, moda, moda_freq, desvio_padrao, variancia, minimo, maximo,\n              q1, q3, iqr, assimetria, curtose, cv]\n})\neda_class_stats[\"Valor\"] = eda_class_stats[\"Valor\"].round(3)\neda_class_stats.to_csv(\"eda_class_distribution_stats.csv\", index=False)\nprint(\"Estatistica descritiva - numero de imagens por classe (104 classes)\")\neda_class_stats\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:07:18.735296Z","iopub.execute_input":"2026-07-02T21:07:18.735677Z","iopub.status.idle":"2026-07-02T21:07:18.762954Z","shell.execute_reply.started":"2026-07-02T21:07:18.735644Z","shell.execute_reply":"2026-07-02T21:07:18.761872Z"}},"outputs":[],"execution_count":null},{"id":"debd0f04-5b41-48b6-8165-6587b31e31aa","cell_type":"code","source":"# Boxplot da distribuicao de imagens por classe (visualiza mediana, quartis e outliers)\nplt.figure(figsize=(6,5))\nplt.boxplot(counts.values, vert=True, patch_artist=True)\nplt.title(\"Boxplot - imagens por classe\")\nplt.ylabel(\"Numero de imagens\")\nplt.axhline(media, color='red', linestyle='--', label=f'Media = {media:.1f}')\nplt.axhline(mediana, color='green', linestyle='--', label=f'Mediana = {mediana:.1f}')\nplt.legend()\nplt.tight_layout()\nplt.savefig(\"class_distribution_boxplot.png\", dpi=120)\nplt.show()\n\n# Histograma com curva de distribuicao\nplt.figure(figsize=(8,5))\nplt.hist(counts.values, bins=20, edgecolor='black', alpha=0.7)\nplt.axvline(media, color='red', linestyle='--', label=f'Media = {media:.1f}')\nplt.axvline(mediana, color='green', linestyle='--', label=f'Mediana = {mediana:.1f}')\nplt.axvline(moda, color='orange', linestyle='--', label=f'Moda = {moda:.1f}')\nplt.title(\"Histograma - numero de imagens por classe\")\nplt.xlabel(\"Numero de imagens\")\nplt.ylabel(\"Frequencia (n. de classes)\")\nplt.legend()\nplt.tight_layout()\nplt.savefig(\"class_distribution_histogram.png\", dpi=120)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:07:22.192973Z","iopub.execute_input":"2026-07-02T21:07:22.193405Z","iopub.status.idle":"2026-07-02T21:07:22.781713Z","shell.execute_reply.started":"2026-07-02T21:07:22.193372Z","shell.execute_reply":"2026-07-02T21:07:22.781032Z"}},"outputs":[],"execution_count":null},{"id":"f3d881fd-a56c-42ab-9201-e4f9b44f3cc6","cell_type":"code","source":"# Exemplos visuais de imagens do dataset\nsample_ds = load_dataset(TRAINING_FILENAMES, labeled=True).take(9)\nplt.figure(figsize=(10,10))\nfor i, (image, label) in enumerate(sample_ds):\n    plt.subplot(3,3,i+1)\n    plt.imshow(image.numpy())\n    plt.title(CLASSES[label.numpy()], fontsize=9)\n    plt.axis('off')\nplt.tight_layout()\nplt.savefig(\"sample_images.png\", dpi=120)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:07:25.212737Z","iopub.execute_input":"2026-07-02T21:07:25.213062Z","iopub.status.idle":"2026-07-02T21:07:27.402279Z","shell.execute_reply.started":"2026-07-02T21:07:25.213034Z","shell.execute_reply":"2026-07-02T21:07:27.401062Z"}},"outputs":[],"execution_count":null},{"id":"b5347873-4977-4f17-ae4d-1af4b47462cd","cell_type":"markdown","source":"### 4.2 Estatistica descritiva - intensidade de pixels\n\nComo as \"variaveis\" de um dataset de imagens sao os valores de pixel (0 a 1 apos normalizacao),\naqui calculamos media, mediana, moda, desvio padrao, quartis, minimo e maximo sobre uma amostra\nde imagens, tanto de forma geral quanto por canal de cor (R, G, B). Tambem calculamos o brilho\nmedio por imagem, que e uma boa medida resumo para comparar entre classes.","metadata":{}},{"id":"c539c59c-f980-4100-a774-df3de8ad034f","cell_type":"code","source":"from scipy import stats as scistats\n\n# Amostragem de imagens para estatisticas de pixel (evita estourar memoria com o dataset inteiro)\nN_SAMPLE = 500\nsample_images = []\nsample_labels = []\nfor image, label in load_dataset(TRAINING_FILENAMES, labeled=True).take(N_SAMPLE):\n    sample_images.append(image.numpy())\n    sample_labels.append(label.numpy())\nsample_images = np.array(sample_images)   # shape: (N, H, W, 3), valores em [0, 1]\nsample_labels = np.array(sample_labels)\n\nflat = sample_images.flatten()\nmode_val = scistats.mode(np.round(flat, 2))\n\npixel_stats = pd.DataFrame({\n    \"Estatistica\": [\"Media\", \"Mediana\", \"Moda (arred. 2 casas)\", \"Desvio Padrao\", \"Variancia\",\n                     \"Minimo\", \"Maximo\", \"Q1 (25%)\", \"Q3 (75%)\", \"Assimetria\", \"Curtose\"],\n    \"Valor\": [flat.mean(), np.median(flat), mode_val.mode[0], flat.std(), flat.var(),\n              flat.min(), flat.max(), np.percentile(flat, 25), np.percentile(flat, 75),\n              pd.Series(flat).skew(), pd.Series(flat).kurtosis()]\n})\npixel_stats[\"Valor\"] = pixel_stats[\"Valor\"].round(4)\nprint(f\"Estatistica descritiva de intensidade de pixel (amostra de {N_SAMPLE} imagens, {len(flat):,} pixels)\")\npixel_stats\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:07:37.234588Z","iopub.execute_input":"2026-07-02T21:07:37.234896Z","iopub.status.idle":"2026-07-02T21:07:45.520162Z","shell.execute_reply.started":"2026-07-02T21:07:37.234870Z","shell.execute_reply":"2026-07-02T21:07:45.519302Z"}},"outputs":[],"execution_count":null},{"id":"1ecaa59b-1305-4681-9721-19eb74097c20","cell_type":"code","source":"# Estatisticas por canal de cor (R, G, B)\nchannel_names = ['R (vermelho)', 'G (verde)', 'B (azul)']\nchannel_rows = []\nfor i, ch in enumerate(channel_names):\n    ch_data = sample_images[:, :, :, i].flatten()\n    ch_mode = scistats.mode(np.round(ch_data, 2))\n    channel_rows.append({\n        'Canal': ch,\n        'Media': round(ch_data.mean(), 4),\n        'Mediana': round(np.median(ch_data), 4),\n        'Moda': round(ch_mode.mode[0], 4),\n        'Desvio Padrao': round(ch_data.std(), 4),\n        'Minimo': round(ch_data.min(), 4),\n        'Maximo': round(ch_data.max(), 4)\n    })\nchannel_df = pd.DataFrame(channel_rows)\nchannel_df.to_csv(\"eda_pixel_channel_stats.csv\", index=False)\nprint(\"Estatistica descritiva por canal de cor\")\nchannel_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:08:41.012180Z","iopub.execute_input":"2026-07-02T21:08:41.012530Z","iopub.status.idle":"2026-07-02T21:08:45.274198Z","shell.execute_reply.started":"2026-07-02T21:08:41.012503Z","shell.execute_reply":"2026-07-02T21:08:45.273363Z"}},"outputs":[],"execution_count":null},{"id":"209e9fc8-f7cb-46f5-8219-ebba6a0b310b","cell_type":"code","source":"# Histograma da intensidade de pixel por canal\nplt.figure(figsize=(9,5))\ncolors = ['red', 'green', 'blue']\nfor i, (ch, color) in enumerate(zip(['R', 'G', 'B'], colors)):\n    ch_data = sample_images[:, :, :, i].flatten()\n    plt.hist(ch_data, bins=50, alpha=0.5, label=ch, color=color, density=True)\nplt.title(\"Distribuicao de intensidade de pixel por canal RGB\")\nplt.xlabel(\"Intensidade normalizada (0-1)\")\nplt.ylabel(\"Densidade\")\nplt.legend()\nplt.tight_layout()\nplt.savefig(\"pixel_channel_histogram.png\", dpi=120)\nplt.show()\n\n# Estatistica de brilho medio por imagem (media dos pixels de cada imagem)\nbrightness = sample_images.mean(axis=(1,2,3))\nbrightness_series = pd.Series(brightness)\nprint(\"Estatistica descritiva - brilho medio por imagem\")\nprint(brightness_series.describe())\nbrightness_mode = scistats.mode(np.round(brightness, 2))\nprint(f\"Moda do brilho (arred. 2 casas): {brightness_mode.mode[0]:.2f}\")\n\nplt.figure(figsize=(8,5))\nplt.hist(brightness, bins=30, edgecolor='black', alpha=0.7, color='gray')\nplt.axvline(brightness.mean(), color='red', linestyle='--', label=f'Media = {brightness.mean():.3f}')\nplt.axvline(np.median(brightness), color='green', linestyle='--', label=f'Mediana = {np.median(brightness):.3f}')\nplt.title(\"Distribuicao do brilho medio por imagem (amostra)\")\nplt.xlabel(\"Brilho medio (0-1)\")\nplt.ylabel(\"Frequencia\")\nplt.legend()\nplt.tight_layout()\nplt.savefig(\"brightness_histogram.png\", dpi=120)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T20:56:23.988455Z","iopub.execute_input":"2026-07-02T20:56:23.988779Z","iopub.status.idle":"2026-07-02T20:56:26.288881Z","shell.execute_reply.started":"2026-07-02T20:56:23.988751Z","shell.execute_reply":"2026-07-02T20:56:26.287977Z"}},"outputs":[],"execution_count":null},{"id":"13e79926-b09b-4c48-bc73-a90be4a1e725","cell_type":"markdown","source":"## 5. Modelagem: 3 arquiteturas via Transfer Learning\n\nVamos treinar e comparar:\n1. **EfficientNetB7** (ou **InceptionV3** como fallback automatico, caso a versao do TensorFlow do ambiente nao tenha o EfficientNet disponivel — o notebook detecta isso sozinho)\n2. **DenseNet201**\n3. **Xception**\n\nTodas pre-treinadas no ImageNet, com fine-tuning na camada final para as 104 classes.","metadata":{}},{"id":"2e5c1984-1130-4d5b-9537-31e36dcbb2d7","cell_type":"code","source":"def build_model(base_model_fn, name):\n    with strategy.scope():\n        base = base_model_fn(weights='imagenet', include_top=False, input_shape=[*IMAGE_SIZE, 3])\n        base.trainable = True  # fine-tuning completo\n        model = tf.keras.Sequential([\n            base,\n            tf.keras.layers.GlobalAveragePooling2D(),\n            tf.keras.layers.Dense(len(CLASSES), activation='softmax')\n        ], name=name)\n        model.compile(\n            optimizer=tf.keras.optimizers.Adam(learning_rate=0.00001),  # valor inicial; quem controla de fato e o LearningRateScheduler\n            loss='sparse_categorical_crossentropy',\n            metrics=['sparse_categorical_accuracy']\n        )\n    return model\n\ndef train_and_eval(model, name):\n    print(f\"\\n===== Treinando {name} =====\")\n\n    # Schedule de LR com warmup + decay exponencial (tecnica classica pra fine-tuning\n    # completo de redes pre-treinadas). Corrige o problema de acuracia baixa causado\n    # por LR constante destravando a rede toda de uma vez.\n    LR_START = 0.00001\n    LR_MAX = 0.0001 * strategy.num_replicas_in_sync\n    LR_MIN = 0.00001\n    LR_RAMPUP_EPOCHS = 5\n    LR_SUSTAIN_EPOCHS = 0\n    LR_EXP_DECAY = .8\n\n    def lrfn(epoch):\n        if epoch < LR_RAMPUP_EPOCHS:\n            lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n        elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n            lr = LR_MAX\n        else:\n            lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY ** (epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n        return lr\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=6, restore_best_weights=True)\n\n    history = model.fit(\n        get_training_dataset(),\n        steps_per_epoch=STEPS_PER_EPOCH,\n        epochs=EPOCHS,\n        validation_data=get_validation_dataset(),\n        callbacks=[lr_callback, early_stop]\n    )\n\n    # F1-score na validacao\n    val_ds = get_validation_dataset(ordered=True)\n    val_images_ds = val_ds.map(lambda image, label: image)\n    val_labels_ds = val_ds.map(lambda image, label: label).unbatch()\n    y_true = next(iter(val_labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy()\n\n    y_prob = model.predict(val_images_ds)\n    y_pred = np.argmax(y_prob, axis=-1)\n\n    f1 = f1_score(y_true, y_pred, average='macro')\n    print(f\"{name} - F1-macro (validacao): {f1:.4f}\")\n\n    return history, f1, y_true, y_pred\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T20:56:29.509072Z","iopub.execute_input":"2026-07-02T20:56:29.509376Z","iopub.status.idle":"2026-07-02T20:56:29.521702Z","shell.execute_reply.started":"2026-07-02T20:56:29.509349Z","shell.execute_reply":"2026-07-02T20:56:29.520854Z"}},"outputs":[],"execution_count":null},{"id":"14a7f431-a397-4e24-81b6-fc2764fef580","cell_type":"code","source":"from tensorflow.keras.applications import DenseNet201, Xception\n\n# EfficientNetB7 so existe em tensorflow.keras.applications a partir do TF 2.3.\n# Em ambientes Kaggle com TF mais antigo (como no erro reportado), fazemos fallback\n# para InceptionV3, que esta disponivel em qualquer versao do TF 1.x/2.x.\nfrom tensorflow.keras.applications import InceptionV3\nMODEL1_FN = InceptionV3\nMODEL1_NAME = \"InceptionV3\"\n\nprint(\"Modelo 1:\", MODEL1_NAME)\n\nresults = {}\nhistories = {}\n\nmodel_eff = build_model(MODEL1_FN, MODEL1_NAME)\nhist_eff, f1_eff, yt_eff, yp_eff = train_and_eval(model_eff, MODEL1_NAME)\nresults[MODEL1_NAME] = f1_eff\nhistories[MODEL1_NAME] = hist_eff\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:13:41.210798Z","iopub.execute_input":"2026-07-02T21:13:41.211111Z","iopub.status.idle":"2026-07-02T21:24:27.629722Z","shell.execute_reply.started":"2026-07-02T21:13:41.211084Z","shell.execute_reply":"2026-07-02T21:24:27.628827Z"}},"outputs":[],"execution_count":null},{"id":"fa908058-4e95-456c-8de1-77f7e53b40aa","cell_type":"code","source":"model_dense = build_model(DenseNet201, \"DenseNet201\")\nhist_dense, f1_dense, yt_dense, yp_dense = train_and_eval(model_dense, \"DenseNet201\")\nresults[\"DenseNet201\"] = f1_dense\nhistories[\"DenseNet201\"] = hist_dense\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:27:28.878619Z","iopub.execute_input":"2026-07-02T21:27:28.879066Z","iopub.status.idle":"2026-07-02T21:50:15.113857Z","shell.execute_reply.started":"2026-07-02T21:27:28.879020Z","shell.execute_reply":"2026-07-02T21:50:15.112897Z"}},"outputs":[],"execution_count":null},{"id":"baeff1dd-c165-455e-acb3-4864ec342500","cell_type":"code","source":"model_xcep = build_model(Xception, \"Xception\")\nhist_xcep, f1_xcep, yt_xcep, yp_xcep = train_and_eval(model_xcep, \"Xception\")\nresults[\"Xception\"] = f1_xcep\nhistories[\"Xception\"] = hist_xcep\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T21:50:29.538493Z","iopub.execute_input":"2026-07-02T21:50:29.538822Z","iopub.status.idle":"2026-07-02T22:15:00.845535Z","shell.execute_reply.started":"2026-07-02T21:50:29.538794Z","shell.execute_reply":"2026-07-02T22:15:00.844473Z"}},"outputs":[],"execution_count":null},{"id":"4696dd05-2f95-46b6-9a4b-dae99e544d3c","cell_type":"markdown","source":"## 6. Comparacao dos resultados","metadata":{}},{"id":"185fecac-3dd2-4ee3-9199-8ea924b80e0a","cell_type":"code","source":"results_df = pd.DataFrame(list(results.items()), columns=[\"Modelo\", \"F1-macro (val)\"]).sort_values(\"F1-macro (val)\", ascending=False)\nprint(results_df)\n\nplt.figure(figsize=(7,4))\nplt.bar(results_df[\"Modelo\"], results_df[\"F1-macro (val)\"])\nplt.title(\"Comparacao de F1-macro entre os modelos (validacao)\")\nplt.ylabel(\"F1-macro\")\nplt.tight_layout()\nplt.savefig(\"model_comparison.png\", dpi=120)\nplt.show()\n\n# Curvas de treino/validacao do melhor modelo\nbest_name = results_df.iloc[0][\"Modelo\"]\nbest_history = histories[best_name]\n\nplt.figure(figsize=(10,4))\nplt.subplot(1,2,1)\nplt.plot(best_history.history['loss'], label='train')\nplt.plot(best_history.history['val_loss'], label='val')\nplt.title(f\"Loss - {best_name}\")\nplt.legend()\n\nplt.subplot(1,2,2)\nplt.plot(best_history.history['sparse_categorical_accuracy'], label='train')\nplt.plot(best_history.history['val_sparse_categorical_accuracy'], label='val')\nplt.title(f\"Accuracy - {best_name}\")\nplt.legend()\nplt.tight_layout()\nplt.savefig(\"best_model_curves.png\", dpi=120)\nplt.show()\n\nprint(f\"\\nMelhor modelo: {best_name} (F1-macro = {results_df.iloc[0]['F1-macro (val)']:.4f})\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T22:15:21.651148Z","iopub.execute_input":"2026-07-02T22:15:21.651476Z","iopub.status.idle":"2026-07-02T22:15:22.273655Z","shell.execute_reply.started":"2026-07-02T22:15:21.651450Z","shell.execute_reply":"2026-07-02T22:15:22.272833Z"}},"outputs":[],"execution_count":null},{"id":"272aaf4f-f633-4164-b62e-ba0a4e1643d3","cell_type":"markdown","source":"## 7. Geracao da submissao (usando o melhor modelo)","metadata":{}},{"id":"92e737e2-93de-4af4-b7eb-71387e939c5e","cell_type":"code","source":"best_model = {MODEL1_NAME: model_eff, \"DenseNet201\": model_dense, \"Xception\": model_xcep}[best_name]\n\ntest_ds = get_test_dataset(ordered=True)\ntest_images_ds = test_ds.map(lambda image, idnum: image)\ntest_ids_ds = test_ds.map(lambda image, idnum: idnum).unbatch()\n\ntest_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES))).numpy().astype('U')\n\nprobs = best_model.predict(test_images_ds)\npreds = np.argmax(probs, axis=-1)\n\nsubmission = pd.DataFrame({'id': test_ids, 'label': preds})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-02T22:17:32.072113Z","iopub.execute_input":"2026-07-02T22:17:32.072501Z","iopub.status.idle":"2026-07-02T22:18:15.870845Z","shell.execute_reply.started":"2026-07-02T22:17:32.072470Z","shell.execute_reply":"2026-07-02T22:18:15.870082Z"}},"outputs":[],"execution_count":null}]}