{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":12631,"databundleVersionId":682729,"sourceType":"competition"},{"sourceId":9858939,"sourceType":"datasetVersion","datasetId":6050405}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from fastai.data.all import *\nfrom fastai.vision.all import *\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport os\nimport torch\nimport zipfile\n\nprint(\"Libraries imported.\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:27:06.736397Z","iopub.execute_input":"2024-11-10T04:27:06.736862Z","iopub.status.idle":"2024-11-10T04:27:18.753954Z","shell.execute_reply.started":"2024-11-10T04:27:06.736809Z","shell.execute_reply":"2024-11-10T04:27:18.753034Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imgs_zip_path = \"../input/hbku2019/imgs.zip\"\nlabels_zip_path = \"../input/hbku2019/labels.zip\"\n\nsubmission_path = \"./submission.csv\"\n\nimgs_extracted_path = \"./\"\nlabels_extracted_path = \"./\"\n\ntrain_img_path = \"./imgs/train/\"\ntest_img_path = \"./imgs/test/\"\n\ncategories_csv_path = \"./labels/categories.csv\"\nlabels_train_csv_path = \"./labels/labels_train.csv\"\n\nprint(\"Constants defined.\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:27:18.755999Z","iopub.execute_input":"2024-11-10T04:27:18.756395Z","iopub.status.idle":"2024-11-10T04:27:18.762412Z","shell.execute_reply.started":"2024-11-10T04:27:18.75635Z","shell.execute_reply":"2024-11-10T04:27:18.761482Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_folder(zip_path, dest_folder):\n    with zipfile.ZipFile(zip_path, \"r\") as zip_ref:\n        namelist = zip_ref.namelist()\n        print(f'The number of files to extract: {len(namelist)}.')\n\n        # tqdm throws an error, that's why the ugly solution\n        for i, file in enumerate(namelist):\n            zip_ref.extract(member=file, path=dest_folder)\n            if i % 5000 == 0:\n                print(i)\n                \nprint(\"Extract images:\")\nextract_folder(imgs_zip_path, imgs_extracted_path)\nprint(\"DONE\")\n\nprint(\"Extract labels:\")\nextract_folder(labels_zip_path, labels_extracted_path)\nprint(\"DONE\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:27:18.7638Z","iopub.execute_input":"2024-11-10T04:27:18.764098Z","iopub.status.idle":"2024-11-10T04:28:42.685678Z","shell.execute_reply.started":"2024-11-10T04:27:18.764066Z","shell.execute_reply":"2024-11-10T04:28:42.684552Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels_train_df = pd.read_csv(labels_train_csv_path, header=None)\nlabels_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:42.688379Z","iopub.execute_input":"2024-11-10T04:28:42.688989Z","iopub.status.idle":"2024-11-10T04:28:43.795759Z","shell.execute_reply.started":"2024-11-10T04:28:42.688945Z","shell.execute_reply":"2024-11-10T04:28:43.794816Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labels_train_df.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T04:28:43.797446Z","iopub.execute_input":"2024-11-10T04:28:43.79822Z","iopub.status.idle":"2024-11-10T04:28:44.861211Z","shell.execute_reply.started":"2024-11-10T04:28:43.798165Z","shell.execute_reply":"2024-11-10T04:28:44.860186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(labels_train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:44.86245Z","iopub.execute_input":"2024-11-10T04:28:44.862864Z","iopub.status.idle":"2024-11-10T04:28:44.868429Z","shell.execute_reply.started":"2024-11-10T04:28:44.862816Z","shell.execute_reply":"2024-11-10T04:28:44.867316Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(os.listdir(train_img_path)))","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:44.869505Z","iopub.execute_input":"2024-11-10T04:28:44.869868Z","iopub.status.idle":"2024-11-10T04:28:44.950947Z","shell.execute_reply.started":"2024-11-10T04:28:44.869828Z","shell.execute_reply":"2024-11-10T04:28:44.949961Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categories_df = pd.read_csv(categories_csv_path, header=None)\ncategories_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:44.952258Z","iopub.execute_input":"2024-11-10T04:28:44.952637Z","iopub.status.idle":"2024-11-10T04:28:44.963183Z","shell.execute_reply.started":"2024-11-10T04:28:44.952596Z","shell.execute_reply":"2024-11-10T04:28:44.962167Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(categories_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:44.964415Z","iopub.execute_input":"2024-11-10T04:28:44.964763Z","iopub.status.idle":"2024-11-10T04:28:44.975016Z","shell.execute_reply.started":"2024-11-10T04:28:44.964724Z","shell.execute_reply":"2024-11-10T04:28:44.974081Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vocab = categories_df[0].tolist()\nprint(vocab)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:44.979597Z","iopub.execute_input":"2024-11-10T04:28:44.980234Z","iopub.status.idle":"2024-11-10T04:28:44.986931Z","shell.execute_reply.started":"2024-11-10T04:28:44.980191Z","shell.execute_reply":"2024-11-10T04:28:44.985808Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(len(os.listdir(test_img_path)))","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:44.98803Z","iopub.execute_input":"2024-11-10T04:28:44.988394Z","iopub.status.idle":"2024-11-10T04:28:45.016595Z","shell.execute_reply.started":"2024-11-10T04:28:44.988363Z","shell.execute_reply":"2024-11-10T04:28:45.015605Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dblock = DataBlock(\n    blocks=(ImageBlock, MultiCategoryBlock(encoded=True, vocab=vocab)),\n    get_x=lambda df: train_img_path + '/' + df[0],\n    get_y=lambda df: df[1:].values.astype(int),\n    item_tfms=RandomResizedCrop(128, min_scale=0.35)\n)\n\nprint(\"Datablock is ready.\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:45.0178Z","iopub.execute_input":"2024-11-10T04:28:45.018185Z","iopub.status.idle":"2024-11-10T04:28:45.027057Z","shell.execute_reply.started":"2024-11-10T04:28:45.018116Z","shell.execute_reply":"2024-11-10T04:28:45.02601Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ds = dblock.datasets(labels_train_df.iloc[[1]])\n\nx, y = ds.train[0]\n\nprint(x)\nprint('--')\nprint(y)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:45.028663Z","iopub.execute_input":"2024-11-10T04:28:45.029012Z","iopub.status.idle":"2024-11-10T04:28:45.269934Z","shell.execute_reply.started":"2024-11-10T04:28:45.028972Z","shell.execute_reply":"2024-11-10T04:28:45.268819Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dls = dblock.dataloaders(labels_train_df)\ndls.show_batch(nrows=3, ncols=3)\n\nprint(\"Dataloaders created.\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:45.271407Z","iopub.execute_input":"2024-11-10T04:28:45.271811Z","iopub.status.idle":"2024-11-10T04:28:48.030806Z","shell.execute_reply.started":"2024-11-10T04:28:45.271765Z","shell.execute_reply":"2024-11-10T04:28:48.029659Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learner = cnn_learner(dls, resnet50)\nlearner.fine_tune(5, base_lr=3e-3, freeze_epochs=6)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T04:28:48.032028Z","iopub.execute_input":"2024-11-10T04:28:48.032373Z","iopub.status.idle":"2024-11-10T05:20:46.493359Z","shell.execute_reply.started":"2024-11-10T04:28:48.032339Z","shell.execute_reply":"2024-11-10T05:20:46.492327Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"valids, targs = learner.get_preds() # predict from the validation set\n\nprint(valids)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:20:46.495Z","iopub.execute_input":"2024-11-10T05:20:46.495422Z","iopub.status.idle":"2024-11-10T05:21:33.139574Z","shell.execute_reply.started":"2024-11-10T05:20:46.495383Z","shell.execute_reply":"2024-11-10T05:21:33.138547Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"valids.shape","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:21:33.140908Z","iopub.execute_input":"2024-11-10T05:21:33.141249Z","iopub.status.idle":"2024-11-10T05:21:33.147589Z","shell.execute_reply.started":"2024-11-10T05:21:33.141215Z","shell.execute_reply":"2024-11-10T05:21:33.146516Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xs = torch.linspace(0.05,0.95,29)\naccs = [accuracy_multi(valids, targs, thresh=i, sigmoid=False) for i in xs]\nplt.plot(xs,accs);","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:21:33.148932Z","iopub.execute_input":"2024-11-10T05:21:33.14926Z","iopub.status.idle":"2024-11-10T05:21:33.472961Z","shell.execute_reply.started":"2024-11-10T05:21:33.149224Z","shell.execute_reply":"2024-11-10T05:21:33.472067Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = 0.5","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:21:33.474444Z","iopub.execute_input":"2024-11-10T05:21:33.474836Z","iopub.status.idle":"2024-11-10T05:21:33.479117Z","shell.execute_reply.started":"2024-11-10T05:21:33.474792Z","shell.execute_reply":"2024-11-10T05:21:33.478151Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_imgs = get_image_files(test_img_path)\n\nprint(\"Test images loaded.\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:21:33.480426Z","iopub.execute_input":"2024-11-10T05:21:33.480791Z","iopub.status.idle":"2024-11-10T05:21:33.684568Z","shell.execute_reply.started":"2024-11-10T05:21:33.48075Z","shell.execute_reply":"2024-11-10T05:21:33.683625Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataloader = learner.dls.test_dl(test_imgs)\npreds, _ = learner.get_preds(dl=test_dataloader)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:21:33.685701Z","iopub.execute_input":"2024-11-10T05:21:33.686006Z","iopub.status.idle":"2024-11-10T05:22:23.804435Z","shell.execute_reply.started":"2024-11-10T05:21:33.685973Z","shell.execute_reply":"2024-11-10T05:22:23.803478Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(preds)","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:22:23.80646Z","iopub.execute_input":"2024-11-10T05:22:23.806866Z","iopub.status.idle":"2024-11-10T05:22:23.814384Z","shell.execute_reply.started":"2024-11-10T05:22:23.806821Z","shell.execute_reply":"2024-11-10T05:22:23.813574Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predicted_classes = [\"\" for i in range(len(test_imgs))]\n\n# i guess it's not the most pythonic, please advise me in the comments how to improve\nfor i, j in torch.nonzero(preds > threshold):\n    predicted_classes[i] += str(j.item())\n    predicted_classes[i] += \" \"\n\ndf = pd.DataFrame()\n\ndf[\"id\"] = [os.path.basename(test_img) for test_img in test_imgs]\ndf[\"values\"] = predicted_classes\n\ndf.info()\n\ndf.to_csv(submission_path, header=[\"id\", \"predictions\"], index=False)\n\nprint(\"Predictions saved to file.\")","metadata":{"execution":{"iopub.status.busy":"2024-11-10T05:22:23.815896Z","iopub.execute_input":"2024-11-10T05:22:23.816263Z","iopub.status.idle":"2024-11-10T05:22:24.660954Z","shell.execute_reply.started":"2024-11-10T05:22:23.816222Z","shell.execute_reply":"2024-11-10T05:22:24.659895Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.learner import load_learner\n\n# Load model\nlearn = load_learner('export.pkl')\n\n# Predict ảnh mới\nimg = PILImage.create('/kaggle/input/abcxyz/Zack_Greinke_on_July_29_2009.jpg')\npred, pred_idx, probs = learn.predict(img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T06:25:03.914817Z","iopub.execute_input":"2024-11-10T06:25:03.915244Z","iopub.status.idle":"2024-11-10T06:25:04.351936Z","shell.execute_reply.started":"2024-11-10T06:25:03.915204Z","shell.execute_reply":"2024-11-10T06:25:04.351078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Định nghĩa các hàm get_x và get_y thông thường thay vì lambda\ndef get_x(row):\n    return train_img_path + '/' + row[0]\n\ndef get_y(row):\n    return row[1:].values.astype(int)\n\n# Tạo lại DataBlock với các hàm thông thường\ndblock = DataBlock(\n    blocks=(ImageBlock, MultiCategoryBlock(encoded=True, vocab=vocab)),\n    get_x=get_x,\n    get_y=get_y,\n    item_tfms=RandomResizedCrop(128, min_scale=0.35)\n)\n\n# Tạo lại dataloaders\ndls = dblock.dataloaders(labels_train_df)\n\n# Tạo và train model như cũ\nlearner = cnn_learner(dls, resnet50)\nlearner.fine_tune(5, base_lr=3e-3, freeze_epochs=6)\n\n# Giờ bạn có thể export model\nlearner.export('export.pkl')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T05:29:42.070279Z","iopub.execute_input":"2024-11-10T05:29:42.07118Z","iopub.status.idle":"2024-11-10T06:22:57.439217Z","shell.execute_reply.started":"2024-11-10T05:29:42.071107Z","shell.execute_reply":"2024-11-10T06:22:57.438196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.learner import load_learner\nfrom fastai.vision.all import PILImage\nimport matplotlib.pyplot as plt\n\n# Load model\nlearn = load_learner('export.pkl')\n\n# Predict ảnh mới\nimg = PILImage.create('/kaggle/input/aaaaaaaaaaa/test.jpg')\npred, pred_idx, probs = learn.predict(img)\n\n# In kết quả\nprint(\"Predicted labels:\", pred)\nprint(\"\\nProbabilities for each class:\")\nfor label, prob in zip(learn.dls.vocab, probs):\n    if prob > 0.5:  # Chỉ in những label có probability > 0.5\n        print(f\"{label}: {prob:.3f}\")\n\n# Hiển thị ảnh\nplt.imshow(img)\nplt.axis('off')  # Ẩn các trục của ảnh\nplt.title(\"Input Image\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T06:53:16.562014Z","iopub.execute_input":"2024-11-10T06:53:16.562913Z","iopub.status.idle":"2024-11-10T06:53:17.182832Z","shell.execute_reply.started":"2024-11-10T06:53:16.562866Z","shell.execute_reply":"2024-11-10T06:53:17.181895Z"}},"outputs":[],"execution_count":null}]}