{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\nimport os\nimport glob\n\nfrom tqdm import tqdm\nimport skimage\nimport PIL\n\nfrom sklearn.model_selection import train_test_split","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a validation, and new test set from the training set","metadata":{}},{"cell_type":"code","source":"# Load the training CSV file\ntrain_data_df = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/train.csv')\n\n# Perform stratified sampling to create the new training, validation, and internal testing sets\ntrain_data_df, temp_data_df = train_test_split(train_data_df, test_size=0.3, stratify=train_data_df['target'], random_state=42)\n\nvalidation_data_df, internal_test_data_df = train_test_split(temp_data_df, test_size=0.5, stratify=temp_data_df['target'], random_state=42)\n\n# Save the updated DataFrames as new .csv files if needed\ntrain_data_df.to_csv('new_train_data.csv', index=False)\nvalidation_data_df.to_csv('validation_data.csv', index=False)\ninternal_test_data_df.to_csv('internal_test_data.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-08T02:29:24.422756Z","iopub.execute_input":"2023-05-08T02:29:24.423737Z","iopub.status.idle":"2023-05-08T02:29:24.664300Z","shell.execute_reply.started":"2023-05-08T02:29:24.423692Z","shell.execute_reply":"2023-05-08T02:29:24.662995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Calculate class distribution of training set","metadata":{}},{"cell_type":"code","source":"# Load the CSV files\ntrain_data_df = pd.read_csv('/kaggle/working/new_train_data.csv')\nvalidation_data_df = pd.read_csv('/kaggle/working/validation_data.csv')\ntest_data_df = pd.read_csv('/kaggle/working/internal_test_data.csv')\n\n# Calculate the class distribution in the training set\ntrain_class_distribution = train_data_df['target'].value_counts(normalize=True)\n\n# Calculate the class distribution in the validation set\nvalidation_class_distribution = validation_data_df['target'].value_counts(normalize=True)\n\n# Calculate the class distribution in the test set\ntest_class_distribution = test_data_df['target'].value_counts(normalize=True)\n\n# Print the class distribution (there are no labels provided in the test set)\nprint(\"Training set class distribution:\\n\", train_class_distribution)\nprint(\"Validation set class distribution:\\n\", validation_class_distribution)\nprint(\"Test set class distribution:\\n\", test_class_distribution)","metadata":{},"execution_count":null,"outputs":[]}]}