{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":4104,"databundleVersionId":46661,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import tensorflow as tf\n\n# Basic libraries\nimport random\nimport math\nimport numpy as np\nfrom PIL import Image\n\n# TensorFlow/Keras for deep learning\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, LeakyReLU, Input\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\n\n# Scikit-learn for train-test split\nfrom sklearn.model_selection import train_test_split\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport cv2\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n# This just prints all the content in the input\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-08-31T19:15:52.684339Z","iopub.execute_input":"2024-08-31T19:15:52.685308Z","iopub.status.idle":"2024-08-31T19:16:05.476415Z","shell.execute_reply.started":"2024-08-31T19:15:52.685263Z","shell.execute_reply":"2024-08-31T19:16:05.47554Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:16:11.523198Z","iopub.execute_input":"2024-08-31T19:16:11.523821Z","iopub.status.idle":"2024-08-31T19:16:11.72839Z","shell.execute_reply.started":"2024-08-31T19:16:11.523791Z","shell.execute_reply":"2024-08-31T19:16:11.727336Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_image_rgb(image_path, target_size=(256, 256), display=False): \n    original_image = cv2.imread(image_path) \n    resized_image = cv2.resize(original_image, target_size) \n    enhanced_image = resized_image \n    # Split the image into its RGB channels \n    b, g, r = cv2.split(resized_image) \n \n    # Apply Gaussian blur to each channel separately \n    blurred_b = cv2.GaussianBlur(b, (5, 5), 0) \n    blurred_g = cv2.GaussianBlur(g, (5, 5), 0) \n    blurred_r = cv2.GaussianBlur(r, (5, 5), 0) \n \n    # Merge the channels back into an RGB image \n    blurred_image = cv2.merge([blurred_b, blurred_g, blurred_r]) \n \n    # Apply contrast enhancement (CLAHE) to each channel separately \n    clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8)) \n    enhanced_b = clahe.apply(blurred_b) \n    enhanced_g = clahe.apply(blurred_g) \n    enhanced_r = clahe.apply(blurred_r) \n \n    # Merge the channels back into an RGB image \n    enhanced_image = cv2.merge([enhanced_b, enhanced_g, enhanced_r]) \n    if display: \n        print(image_path) \n        # Display the images \n        plt.figure(figsize=(12, 6)) \n \n        plt.subplot(1, 3, 1) \n        plt.imshow(cv2.cvtColor(original_image, cv2.COLOR_BGR2RGB)) \n        plt.title('Original Image') \n \n        plt.subplot(1, 3, 2) \n        plt.imshow(cv2.cvtColor(blurred_image, cv2.COLOR_BGR2RGB)) \n        plt.title('Blurred Image') \n \n        plt.subplot(1, 3, 3) \n        plt.imshow(cv2.cvtColor(enhanced_image, cv2.COLOR_BGR2RGB)) \n        plt.title('Enhanced Image') \n \n        plt.show() \n    image_array = np.array(enhanced_image) \n    image_array = image_array / 255.0 \n    return image_array\n","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:16:14.552478Z","iopub.execute_input":"2024-08-31T19:16:14.553164Z","iopub.status.idle":"2024-08-31T19:16:14.564002Z","shell.execute_reply.started":"2024-08-31T19:16:14.553131Z","shell.execute_reply":"2024-08-31T19:16:14.563085Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cv2_imshow(img):\n    plt.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:16:19.165903Z","iopub.execute_input":"2024-08-31T19:16:19.166587Z","iopub.status.idle":"2024-08-31T19:16:19.170802Z","shell.execute_reply.started":"2024-08-31T19:16:19.166547Z","shell.execute_reply":"2024-08-31T19:16:19.169847Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This is to get all the images\n!7z x /kaggle/input/diabetic-retinopathy-detection/train.zip.001","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:16:20.949215Z","iopub.execute_input":"2024-08-31T19:16:20.949842Z","iopub.status.idle":"2024-08-31T19:17:03.961531Z","shell.execute_reply.started":"2024-08-31T19:16:20.94981Z","shell.execute_reply":"2024-08-31T19:17:03.960636Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This is to get the CSV File\n!unzip /kaggle/input/diabetic-retinopathy-detection/trainLabels.csv.zip","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:17:10.731306Z","iopub.execute_input":"2024-08-31T19:17:10.732008Z","iopub.status.idle":"2024-08-31T19:17:11.742032Z","shell.execute_reply.started":"2024-08-31T19:17:10.731969Z","shell.execute_reply":"2024-08-31T19:17:11.740855Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Add the image path to the CSV File\n\ntrain_labels = pd.read_csv(\"/kaggle/working/trainLabels.csv\").copy()\ntrain_labels[\"image_path\"] = train_labels[\"image\"].apply(lambda x : os.path.join(\"/kaggle/working/train\", f\"{x}.jpeg\")).copy()\ntrain_labels.head()","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:17:13.304407Z","iopub.execute_input":"2024-08-31T19:17:13.304777Z","iopub.status.idle":"2024-08-31T19:17:13.434941Z","shell.execute_reply.started":"2024-08-31T19:17:13.304748Z","shell.execute_reply":"2024-08-31T19:17:13.434048Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This is to mainly check if the image exists\n\ndef image_exists(image, option = False):\n    filepath = f\"/kaggle/working/train/{image}.jpeg\"\n    if (option == True):\n        print(filepath)\n    return os.path.isfile(filepath)","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:17:18.255224Z","iopub.execute_input":"2024-08-31T19:17:18.255931Z","iopub.status.idle":"2024-08-31T19:17:18.260711Z","shell.execute_reply.started":"2024-08-31T19:17:18.255899Z","shell.execute_reply":"2024-08-31T19:17:18.259728Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Boolean values are choosen to fill the train_labels list\nimage_labels_train = train_labels[train_labels[\"image\"].apply(lambda x : image_exists(x))]\nprint(image_labels_train.shape)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:17:21.587179Z","iopub.execute_input":"2024-08-31T19:17:21.587958Z","iopub.status.idle":"2024-08-31T19:17:21.83143Z","shell.execute_reply.started":"2024-08-31T19:17:21.587928Z","shell.execute_reply":"2024-08-31T19:17:21.830546Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"temp = pd.DataFrame(image_labels_train)\ntemp.groupby('level').count()","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:17:24.888742Z","iopub.execute_input":"2024-08-31T19:17:24.889086Z","iopub.status.idle":"2024-08-31T19:17:24.903396Z","shell.execute_reply.started":"2024-08-31T19:17:24.889063Z","shell.execute_reply":"2024-08-31T19:17:24.90258Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image_labels_train.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-08-31T03:46:16.205648Z","iopub.execute_input":"2024-08-31T03:46:16.206037Z","iopub.status.idle":"2024-08-31T03:46:16.21691Z","shell.execute_reply.started":"2024-08-31T03:46:16.206009Z","shell.execute_reply":"2024-08-31T03:46:16.215791Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This function is goated\n\ndef load_data(df):\n    images = []\n    labels = []\n    for _, row in df.iterrows():\n        image = preprocess_image_rgb(row['image_path'], display=False)\n        images.append(image)\n        labels.append(row['level'])\n\n    images = np.array(images) \n    labels = np.array(labels)\n    \n    return images, labels\n\n# Load training and testing data\ntrain_images, train_labels = load_data(image_labels_train)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:17:32.109434Z","iopub.execute_input":"2024-08-31T19:17:32.109809Z","iopub.status.idle":"2024-08-31T19:19:13.156477Z","shell.execute_reply.started":"2024-08-31T19:17:32.109782Z","shell.execute_reply":"2024-08-31T19:19:13.155396Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_labels.shape\ntrain_images.shape","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:19:15.284237Z","iopub.execute_input":"2024-08-31T19:19:15.284958Z","iopub.status.idle":"2024-08-31T19:19:15.290792Z","shell.execute_reply.started":"2024-08-31T19:19:15.284926Z","shell.execute_reply":"2024-08-31T19:19:15.289897Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Adding Class weights to help the skew effect\n\nfrom sklearn.utils import class_weight\n\nclass_weights = class_weight.compute_class_weight('balanced',classes= np.unique(image_labels_train['level']), y=image_labels_train['level'])\n\n# Convert class weights to dictionary\nclass_weights = dict(enumerate(class_weights))\n\nclass_weights\n\n# Dont add weights as of yet, it's fudgin up the results","metadata":{"execution":{"iopub.status.busy":"2024-08-31T05:10:05.839657Z","iopub.execute_input":"2024-08-31T05:10:05.840281Z","iopub.status.idle":"2024-08-31T05:10:05.84914Z","shell.execute_reply.started":"2024-08-31T05:10:05.840249Z","shell.execute_reply":"2024-08-31T05:10:05.848207Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#always use datagen for image data, it's so much easier\n\ndatagen = ImageDataGenerator(\n    rotation_range=10,\n#     width_shift_range=0.2,\n#     height_shift_range=0.2,\n#     shear_range=0.2,\n    zoom_range=0.1,\n    horizontal_flip=True,\n    fill_mode='nearest',\n    validation_split=0.2\n)\n\ndatagen.fit(train_images)\n\ntrain_data = datagen.flow(train_images,train_labels,batch_size=8,subset='training')\nval_data = datagen.flow(train_images,train_labels,batch_size=8,subset='validation')\n\n'''\nhistory = model.fit(train_data,validation_data=val_data,epochs=50,callbacks=10,verbose=0) \nYou can use the above technique, if you have datagen\n\n\nUse the bottom technique, if you want to not use datagen\nhistory = model.fit(train_images,train_labels,epochs=10,batch_size=32, validation_split=0.2)\n'''\n\n# train_data and val_data","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:19:21.045926Z","iopub.execute_input":"2024-08-31T19:19:21.046565Z","iopub.status.idle":"2024-08-31T19:19:22.926661Z","shell.execute_reply.started":"2024-08-31T19:19:21.046527Z","shell.execute_reply":"2024-08-31T19:19:22.925681Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This is the complicated model\n\ndef create_model(params): \n   \n    # input_shape = (256, 256, 3) \n    # Layer 1 \n    model = Sequential()\n    model.add(Input(shape=(256, 256, 3)))\n    # Adding an input layer explicitly will help\n    \n    model.add(Conv2D(params['num_filters'], (3, 3), activation=None, padding='same')) \n    model.add(LeakyReLU()) \n    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2))) \n \n    # Layer 2 \n    model.add(Conv2D(2*params['num_filters'], (3, 3), activation=None, padding='same')) \n    model.add(Dropout(0.1)) \n    model.add(LeakyReLU()) \n    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2))) \n \n    # Layer 3 \n    model.add(Conv2D(2*params['num_filters'], (3, 3), activation=None, padding='same')) \n    model.add(Dropout(0.1)) \n    model.add(LeakyReLU()) \n    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2))) \n \n    # Layer 4 \n    model.add(Conv2D(4*params['num_filters'], (3, 3), activation=None, padding='same')) \n    model.add(Dropout(0.1)) \n    model.add(LeakyReLU()) \n    model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2)))\n    \n    \n    model.add(Flatten()) \n    \n    for _ in range(params['num_dense_layers']): \n        model.add(Dense(2048, activation=None, kernel_regularizer = tf.keras.regularizers.l2( 0.01))) \n        model.add(Dropout(params['dropout_rate'])) \n        model.add(LeakyReLU()) \n    \n    # Output Layer \n    \n    # We are adding 5 dense layers for the classification problem\n    \n    model.add(Dense(5, activation='softmax')) \n    optimizer = tf.keras.optimizers.SGD(learning_rate=params['learning_rate'], momentum=0.9) \n    model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) \n    model.batch_size = params['batch_size']\n\n    \n    return model\n","metadata":{"execution":{"iopub.status.busy":"2024-08-31T03:57:24.288495Z","iopub.execute_input":"2024-08-31T03:57:24.288886Z","iopub.status.idle":"2024-08-31T03:57:24.30298Z","shell.execute_reply.started":"2024-08-31T03:57:24.288856Z","shell.execute_reply":"2024-08-31T03:57:24.301862Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This is a small model\n\ndef create_model(params): \n    model = Sequential() \n    model.add(Input(shape=(256, 256, 3)))\n    # Layer 1 \n    model.add(Conv2D(params['num_filters'], (3, 3), activation=None, padding='same')) \n    model.add(LeakyReLU()) \n    model.add(MaxPooling2D(pool_size=(2, 2))) \n \n    # Layer 2 \n    model.add(Conv2D(2*params['num_filters'], (3, 3), activation=None, padding='same')) \n    model.add(Dropout(0.1)) \n    model.add(LeakyReLU()) \n    model.add(MaxPooling2D(pool_size=(2, 2)))\n \n    # Layer 3 \n    model.add(Conv2D(2*params['num_filters'], (3, 3), activation=None, padding='same')) \n    model.add(Dropout(0.1)) \n    model.add(LeakyReLU()) \n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    \n    model.add(Flatten())\n    \n    for _ in range(params['num_dense_layers']): \n        model.add(Dense(128, activation='relu')) \n        model.add(Dropout(params['dropout_rate'])) \n    \n    # Output Layer \n    model.add(Dense(5, activation='softmax')) \n    optimizer = tf.keras.optimizers.SGD(learning_rate=params['learning_rate'], momentum=0.9) \n    model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) \n    model.batch_size = params['batch_size']\n    return model ","metadata":{"execution":{"iopub.status.busy":"2024-08-31T19:19:30.683072Z","iopub.execute_input":"2024-08-31T19:19:30.68395Z","iopub.status.idle":"2024-08-31T19:19:30.694315Z","shell.execute_reply.started":"2024-08-31T19:19:30.683917Z","shell.execute_reply":"2024-08-31T19:19:30.693365Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This function is just for your reference purpose\nparams = {\n            'learning_rate': 10**random.uniform(-3, -1),  \n            'batch_size': random.randint(8,64), \n            'num_filters': random.randint(32,128),\n            'dropout_rate': random.uniform(0.2, 0.8),\n            'num_dense_layers': random.randint(1,3)  \n        }\n\n\nmodel = create_model(params)\nhistory = model.fit(train_data,validation_data=val_data,epochs=5)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-14T13:32:20.529555Z","iopub.execute_input":"2024-08-14T13:32:20.529916Z","iopub.status.idle":"2024-08-14T13:32:20.567619Z","shell.execute_reply.started":"2024-08-14T13:32:20.529888Z","shell.execute_reply":"2024-08-14T13:32:20.566172Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_hyperparams = jade_hyperparameter_tuning()\n\nprint('Best Hyperparameters: ', best_hyperparams)\n\nmodel = create_model(best_hyperparams)\n\nhistory = model.fit(train_data,validation_data=val_data,epochs=50,callbacks=call)\n                    \ntraining_loss = history.history['loss'][1:]\ntraining_accuracy = history.history['accuracy'][1:]\nvalidation_loss = history.history['val_loss'][1:]\nvalidation_accuracy = history.history['val_accuracy'][1:]\nprint(validation_accuracy)\n                    \n# Plot training and validation loss\nplt.figure(figsize=(12, 5))\nplt.subplot(1, 2, 1)\nplt.plot(training_loss, label='Training Loss')\nplt.plot(validation_loss, label='Validation Loss')\nplt.title('Training and Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\n                    \n# Plot training and validation accuracy\nplt.subplot(1, 2, 2)\nplt.plot(training_accuracy, label='Training Accuracy')\nplt.plot(validation_accuracy, label='Validation Accuracy')\nplt.title('Training and Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2024-08-31T05:35:39.904126Z","iopub.execute_input":"2024-08-31T05:35:39.905133Z","iopub.status.idle":"2024-08-31T05:36:10.122085Z","shell.execute_reply.started":"2024-08-31T05:35:39.905096Z","shell.execute_reply":"2024-08-31T05:36:10.120656Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# You have to create this df_metamodel csv file\n","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clip_params(params):\n    # Define parameter bounds\n    bounds = {\n        'learning_rate': {'min': 10**(-3), 'max': 10**(-1)},\n        'batch_size': {'min': 16, 'max': 64},\n        'num_filters': {'min': 32, 'max': 128},\n        'num_dense_layers': {'min': 1, 'max': 3}\n    }\n    # Clip the parameters to the specified bounds\n    clipped_params = {\n        param: max(bounds[param]['min'], min(params[param], bounds[param]['max']))\n        for param in params\n    }\n    return clipped_params\n           \n# This is used to retrain your meta model\ndef retrain_meta(params):\n    model = create_model(params)\n    history = model.fit(train_data,validation_data=val_data,epochs=50,callbacks=10,verbose=0) \n    print(history.history['accuracy'][-1])\n    params.accuracy = (np.max(history.history['accuracy']))\n    df_metamodel.append(params)\n    X = df_metamodel.drop(['accuracy'], axis=1)\n    y = df_metamodel['accuracy']\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n    metamodel.fit(X_train, y_train)\n    pickle.dump(metamodel_regressor, open('surrogate.pkl', 'wb'))\n    metamodel = pickle.load(open('./surrogate.pkl', 'rb'))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def initialize_population(population_size):\n    population = []\n    for _ in range(population_size):\n        params = {\n            'learning_rate': 10**random.uniform(-4, -2),  # Example range for learning rate\n            'batch_size': random.choice([16, 32, 64, 128]),\n            'dropout_rate': random.uniform(0.2, 0.8),  # Example range for dropout rate\n            'num_filters': random.choice([32, 64, 128]),  # Example choices for the number of filters\n            'num_dense_layers': random.choice([1, 2, 3])  # Example choices for the number of dense layers\n        }\n        population.append(params)\n    return population\n\ndef evaluate_model(model):\n    history = model.fit(train_images,train_labels,epochs=10,batch_size=32, validation_split=0.2)\n    return history.history['accuracy'][-1]\n\ndef direct_eval(params):\n    model = create_model(params)\n    history = model.fit(train_data,validation_data=val_data,epochs=5, class_weight=class_weights)\n    return history.history['accuracy'][-1]    \n\ndef evaluate_model(params):\n    fitness = metamodel.predict(dataframe_like(params))\n    return fitness\n\ndef dataframe_like(params):\n    d = {param:[params[param]] for param in params.keys()}\n    return pd.DataFrame(d)\n    \ndef array_like(X):\n    x=[]\n    for v in X.values():\n        x.append(v)\n    x = np.array(x)\n    return x\n\ndef dict_like(x):\n    p = {}\n    for k, v in zip(param_names, x):\n        p[k] = v\n        if k not in ['learning_rate']:\n            p[k] = int(p[k])\n    return p\n\n\ndef mutate_params(X, r1, r2, f, best=None):\n    x = array_like(X)\n    r1 = array_like(r1)\n    r2 = array_like(r2)\n    xb = array_like(best) if best is not None else x\n    v = x + f*(xb-x)+f*(r1-r2)\n    return v\n\ndef crossover(x, v, cr):\n    jrand = np.random.randint(0, 4)\n    u=np.zeros(4)\n    for j in range(0, 4):\n        if j==jrand or np.random.rand()<cr:\n            u[j]=v[j]\n        else:\n            u[j]=x[j]\n    return u\n\ndef lehmer(F):\n    F = np.array(F)\n    L = np.sum(np.square(F))/np.sum(F)\n    return L\n\n","metadata":{"execution":{"iopub.status.busy":"2024-08-31T05:11:23.042412Z","iopub.execute_input":"2024-08-31T05:11:23.043182Z","iopub.status.idle":"2024-08-31T05:11:23.06254Z","shell.execute_reply.started":"2024-08-31T05:11:23.043144Z","shell.execute_reply":"2024-08-31T05:11:23.061502Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nimport pickle\n\n# Assuming you have collected hyperparameter performance data\n# Example dataframe\ndf_metamodel = pd.DataFrame({\n    'learning_rate': [0.001, 0.01, 0.1],\n    'batch_size': [32, 64, 128],\n    'num_filters': [32, 64, 128],\n    'num_dense_layers': [1, 2, 3],\n    'accuracy': [0.75, 0.80, 0.85]\n})\n\n# Define features and target variable\nX = df_metamodel.drop(['accuracy'], axis=1)\ny = df_metamodel['accuracy']\n\n# Train the metamodel\nmetamodel = RandomForestRegressor()\nmetamodel.fit(X, y)\n\n# Save the metamodel\nwith open('surrogate.pkl', 'wb') as f:\n    pickle.dump(metamodel, f)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-11T06:00:54.470998Z","iopub.execute_input":"2024-08-11T06:00:54.471371Z","iopub.status.idle":"2024-08-11T06:00:56.618143Z","shell.execute_reply.started":"2024-08-11T06:00:54.471342Z","shell.execute_reply":"2024-08-11T06:00:56.617158Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def jade_hyperparameter_tuning(NP=10, G=10):  #NP - population size, G - is the number of generations  300, 500\n    fitnesses = [] \n    population = initialize_population(NP) \n    for i in range(NP): \n        population[i]['fitness'] = direct_eval(population[i]) \n    population = sorted(population, key=lambda item: item['fitness'], reverse=True) \n    M = {'cr':0.7, 'f':0.3, 'p':0.2} \n    fbest=0 \n    best_params=population[0] \n    fbest = population[0]['fitness'] \n    retrain_chance = 0 \n    for i in range(G): \n        if i%20==0: \n            print('Generation ', i) \n        cr = np.random.normal(M['cr'], 0.1, size=(NP)) \n        f = np.concatenate((np.random.standard_cauchy(2*NP//3)*0.1+M['f'],np.random.random(NP - 2*NP//3)*1.2)) \n        s = {'cr':[] ,'f':[]} \n        for k in range(NP): \n            population = sorted(population, key=lambda item: item['fitness'], reverse=True) \n            r1 = random.choice(range(0, NP)) \n            while r1==k: \n                r1 = random.choice(range(0, NP)) \n            r2 = random.choice(range(0, NP)) \n            while r1==r2 or r2==k: \n                r2 = random.choice(range(0, NP)) \n            pbest = random.choice(range(0, int(NP*M['p']))) \n            v = mutate_params(population[k], population[r1], population[r2], f[k], k, population[pbest]) \n            x = array_like(population[k]) \n            u = crossover(x, v, cr[k]) \n            U = dict_like(u) \n            U = clip_params(U) \n            #if random.random() < retrain_chance: \n                #print(k) \n                #retrain_meta(population[k]) \n            fu = direct_eval(U) \n            U['fitness'] = fu[0] \n            if fu>=fbest: \n                fbest=fu \n                best_params=U \n            if fu>population[k]['fitness']: \n                population[k] = U \n                s['cr'].append(cr[k]) \n                s['f'].append(f) \n                c=0.3 \n                M['cr'] = (1-c)*M['cr'] + c* np.mean(s['cr']) \n                M['f'] = (1-c)*M['f'] + c* lehmer(s['f']) \n        fitnesses.append(np.average([p['fitness'] for p in population])) \n        fbest=0 \n    population = sorted(population, key=lambda item: item['fitness'], reverse=True) \n    best = population[random.choice(range(0, int(NP*M['p'])))] \n    return best, fitnesses","metadata":{"execution":{"iopub.status.busy":"2024-08-31T03:50:05.892403Z","iopub.execute_input":"2024-08-31T03:50:05.892818Z","iopub.status.idle":"2024-08-31T03:50:05.910068Z","shell.execute_reply.started":"2024-08-31T03:50:05.89279Z","shell.execute_reply":"2024-08-31T03:50:05.909082Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#This is the original JADE\n\ndef jade_hyperparameter_tuning(NP=10, G=500):  #NP - population size, G - is the number of generations  \n    fitnesses = [] \n    population = initialize_population(NP) \n    for i in range(NP): \n        population[i]['fitness'] = evaluate_model(population[i])[0] \n    population = sorted(population, key=lambda item: item['fitness'], reverse=True) \n    M = {'cr':0.7, 'f':0.3, 'p':0.2} \n    fbest=0 \n    best_params=population[0] \n    fbest = population[0]['fitness'] \n    retrain_chance = 0 \n    for i in range(G): \n        if i%20==0: \n            print('Generation ', i) \n        cr = np.random.normal(M['cr'], 0.1, size=(NP)) \n        f = np.concatenate((np.random.standard_cauchy(2*NP//3)*0.1+M['f'],np.random.random(NP - 2*NP//3)*1.2)) \n        s = {'cr':[] ,'f':[]} \n        for k in range(NP): \n            population = sorted(population, key=lambda item: item['fitness'], reverse=True) \n            r1 = random.choice(range(0, NP)) \n            while r1==k: \n                r1 = random.choice(range(0, NP)) \n            r2 = random.choice(range(0, NP)) \n            while r1==r2 or r2==k: \n                r2 = random.choice(range(0, NP)) \n            pbest = random.choice(range(0, int(NP*M['p']))) \n            v = mutate_params(population[k], population[r1], population[r2], f[k], k, population[pbest]) \n            x = array_like(population[k]) \n            u = crossover(x, v, cr[k]) \n            U = dict_like(u) \n            U = clip_params(U) \n            if random.random() < retrain_chance: \n                print(k) \n                retrain_meta(population[k]) \n            fu = evaluate_model(U) \n            U['fitness'] = fu[0] \n            if fu>=fbest: \n                fbest=fu \n                best_params=U \n            if fu>population[k]['fitness']: \n                population[k] = U \n                s['cr'].append(cr[k]) \n                s['f'].append(f) \n                c=0.3 \n                M['cr'] = (1-c)*M['cr'] + c* np.mean(s['cr']) \n                M['f'] = (1-c)*M['f'] + c* lehmer(s['f']) \n        fitnesses.append(np.average([p['fitness'] for p in population])) \n        fbest=0 \n    population = sorted(population, key=lambda item: item['fitness'], reverse=True) \n    best = population[random.choice(range(0, int(NP*M['p'])))] \n    return best, fitnesses","metadata":{"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train =accuracies\ny_train = \ngp_model = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, random_state=42) \ngp_model.fit(X_train, y_train)","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n    except RuntimeError as e:\n        print(e)\n","metadata":{"execution":{"iopub.status.busy":"2024-08-14T13:39:56.997524Z","iopub.execute_input":"2024-08-14T13:39:56.998348Z","iopub.status.idle":"2024-08-14T13:39:57.0041Z","shell.execute_reply.started":"2024-08-14T13:39:56.998312Z","shell.execute_reply":"2024-08-14T13:39:57.003193Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":" def initialize_population(population_size):\n    population = []\n    for _ in range(population_size):\n        params = {\n            'learning_rate': 10**random.uniform(-3, -1),  \n            'batch_size': random.randint(8,64), \n            'num_filters': random.randint(32,128),\n            'dropout_rate': random.uniform(0.2, 0.8),\n            'num_dense_layers': random.randint(1,3)  \n        }\n        population.append(params)\n    return population\n\ndef evaluate_model(model):\n    history = model.fit(train_data, epochs=25, validation_data=val_data, class_weight=class_weights, callbacks=[reduce_lr])\n    return history.history['accuracy'][-1]\n\n# Creating the metamodel\nmodel_list = initialize_population(50) \naccuracies = [] \nm=0 \nfor model in model_list: \n    print('Model', m) \n    m+=1 \n    fitness = evaluate_model(create_model(model)) \n    model['accuracy'] = fitness \n    print(model) \n    accuracies.append(model) \n    \naccuracies = pd.DataFrame(accuracies) \n","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:29:01.052575Z","iopub.execute_input":"2024-08-14T18:29:01.053475Z","iopub.status.idle":"2024-08-14T23:03:34.53355Z","shell.execute_reply.started":"2024-08-14T18:29:01.05342Z","shell.execute_reply":"2024-08-14T23:03:34.532107Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2024-08-14T17:09:55.576607Z","iopub.execute_input":"2024-08-14T17:09:55.576958Z","iopub.status.idle":"2024-08-14T17:09:55.611954Z","shell.execute_reply.started":"2024-08-14T17:09:55.576929Z","shell.execute_reply":"2024-08-14T17:09:55.610753Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This is the Shade Section\n\ndef initialize_population(population_size):\n    population = []\n    for _ in range(population_size):\n        params = {\n        'learning_rate': 10**random.uniform(-4, -2),\n        'batch_size': random.choice([16, 32, 64, 128, 256]),\n        'num_filters': random.choice([8, 16, 32, 64]),\n        'num_dense_layers': random.choice([1, 2, 3])\n        }\n        population.append(params)\n    return population\n\ndef evaluate_model(model, params, train_data, val_data):\n    history = model.fit(train_data,validation_data=val_data,epochs=10,callbacks=[reduce_lr])\n    print(np.max(history.history['accuracy']))\n    return np.max(history.history['accuracy'])\n\ndef mutate_params(current_params, best_params, p_best, scaling_factor):\n    mutant_params = {}\n    for param, value in current_params.items():\n        if random.random() < p_best:\n            mutant_params = best_params\n        else:\n            if param == 'num_filters':\n                mutant_params[param] = random.choice([8, 16, 32, 64])\n            elif param == 'num_dense_layers':\n                mutant_params[param] = random.choice([1, 2, 3])\n            elif param=='batch_size':\n                mutant_params[param] = random.choice([16, 32, 64, 128])\n            else:\n                mutant_params[param] = current_params[param] + scaling_factor * current_params[param]\n                mutant_params[param] = np.clip(mutant_params[param], 0, 1)\n    return mutant_params\n\ndef crossover(current_params, mutant_params, crossover_rate):\n    trial_params = {}\n    for param, value in current_params.items():\n        if random.uniform(0, 1) < crossover_rate:\n            trial_params[param] = mutant_params[param]\n        else:\n            trial_params[param] = value\n    return trial_params\n\ndef update_histories(M, S, delf, K):\n    mean_wa = 0\n    mean_wl = 0\n    t = 0\n    sum_delf = np.sum(delf)\n    w = np.array(delf)/sum_delf\n    if len(S['cr'])>0:   \n        for k in range(len(S['cr'])):\n            mean_wa+=w[k]*S['cr'][k]\n        M['cr'][K] = mean_wa\n    if len(S['f'])>0:  \n        for k in range(len(S['f'])):\n            t += w[k]*S['f'][k]*S['f'][k]\n            mean_wl += w[k]*S['f'][k]\n        mean_wl = t/mean_wl\n        M['f'][K] = mean_wl\n    return M\n\ndef shade_hyperparameter_tuning(N=100, G=100, H=50): \n    x = initialize_population(N) \n    M = {'cr':np.ones(N)*0.5, 'f':np.ones(N)*0.5} \n    A = [] \n    K = 0 \n    for i in range(N): \n        x[i]['fitness'] = evaluate_model(x[i])[0] \n    x = sorted(x, key=lambda item: item['fitness'], reverse=True) \n    best_params = x[0] \n    best_fitness = x[0]['fitness'] \n    fitnesses = [] \n     \n    for g in range(G): \n        S = {'cr':[], 'f':[]} \n        u = [] \n        w = [] \n        for i in range(N): \n            r = random.choice(range(0, H)) \n            cr = np.random.normal(M['cr'][r], 0.1) \n            f = np.random.standard_cauchy(1)*0.1+M['f'][r] \n            p = 0.2 \n            r1 = random.choice(range(0, N)) \n            while r1==i: \n                r1 = random.choice(range(0, N)) \n            r2 = random.choice(range(0, N)) \n            while r1==r2 or r2==i: \n                r2 = random.choice(range(0, N)) \n            pbest = random.choice(range(0, int(N*p))) \n            v = mutate_params(x[i], x[r1], x[r2], f, i, x[pbest]) \n            X = array_like(x[i]) \n            _u = crossover(X, v, cr) \n            U = dict_like(_u) \n            U = clip_params(U) \n            u.append(U) \n        for i in range(N): \n            fu = evaluate_model(u[i]) \n            u[i]['fitness'] = fu[0] \n            fx = x[i]['fitness'] \n            if fu>best_fitness: \n                best_params = u[i] \n                best_fitness = fu \n            if fu>=fx: \n                x[i] = u[i] \n            if fu<fx: \n                A.append(x[i]) \n                S['cr'].append(cr), S['f'].append(f) \n                w.append(np.abs(fu-fx)) \n        w = np.array(w)/np.sum(w) \n        while len(A)>N: \n            A.pop(random.randint(1, N)) \n        if len(S['f'])>0 and len(S['cr'])>0: \n            M = update_histories(M, S, w, K) \n            K = (K+1)%H \n            \n        x = sorted(x, key=lambda item: item['fitness'], reverse=True) \n        best_fitness=0 \n        fitnesses.append(np.average([p['fitness'] for p in x])) \n    best = x[random.choice(range(0, int(N*0.1)))] \n    return best, fitnesses ","metadata":{"execution":{"iopub.status.busy":"2024-08-14T18:28:23.316206Z","iopub.execute_input":"2024-08-14T18:28:23.316693Z","iopub.status.idle":"2024-08-14T18:28:23.349902Z","shell.execute_reply.started":"2024-08-14T18:28:23.316654Z","shell.execute_reply":"2024-08-14T18:28:23.348961Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The progress, all the code has been written","metadata":{},"outputs":[],"execution_count":null}]}