Use third-party optimizers¶

import comet_ml

# Define static parameters which are not returned by the optimizer
static_parameters = {}

def trial(...):

    # Create a Comet Experiment
    exp = comet_ml.Experiment(...)

    # Obtain optimization parameters from your custom or third-party optimizer
    learning_rate, n_layers = my_optimizer.get_parameters(...)

    # Train and evaluate the model
    loss, accuracy = train_and_evaluate_model(learning_rate, n_layers, static_parameters, ...)

    # Log the optimization data to Comet
    exp.log_optimization(
        optimization_id=OPTIMIZATION_ID,  # This should be unique across all trials of the same sweep
        metric_name="loss",
        metric_value=loss,
        parameters={"learning_rate": learning_rate, "n_layers": n_layers},
        objective="minimize",
    )

    # Log relevant metrics, other parameters, and assets to Comet
    exp.log_metric("accuracy", accuracy)
    exp.log_parameters(static_parameters)

import comet_ml
import itertools
import json
import random
import os

# Initialize the Comet SDK
comet_ml.login()

# Define your search space
hyperparameters = {
    "learning-rate": [0.001, 0.1, 0.2, 0.5, 0.9],
    "batch-size": [16, 32, 64],
    "hidden-layer-size": [5, 10, 15],
    "optimizer": ["adam", "sgd"],
}

# Define your tuning config
max_n_trials = 5

# Define support methods for tuning
def my_optimizer(params, shuffle=True):
    if shuffle:
        combinations = []
        for values in itertools.product(*params.values()):
            combinations.append(dict(zip(params, values)))
        random.shuffle(combinations)
        for combo in combinations:
            yield combo
    else:
        for values in itertools.product(*params.values()):
            yield dict(zip(params, values))


def train(**params):
    return random.random()


# Run tuning with the custom optimizer
for count in range(max_n_trials):

    # Create a Comet Experiment
    exp = Experiment(project_name="dummy-custom-optimization")

    # Obtain optimization parameters from your custom or third-party optimizer
    parameters = list(my_optimizer(hyperparameters))[count]

    # Train and evaluate the model
    metric_value = train(**parameters)

    # Log the optimization data to Comet
    exp.log_optimization(
        optimization_id=f"my_run_{count}",
        metric_name="dummy_metric",
        metric_value=metric_value,
        parameters=parameters,
        objective="minimize",
    )

    # Log relevant metrics, other parameters, and assets to Comet
    exp.log_other("optimizer_name", "dummy-optimizer-001")
    exp.log_other("optimizer_version", "dummy-1.0")
    exp.log_other("optimizer_process", os.getpid())
    exp.log_other("optimizer_run_n", count)