def evaluate_emulator_performance(emulator_type, test_function, x_range, n_train=50, n_test=100, **emulator_kwargs):
"""
Comprehensive emulator evaluation framework.
Args:
emulator_type: 'linear', 'gpr', or 'neural_network'
test_function: Function to emulate
x_range: (min, max) range for input values
n_train: Number of training samples
n_test: Number of test samples
**emulator_kwargs: Additional emulator configuration
Returns:
Dictionary with performance metrics and results
"""
# Generate training data
x_train = np.random.uniform(x_range[0], x_range[1], n_train)
y_train = test_function(x_train)
# Generate test data
x_test = np.linspace(x_range[0], x_range[1], n_test)
y_test_true = test_function(x_test, noise_level=0) # True function without noise
# Prepare data for history matching format
X_train = pd.DataFrame({'x': x_train})
y_train_df = pd.DataFrame({'y': y_train})
X_test = pd.DataFrame({'x': x_test})
# Create emulator using EmulatorFactory
factory = hm.EmulatorFactory(default_type=emulator_type, **emulator_kwargs)
# Time the training
start_time = time.time()
emulators = factory.create_emulators_for_features(
X_train, y_train_df, ['y']
)
emulator = emulators['y']
training_time = time.time() - start_time
# Time the prediction
start_time = time.time()
predictions = emulator.predict(X_test)
prediction_time = time.time() - start_time
y_pred = predictions.get_mean().values
y_var = predictions.get_variance().values
# Calculate performance metrics
mse = mean_squared_error(y_test_true, y_pred)
rmse = np.sqrt(mse)
r2 = r2_score(y_test_true, y_pred)
# Mean absolute error
mae = np.mean(np.abs(y_test_true - y_pred))
# Coverage metrics (if uncertainty available)
coverage_95 = None
if np.any(y_var > 0):
y_std = np.sqrt(y_var)
lower_bound = y_pred - 1.96 * y_std
upper_bound = y_pred + 1.96 * y_std
coverage_95 = np.mean((y_test_true >= lower_bound) & (y_test_true <= upper_bound))
return {
'emulator_type': emulator_type,
'emulator': emulator,
'training_time': training_time,
'prediction_time': prediction_time,
'mse': mse,
'rmse': rmse,
'mae': mae,
'r2': r2,
'coverage_95': coverage_95,
'x_train': x_train,
'y_train': y_train,
'x_test': x_test,
'y_test_true': y_test_true,
'y_pred': y_pred,
'y_var': y_var
}
def plot_emulator_comparison(results_list, title="Emulator Comparison"):
"""
Plot comparison of multiple emulator results.
"""
fig, axes = plt.subplots(2, 2, figsize=(15, 12))
colors = ['blue', 'red', 'green', 'purple', 'orange']
# Predictions plot
ax = axes[0, 0]
for i, result in enumerate(results_list):
color = colors[i % len(colors)]
# Plot true function
if i == 0:
ax.plot(result['x_test'], result['y_test_true'], 'k-', linewidth=2, label='True function')
ax.scatter(result['x_train'], result['y_train'], color='gray', alpha=0.6, s=20, label='Training data')
# Plot predictions
ax.plot(result['x_test'], result['y_pred'], color=color, linestyle='--', linewidth=2,
label=f"{result['emulator_type']} (R²={result['r2']:.3f})")
# Plot uncertainty if available
if np.any(result['y_var'] > 0):
y_std = np.sqrt(result['y_var'])
ax.fill_between(result['x_test'],
result['y_pred'] - 1.96 * y_std,
result['y_pred'] + 1.96 * y_std,
color=color, alpha=0.2)
ax.set_title('Predictions Comparison')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.legend()
ax.grid(True, alpha=0.3)
# Performance metrics
ax = axes[0, 1]
metrics = ['rmse', 'mae', 'r2']
metric_names = ['RMSE', 'MAE', 'R²']
x_pos = np.arange(len(metrics))
width = 0.8 / len(results_list)
for i, result in enumerate(results_list):
values = [result[metric] for metric in metrics]
ax.bar(x_pos + i * width, values, width, label=result['emulator_type'],
color=colors[i % len(colors)], alpha=0.7)
ax.set_xlabel('Metrics')
ax.set_ylabel('Value')
ax.set_title('Performance Metrics')
ax.set_xticks(x_pos + width * (len(results_list) - 1) / 2)
ax.set_xticklabels(metric_names)
ax.legend()
ax.grid(True, alpha=0.3)
# Timing comparison
ax = axes[1, 0]
emulator_names = [r['emulator_type'] for r in results_list]
training_times = [r['training_time'] for r in results_list]
prediction_times = [r['prediction_time'] for r in results_list]
x_pos = np.arange(len(emulator_names))
width = 0.35
ax.bar(x_pos - width/2, training_times, width, label='Training time', alpha=0.7)
ax.bar(x_pos + width/2, prediction_times, width, label='Prediction time', alpha=0.7)
ax.set_xlabel('Emulator Type')
ax.set_ylabel('Time (seconds)')
ax.set_title('Training and Prediction Times')
ax.set_xticks(x_pos)
ax.set_xticklabels(emulator_names)
ax.legend()
ax.grid(True, alpha=0.3)
# Residuals plot
ax = axes[1, 1]
for i, result in enumerate(results_list):
residuals = result['y_test_true'] - result['y_pred']
ax.scatter(result['y_pred'], residuals, alpha=0.6, s=20,
color=colors[i % len(colors)], label=result['emulator_type'])
ax.axhline(y=0, color='black', linestyle='--', alpha=0.5)
ax.set_xlabel('Predicted values')
ax.set_ylabel('Residuals')
ax.set_title('Residuals vs Predicted')
ax.legend()
ax.grid(True, alpha=0.3)
plt.suptitle(title, fontsize=16)
plt.tight_layout()
plt.show()
print("Emulator evaluation framework ready!")