diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go
index 93eb9f38ee5b..8867b8c8fed8 100644
--- a/api/v1alpha1/envoygateway_types.go
+++ b/api/v1alpha1/envoygateway_types.go
@@ -500,6 +500,18 @@ type ExtensionManager struct {
//
// +kubebuilder:validation:Required
Service *ExtensionService `json:"service,omitempty"`
+
+ // FailOpen defines if Envoy Gateway should ignore errors returned from the Extension Service hooks.
+ // The default is false, which means Envoy Gateway will fail closed if the Extension Service returns an error.
+ //
+ // Fail-close means that if the Extension Service hooks return an error, the relevant route/listener/resource
+ // will be replaced with a default configuration returning Internal Server Error (HTTP 500).
+ //
+ // Fail-open means that if the Extension Service hooks return an error, no changes will be applied to the
+ // source of the configuration which was sent to the extension server.
+ //
+ // +optional
+ FailOpen bool `json:"failOpen,omitempty"`
}
// ExtensionHooks defines extension hooks across all supported runners
diff --git a/internal/extension/registry/extension_manager.go b/internal/extension/registry/extension_manager.go
index cf4b86d3d083..67beedc4d56c 100644
--- a/internal/extension/registry/extension_manager.go
+++ b/internal/extension/registry/extension_manager.go
@@ -107,6 +107,11 @@ func NewInMemoryManager(cfg egv1a1.ExtensionManager, server extension.EnvoyGatew
}, c, nil
}
+// FailOpen returns true if the extension manager is configured to fail open, and false otherwise.
+func (m *Manager) FailOpen() bool {
+ return m.extension.FailOpen
+}
+
// HasExtension checks to see whether a given Group and Kind has an
// associated extension registered for it.
func (m *Manager) HasExtension(g gwapiv1.Group, k gwapiv1.Kind) bool {
@@ -139,15 +144,15 @@ func getExtensionServerAddress(service *egv1a1.ExtensionService) string {
// that are used to generate an xDS resource.
// If the extension makes use of the hook then the XDS Hook Client is returned. If it does not support
// the hook type then nil is returned
-func (m *Manager) GetPreXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) extTypes.XDSHookClient {
+func (m *Manager) GetPreXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) (extTypes.XDSHookClient, error) {
ctx := context.Background()
ext := m.extension
if ext.Hooks == nil {
- return nil
+ return nil, nil
}
if ext.Hooks.XDSTranslator == nil {
- return nil
+ return nil, nil
}
hookUsed := false
@@ -158,7 +163,7 @@ func (m *Manager) GetPreXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) extT
}
}
if !hookUsed {
- return nil
+ return nil, nil
}
if m.extensionConnCache == nil {
@@ -166,12 +171,12 @@ func (m *Manager) GetPreXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) extT
opts, err := setupGRPCOpts(ctx, m.k8sClient, &ext, m.namespace)
if err != nil {
- return nil
+ return nil, err
}
conn, err := grpc.Dial(serverAddr, opts...)
if err != nil {
- return nil
+ return nil, err
}
m.extensionConnCache = conn
@@ -181,22 +186,22 @@ func (m *Manager) GetPreXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) extT
xdsHookClient := &XDSHook{
grpcClient: client,
}
- return xdsHookClient
+ return xdsHookClient, nil
}
// GetPostXDSHookClient checks if the registered extension makes use of a particular hook type that modifies
// xDS resources after they are generated by Envoy Gateway.
// If the extension makes use of the hook then the XDS Hook Client is returned. If it does not support
// the hook type then nil is returned
-func (m *Manager) GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) extTypes.XDSHookClient {
+func (m *Manager) GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) (extTypes.XDSHookClient, error) {
ctx := context.Background()
ext := m.extension
if ext.Hooks == nil {
- return nil
+ return nil, nil
}
if ext.Hooks.XDSTranslator == nil {
- return nil
+ return nil, nil
}
hookUsed := false
@@ -207,7 +212,7 @@ func (m *Manager) GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) ext
}
}
if !hookUsed {
- return nil
+ return nil, nil
}
if m.extensionConnCache == nil {
@@ -215,12 +220,12 @@ func (m *Manager) GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) ext
opts, err := setupGRPCOpts(ctx, m.k8sClient, &ext, m.namespace)
if err != nil {
- return nil
+ return nil, err
}
conn, err := grpc.Dial(serverAddr, opts...)
if err != nil {
- return nil
+ return nil, err
}
m.extensionConnCache = conn
@@ -230,7 +235,7 @@ func (m *Manager) GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) ext
xdsHookClient := &XDSHook{
grpcClient: client,
}
- return xdsHookClient
+ return xdsHookClient, nil
}
func (m *Manager) CleanupHookConns() {
diff --git a/internal/extension/types/manager.go b/internal/extension/types/manager.go
index f761cbe8f66d..287a6bba7850 100644
--- a/internal/extension/types/manager.go
+++ b/internal/extension/types/manager.go
@@ -25,11 +25,14 @@ type Manager interface {
// that are used to generate an xDS resource.
// If the extension makes use of the hook then the XDS Hook Client is returned. If it does not support
// the hook type then nil is returned
- GetPreXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) XDSHookClient
+ GetPreXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) (XDSHookClient, error)
// GetPostXDSHookClient checks if the registered extension makes use of a particular hook type that modifies
// xDS resources after they are generated by Envoy Gateway.
// If the extension makes use of the hook then the XDS Hook Client is returned. If it does not support
// the hook type then nil is returned
- GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) XDSHookClient
+ GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) (XDSHookClient, error)
+
+ // FailOpen returns true if the extension manager is configured to fail open, and false otherwise.
+ FailOpen() bool
}
diff --git a/internal/xds/translator/extension.go b/internal/xds/translator/extension.go
index 5a14bd2f7cb2..1fdc37d9b40b 100644
--- a/internal/xds/translator/extension.go
+++ b/internal/xds/translator/extension.go
@@ -35,7 +35,10 @@ func processExtensionPostRouteHook(route *routev3.Route, vHost *routev3.VirtualH
// Check if an extension want to modify the route that was just configured/created
extManager := *em
- extRouteHookClient := extManager.GetPostXDSHookClient(egv1a1.XDSRoute)
+ extRouteHookClient, err := extManager.GetPostXDSHookClient(egv1a1.XDSRoute)
+ if err != nil {
+ return err
+ }
if extRouteHookClient == nil {
return nil
}
@@ -71,7 +74,10 @@ func processExtensionPostVHostHook(vHost *routev3.VirtualHost, em *extensionType
// Check if an extension want to modify the route that was just configured/created
extManager := *em
- extVHHookClient := extManager.GetPostXDSHookClient(egv1a1.XDSVirtualHost)
+ extVHHookClient, err := extManager.GetPostXDSHookClient(egv1a1.XDSVirtualHost)
+ if err != nil {
+ return err
+ }
if extVHHookClient == nil {
return nil
}
@@ -100,7 +106,10 @@ func processExtensionPostListenerHook(tCtx *types.ResourceVersionTable, xdsListe
// Check if an extension want to modify the listener that was just configured/created
extManager := *em
- extListenerHookClient := extManager.GetPostXDSHookClient(egv1a1.XDSHTTPListener)
+ extListenerHookClient, err := extManager.GetPostXDSHookClient(egv1a1.XDSHTTPListener)
+ if err != nil {
+ return err
+ }
if extListenerHookClient != nil {
unstructuredResources := make([]*unstructured.Unstructured, len(extensionRefs))
for refIdx, ref := range extensionRefs {
@@ -140,7 +149,10 @@ func processExtensionPostTranslationHook(tCtx *types.ResourceVersionTable, em *e
// that is non-static. If a cluster definition is unlikely to change over the course of an extension's lifetime then the custom bootstrap config
// is the preferred way of adding it.
extManager := *em
- extensionInsertHookClient := extManager.GetPostXDSHookClient(egv1a1.XDSTranslation)
+ extensionInsertHookClient, err := extManager.GetPostXDSHookClient(egv1a1.XDSTranslation)
+ if err != nil {
+ return err
+ }
if extensionInsertHookClient == nil {
return nil
}
diff --git a/internal/xds/translator/extensionserver_test.go b/internal/xds/translator/extensionserver_test.go
index ef34d3f8d50d..bb61ace265e5 100644
--- a/internal/xds/translator/extensionserver_test.go
+++ b/internal/xds/translator/extensionserver_test.go
@@ -202,6 +202,18 @@ func (t *testingExtensionServer) PostHTTPListenerModify(_ context.Context, req *
// PostTranslateModifyHook inserts and overrides some clusters/secrets
func (t *testingExtensionServer) PostTranslateModify(_ context.Context, req *pb.PostTranslateModifyRequest) (*pb.PostTranslateModifyResponse, error) {
+ for _, cluster := range req.Clusters {
+ // This simulates an extension server that returns an error. It allows verifying that fail-close is working.
+ if edsConfig := cluster.GetEdsClusterConfig(); edsConfig != nil {
+ if strings.Contains(edsConfig.ServiceName, "fail-close-error") {
+ return &pb.PostTranslateModifyResponse{
+ Clusters: req.Clusters,
+ Secrets: req.Secrets,
+ }, fmt.Errorf("cluster hook resource error: %s", edsConfig.ServiceName)
+ }
+ }
+ }
+
extensionSvcEndpoint := &endpointV3.LbEndpoint_Endpoint{
Endpoint: &endpointV3.Endpoint{
Address: &coreV3.Address{
diff --git a/internal/xds/translator/runner/runner_test.go b/internal/xds/translator/runner/runner_test.go
index dba7ddeb6224..a154b9f82c63 100644
--- a/internal/xds/translator/runner/runner_test.go
+++ b/internal/xds/translator/runner/runner_test.go
@@ -174,12 +174,16 @@ type extManagerMock struct {
types.Manager
}
-func (m *extManagerMock) GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) types.XDSHookClient {
+func (m *extManagerMock) GetPostXDSHookClient(xdsHookType egv1a1.XDSTranslatorHook) (types.XDSHookClient, error) {
if xdsHookType == egv1a1.XDSHTTPListener {
- return &xdsHookClientMock{}
+ return &xdsHookClientMock{}, nil
}
- return nil
+ return nil, nil
+}
+
+func (m *extManagerMock) FailOpen() bool {
+ return false
}
type xdsHookClientMock struct {
diff --git a/internal/xds/translator/testdata/in/extension-xds-ir/http-route-extension-translate-error.yaml b/internal/xds/translator/testdata/in/extension-xds-ir/http-route-extension-translate-error.yaml
new file mode 100644
index 000000000000..792e5b1f7182
--- /dev/null
+++ b/internal/xds/translator/testdata/in/extension-xds-ir/http-route-extension-translate-error.yaml
@@ -0,0 +1,20 @@
+http:
+- name: "extension-post-xdstranslate-hook-error"
+ address: "0.0.0.0"
+ port: 10080
+ hostnames:
+ - "*"
+ path:
+ mergeSlashes: true
+ escapedSlashesAction: UnescapeAndRedirect
+ routes:
+ - name: "first-route"
+ hostname: "*"
+ pathMatch:
+ prefix: "/"
+ destination:
+ name: "fail-close-error"
+ settings:
+ - endpoints:
+ - host: "1.2.3.4"
+ port: 50000
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.clusters.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.clusters.yaml
new file mode 100644
index 000000000000..0dcc3b48190b
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.clusters.yaml
@@ -0,0 +1,27 @@
+- circuitBreakers:
+ thresholds:
+ - maxRetries: 1024
+ commonLbConfig:
+ localityWeightedLbConfig: {}
+ connectTimeout: 10s
+ dnsLookupFamily: V4_PREFERRED
+ edsClusterConfig:
+ edsConfig:
+ ads: {}
+ resourceApiVersion: V3
+ serviceName: first-route-dest
+ ignoreHealthOnHostRemoval: true
+ lbPolicy: LEAST_REQUEST
+ name: first-route-dest
+ perConnectionBufferLimitBytes: 32768
+ type: EDS
+- loadAssignment:
+ clusterName: mock-extension-injected-cluster
+ endpoints:
+ - lbEndpoints:
+ - endpoint:
+ address:
+ socketAddress:
+ address: exampleservice.examplenamespace.svc.cluster.local
+ portValue: 5000
+ name: mock-extension-injected-cluster
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.endpoints.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.endpoints.yaml
new file mode 100644
index 000000000000..3b3f2d09076e
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.endpoints.yaml
@@ -0,0 +1,12 @@
+- clusterName: first-route-dest
+ endpoints:
+ - lbEndpoints:
+ - endpoint:
+ address:
+ socketAddress:
+ address: 1.2.3.4
+ portValue: 50000
+ loadBalancingWeight: 1
+ loadBalancingWeight: 1
+ locality:
+ region: first-route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.listeners.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.listeners.yaml
new file mode 100644
index 000000000000..098d34d564e6
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.listeners.yaml
@@ -0,0 +1,41 @@
+- address:
+ socketAddress:
+ address: 0.0.0.0
+ portValue: 10080
+ defaultFilterChain:
+ filters:
+ - name: envoy.filters.network.http_connection_manager
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+ commonHttpProtocolOptions:
+ headersWithUnderscoresAction: REJECT_REQUEST
+ http2ProtocolOptions:
+ initialConnectionWindowSize: 1048576
+ initialStreamWindowSize: 65536
+ maxConcurrentStreams: 100
+ httpFilters:
+ - name: envoy.filters.http.router
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+ suppressEnvoyHeaders: true
+ mergeSlashes: true
+ normalizePath: true
+ pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT
+ routeConfig:
+ name: error_route_configuration
+ virtualHosts:
+ - domains:
+ - '*'
+ name: error_vhost
+ routes:
+ - directResponse:
+ status: 500
+ match:
+ prefix: /
+ name: error_route
+ serverHeaderTransformation: PASS_THROUGH
+ statPrefix: http-10080
+ useRemoteAddress: true
+ name: extension-post-xdslistener-hook-error
+ name: extension-post-xdslistener-hook-error
+ perConnectionBufferLimitBytes: 32768
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.routes.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.routes.yaml
new file mode 100644
index 000000000000..3010873c902a
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.routes.yaml
@@ -0,0 +1,14 @@
+- ignorePortInHostMatching: true
+ name: extension-post-xdslistener-hook-error
+ virtualHosts:
+ - domains:
+ - '*'
+ name: extension-post-xdslistener-hook-error/*
+ routes:
+ - match:
+ prefix: /
+ name: first-route
+ route:
+ cluster: first-route-dest
+ upgradeConfigs:
+ - upgradeType: websocket
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.secrets.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.secrets.yaml
new file mode 100644
index 000000000000..b04034756d9b
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-listener-error.secrets.yaml
@@ -0,0 +1,4 @@
+- genericSecret:
+ secret:
+ inlineString: super-secret-extension-secret
+ name: mock-extension-injected-secret
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.clusters.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.clusters.yaml
new file mode 100644
index 000000000000..f8f489d240e6
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.clusters.yaml
@@ -0,0 +1,27 @@
+- circuitBreakers:
+ thresholds:
+ - maxRetries: 1024
+ commonLbConfig:
+ localityWeightedLbConfig: {}
+ connectTimeout: 10s
+ dnsLookupFamily: V4_PREFERRED
+ edsClusterConfig:
+ edsConfig:
+ ads: {}
+ resourceApiVersion: V3
+ serviceName: extension-post-xdsroute-hook-error-dest
+ ignoreHealthOnHostRemoval: true
+ lbPolicy: LEAST_REQUEST
+ name: extension-post-xdsroute-hook-error-dest
+ perConnectionBufferLimitBytes: 32768
+ type: EDS
+- loadAssignment:
+ clusterName: mock-extension-injected-cluster
+ endpoints:
+ - lbEndpoints:
+ - endpoint:
+ address:
+ socketAddress:
+ address: exampleservice.examplenamespace.svc.cluster.local
+ portValue: 5000
+ name: mock-extension-injected-cluster
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.endpoints.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.endpoints.yaml
new file mode 100644
index 000000000000..c706470026d9
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.endpoints.yaml
@@ -0,0 +1,12 @@
+- clusterName: extension-post-xdsroute-hook-error-dest
+ endpoints:
+ - lbEndpoints:
+ - endpoint:
+ address:
+ socketAddress:
+ address: 1.2.3.4
+ portValue: 50000
+ loadBalancingWeight: 1
+ loadBalancingWeight: 1
+ locality:
+ region: extension-post-xdsroute-hook-error-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.listeners.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.listeners.yaml
new file mode 100644
index 000000000000..c3fb113017a9
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.listeners.yaml
@@ -0,0 +1,34 @@
+- address:
+ socketAddress:
+ address: 0.0.0.0
+ portValue: 10080
+ defaultFilterChain:
+ filters:
+ - name: envoy.filters.network.http_connection_manager
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+ commonHttpProtocolOptions:
+ headersWithUnderscoresAction: REJECT_REQUEST
+ http2ProtocolOptions:
+ initialConnectionWindowSize: 1048576
+ initialStreamWindowSize: 65536
+ maxConcurrentStreams: 100
+ httpFilters:
+ - name: envoy.filters.http.router
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+ suppressEnvoyHeaders: true
+ mergeSlashes: true
+ normalizePath: true
+ pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT
+ rds:
+ configSource:
+ ads: {}
+ resourceApiVersion: V3
+ routeConfigName: first-listener
+ serverHeaderTransformation: PASS_THROUGH
+ statPrefix: http-10080
+ useRemoteAddress: true
+ name: first-listener
+ name: first-listener
+ perConnectionBufferLimitBytes: 32768
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.routes.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.routes.yaml
new file mode 100644
index 000000000000..111ce49e0f6d
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.routes.yaml
@@ -0,0 +1,12 @@
+- ignorePortInHostMatching: true
+ name: first-listener
+ virtualHosts:
+ - domains:
+ - '*'
+ name: first-listener/*
+ routes:
+ - directResponse:
+ status: 500
+ match:
+ prefix: /
+ name: extension-post-xdsroute-hook-error
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.secrets.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.secrets.yaml
new file mode 100644
index 000000000000..b04034756d9b
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-route-error.secrets.yaml
@@ -0,0 +1,4 @@
+- genericSecret:
+ secret:
+ inlineString: super-secret-extension-secret
+ name: mock-extension-injected-secret
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.clusters.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.clusters.yaml
new file mode 100644
index 000000000000..46fd544dc175
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.clusters.yaml
@@ -0,0 +1,17 @@
+- circuitBreakers:
+ thresholds:
+ - maxRetries: 1024
+ commonLbConfig:
+ localityWeightedLbConfig: {}
+ connectTimeout: 10s
+ dnsLookupFamily: V4_PREFERRED
+ edsClusterConfig:
+ edsConfig:
+ ads: {}
+ resourceApiVersion: V3
+ serviceName: fail-close-error
+ ignoreHealthOnHostRemoval: true
+ lbPolicy: LEAST_REQUEST
+ name: fail-close-error
+ perConnectionBufferLimitBytes: 32768
+ type: EDS
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.endpoints.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.endpoints.yaml
new file mode 100644
index 000000000000..4d832fbf0377
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.endpoints.yaml
@@ -0,0 +1,12 @@
+- clusterName: fail-close-error
+ endpoints:
+ - lbEndpoints:
+ - endpoint:
+ address:
+ socketAddress:
+ address: 1.2.3.4
+ portValue: 50000
+ loadBalancingWeight: 1
+ loadBalancingWeight: 1
+ locality:
+ region: fail-close-error/backend/0
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.listeners.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.listeners.yaml
new file mode 100644
index 000000000000..02769a8c0d9f
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.listeners.yaml
@@ -0,0 +1,41 @@
+- address:
+ socketAddress:
+ address: 0.0.0.0
+ portValue: 10080
+ defaultFilterChain:
+ filters:
+ - name: envoy.filters.network.http_connection_manager
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+ commonHttpProtocolOptions:
+ headersWithUnderscoresAction: REJECT_REQUEST
+ http2ProtocolOptions:
+ initialConnectionWindowSize: 1048576
+ initialStreamWindowSize: 65536
+ maxConcurrentStreams: 100
+ httpFilters:
+ - name: envoy.filters.http.router
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+ suppressEnvoyHeaders: true
+ mergeSlashes: true
+ normalizePath: true
+ pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT
+ routeConfig:
+ name: error_route_configuration
+ virtualHosts:
+ - domains:
+ - '*'
+ name: error_vhost
+ routes:
+ - directResponse:
+ status: 500
+ match:
+ prefix: /
+ name: error_route
+ serverHeaderTransformation: PASS_THROUGH
+ statPrefix: http-10080
+ useRemoteAddress: true
+ name: extension-post-xdstranslate-hook-error
+ name: extension-post-xdstranslate-hook-error
+ perConnectionBufferLimitBytes: 32768
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.routes.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.routes.yaml
new file mode 100644
index 000000000000..489bb7edc882
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-translate-error.routes.yaml
@@ -0,0 +1,14 @@
+- ignorePortInHostMatching: true
+ name: extension-post-xdstranslate-hook-error
+ virtualHosts:
+ - domains:
+ - '*'
+ name: extension-post-xdstranslate-hook-error/*
+ routes:
+ - match:
+ prefix: /
+ name: first-route
+ route:
+ cluster: fail-close-error
+ upgradeConfigs:
+ - upgradeType: websocket
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.clusters.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.clusters.yaml
new file mode 100644
index 000000000000..0dcc3b48190b
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.clusters.yaml
@@ -0,0 +1,27 @@
+- circuitBreakers:
+ thresholds:
+ - maxRetries: 1024
+ commonLbConfig:
+ localityWeightedLbConfig: {}
+ connectTimeout: 10s
+ dnsLookupFamily: V4_PREFERRED
+ edsClusterConfig:
+ edsConfig:
+ ads: {}
+ resourceApiVersion: V3
+ serviceName: first-route-dest
+ ignoreHealthOnHostRemoval: true
+ lbPolicy: LEAST_REQUEST
+ name: first-route-dest
+ perConnectionBufferLimitBytes: 32768
+ type: EDS
+- loadAssignment:
+ clusterName: mock-extension-injected-cluster
+ endpoints:
+ - lbEndpoints:
+ - endpoint:
+ address:
+ socketAddress:
+ address: exampleservice.examplenamespace.svc.cluster.local
+ portValue: 5000
+ name: mock-extension-injected-cluster
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.endpoints.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.endpoints.yaml
new file mode 100644
index 000000000000..3b3f2d09076e
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.endpoints.yaml
@@ -0,0 +1,12 @@
+- clusterName: first-route-dest
+ endpoints:
+ - lbEndpoints:
+ - endpoint:
+ address:
+ socketAddress:
+ address: 1.2.3.4
+ portValue: 50000
+ loadBalancingWeight: 1
+ loadBalancingWeight: 1
+ locality:
+ region: first-route-dest/backend/0
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.listeners.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.listeners.yaml
new file mode 100644
index 000000000000..96c45d3ee2ea
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.listeners.yaml
@@ -0,0 +1,34 @@
+- address:
+ socketAddress:
+ address: 0.0.0.0
+ portValue: 10080
+ defaultFilterChain:
+ filters:
+ - name: envoy.filters.network.http_connection_manager
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+ commonHttpProtocolOptions:
+ headersWithUnderscoresAction: REJECT_REQUEST
+ http2ProtocolOptions:
+ initialConnectionWindowSize: 1048576
+ initialStreamWindowSize: 65536
+ maxConcurrentStreams: 100
+ httpFilters:
+ - name: envoy.filters.http.router
+ typedConfig:
+ '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+ suppressEnvoyHeaders: true
+ mergeSlashes: true
+ normalizePath: true
+ pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT
+ rds:
+ configSource:
+ ads: {}
+ resourceApiVersion: V3
+ routeConfigName: extension-post-xdsvirtualhost-hook-error
+ serverHeaderTransformation: PASS_THROUGH
+ statPrefix: http-10080
+ useRemoteAddress: true
+ name: extension-post-xdsvirtualhost-hook-error
+ name: extension-post-xdsvirtualhost-hook-error
+ perConnectionBufferLimitBytes: 32768
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.routes.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.routes.yaml
new file mode 100644
index 000000000000..8433c9b4035c
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.routes.yaml
@@ -0,0 +1,12 @@
+- ignorePortInHostMatching: true
+ name: extension-post-xdsvirtualhost-hook-error
+ virtualHosts:
+ - domains:
+ - '*'
+ name: extension-post-xdsvirtualhost-hook-error/*
+ routes:
+ - directResponse:
+ status: 500
+ match:
+ prefix: /
+ name: error_route
diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.secrets.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.secrets.yaml
new file mode 100644
index 000000000000..b04034756d9b
--- /dev/null
+++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-extension-virtualhost-error.secrets.yaml
@@ -0,0 +1,4 @@
+- genericSecret:
+ secret:
+ inlineString: super-secret-extension-secret
+ name: mock-extension-injected-secret
diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go
index 79f16d5d1b52..eaf9625f092a 100644
--- a/internal/xds/translator/translator.go
+++ b/internal/xds/translator/translator.go
@@ -8,6 +8,7 @@ package translator
import (
"errors"
"fmt"
+ "net/http"
"strings"
"time"
@@ -25,6 +26,7 @@ import (
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/wrapperspb"
"k8s.io/apimachinery/pkg/util/sets"
+ "k8s.io/utils/ptr"
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
extensionTypes "github.com/envoyproxy/gateway/internal/extension/types"
@@ -120,6 +122,13 @@ func (t *Translator) Translate(xdsIR *ir.Xds) (*types.ResourceVersionTable, erro
// If no extension exists (or it doesn't subscribe to this hook) then this is a quick no-op
if err := processExtensionPostTranslationHook(tCtx, t.ExtensionManager); err != nil {
errs = errors.Join(errs, err)
+ // Setting the configuration to fail open will mean that Envoy Gateway ignores the error and keeps the resources
+ // as they were before the extension server was called.
+ if t.ExtensionManager != nil && !(*t.ExtensionManager).FailOpen() {
+ for _, listener := range tCtx.XdsResources[resourcev3.ListenerType] {
+ errs = errors.Join(errs, clearListenerRoutes(listener.(*listenerv3.Listener)))
+ }
+ }
}
return tCtx, errs
@@ -159,7 +168,7 @@ func (t *Translator) notifyExtensionServerAboutListeners(
if t.ExtensionManager == nil {
return nil
}
- if (*t.ExtensionManager).GetPostXDSHookClient(egv1a1.XDSHTTPListener) == nil {
+ if postHookClient, err := (*t.ExtensionManager).GetPostXDSHookClient(egv1a1.XDSHTTPListener); postHookClient == nil && err == nil {
return nil
}
@@ -179,11 +188,75 @@ func (t *Translator) notifyExtensionServerAboutListeners(
}
if err := processExtensionPostListenerHook(tCtx, listener, policies, t.ExtensionManager); err != nil {
errs = errors.Join(errs, err)
+ // If the extension server returns an error, and the extension server is not configured to fail open,
+ // then replace all of the routes in the virtual host with a single route that returns an InternalServerError result.
+ // Setting the configuration to fail open will mean that Envoy Gateway ignores the error and keeps the routes
+ // as they were before the extension server was called.
+ if !(*t.ExtensionManager).FailOpen() {
+ errs = errors.Join(errs, clearListenerRoutes(listener))
+ }
+ }
+ }
+ return errs
+}
+
+func clearListenerRoutes(listener *listenerv3.Listener) error {
+ var errs error
+ hcm, err := findHCMinFilterChain(listener.DefaultFilterChain)
+ if err != nil {
+ // no HCM found, skip
+ } else {
+ clearAllRoutes(hcm)
+ if err := replaceHCMInFilterChain(hcm, listener.DefaultFilterChain); err != nil {
+ errs = errors.Join(errs, err)
+ }
+ }
+ for _, filter := range listener.FilterChains {
+ hcm, err := findHCMinFilterChain(filter)
+ if err != nil {
+ // no HCM found, skip
+ continue
+ }
+ clearAllRoutes(hcm)
+ if err := replaceHCMInFilterChain(hcm, filter); err != nil {
+ errs = errors.Join(errs, err)
}
}
+
return errs
}
+func clearAllRoutes(hcm *hcmv3.HttpConnectionManager) {
+ // Discard all of the routes configured on this HCM and replace them
+ // with a single route that returns an InternalServerError result.
+ hcm.RouteSpecifier = &hcmv3.HttpConnectionManager_RouteConfig{
+ RouteConfig: &routev3.RouteConfiguration{
+ Name: "error_route_configuration",
+ VirtualHosts: []*routev3.VirtualHost{
+ {
+ Name: "error_vhost",
+ Domains: []string{"*"},
+ Routes: []*routev3.Route{
+ {
+ Name: "error_route",
+ Match: &routev3.RouteMatch{
+ PathSpecifier: &routev3.RouteMatch_Prefix{
+ Prefix: "/",
+ },
+ },
+ Action: &routev3.Route_DirectResponse{
+ DirectResponse: buildXdsDirectResponseAction(&ir.CustomResponse{
+ StatusCode: ptr.To(uint32(http.StatusInternalServerError)),
+ }),
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
func (t *Translator) processHTTPListenerXdsTranslation(
tCtx *types.ResourceVersionTable,
httpListeners []*ir.HTTPListener,
@@ -449,6 +522,15 @@ func (t *Translator) addRouteToRouteConfig(
// If no extension exists (or it doesn't subscribe to this hook) then this is a quick no-op.
if err = processExtensionPostRouteHook(xdsRoute, vHost, httpRoute, t.ExtensionManager); err != nil {
errs = errors.Join(errs, err)
+ // If the extension server returns an error, and the extension server is not configured to fail open,
+ // then replace the route with one that returns an InternalServerError result.
+ // Setting the configuration to fail open will mean that Envoy Gateway ignores the error and keeps the route
+ // as it was before the extension server was called.
+ if t.ExtensionManager != nil && !(*t.ExtensionManager).FailOpen() {
+ xdsRoute.Action = &routev3.Route_DirectResponse{DirectResponse: buildXdsDirectResponseAction(&ir.CustomResponse{
+ StatusCode: ptr.To(uint32(http.StatusInternalServerError)),
+ })}
+ }
}
if http3Enabled {
@@ -500,6 +582,25 @@ func (t *Translator) addRouteToRouteConfig(
// If no extension exists (or it doesn't subscribe to this hook) then this is a quick no-op.
if err = processExtensionPostVHostHook(vHost, t.ExtensionManager); err != nil {
errs = errors.Join(errs, err)
+ // If the extension server returns an error, and the extension server is not configured to fail open,
+ // then replace all of the virtual hosts such that accessing them returns an InternalServerError result.
+ // Setting the configuration to fail open will mean that Envoy Gateway ignores the error and keeps the routes
+ // as they were before the extension server was called.
+ if t.ExtensionManager != nil && !(*t.ExtensionManager).FailOpen() {
+ vHost.Routes = []*routev3.Route{
+ {
+ Name: "error_route",
+ Match: &routev3.RouteMatch{
+ PathSpecifier: &routev3.RouteMatch_Prefix{
+ Prefix: "/",
+ },
+ },
+ Action: &routev3.Route_DirectResponse{
+ DirectResponse: buildXdsDirectResponseAction(&ir.CustomResponse{StatusCode: ptr.To(uint32(http.StatusInternalServerError))}),
+ },
+ },
+ }
+ }
}
}
xdsRouteCfg.VirtualHosts = append(xdsRouteCfg.VirtualHosts, vHostList...)
@@ -521,7 +622,11 @@ func (t *Translator) addHTTPFiltersToHCM(filterChain *listenerv3.FilterChain, ht
if err = t.patchHCMWithFilters(hcm, httpListener); err != nil {
return err
}
+ return replaceHCMInFilterChain(hcm, filterChain)
+}
+func replaceHCMInFilterChain(hcm *hcmv3.HttpConnectionManager, filterChain *listenerv3.FilterChain) error {
+ var err error
for i, filter := range filterChain.Filters {
if filter.Name == wellknown.HTTPConnectionManager {
var mgrAny *anypb.Any
diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go
index 4d41b865afa8..507316e3371c 100644
--- a/internal/xds/translator/translator_test.go
+++ b/internal/xds/translator/translator_test.go
@@ -219,6 +219,9 @@ func TestTranslateXdsWithExtension(t *testing.T) {
"http-route-extension-listener-error": {
errMsg: "rpc error: code = Unknown desc = extension post xds listener hook error",
},
+ "http-route-extension-translate-error": {
+ errMsg: "rpc error: code = Unknown desc = cluster hook resource error: fail-close-error",
+ },
}
inputFiles, err := filepath.Glob(filepath.Join("testdata", "in", "extension-xds-ir", "*.yaml"))
@@ -282,28 +285,28 @@ func TestTranslateXdsWithExtension(t *testing.T) {
require.EqualError(t, err, cfg.errMsg)
} else {
require.NoError(t, err)
- listeners := tCtx.XdsResources[resourcev3.ListenerType]
- routes := tCtx.XdsResources[resourcev3.RouteType]
- clusters := tCtx.XdsResources[resourcev3.ClusterType]
- endpoints := tCtx.XdsResources[resourcev3.EndpointType]
- if *overrideTestData {
- require.NoError(t, file.Write(requireResourcesToYAMLString(t, listeners), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".listeners.yaml")))
- require.NoError(t, file.Write(requireResourcesToYAMLString(t, routes), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".routes.yaml")))
- require.NoError(t, file.Write(requireResourcesToYAMLString(t, clusters), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".clusters.yaml")))
- require.NoError(t, file.Write(requireResourcesToYAMLString(t, endpoints), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".endpoints.yaml")))
- }
- require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".listeners.yaml"), requireResourcesToYAMLString(t, listeners))
- require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".routes.yaml"), requireResourcesToYAMLString(t, routes))
- require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".clusters.yaml"), requireResourcesToYAMLString(t, clusters))
- require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".endpoints.yaml"), requireResourcesToYAMLString(t, endpoints))
+ }
+ listeners := tCtx.XdsResources[resourcev3.ListenerType]
+ routes := tCtx.XdsResources[resourcev3.RouteType]
+ clusters := tCtx.XdsResources[resourcev3.ClusterType]
+ endpoints := tCtx.XdsResources[resourcev3.EndpointType]
+ if *overrideTestData {
+ require.NoError(t, file.Write(requireResourcesToYAMLString(t, listeners), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".listeners.yaml")))
+ require.NoError(t, file.Write(requireResourcesToYAMLString(t, routes), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".routes.yaml")))
+ require.NoError(t, file.Write(requireResourcesToYAMLString(t, clusters), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".clusters.yaml")))
+ require.NoError(t, file.Write(requireResourcesToYAMLString(t, endpoints), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".endpoints.yaml")))
+ }
+ require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".listeners.yaml"), requireResourcesToYAMLString(t, listeners))
+ require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".routes.yaml"), requireResourcesToYAMLString(t, routes))
+ require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".clusters.yaml"), requireResourcesToYAMLString(t, clusters))
+ require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".endpoints.yaml"), requireResourcesToYAMLString(t, endpoints))
- secrets, ok := tCtx.XdsResources[resourcev3.SecretType]
- if ok {
- if *overrideTestData {
- require.NoError(t, file.Write(requireResourcesToYAMLString(t, secrets), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".secrets.yaml")))
- }
- require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".secrets.yaml"), requireResourcesToYAMLString(t, secrets))
+ secrets, ok := tCtx.XdsResources[resourcev3.SecretType]
+ if ok {
+ if *overrideTestData {
+ require.NoError(t, file.Write(requireResourcesToYAMLString(t, secrets), filepath.Join("testdata", "out", "extension-xds-ir", inputFileName+".secrets.yaml")))
}
+ require.Equal(t, requireTestDataOutFile(t, "extension-xds-ir", inputFileName+".secrets.yaml"), requireResourcesToYAMLString(t, secrets))
}
})
}
diff --git a/release-notes/current.yaml b/release-notes/current.yaml
index 5bd2fe46230d..54447797734d 100644
--- a/release-notes/current.yaml
+++ b/release-notes/current.yaml
@@ -7,6 +7,7 @@ breaking changes: |
An empty TLS ALPNProtocols list is now treated as user-defined disablement of the TLS ALPN extension.
Outlier detection (passive health check) is now disabled by default.
refer to https://gateway.envoyproxy.io/docs/api/extension_types/#backendtrafficpolicy for working with passive health checks.
+ Envoy Gateway treats errors in calls to an extension service as fail-closed by default. Any error returned from the extension server will replace the affected resource with an "Internal Server Error" immediate response. The previous behavior can be enabled by setting the `failOpen` field to `true` in the extension service configuration.
# Updates addressing vulnerabilities, security flaws, or compliance requirements.
security updates: |
diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md
index b7398e09b657..4e3e89da5dba 100644
--- a/site/content/en/latest/api/extension_types.md
+++ b/site/content/en/latest/api/extension_types.md
@@ -1597,6 +1597,7 @@ _Appears in:_
| `policyResources` | _[GroupVersionKind](#groupversionkind) array_ | false | PolicyResources defines the set of K8S resources the extension server will handle
as directly attached GatewayAPI policies |
| `hooks` | _[ExtensionHooks](#extensionhooks)_ | true | Hooks defines the set of hooks the extension supports |
| `service` | _[ExtensionService](#extensionservice)_ | true | Service defines the configuration of the extension service that the Envoy
Gateway Control Plane will call through extension hooks. |
+| `failOpen` | _boolean_ | false | FailOpen defines if Envoy Gateway should ignore errors returned from the Extension Service hooks.
The default is false, which means Envoy Gateway will fail closed if the Extension Service returns an error.
Fail-close means that if the Extension Service hooks return an error, the relevant route/listener/resource
will be replaced with a default configuration returning Internal Server Error (HTTP 500).
Fail-open means that if the Extension Service hooks return an error, no changes will be applied to the
source of the configuration which was sent to the extension server. |
#### ExtensionService
diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md
index b7398e09b657..4e3e89da5dba 100644
--- a/site/content/zh/latest/api/extension_types.md
+++ b/site/content/zh/latest/api/extension_types.md
@@ -1597,6 +1597,7 @@ _Appears in:_
| `policyResources` | _[GroupVersionKind](#groupversionkind) array_ | false | PolicyResources defines the set of K8S resources the extension server will handle
as directly attached GatewayAPI policies |
| `hooks` | _[ExtensionHooks](#extensionhooks)_ | true | Hooks defines the set of hooks the extension supports |
| `service` | _[ExtensionService](#extensionservice)_ | true | Service defines the configuration of the extension service that the Envoy
Gateway Control Plane will call through extension hooks. |
+| `failOpen` | _boolean_ | false | FailOpen defines if Envoy Gateway should ignore errors returned from the Extension Service hooks.
The default is false, which means Envoy Gateway will fail closed if the Extension Service returns an error.
Fail-close means that if the Extension Service hooks return an error, the relevant route/listener/resource
will be replaced with a default configuration returning Internal Server Error (HTTP 500).
Fail-open means that if the Extension Service hooks return an error, no changes will be applied to the
source of the configuration which was sent to the extension server. |
#### ExtensionService