Flaky Test: Data race in TestOutlierDetectionConfigPropagationToChildPolicy
Author: arvindbr8Created Jun 30, 2023Updated Sep 10, 2026
LabelsP1Type: TestingArea: xDS
WARNING: DATA RACE
Write at 0x00c0000b11a0 by goroutine 2937:
runtime.mapassign_faststr()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/map_faststr.go:203 +0x0
google.golang.org/grpc/balancer.Register()
/home/runner/work/grpc-go/grpc-go/balancer/balancer.go:54 +0x77
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.s.TestOutlierDetectionConfigPropagationToChildPolicy.func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go:440 +0x47
runtime.deferreturn()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/panic.go:476 +0x32
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.(*s).TestOutlierDetectionConfigPropagationToChildPolicy()
<autogenerated>:1 +0x44
runtime.call16()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/asm_amd64.s:728 +0x48
reflect.callMethod()
<autogenerated>:1 +0x64
reflect.methodValueCall()
/opt/hostedtoolcache/go/1.20.5/x64/src/reflect/asm_amd64.s:76 +0x79
testing.tRunner()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1576 +0x[216](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:217)
testing.(*T).Run.func1()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1629 +0x47
Previous read at 0x00c0000b11a0 by goroutine 2964:
runtime.mapaccess2_faststr()
/opt/hostedtoolcache/go/1.20.5/x64/src/runtime/map_faststr.go:108 +0x0
google.golang.org/grpc/balancer.Get()
/home/runner/work/grpc-go/grpc-go/balancer/balancer.go:73 +0xa7
google.golang.org/grpc/xds/internal/balancer/wrrlocality.bb.Build()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/wrrlocality/balancer.go:63 +0x46
google.golang.org/grpc/xds/internal/balancer/wrrlocality.(*bb).Build()
<autogenerated>:1 +0xa7
google.golang.org/grpc/xds/internal/balancer/clusterimpl.(*clusterImplBalancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterimpl/clusterimpl.go:259 +0x6b8
google.golang.org/grpc/internal/balancer/gracefulswitch.(*Balancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancer/gracefulswitch/gracefulswitch.go:164 +0x10d
google.golang.org/grpc/xds/internal/balancer/outlierdetection.(*outlierDetectionBalancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/outlierdetection/balancer.go:326 +0xe89
google.golang.org/grpc/internal/balancer/gracefulswitch.(*Balancer).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancer/gracefulswitch/gracefulswitch.go:164 +0x10d
google.golang.org/grpc/internal/balancergroup.(*subBalancerWrapper).updateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancergroup/balancergroup.go:144 +0x2da
google.golang.org/grpc/internal/balancergroup.(*BalancerGroup).UpdateClientConnState()
/home/runner/work/grpc-go/grpc-go/internal/balancergroup/balancergroup.go:478 +0x136
google.golang.org/grpc/xds/internal/balancer/priority.(*childBalancer).sendUpdate()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_child.go:113 +0x1b8
google.golang.org/grpc/xds/internal/balancer/priority.(*childBalancer).start()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_child.go:106 +0x15e
google.golang.org/grpc/xds/internal/balancer/priority.(*priorityBalancer).switchToChild()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_priority.go:165 +0x157
google.golang.org/grpc/xds/internal/balancer/priority.(*priorityBalancer).syncPriority()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer_priority.go:108 +0x62c
google.golang.org/grpc/xds/internal/balancer/priority.(*priorityBalancer).run()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer.go:268 +0x2f9
google.golang.org/grpc/xds/internal/balancer/priority.bb.Build.func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer.go:65 +0x39
Goroutine 2937 (running) created at:
testing.(*T).Run()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1629 +0x805
google.golang.org/grpc/internal/grpctest.RunSubTests()
/home/runner/work/grpc-go/grpc-go/internal/grpctest/grpctest.go:101 +0x50e
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.Test()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go:74 +0x3c
testing.tRunner()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1576 +0x216
testing.(*T).Run.func1()
/opt/hostedtoolcache/go/1.20.5/x64/src/testing/testing.go:1629 +0x47
Goroutine 2964 (running) created at:
google.golang.org/grpc/xds/internal/balancer/priority.bb.Build()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/priority/balancer.go:65 +0x925
google.golang.org/grpc/xds/internal/balancer/priority.(*bb).Build()
<autogenerated>:1 +0xa7
google.golang.org/grpc/xds/internal/balancer/clusterresolver/e2e_test_test.(*wrappedPriorityBuilder).Build()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go:342 +0xc7
google.golang.org/grpc/xds/internal/balancer/clusterresolver.glob..func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:53 +0xaf
google.golang.org/grpc/xds/internal/balancer/clusterresolver.(*clusterResolverBalancer).updateChildConfig()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:241 +0x177
google.golang.org/grpc/xds/internal/balancer/clusterresolver.(*clusterResolverBalancer).handleResourceUpdate()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:230 +0x109
google.golang.org/grpc/xds/internal/balancer/clusterresolver.(*clusterResolverBalancer).run()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:331 +0x512
google.golang.org/grpc/xds/internal/balancer/clusterresolver.bb.Build.func1()
/home/runner/work/grpc-go/grpc-go/xds/internal/balancer/clusterresolver/clusterresolver.go:94 +0x39
==================
--- FAIL: Test (5.40s)
--- FAIL: Test/OutlierDetectionConfigPropagationToChildPolicy (0.02s)
tlogger.go:116: INFO server.go:132 [xds-e2e] Created new snapshot cache... (t=+59.5µs)
tlogger.go:116: INFO server.go:652 [core] [Server #573] Server created (t=+419.102µs)
tlogger.go:116: INFO server.go:158 [xds-e2e] Registered Aggregated Discovery Service (ADS)... (t=+479.303µs)
tlogger.go:116: INFO server.go:177 [xds-e2e] xDS management server serving at: 127.0.0.1:45339... (t=+526.903µs)
tlogger.go:116: INFO server.go:840 [core] [Server #573 ListenSocket #574] ListenSocket created (t=+1.42921ms)
tlogger.go:116: INFO server.go:652 [core] [Server #575] Server created (t=+2.137015ms)
stubserver.go:222: Started test service backend at "127.0.0.1:40833"
tlogger.go:116: INFO server.go:[220](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:221) [xds-e2e] Created new resource snapshot... (t=+2.281516ms)
tlogger.go:116: INFO server.go:226 [xds-e2e] Updated snapshot cache with resource snapshot... (t=+2.327116ms)
tlogger.go:116: INFO server.go:840 [core] [Server #575 ListenSocket #576] ListenSocket created (t=+3.686626ms)
tlogger.go:116: INFO client_new.go:80 [xds] [xds-client 0xc0005769b0] Created client to xDS management server: 127.0.0.1:45339-insecure-xds_v3 (t=+5.409738ms)
tlogger.go:116: INFO clientconn.go:316 [core] [Channel #577] Channel created (t=+5.652939ms)
tlogger.go:116: INFO clientconn.go:1770 [core] [Channel #577] original dial target is: "whatever:///test.service" (t=+5.70334ms)
tlogger.go:116: INFO clientconn.go:1777 [core] [Channel #577] parsed dial target is: {Scheme:whatever Authority: URL:{Scheme:whatever Opaque: User: Host: Path:/test.service RawPath: OmitHost:false ForceQuery:false RawQuery: Fragment: RawFragment:}} (t=+5.78894ms)
tlogger.go:116: INFO clientconn.go:1877 [core] [Channel #577] Channel authority set to "test.service" (t=+5.832841ms)
tlogger.go:116: INFO resolver_conn_wrapper.go:238 [core] [Channel #577] Resolver state updated: {
"Addresses": null,
"ServiceConfig": {
"Config": {
"Config": null,
"LB": null,
"Methods": {}
},
"Err": null
},
"Attributes": {}
} (service config updated) (t=+6.026142ms)
tlogger.go:116: INFO balancer_conn_wrappers.go:192 [core] [Channel #577] Channel switches to new LB policy "cds_experimental" (t=+6.124043ms)
tlogger.go:116: INFO cdsbalancer.go:99 [xds] [cds-lb 0xc0007c9860] Created (t=+6.188143ms)
tlogger.go:116: INFO cdsbalancer.go:110 [xds] [cds-lb 0xc0007c9860] xDS credentials in use: false (t=+6.[224](https://github.com/grpc/grpc-go/actions/runs/5425936248/jobs/9867334312?pr=6425#step:8:225)543ms)
tlogger.go:116: INFO cdsbalancer.go:514 [xds] [cds-lb 0xc0007c9860] Received balancer config update: {
"LoadBalancingConfig": null,
"Cluster": "cluster-my-service-client-side-xds"
} (t=+6.338744ms)
tlogger.go:116: INFO cluster_handler.go:190 [xds] [cds-lb 0xc0007c9860] CDS watch started on cluster-my-service-client-side-xds (t=+6.421145ms)
tlogger.go:116: INFO clientconn.go:316 [core] [Channel #578] Channel created (t=+6.572946ms)
tlogger.go:116: INFO clientconn.go:1770 [core] [Channel #578] original dial target is: "127.0.0.1:45339" (t=+6.615646ms)
tlogger.go:116: INFO clientconn.go:1775 [core] [Channel #578] dial target "127.0.0.1:45339" parse failed: parse "127.0.0.1:45339": first path segment in URL cannot contain colon (t=+6.666547ms)
tlogger.go:116: INFO clientconn.go:1791 [core] [Channel #578] fallback to scheme "passthrough" (t=+6.699047ms)
tlogger.go:116: INFO clientconn.go:1799 [core] [Channel #578] parsed dial target is: {Scheme:passthrough Authority: URL:{Scheme:passthrough Opaque: User: Host: Path:/127.0.0.1:45339 RawPath: OmitHost:false ForceQuery:false RawQuery: Fragment: RawFragment:}} (t=+6.758147ms)
tlogger.go:116: INFO clientconn.go:1877 [core] [Channel #578] Channel authority set to "127.0.0.1:45339" (t=+6.799348ms)
tlogger.go:116: INFO resolver_conn_wrapper.go:238 [core] [Channel #578] Resolver state updated: {
"Addresses": [
{
"Addr": "127.0.0.1:45339",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Type": 0,
"Metadata": null
}
],
"ServiceConfig": null,
"Attributes": null
} (resolver returned new addresses) (t=+7.038749ms)
tlogger.go:116: INFO balancer_conn_wrappers.go:192 [core] [Channel #578] Channel switches to new LB policy "pick_first" (t=+7.16305ms)
tlogger.go:116: INFO clientconn.go:897 [core] [Channel #578 SubChannel #579] Subchannel created (t=+7.260951ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #578] Channel Connectivity change to CONNECTING (t=+7.311151ms)
tlogger.go:116: INFO transport.go:238 [xds] [xds-client 0xc0005769b0] [127.0.0.1:45339] Created transport to server "127.0.0.1:45339" (t=+7.578853ms)
tlogger.go:116: INFO clientconn.go:1285 [core] [Channel #578 SubChannel #579] Subchannel Connectivity change to CONNECTING (t=+7.655654ms)
tlogger.go:116: INFO clientconn.go:1398 [core] [Channel #578 SubChannel #579] Subchannel picks a new address "127.0.0.1:45339" to connect (t=+7.737954ms)
tlogger.go:116: INFO clientconn.go:1285 [core] [Channel #578 SubChannel #579] Subchannel Connectivity change to READY (t=+8.814462ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #578] Channel Connectivity change to READY (t=+8.909362ms)
tlogger.go:116: INFO transport.go:347 [xds] [xds-client 0xc0005769b0] [127.0.0.1:45339] ADS stream created (t=+9.035763ms)
tlogger.go:116: INFO simple.go:359 [xds-e2e] nodeID "7a165661-0c37-4992-9315-770a7dc9bd19" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] and known map[]. Diff [cluster-my-service-client-side-xds] (t=+9.882069ms)
tlogger.go:116: INFO simple.go:428 [xds-e2e] respond type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] version "" with version "1" (t=+9.94087ms)
tlogger.go:116: INFO cdsbalancer.go:297 [xds] [cds-lb 0xc0007c9860] Received Cluster resource contains content: [
{
"ClusterType": 0,
"ClusterName": "cluster-my-service-client-side-xds",
"EDSServiceName": "endpoints-my-service-client-side-xds",
"LRSServerConfig": 0,
"SecurityCfg": null,
"MaxRequests": null,
"DNSHostName": "",
"PrioritizedClusterNames": null,
"LBPolicy": [
{
"xds_wrr_locality_experimental": {
"childPolicy": [
{
"round_robin": {}
}
]
}
}
],
"OutlierDetection": {
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 2000,
"enforcementPercentage": 50,
"minimumHosts": 10,
"requestVolume": 50
}
},
"Raw": {
"type_url": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"value": "CiJjbHVzdGVyLW15LXNlcnZpY2UtY2xpZW50LXNpZGUteGRzGioKAhoAEiRlbmRwb2ludHMtbXktc2VydmljZS1jbGllbnQtc2lkZS14ZHOaASMSAggKGgIIHiICCAoyAggyOgIICkICCDJKAwjQD6oBAwisAhAD"
}
}
], security config: null (t=+10.856176ms)
tlogger.go:116: INFO clusterresolver.go:86 [xds] [xds-cluster-resolver-lb 0xc0002ef6c0] Created (t=+10.941277ms)
tlogger.go:116: INFO cdsbalancer.go:323 [xds] [cds-lb 0xc0007c9860] Created child policy 0xc0002ef6c0 of type cluster_resolver_experimental (t=+10.984877ms)
tlogger.go:116: INFO clusterresolver.go:196 [xds] [xds-cluster-resolver-lb 0xc0002ef6c0] Received new balancer config: {
"discoveryMechanisms": [
{
"cluster": "cluster-my-service-client-side-xds",
"edsServiceName": "endpoints-my-service-client-side-xds",
"outlierDetection": {
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 2000,
"enforcementPercentage": 50,
"minimumHosts": 10,
"requestVolume": 50
}
}
}
],
"xdsLbPolicy": [
{
"xds_wrr_locality_experimental": {
"childPolicy": [
{
"round_robin": {}
}
]
}
}
]
} (t=+11.588281ms)
tlogger.go:116: INFO simple.go:359 [xds-e2e] nodeID "7a165661-0c37-4992-9315-770a7dc9bd19" requested type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] and known map[cluster-my-service-client-side-xds:{}]. Diff [] (t=+12.113285ms)
tlogger.go:116: INFO simple.go:380 [xds-e2e] open watch 1 for type.googleapis.com/envoy.config.cluster.v3.Cluster[cluster-my-service-client-side-xds] from nodeID "7a165661-0c37-4992-9315-770a7dc9bd19", version "1" (t=+12.176885ms)
tlogger.go:116: INFO simple.go:359 [xds-e2e] nodeID "7a165661-0c37-4992-9315-770a7dc9bd19" requested type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment[endpoints-my-service-client-side-xds] and known map[]. Diff [endpoints-my-service-client-side-xds] (t=+12.324487ms)
tlogger.go:116: INFO simple.go:428 [xds-e2e] respond type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment[endpoints-my-service-client-side-xds] version "" with version "1" (t=+12.405587ms)
tlogger.go:116: INFO balancer.go:66 [xds] [priority-lb 0xc000146cb0] Created (t=+13.208593ms)
tlogger.go:116: INFO clusterresolver.go:254 [xds] [xds-cluster-resolver-lb 0xc0002ef6c0] Built child policy config: {
"children": {
"priority-0-0": {
"config": [
{
"outlier_detection_experimental": {
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 2000,
"enforcementPercentage": 50,
"minimumHosts": 10,
"requestVolume": 50
},
"childPolicy": [
{
"xds_cluster_impl_experimental": {
"cluster": "cluster-my-service-client-side-xds",
"edsServiceName": "endpoints-my-service-client-side-xds",
"childPolicy": [
{
"xds_wrr_locality_experimental": {
"childPolicy": [
{
"round_robin": {}
}
]
}
}
]
}
}
]
}
}
],
"ignoreReresolutionRequests": true
}
},
"priorities": [
"priority-0-0"
]
} (t=+14.477402ms)
tlogger.go:116: INFO clientconn.go:564 [core] [Channel #577] Channel Connectivity change to CONNECTING (t=+15.018606ms)
tlogger.go:116: INFO balancergroup.go:311 [xds] [priority-lb 0xc000146cb0] Adding child policy of type "outlier_detection_experimental" for locality "priority-0-0" (t=+15.070506ms)
tlogger.go:116: INFO balancergroup.go:100 [xds] [priority-lb 0xc000146cb0] Creating child policy of type "outlier_detection_experimental" for locality "priority-0-0" (t=+15.131906ms)
tlogger.go:116: WARSource: grpc/grpc-go