Aurora: fully_consistent and at_least_as_fresh reads intermittently miss the immediately-preceding write.

exec spicedb serve \
  --http-enabled \
  --grpc-preshared-key=$SPICEDB_GRPC_PRESHARED_KEY \
  --datastore-engine=postgres \
  --grpc-max-workers $MAX_WORKERS \
  --dispatch-cache-max-cost=$DISPATCH_CACHE_MAX_COST \
  --datastore-conn-pool-read-max-open $MAX_OPEN_CONNECTIONS \
  --datastore-conn-uri=$DB_URL \
  --skip-release-check=true \
  --datastore-read-replica-conn-uri=$REPLICA_DB_URL \
  --datastore-read-replica-conn-pool-max-open=$REPLICA_MAX_OPEN_CONNECTIONS \
  --grpc-max-conn-age=10m \
  --max-lookup-resources-limit=10000 \
  --datastore-revision-quantization-interval=100ms \
  --log-level=panic \
  --log-format=none

require 'securerandom'

resource_type = 'team'
relation      = 'creator'
subject_type  = 'workspace_user'
subject_id    = SecureRandom.uuid

misses_fully_consistent  = []
misses_at_least_as_fresh = []

client_options = {
  target: 'auth:50051',
  interceptors: [Authzed::GrpcUtil::BearerToken.new(token: 't_your_token_here_1234567deadbeef'), []].compact.flatten
}

# Only use insecure channel in development environment
client_options[:credentials] = :this_channel_is_insecure

client = Authzed::Api::V1::Client.new(**client_options)

(1..1000).each do |i|
  resource_id = "res_#{i}"

  update = Authzed::Api::V1::RelationshipUpdate.new(
    operation: Authzed::Api::V1::RelationshipUpdate::Operation::OPERATION_TOUCH,
    relationship: Authzed::Api::V1::Relationship.new(
      resource: Authzed::Api::V1::ObjectReference.new(object_type: resource_type, object_id: resource_id),
      relation: relation,
      subject: Authzed::Api::V1::SubjectReference.new(
        object: Authzed::Api::V1::ObjectReference.new(object_type: subject_type, object_id: subject_id)
      )
    )
  )

  write_response = client.permissions_service.write_relationships(
    Authzed::Api::V1::WriteRelationshipsRequest.new(updates: [update])
  )
  token = write_response.written_at

  # fully_consistent check
  fc_hit = client.permissions_service.read_relationships(
    Authzed::Api::V1::ReadRelationshipsRequest.new(
      relationship_filter: Authzed::Api::V1::RelationshipFilter.new(
        resource_type: resource_type,
        optional_resource_id: resource_id,
        optional_relation: relation
      ),
      consistency: Authzed::Api::V1::Consistency.new(fully_consistent: true)
    )
  ).any?
  misses_fully_consistent << resource_id unless fc_hit

  # at_least_as_fresh check, using the token from this exact write
  af_hit = client.permissions_service.read_relationships(
    Authzed::Api::V1::ReadRelationshipsRequest.new(
      relationship_filter: Authzed::Api::V1::RelationshipFilter.new(
        resource_type: resource_type,
        optional_resource_id: resource_id,
        optional_relation: relation
      ),
      consistency: Authzed::Api::V1::Consistency.new(at_least_as_fresh: token)
    )
  ).any?
  misses_at_least_as_fresh << resource_id unless af_hit
end

puts "fully_consistent misses:  #{misses_fully_consistent.size}/1000  -> #{misses_fully_consistent}"
puts "at_least_as_fresh misses: #{misses_at_least_as_fresh.size}/1000 -> #{misses_at_least_as_fresh}"


#### OUTPUT ####
fully_consistent misses:  13/1000  -> ["res_103", "res_138", "res_169", "res_194", "res_261", "res_341", "res_423", "res_583", "res_605", "res_695", "res_696", "res_747", "res_786"]
at_least_as_fresh misses: 13/1000 -> ["res_38", "res_86", "res_328", "res_349", "res_414", "res_496", "res_581", "res_697", "res_711", "res_762", "res_837", "res_902", "res_985"]
##############