TUIC/BBR: upload throughput drops after a download on a reused connection (local CLI reproducer)

sing-box version 1.14.1

Environment: go1.26.8 linux/amd64
Tags: with_gvisor,with_quic,with_dhcp,with_wireguard,with_utls,with_acme,with_clash_api,with_tailscale,with_ccm,with_ocm,with_cloudflared,with_naive_outbound,with_usbip,with_openvpn,with_openconnect,badlinkname,tfogo_checklinkname0,with_musl
Revision: 1ac1a339cb1223e9c70eae14c44411c75033c02d
CGO: enabled

Python workload -> SOCKS 127.0.0.1:18890 -> sing-box TUIC client
        isolated client namespace, 192.0.2.1
               <veth; HTB 100 Mbps + netem 5 ms each way>
        isolated server namespace, 192.0.2.2
sing-box TUIC server :14443 -> Python receiver 127.0.0.1:18080

#!/usr/bin/env python3
"""Local TUIC connection-history benchmark. Requires root, Linux ip/tc, openssl.
Only creates disposable network namespaces; no production config or host qdisc changes.
Usage: sudo python3 repro.py --binary /absolute/path/sing-box --output ./repro-output
"""
import argparse, concurrent.futures, hashlib, json, os, pathlib, shutil, socket, socketserver, struct, subprocess, sys, tempfile, threading, time, uuid

def exact(s,n):
    b=b''
    while len(b)<n:
        x=s.recv(n-len(b))
        if not x:raise EOFError('short transfer')
        b+=x
    return b

class Receiver(socketserver.BaseRequestHandler):
    def handle(self):
        s=self.request;s.settimeout(60);s.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
        line=b''
        while not line.endswith(b'\n'):
            line+=exact(s,1)
            if len(line)>100:raise ValueError('long header')
        op,n=line.decode().split();n=int(n)
        if op=='UP':
            count=0
            while count<n:
                data=s.recv(min(65536,n-count))
                if not data:raise EOFError('short upload')
                if data!=b'x'*len(data):raise ValueError('wrong upload data')
                count+=len(data)
            s.sendall((str(count)+'\n').encode())
        elif op=='DOWN':
            remaining=n
            while remaining:
                data=b'x'*min(65536,remaining);s.sendall(data);remaining-=len(data)

def transfer(op,size,barrier):
    with socket.create_connection(('127.0.0.1',18890),timeout=60) as s:
        s.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
        s.sendall(b'\5\1\0');assert exact(s,2)==b'\5\0'
        s.sendall(b'\5\1\0\1'+socket.inet_aton('127.0.0.1')+struct.pack('!H',18080))
        h=exact(s,4);assert h[1]==0
        if h[3]==1:exact(s,6)
        elif h[3]==4:exact(s,18)
        else:exact(s,exact(s,1)[0]+2)
        barrier.wait(60);s.sendall(f'{op} {size}\n'.encode());count=0
        while count<size:
            if op=='UP':
                data=b'x'*min(65536,size-count);s.sendall(data)
            else:
                data=s.recv(min(65536,size-count))
                if not data:raise EOFError('short download')
                if data!=b'x'*len(data):raise ValueError('wrong download data')
            count+=len(data)
        if op=='UP':
            ack=b''
            while not ack.endswith(b'\n'):ack+=exact(s,1)
            assert int(ack)==size
        return count

def worker(phases):
    results=[]
    for op,mib,streams in phases:
        size=mib*1048576;barrier=threading.Barrier(streams);start=time.monotonic()
        with concurrent.futures.ThreadPoolExecutor(max_workers=streams) as pool:
            jobs=[pool.submit(transfer,op,size//streams,barrier) for _ in range(streams)]
            total=sum(j.result() for j in jobs)
        elapsed=time.monotonic()-start;assert total==size
        results.append({'direction':op,'MiB':mib,'streams':streams,'bytes':total,'seconds':round(elapsed,6),'Mbps':round(size*8/elapsed/1e6,3)})
    print(json.dumps(results),flush=True)

def main():
    ap=argparse.ArgumentParser();ap.add_argument('--binary',default='/usr/bin/sing-box');ap.add_argument('--output',required=True);ap.add_argument('--delay-ms',type=float,default=5);ap.add_argument('--rate-mbps',type=int,default=100);ap.add_argument('--rounds',type=int,default=2);a=ap.parse_args()
    if os.geteuid()!=0:raise SystemExit('Run as root: disposable network namespaces require it.')
    binary=str(pathlib.Path(a.binary).resolve());script=str(pathlib.Path(__file__).resolve());out=pathlib.Path(a.output).resolve()
    if out.exists():raise SystemExit('Output directory already exists; choose a new path.')
    out.mkdir(mode=0o700,parents=True)
    for tool in ['ip','tc','openssl']:
        if not shutil.which(tool):raise SystemExit('Missing required tool: '+tool)
    def run(args,**kw):return subprocess.run(args,check=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,**kw)
    suffix=uuid.uuid4().hex[:8];cn='tuic-c-'+suffix;sn='tuic-s-'+suffix;names=[];processes=[];logs=[]
    def ns(n,*args):return ['ip','netns','exec',n,*args]
    def launch(args,name):
        f=(out/name).open('w');logs.append(f);env=os.environ.copy();env['QUIC_GO_DISABLE_GSO']='true'
        p=subprocess.Popen(args,stdout=f,stderr=f,env=env);processes.append(p);return p
    def stop(p):
        if p.poll() is None:
            p.terminate()
            try:p.wait(5)
            except subprocess.TimeoutExpired:p.kill();p.wait()
    result={'environment':{'uname':run(['uname','-srmo']).stdout.decode().strip(),'os_release':pathlib.Path('/etc/os-release').read_text(),'version':run([binary,'version']).stdout.decode(),'binary_sha256':hashlib.sha256(pathlib.Path(binary).read_bytes()).hexdigest()},'link':{'rate_mbps_each_direction':a.rate_mbps,'delay_ms_each_direction':a.delay_ms,'loss_percent':0,'netem_limit_packets':3000,'quic_gso_disabled_in_lab':True},'cases':[]}
    try:
        for n in [cn,sn]:run(['ip','netns','add',n]);names.append(n);run(ns(n,'ip','link','set','lo','up'))
        run(ns(cn,'ip','link','add','eth0','type','veth','peer','name','peer0'))
        run(ns(cn,'ip','link','set','peer0','netns',sn));run(ns(sn,'ip','link','set','peer0','name','eth0'))
        for n,addr in [(cn,'192.0.2.1/30'),(sn,'192.0.2.2/30')]:
            run(ns(n,'ip','addr','add',addr,'dev','eth0'));run(ns(n,'ip','link','set','eth0','up'))
            # Both namespaces remain isolated; this route only satisfies the
            # core's default-interface monitor and has no path to the Internet.
            run(ns(n,'ip','route','add','default','dev','eth0'))
            run(ns(n,'tc','qdisc','add','dev','eth0','root','handle','1:','htb','default','10'))
            run(ns(n,'tc','class','add','dev','eth0','parent','1:','classid','1:10','htb','rate',f'{a.rate_mbps}mbit','ceil',f'{a.rate_mbps}mbit','quantum','1514'))
            run(ns(n,'tc','qdisc','add','dev','eth0','parent','1:10','handle','10:','netem','limit','3000','delay',f'{a.delay_ms}ms'))
        with tempfile.TemporaryDirectory(prefix='tuic-repro-key-') as td:
            key=pathlib.Path(td)/'key.pem';cert=out/'certificate.pem'
            run(['openssl','req','-x509','-newkey','rsa:2048','-nodes','-keyout',str(key),'-out',str(cert),'-days','1','-subj','/CN=tuic-repro.test','-addext','subjectAltName=DNS:tuic-repro.test'])
            user=str(uuid.uuid4());password=uuid.uuid4().hex
            server={'log':{'level':'debug','timestamp':True},'inbounds':[{'type':'tuic','tag':'tuic-in','listen':'192.0.2.2','listen_port':14443,'users':[{'uuid':user,'password':password}],'congestion_control':'bbr','initial_packet_size':1200,'disable_path_mtu_discovery':True,'tls':{'enabled':True,'alpn':['h3'],'certificate_path':str(cert),'key_path':str(key)}}],'outbounds':[{'type':'direct','tag':'direct'}]}
            client={'log':{'level':'debug','timestamp':True},'inbounds':[{'type':'mixed','listen':'127.0.0.1','listen_port':18890}],'outbounds':[{'type':'tuic','tag':'tuic-out','server':'192.0.2.2','server_port':14443,'uuid':user,'password':password,'congestion_control':'bbr','udp_relay_mode':'native','initial_packet_size':1200,'disable_path_mtu_discovery':True,'tls':{'enabled':True,'server_name':'tuic-repro.test','alpn':['h3'],'certificate_path':str(cert)}}],'route':{'final':'tuic-out'}}
            for name,cfg in [('server.json',server),('client.json',client)]:
                p=out/name;p.write_text(json.dumps(cfg,indent=2)+'\n');p.chmod(0o600);run([binary,'check','-c',str(p)])
            sp=launch(ns(sn,binary,'run','-c',str(out/'server.json')),'server.log')
            rp=launch(ns(sn,sys.executable,script,'receiver'),'receiver.log');time.sleep(.5)
            if sp.poll() is not None or rp.poll() is not None:raise RuntimeError('receiver/server did not start; inspect logs')
            for rnd in range(1,a.rounds+1):
                variants=['fresh','after-download'] if rnd%2 else ['after-download','fresh']
                for variant in variants:
                    cp=launch(ns(cn,binary,'run','-c',str(out/'client.json')),f'client-r{rnd}-{variant}.log');time.sleep(.5)
                    if cp.poll() is not None:raise RuntimeError('client did not start; inspect logs')
                    phases=[['UP',1,1]]
                    if variant=='after-download':phases += [['UP',4,1],['DOWN',4,1],['UP',16,1],['DOWN',16,1]]
                    phases += [['UP',16,4]]
                    try:
                        data=run(ns(cn,sys.executable,script,'worker',json.dumps(phases)),timeout=240)
                        row={'round':rnd,'variant':variant,'phases':json.loads(data.stdout)};result['cases'].append(row);print(json.dumps(row),flush=True)
                    finally:stop(cp)
            result['qdisc_end']={n:json.loads(run(ns(n,'tc','-s','-j','qdisc','show','dev','eth0')).stdout) for n in [cn,sn]}
    except Exception as e:
        result['error']=type(e).__name__+': '+str(e);raise
    finally:
        for p in reversed(processes):stop(p)
        for f in logs:f.close()
        for n in reversed(names):run(['ip','netns','del',n])
        (out/'results.json').write_text(json.dumps(result,indent=2)+'\n')
        print('Results/logs saved to '+str(out)+'; temporary namespaces and key removed.',flush=True)
if __name__=='__main__':
    if len(sys.argv)>1 and sys.argv[1]=='receiver':
        class S(socketserver.ThreadingTCPServer):
            allow_reuse_address=True;daemon_threads=True
        S(('127.0.0.1',18080),Receiver).serve_forever()
    elif len(sys.argv)>1 and sys.argv[1]=='worker':worker(json.loads(sys.argv[2]))
    else:main()

{
  "log": {
    "level": "debug",
    "timestamp": true
  },
  "inbounds": [
    {
      "type": "tuic",
      "tag": "tuic-in",
      "listen": "192.0.2.2",
      "listen_port": 14443,
      "users": [
        {
          "uuid": "d1f38bf5-4568-40a1-9c0f-0ed78b26267b",
          "password": "c1984724332e4972b4c3a709a7a7a6b3"
        }
      ],
      "congestion_control": "bbr",
      "initial_packet_size": 1200,
      "disable_path_mtu_discovery": true,
      "tls": {
        "enabled": true,
        "alpn": [
          "h3"
        ],
        "certificate_path": "/tmp/tuic-issue-work/run-final/certificate.pem",
        "key_path": "/tmp/tuic-repro-key-_kj6gbye/key.pem"
      }
    }
  ],
  "outbounds": [
    {
      "type": "direct",
      "tag": "direct"
    }
  ]
}

{
  "log": {
    "level": "debug",
    "timestamp": true
  },
  "inbounds": [
    {
      "type": "mixed",
      "listen": "127.0.0.1",
      "listen_port": 18890
    }
  ],
  "outbounds": [
    {
      "type": "tuic",
      "tag": "tuic-out",
      "server": "192.0.2.2",
      "server_port": 14443,
      "uuid": "d1f38bf5-4568-40a1-9c0f-0ed78b26267b",
      "password": "c1984724332e4972b4c3a709a7a7a6b3",
      "congestion_control": "bbr",
      "udp_relay_mode": "native",
      "initial_packet_size": 1200,
      "disable_path_mtu_discovery": true,
      "tls": {
        "enabled": true,
        "server_name": "tuic-repro.test",
        "alpn": [
          "h3"
        ],
        "certificate_path": "/tmp/tuic-issue-work/run-final/certificate.pem"
      }
    }
  ],
  "route": {
    "final": "tuic-out"
  }
}

{
  "environment": {
    "uname": "Linux 6.18.35-x64v1-xanmod1 x86_64 GNU/Linux",
    "os_release": "PRETTY_NAME=\"Debian GNU/Linux forky/sid\"\nNAME=\"Debian GNU/Linux\"\nVERSION_CODENAME=forky\nID=debian\nHOME_URL=\"https://www.debian.org/\"\nSUPPORT_URL=\"https://www.debian.org/support\"\nBUG_REPORT_URL=\"https://bugs.debian.org/\"\n",
    "version": "sing-box version 1.14.1\n\nEnvironment: go1.26.8 linux/amd64\nTags: with_gvisor,with_quic,with_dhcp,with_wireguard,with_utls,with_acme,with_clash_api,with_tailscale,with_ccm,with_ocm,with_cloudflared,with_naive_outbound,with_usbip,with_openvpn,with_openconnect,badlinkname,tfogo_checklinkname0,with_musl\nRevision: 1ac1a339cb1223e9c70eae14c44411c75033c02d\nCGO: enabled\n",
    "binary_sha256": "f766eb1d1733940b332cc65f38f899c6b5392c3f9e4a35d424da2d469510236e"
  },
  "link": {
    "rate_mbps_each_direction": 100,
    "delay_ms_each_direction": 5,
    "loss_percent": 0,
    "netem_limit_packets": 3000,
    "quic_gso_disabled_in_lab": true
  },
  "cases": [
    {
      "round": 1,
      "variant": "fresh",
      "phases": [
        {
          "direction": "UP",
          "MiB": 1,
          "streams": 1