server/group: refactor with shared abstractions and fix concurrency issues (#5222)

* server/group: refactor group package with shared abstractions and fix concurrency issues

Extract common patterns into reusable components:
- groupRegistry[G]: generic concurrent map for group lifecycle management
- baseGroup: shared plumbing for listener-based groups (TCP, HTTPS, TCPMux)
- Listener: unified virtual listener replacing 3 identical implementations

Fix concurrency issues:
- Stale-pointer race: isCurrent check + errGroupStale + controller retry loops
- Worker generation safety: pass realLn and acceptCh as params instead of reading mutable fields
- Connection leak: close conn on worker panic recovery path
- ABBA deadlock in HTTP UnRegister: consistent lock ordering (group.mu -> registry.mu)
- Round-robin overflow in HTTPGroup: use unsigned modulo

Add unit tests (17 tests) for registry, listener, and baseGroup.
Add TCPMux group load balancing e2e test.

* server/group: replace tautological assertion with require.NotPanics

* server/group: remove blank line between doc comment and type declaration
This commit is contained in:
fatedier
2026-03-08 18:57:21 +08:00
committed by GitHub
parent eeb0dacfc1
commit c7ac12ea0f
12 changed files with 720 additions and 436 deletions

View File

@@ -17,83 +17,67 @@ package group
import (
"net"
"strconv"
"sync"
gerr "github.com/fatedier/golib/errors"
"github.com/fatedier/frp/server/ports"
)
// TCPGroupCtl manage all TCPGroups
// TCPGroupCtl manages all TCPGroups.
type TCPGroupCtl struct {
groups map[string]*TCPGroup
// portManager is used to manage port
groupRegistry[*TCPGroup]
portManager *ports.Manager
mu sync.Mutex
}
// NewTCPGroupCtl return a new TcpGroupCtl
// NewTCPGroupCtl returns a new TCPGroupCtl.
func NewTCPGroupCtl(portManager *ports.Manager) *TCPGroupCtl {
return &TCPGroupCtl{
groups: make(map[string]*TCPGroup),
portManager: portManager,
groupRegistry: newGroupRegistry[*TCPGroup](),
portManager: portManager,
}
}
// Listen is the wrapper for TCPGroup's Listen
// If there are no group, we will create one here
// Listen is the wrapper for TCPGroup's Listen.
// If there is no group, one will be created.
func (tgc *TCPGroupCtl) Listen(proxyName string, group string, groupKey string,
addr string, port int,
) (l net.Listener, realPort int, err error) {
tgc.mu.Lock()
tcpGroup, ok := tgc.groups[group]
if !ok {
tcpGroup = NewTCPGroup(tgc)
tgc.groups[group] = tcpGroup
for {
tcpGroup := tgc.getOrCreate(group, func() *TCPGroup {
return NewTCPGroup(tgc)
})
l, realPort, err = tcpGroup.Listen(proxyName, group, groupKey, addr, port)
if err == errGroupStale {
continue
}
return
}
tgc.mu.Unlock()
return tcpGroup.Listen(proxyName, group, groupKey, addr, port)
}
// RemoveGroup remove TCPGroup from controller
func (tgc *TCPGroupCtl) RemoveGroup(group string) {
tgc.mu.Lock()
defer tgc.mu.Unlock()
delete(tgc.groups, group)
}
// TCPGroup route connections to different proxies
// TCPGroup routes connections to different proxies.
type TCPGroup struct {
group string
groupKey string
baseGroup
addr string
port int
realPort int
acceptCh chan net.Conn
tcpLn net.Listener
lns []*TCPGroupListener
ctl *TCPGroupCtl
mu sync.Mutex
}
// NewTCPGroup return a new TCPGroup
// NewTCPGroup returns a new TCPGroup.
func NewTCPGroup(ctl *TCPGroupCtl) *TCPGroup {
return &TCPGroup{
lns: make([]*TCPGroupListener, 0),
ctl: ctl,
acceptCh: make(chan net.Conn),
ctl: ctl,
}
}
// Listen will return a new TCPGroupListener
// if TCPGroup already has a listener, just add a new TCPGroupListener to the queues
// otherwise, listen on the real address
func (tg *TCPGroup) Listen(proxyName string, group string, groupKey string, addr string, port int) (ln *TCPGroupListener, realPort int, err error) {
// Listen will return a new Listener.
// If TCPGroup already has a listener, just add a new Listener to the queues,
// otherwise listen on the real address.
func (tg *TCPGroup) Listen(proxyName string, group string, groupKey string, addr string, port int) (ln *Listener, realPort int, err error) {
tg.mu.Lock()
defer tg.mu.Unlock()
if !tg.ctl.isCurrent(group, func(cur *TCPGroup) bool { return cur == tg }) {
return nil, 0, errGroupStale
}
if len(tg.lns) == 0 {
// the first listener, listen on the real address
realPort, err = tg.ctl.portManager.Acquire(proxyName, port)
@@ -106,19 +90,18 @@ func (tg *TCPGroup) Listen(proxyName string, group string, groupKey string, addr
err = errRet
return
}
ln = newTCPGroupListener(group, tg, tcpLn.Addr())
tg.group = group
tg.groupKey = groupKey
tg.addr = addr
tg.port = port
tg.realPort = realPort
tg.tcpLn = tcpLn
tg.lns = append(tg.lns, ln)
if tg.acceptCh == nil {
tg.acceptCh = make(chan net.Conn)
}
go tg.worker()
tg.initBase(group, groupKey, tcpLn, func() {
tg.ctl.portManager.Release(tg.realPort)
tg.ctl.removeIf(tg.group, func(cur *TCPGroup) bool {
return cur == tg
})
})
ln = tg.newListener(tcpLn.Addr())
go tg.worker(tcpLn, tg.acceptCh)
} else {
// address and port in the same group must be equal
if tg.group != group || tg.addr != addr {
@@ -133,92 +116,8 @@ func (tg *TCPGroup) Listen(proxyName string, group string, groupKey string, addr
err = ErrGroupAuthFailed
return
}
ln = newTCPGroupListener(group, tg, tg.lns[0].Addr())
ln = tg.newListener(tg.lns[0].Addr())
realPort = tg.realPort
tg.lns = append(tg.lns, ln)
}
return
}
// worker is called when the real tcp listener has been created
func (tg *TCPGroup) worker() {
for {
c, err := tg.tcpLn.Accept()
if err != nil {
return
}
err = gerr.PanicToError(func() {
tg.acceptCh <- c
})
if err != nil {
return
}
}
}
func (tg *TCPGroup) Accept() <-chan net.Conn {
return tg.acceptCh
}
// CloseListener remove the TCPGroupListener from the TCPGroup
func (tg *TCPGroup) CloseListener(ln *TCPGroupListener) {
tg.mu.Lock()
defer tg.mu.Unlock()
for i, tmpLn := range tg.lns {
if tmpLn == ln {
tg.lns = append(tg.lns[:i], tg.lns[i+1:]...)
break
}
}
if len(tg.lns) == 0 {
close(tg.acceptCh)
tg.tcpLn.Close()
tg.ctl.portManager.Release(tg.realPort)
tg.ctl.RemoveGroup(tg.group)
}
}
// TCPGroupListener
type TCPGroupListener struct {
groupName string
group *TCPGroup
addr net.Addr
closeCh chan struct{}
}
func newTCPGroupListener(name string, group *TCPGroup, addr net.Addr) *TCPGroupListener {
return &TCPGroupListener{
groupName: name,
group: group,
addr: addr,
closeCh: make(chan struct{}),
}
}
// Accept will accept connections from TCPGroup
func (ln *TCPGroupListener) Accept() (c net.Conn, err error) {
var ok bool
select {
case <-ln.closeCh:
return nil, ErrListenerClosed
case c, ok = <-ln.group.Accept():
if !ok {
return nil, ErrListenerClosed
}
return c, nil
}
}
func (ln *TCPGroupListener) Addr() net.Addr {
return ln.addr
}
// Close close the listener
func (ln *TCPGroupListener) Close() (err error) {
close(ln.closeCh)
// remove self from TcpGroup
ln.group.CloseListener(ln)
return
}