Refactor Binance Futures Websocket: enhance configuration management, implement shard handling, and improve subscription logic
This commit is contained in:
@@ -1,119 +1,179 @@
|
||||
// Package test implements a configurable synthetic data provider.
|
||||
//
|
||||
// Config via subject string. Two syntaxes are accepted:
|
||||
//
|
||||
// Query style: "foo?period=7us&size=64&mode=const&burst=1&jitter=0.02&drop=1&ts=1&log=1"
|
||||
// Path style: "foo/period/7us/size/64/mode/poisson/rate/120000/jitter/0.05/drop/0/ts/1/log/1"
|
||||
//
|
||||
// Parameters:
|
||||
//
|
||||
// period: Go duration. Inter-message target (wins over rate).
|
||||
// rate: Messages per second. Used if period absent.
|
||||
// mode: const | poisson | onoff
|
||||
// burst: Messages emitted per tick (>=1).
|
||||
// jitter: ±fraction jitter on period (e.g., 0.05 = ±5%).
|
||||
// on/off: Durations for onoff mode (e.g., on=5ms&off=1ms).
|
||||
// size: Payload bytes (>=1). If ts=1 and size<16, auto-extends to 16.
|
||||
// ptype: bytes | counter | json (payload content generator)
|
||||
// drop: 1=non-blocking send (drop on backpressure), 0=block.
|
||||
// ts: 1=prepend 16B header: [sendUnixNano int64][seq int64].
|
||||
// log: 1=emit per-second metrics via slog.
|
||||
//
|
||||
// Notes:
|
||||
// - Constant mode uses sleep-then-spin pacer for sub-10µs.
|
||||
// - Poisson mode draws inter-arrivals from Exp(rate).
|
||||
// - On/Off emits at period during "on", silent during "off" windows.
|
||||
// - Metrics include msgs/s, bytes/s, drops/s per stream.
|
||||
// - Fetch is unsupported (returns error).
|
||||
package test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"gitlab.michelsen.id/phillmichelsen/tessera/services/data_service/internal/domain"
|
||||
)
|
||||
|
||||
type TestProvider struct {
|
||||
mu sync.Mutex
|
||||
streams map[string]*stream
|
||||
outputChannel chan<- domain.Message
|
||||
tickDuration time.Duration
|
||||
mu sync.Mutex
|
||||
streams map[string]*stream
|
||||
out chan<- domain.Message
|
||||
defaults cfg
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
cancel context.CancelFunc
|
||||
done chan struct{}
|
||||
stats *metrics
|
||||
}
|
||||
|
||||
// NewTestProvider wires the outbound channel.
|
||||
func NewTestProvider(out chan<- domain.Message, tickDuration time.Duration) *TestProvider {
|
||||
type metrics struct {
|
||||
sent, dropped atomic.Uint64
|
||||
prevSent uint64
|
||||
prevDropped uint64
|
||||
startUnix int64
|
||||
}
|
||||
|
||||
type mode int
|
||||
|
||||
const (
|
||||
modeConst mode = iota
|
||||
modePoisson
|
||||
modeOnOff
|
||||
)
|
||||
|
||||
type ptype int
|
||||
|
||||
const (
|
||||
ptBytes ptype = iota
|
||||
ptCounter
|
||||
ptJSON
|
||||
)
|
||||
|
||||
type cfg struct {
|
||||
period time.Duration // inter-arrival target
|
||||
rate float64 // msgs/sec if period == 0
|
||||
jitter float64 // ±fraction
|
||||
mode mode
|
||||
onDur time.Duration // for onoff
|
||||
offDur time.Duration // for onoff
|
||||
burst int
|
||||
size int
|
||||
pType ptype
|
||||
dropIfSlow bool
|
||||
embedTS bool
|
||||
logEverySec bool
|
||||
}
|
||||
|
||||
// NewTestProvider returns a provider with sane defaults.
|
||||
func NewTestProvider(out chan<- domain.Message, defaultPeriod time.Duration) *TestProvider {
|
||||
if defaultPeriod <= 0 {
|
||||
defaultPeriod = 100 * time.Microsecond
|
||||
}
|
||||
return &TestProvider{
|
||||
streams: make(map[string]*stream),
|
||||
outputChannel: out,
|
||||
tickDuration: tickDuration,
|
||||
streams: make(map[string]*stream),
|
||||
out: out,
|
||||
defaults: cfg{
|
||||
period: defaultPeriod,
|
||||
rate: 0,
|
||||
jitter: 0,
|
||||
mode: modeConst,
|
||||
onDur: 5 * time.Millisecond,
|
||||
offDur: 1 * time.Millisecond,
|
||||
burst: 1,
|
||||
size: 32,
|
||||
pType: ptBytes,
|
||||
dropIfSlow: true,
|
||||
embedTS: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TestProvider) Start() error { return nil }
|
||||
func (p *TestProvider) Start() error { return nil }
|
||||
|
||||
func (t *TestProvider) Stop() {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
for key, s := range t.streams {
|
||||
func (p *TestProvider) Stop() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
for key, s := range p.streams {
|
||||
s.cancel()
|
||||
<-s.done
|
||||
delete(t.streams, key)
|
||||
delete(p.streams, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TestProvider) Subscribe(subject string) <-chan error {
|
||||
func (p *TestProvider) Subscribe(subject string) <-chan error {
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
if !t.IsValidSubject(subject, false) {
|
||||
if !p.IsValidSubject(subject, false) {
|
||||
errCh <- errors.New("invalid subject")
|
||||
close(errCh)
|
||||
return errCh
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
// Already active: treat as success.
|
||||
if _, ok := t.streams[subject]; ok {
|
||||
t.mu.Unlock()
|
||||
p.mu.Lock()
|
||||
if _, exists := p.streams[subject]; exists {
|
||||
p.mu.Unlock()
|
||||
errCh <- nil
|
||||
return errCh
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := &stream{cancel: cancel, done: make(chan struct{})}
|
||||
t.streams[subject] = s
|
||||
out := t.outputChannel
|
||||
t.mu.Unlock()
|
||||
s := &stream{
|
||||
cancel: cancel,
|
||||
done: make(chan struct{}),
|
||||
stats: &metrics{startUnix: time.Now().Unix()},
|
||||
}
|
||||
p.streams[subject] = s
|
||||
out := p.out
|
||||
conf := p.parseCfg(subject)
|
||||
p.mu.Unlock()
|
||||
|
||||
// Stream goroutine.
|
||||
go func(subj string, s *stream) {
|
||||
slog.Default().Debug("new stream routine started", slog.String("cmp", "test_provider"), slog.String("subject", subj))
|
||||
ticker := time.NewTicker(t.tickDuration)
|
||||
ident, _ := domain.RawID("test_provider", subj)
|
||||
defer func() {
|
||||
ticker.Stop()
|
||||
close(s.done)
|
||||
}()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if out != nil {
|
||||
msg := domain.Message{
|
||||
Identifier: ident,
|
||||
Payload: []byte(time.Now().UTC().Format(time.RFC3339Nano)),
|
||||
}
|
||||
// Non-blocking send avoids deadlock if caller stops reading.
|
||||
select {
|
||||
case out <- msg:
|
||||
default:
|
||||
slog.Default().Warn("dropping message due to backpressure", "cmp", "test_provider", "subject", subj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}(subject, s)
|
||||
go run(ctx, s, out, subject, conf)
|
||||
|
||||
// Signal successful subscription.
|
||||
errCh <- nil
|
||||
return errCh
|
||||
}
|
||||
|
||||
func (t *TestProvider) Unsubscribe(subject string) <-chan error {
|
||||
func (p *TestProvider) Unsubscribe(subject string) <-chan error {
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
t.mu.Lock()
|
||||
s, ok := t.streams[subject]
|
||||
p.mu.Lock()
|
||||
s, ok := p.streams[subject]
|
||||
if !ok {
|
||||
t.mu.Unlock()
|
||||
p.mu.Unlock()
|
||||
errCh <- errors.New("not subscribed")
|
||||
return errCh
|
||||
}
|
||||
delete(t.streams, subject)
|
||||
t.mu.Unlock()
|
||||
delete(p.streams, subject)
|
||||
p.mu.Unlock()
|
||||
|
||||
go func() {
|
||||
s.cancel()
|
||||
@@ -123,27 +183,360 @@ func (t *TestProvider) Unsubscribe(subject string) <-chan error {
|
||||
return errCh
|
||||
}
|
||||
|
||||
func (t *TestProvider) Fetch(subject string) (domain.Message, error) {
|
||||
func (p *TestProvider) Fetch(_ string) (domain.Message, error) {
|
||||
return domain.Message{}, fmt.Errorf("fetch not supported by provider")
|
||||
}
|
||||
|
||||
func (t *TestProvider) GetActiveStreams() []string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
keys := make([]string, 0, len(t.streams))
|
||||
for k := range t.streams {
|
||||
func (p *TestProvider) GetActiveStreams() []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
keys := make([]string, 0, len(p.streams))
|
||||
for k := range p.streams {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func (t *TestProvider) IsStreamActive(key string) bool {
|
||||
t.mu.Lock()
|
||||
_, ok := t.streams[key]
|
||||
t.mu.Unlock()
|
||||
func (p *TestProvider) IsStreamActive(key string) bool {
|
||||
p.mu.Lock()
|
||||
_, ok := p.streams[key]
|
||||
p.mu.Unlock()
|
||||
return ok
|
||||
}
|
||||
|
||||
func (t *TestProvider) IsValidSubject(key string, _ bool) bool {
|
||||
return key != ""
|
||||
func (p *TestProvider) IsValidSubject(key string, _ bool) bool {
|
||||
if key == "" {
|
||||
return false
|
||||
}
|
||||
// Accept anything parseable via parseCfg; fallback true.
|
||||
return true
|
||||
}
|
||||
|
||||
// --- core ---
|
||||
|
||||
func run(ctx context.Context, s *stream, out chan<- domain.Message, subject string, c cfg) {
|
||||
defer close(s.done)
|
||||
|
||||
ident, _ := domain.RawID("test_provider", subject)
|
||||
|
||||
// Sanitize
|
||||
if c.burst < 1 {
|
||||
c.burst = 1
|
||||
}
|
||||
if c.size < 1 {
|
||||
c.size = 1
|
||||
}
|
||||
if c.embedTS && c.size < 16 {
|
||||
c.size = 16
|
||||
}
|
||||
if c.period <= 0 {
|
||||
if c.rate > 0 {
|
||||
c.period = time.Duration(float64(time.Second) / c.rate)
|
||||
} else {
|
||||
c.period = 10 * time.Microsecond
|
||||
}
|
||||
}
|
||||
if c.jitter < 0 {
|
||||
c.jitter = 0
|
||||
}
|
||||
if c.jitter > 0.95 {
|
||||
c.jitter = 0.95
|
||||
}
|
||||
|
||||
// Per-second logging
|
||||
var logTicker *time.Ticker
|
||||
if c.logEverySec {
|
||||
logTicker = time.NewTicker(time.Second)
|
||||
defer logTicker.Stop()
|
||||
}
|
||||
|
||||
var seq uint64
|
||||
base := make([]byte, c.size)
|
||||
|
||||
// On/Off state
|
||||
onUntil := time.Time{}
|
||||
offUntil := time.Time{}
|
||||
inOn := true
|
||||
now := time.Now()
|
||||
onUntil = now.Add(c.onDur)
|
||||
|
||||
// Scheduling
|
||||
next := time.Now()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
switch c.mode {
|
||||
case modeConst:
|
||||
// sleep-then-spin to hit sub-10µs with isolated core
|
||||
if d := time.Until(next); d > 0 {
|
||||
if d > 30*time.Microsecond {
|
||||
time.Sleep(d - 30*time.Microsecond)
|
||||
}
|
||||
for time.Now().Before(next) {
|
||||
}
|
||||
}
|
||||
case modePoisson:
|
||||
// draw from exponential with mean=period
|
||||
lam := 1.0 / float64(c.period)
|
||||
ia := time.Duration(rand.ExpFloat64() / lam)
|
||||
next = time.Now().Add(ia)
|
||||
// No pre-wait here; emit immediately then sleep to next
|
||||
case modeOnOff:
|
||||
now = time.Now()
|
||||
if inOn {
|
||||
if now.After(onUntil) {
|
||||
inOn = false
|
||||
offUntil = now.Add(c.offDur)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if now.After(offUntil) {
|
||||
inOn = true
|
||||
onUntil = now.Add(c.onDur)
|
||||
}
|
||||
// While off, push next and wait
|
||||
// Small sleep to avoid busy loop during off
|
||||
time.Sleep(minDur(c.offDur/4, 200*time.Microsecond))
|
||||
continue
|
||||
}
|
||||
// For on state, behave like const
|
||||
if d := time.Until(next); d > 0 {
|
||||
if d > 30*time.Microsecond {
|
||||
time.Sleep(d - 30*time.Microsecond)
|
||||
}
|
||||
for time.Now().Before(next) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit burst
|
||||
for i := 0; i < c.burst; i++ {
|
||||
seq++
|
||||
payload := base[:c.size]
|
||||
switch c.pType {
|
||||
case ptBytes:
|
||||
fillPattern(payload, uint64(seq))
|
||||
case ptCounter:
|
||||
fillCounter(payload, uint64(seq))
|
||||
case ptJSON:
|
||||
// build minimal, fixed-size-ish JSON into payload
|
||||
n := buildJSON(payload, uint64(seq))
|
||||
payload = payload[:n]
|
||||
}
|
||||
|
||||
if c.embedTS {
|
||||
ensureCap(&payload, 16)
|
||||
ts := time.Now().UnixNano()
|
||||
putInt64(payload[0:8], ts)
|
||||
putInt64(payload[8:16], int64(seq))
|
||||
}
|
||||
|
||||
msg := domain.Message{
|
||||
Identifier: ident,
|
||||
Payload: payload,
|
||||
}
|
||||
|
||||
if out != nil {
|
||||
if c.dropIfSlow {
|
||||
select {
|
||||
case out <- msg:
|
||||
s.stats.sent.Add(1)
|
||||
default:
|
||||
s.stats.dropped.Add(1)
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case out <- msg:
|
||||
s.stats.sent.Add(1)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule next
|
||||
adj := c.period
|
||||
if c.mode == modePoisson {
|
||||
// next already chosen
|
||||
} else {
|
||||
if c.jitter > 0 {
|
||||
j := (rand.Float64()*2 - 1) * c.jitter
|
||||
adj = time.Duration(float64(c.period) * (1 + j))
|
||||
if adj < 0 {
|
||||
adj = 0
|
||||
}
|
||||
}
|
||||
next = next.Add(adj)
|
||||
}
|
||||
|
||||
// For poisson, actively wait to next
|
||||
if c.mode == modePoisson {
|
||||
if d := time.Until(next); d > 0 {
|
||||
if d > 30*time.Microsecond {
|
||||
time.Sleep(d - 30*time.Microsecond)
|
||||
}
|
||||
for time.Now().Before(next) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- config parsing ---
|
||||
|
||||
func (p *TestProvider) parseCfg(subject string) cfg {
|
||||
c := p.defaults
|
||||
|
||||
// Query style first
|
||||
if i := strings.Index(subject, "?"); i >= 0 && i < len(subject)-1 {
|
||||
if qv, err := url.ParseQuery(subject[i+1:]); err == nil {
|
||||
c = applyQuery(c, qv)
|
||||
}
|
||||
}
|
||||
|
||||
// Path segments like /key/value/ pairs
|
||||
parts := strings.Split(subject, "/")
|
||||
for i := 0; i+1 < len(parts); i += 2 {
|
||||
k := strings.ToLower(parts[i])
|
||||
v := parts[i+1]
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
applyKV(&c, k, v)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func applyQuery(c cfg, v url.Values) cfg {
|
||||
for k, vals := range v {
|
||||
if len(vals) == 0 {
|
||||
continue
|
||||
}
|
||||
applyKV(&c, strings.ToLower(k), vals[0])
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func applyKV(c *cfg, key, val string) {
|
||||
switch key {
|
||||
case "period":
|
||||
if d, err := time.ParseDuration(val); err == nil && d > 0 {
|
||||
c.period = d
|
||||
}
|
||||
case "rate":
|
||||
if f, err := strconv.ParseFloat(val, 64); err == nil && f > 0 {
|
||||
c.rate = f
|
||||
c.period = 0 // let rate take effect if period unset later
|
||||
}
|
||||
case "mode":
|
||||
switch strings.ToLower(val) {
|
||||
case "const", "steady":
|
||||
c.mode = modeConst
|
||||
case "poisson":
|
||||
c.mode = modePoisson
|
||||
case "onoff", "burst":
|
||||
c.mode = modeOnOff
|
||||
}
|
||||
case "on":
|
||||
if d, err := time.ParseDuration(val); err == nil && d >= 0 {
|
||||
c.onDur = d
|
||||
}
|
||||
case "off":
|
||||
if d, err := time.ParseDuration(val); err == nil && d >= 0 {
|
||||
c.offDur = d
|
||||
}
|
||||
case "burst":
|
||||
if n, err := strconv.Atoi(val); err == nil && n > 0 {
|
||||
c.burst = n
|
||||
}
|
||||
case "jitter":
|
||||
if f, err := strconv.ParseFloat(val, 64); err == nil && f >= 0 && f < 1 {
|
||||
c.jitter = f
|
||||
}
|
||||
case "size":
|
||||
if n, err := strconv.Atoi(val); err == nil && n > 0 {
|
||||
c.size = n
|
||||
}
|
||||
case "ptype":
|
||||
switch strings.ToLower(val) {
|
||||
case "bytes":
|
||||
c.pType = ptBytes
|
||||
case "counter":
|
||||
c.pType = ptCounter
|
||||
case "json":
|
||||
c.pType = ptJSON
|
||||
}
|
||||
case "drop":
|
||||
c.dropIfSlow = val == "1" || strings.EqualFold(val, "true")
|
||||
case "ts":
|
||||
c.embedTS = val == "1" || strings.EqualFold(val, "true")
|
||||
case "log":
|
||||
c.logEverySec = val == "1" || strings.EqualFold(val, "true")
|
||||
}
|
||||
}
|
||||
|
||||
// --- payload builders ---
|
||||
|
||||
func fillPattern(b []byte, seed uint64) {
|
||||
// xorshift for deterministic but non-trivial bytes
|
||||
if len(b) == 0 {
|
||||
return
|
||||
}
|
||||
x := seed | 1
|
||||
for i := range b {
|
||||
x ^= x << 13
|
||||
x ^= x >> 7
|
||||
x ^= x << 17
|
||||
b[i] = byte(x)
|
||||
}
|
||||
}
|
||||
|
||||
func fillCounter(b []byte, seq uint64) {
|
||||
for i := range b {
|
||||
b[i] = byte((seq + uint64(i)) & 0xFF)
|
||||
}
|
||||
}
|
||||
|
||||
func buildJSON(buf []byte, seq uint64) int {
|
||||
// Small fixed fields. Truncate if buffer small.
|
||||
// Example: {"t":1694490000000000,"s":12345,"p":100.12}
|
||||
ts := time.Now().UnixNano()
|
||||
price := 10000 + float64(seq%1000)*0.01
|
||||
str := fmt.Sprintf(`{"t":%d,"s":%d,"p":%.2f}`, ts, seq, price)
|
||||
n := copy(buf, str)
|
||||
return n
|
||||
}
|
||||
|
||||
func ensureCap(b *[]byte, need int) {
|
||||
if len(*b) >= need {
|
||||
return
|
||||
}
|
||||
nb := make([]byte, need)
|
||||
copy(nb, *b)
|
||||
*b = nb
|
||||
}
|
||||
|
||||
func putInt64(b []byte, v int64) {
|
||||
_ = b[7]
|
||||
b[0] = byte(v >> 56)
|
||||
b[1] = byte(v >> 48)
|
||||
b[2] = byte(v >> 40)
|
||||
b[3] = byte(v >> 32)
|
||||
b[4] = byte(v >> 24)
|
||||
b[5] = byte(v >> 16)
|
||||
b[6] = byte(v >> 8)
|
||||
b[7] = byte(v)
|
||||
}
|
||||
|
||||
func minDur(a, b time.Duration) time.Duration {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user