WIP Reworking data module

pkg/data/component.go

@@ -0,0 +1,33 @@
+package data
+
+import "context"
+
+type Component interface {
+	Start(ctx context.Context) error
+	Stop(ctx context.Context) error
+
+	ValidateConfig(cfg ComponentConfig) (ComponentConfigValidation, error)
+	ApplyConfig(cfg ComponentConfig) error
+
+	Status() ComponentStatus
+}
+
+type ComponentRuntime interface {
+	OpenStream(id StreamID) (Stream, error)
+	ReportError(fatal bool, err error)
+}
+
+type ComponentStatus struct {
+	Active   bool
+	Patching bool
+	Config   any
+	Error    error
+}
+
+type ComponentConfig any
+
+type ComponentConfigValidation struct {
+	Valid           bool
+	RequiresRestart bool
+	Warnings        []string
+}

pkg/data/coordinator/coordinator.go

@@ -1,2 +0,0 @@
-// Package coordinator ...
-package coordinator

pkg/data/egress.go

@@ -1,18 +0,0 @@
-package data
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-type Egress interface {
-	Name() string
-	Configure(cfg map[string]any) error
-	Connect(ctx context.Context, actions EgresActions) error
-	Disconnect(ctx context.Context) error
-}
-
-type EgresActions interface {
-	Subscribe(ctx context.Context, namespace string, id uuid.UUID) (<-chan Envelope, func(), error)
-}

pkg/data/ingress.go

@@ -1,34 +0,0 @@
-package data
-
-import (
-	"context"
-	"time"
-
-	"github.com/google/uuid"
-)
-
-type Ingress interface {
-	Name() string
-	Configure(cfg map[string]any) error
-	Connect(ctx context.Context, actions IngressActions) error
-	Disconnect(ctx context.Context) error
-}
-
-type IngressActions interface {
-	Emit(namespace string, id uuid.UUID, envelope Envelope) error
-	EmitBatch(namespace string, id uuid.UUID, envelopes []Envelope) error
-}
-
-type LiveIngress interface {
-	Subscribe(ctx context.Context, key ...string) error
-	Unsubscribe(ctx context.Context, key ...string) error
-	ListAvailableKeys(ctx context.Context) []string
-	IsValidKey(ctx context.Context, key string) bool
-}
-
-type HistoricalIngress interface {
-	Fetch(ctx context.Context, key string, start time.Time, end time.Time) ([]Envelope, error)
-	ListAvailableKeys(ctx context.Context) []string
-	IsValidKey(ctx context.Context, key string) bool
-	// some other method that gives the avalible time period for a key
-}

pkg/data/processor.go

@@ -1,14 +0,0 @@
-package data
-
-import "context"
-
-type Processor interface {
-	Name() string
-	Configure(cfg map[string]any) error
-	Run(ctx context.Context, actions ProcessorActions) error
-}
-
-type ProcessorActions interface {
-	IngressActions
-	EgresActions
-}

pkg/data/router/router.go

@@ -1,2 +0,0 @@
-// Package router ...
-package router

pkg/data/routing/inproc.go

@@ -0,0 +1,130 @@
+package routing
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	"gitlab.michelsen.id/phillmichelsen/tessera/pkg/data"
+)
+
+type InprocRouter struct {
+	mu      sync.RWMutex
+	streams map[data.StreamID]*inprocStream
+}
+
+func NewInprocRouter() *InprocRouter {
+	return &InprocRouter{
+		streams: make(map[data.StreamID]*inprocStream),
+	}
+}
+
+func (r *InprocRouter) OpenStream(id data.StreamID) (data.Stream, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	s := r.streams[id]
+	if s != nil {
+		return s, nil
+	}
+
+	s = newInprocStream(id)
+	r.streams[id] = s
+	return s, nil
+}
+
+type inprocStream struct {
+	id data.StreamID
+
+	seq    uint64
+	latest data.Envelope
+
+	streamClosed bool
+	mu           sync.RWMutex
+}
+
+func newInprocStream(id data.StreamID) *inprocStream {
+	return &inprocStream{
+		id: id,
+	}
+}
+
+func (s *inprocStream) ID() data.StreamID {
+	return s.id
+}
+
+func (s *inprocStream) Sender() data.Sender {
+	return &inprocSender{stream: s}
+}
+
+func (s *inprocStream) Receiver() data.Receiver {
+	s.mu.RLock()
+	cur := s.seq
+	s.mu.RUnlock()
+
+	return &inprocReceiver{
+		stream:      s,
+		lastSeenSeq: cur,
+	}
+
+}
+
+type inprocSender struct {
+	stream *inprocStream
+}
+
+func (tx *inprocSender) Send(ctx context.Context, env data.Envelope) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	s := tx.stream
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if s.streamClosed {
+		return errors.New("stream closed")
+	}
+
+	env.SendTime = time.Now().UTC()
+
+	s.seq++
+	s.latest = env
+
+	return nil
+}
+
+func (tx *inprocSender) SendBatch(ctx context.Context, envs []data.Envelope) error {
+	panic("unimplemented")
+}
+
+type inprocReceiver struct {
+	stream      *inprocStream
+	lastSeenSeq uint64
+}
+
+func (rx *inprocReceiver) TryReceive() (data.Envelope, bool, error) {
+	s := rx.stream
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	if s.streamClosed {
+		return data.Envelope{}, false, errors.New("stream closed")
+	}
+
+	if s.seq == 0 || s.seq == rx.lastSeenSeq {
+		return data.Envelope{}, false, nil
+	}
+
+	rx.lastSeenSeq = s.seq
+	return s.latest, true, nil
+}
+
+func (rx *inprocReceiver) ReceiveNext(ctx context.Context) (data.Envelope, error) {
+	panic("unimplemented")
+}
+
+func (rx *inprocReceiver) Seq() uint64 {
+	return rx.lastSeenSeq
+}

pkg/data/stream.go

@@ -0,0 +1,27 @@
+package data
+
+import (
+	"context"
+
+	"github.com/google/uuid"
+)
+
+type StreamID uuid.UUID
+
+type Stream interface {
+	ID() StreamID
+
+	Sender() Sender
+	Receiver() Receiver
+}
+
+type Sender interface {
+	Send(context.Context, Envelope) error
+	SendBatch(context.Context, []Envelope) error
+}
+
+type Receiver interface {
+	TryReceive() (Envelope, bool, error)
+	ReceiveNext(context.Context) (Envelope, error)
+	Seq() uint64
+}