Switched to a DAG ledger

cmd/chron-cli/main.go

@@ -84,7 +84,7 @@ func main() {
 		return
 	}
 
-	for i := range 500 {
+	for i := range 1000000 {
 		data := fmt.Sprintf("test%d", i)
 
 		err = ledger.Append(ctx, []byte(data))
@@ -107,34 +107,37 @@ func main() {
 		return entries[i].Timestamp.Before(entries[j].Timestamp)
 	})
 
-	fmt.Println("Entries:")
+	headID, ok, err := referenceStore.Get(ctx, "HEAD")
-	if len(entries) == 0 {
+	if err != nil {
-		fmt.Println("  (none)")
+		fmt.Println("get HEAD:", err)
-	} else {
+		return
-		for _, e := range entries {
-			fmt.Printf("  ts=%d id=%s prev=%s payload=%q\n",
-				e.Timestamp.UnixNano(),
-				hex.EncodeToString(e.EntryID[:]),
-				hex.EncodeToString(e.Previous[:]),
-				e.Payload,
-			)
 	}
+	if !ok {
+		fmt.Println("HEAD reference not set")
+		return
 	}
 
-	// ---- Print references (name -> EntryID hex) ----
+	isZero32 := func(b [32]byte) bool {
-	names := make([]string, 0, len(referenceStore.references))
+		for _, v := range b {
-	for name := range referenceStore.references {
+			if v != 0 {
-		names = append(names, name)
+				return false
+			}
+		}
+		return true
 	}
-	sort.Strings(names)
 
-	fmt.Println("References:")
+	curID := headID
-	if len(names) == 0 {
+	for {
-		fmt.Println("  (none)")
+		ent, err := entryStore.Load(ctx, curID)
-	} else {
+		if err != nil {
-		for _, name := range names {
+			return
-			id := referenceStore.references[name]
+		}
-			fmt.Printf("  %s -> %s\n", name, hex.EncodeToString(id[:]))
+
-		}
+		if isZero32(ent.Parents) {
+			break
+		}
+
+		// Follow the linked list backwards
+		curID = core.EntryID(ent.Parents)
 	}
 }

core/interfaces.go

@@ -1,18 +1,24 @@
 package core
 
-import (
+import "context"
-	"context"
-)
 
 type EntryStore interface {
 	Store(ctx context.Context, entry Entry) error
 	Load(ctx context.Context, id EntryID) (Entry, error)
 	Exists(ctx context.Context, id EntryID) (bool, error)
 	Delete(ctx context.Context, id EntryID) error
+
+	StoreBatch(ctx context.Context, entries []Entry) error
+	LoadBatch(ctx context.Context, ids []EntryID) ([]Entry, error)
 }
 
 type ReferenceStore interface {
 	Set(ctx context.Context, name string, entryID EntryID) error
 	Get(ctx context.Context, name string) (EntryID, bool, error)
 	Delete(ctx context.Context, name string) error
+
+	List(ctx context.Context, prefix string) (map[string]EntryID, error)
+
+	SetBatch(ctx context.Context, refs map[string]EntryID) error
+	GetBatch(ctx context.Context, names []string) (map[string]EntryID, error)
 }

core/ledger.go

@@ -1,11 +1,19 @@
 package core
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"sort"
 	"time"
 )
 
+var (
+	ErrHeadNotSet      = errors.New("HEAD not set")
+	ErrEntryIDMismatch = errors.New("entry id mismatch")
+	ErrBatchLoadSize   = errors.New("LoadBatch returned unexpected number of entries")
+)
+
 type Ledger struct {
 	entryStore     EntryStore
 	referenceStore ReferenceStore
@@ -18,111 +26,278 @@ func NewLedger(entryStore EntryStore, referenceStore ReferenceStore) (*Ledger, e
 	}, nil
 }
 
-func (l *Ledger) Append(ctx context.Context, payload []byte) error {
+func (l *Ledger) Add(ctx context.Context, parents []EntryID, payload []byte) (EntryID, error) {
-	currentHeadEntryID, ok, err := l.referenceStore.Get(ctx, "HEAD")
+	ps := normalizeParents(parents)
-	if err != nil {
-		return err
-	}
 
-	if !ok {
+	ts := time.Now()
-		l.referenceStore.Set(ctx, "HEAD", EntryID{})
+	id := ComputeEntryID(ps, ts, payload)
-		currentHeadEntryID = EntryID{}
-	}
 
-	entryTime := time.Now()
+	e := Entry{
-	entryID := ComputeEntryID(currentHeadEntryID, entryTime, payload)
+		EntryID:   id,
-
+		Parents:   ps,
-	entry := Entry{
+		Timestamp: ts,
-		EntryID:   entryID,
-		Previous:  currentHeadEntryID,
-		Timestamp: entryTime,
 		Payload:   payload,
 	}
 
-	if ComputeEntryID(entry.Previous, entry.Timestamp, entry.Payload) != entryID {
+	if err := l.VerifyEntry(e); err != nil {
-		panic("EntryID hash mismatch fuckup")
+		return EntryID{}, err
 	}
 
-	err = l.entryStore.Store(ctx, entry)
+	if err := l.entryStore.Store(ctx, e); err != nil {
+		return EntryID{}, err
+	}
+
+	return id, nil
+}
+
+func (l *Ledger) Append(ctx context.Context, payload []byte) (EntryID, error) {
+	head, ok, err := l.referenceStore.Get(ctx, "HEAD")
+	if err != nil {
+		return EntryID{}, err
+	}
+
+	var parents []EntryID
+	if ok {
+		parents = []EntryID{head}
+	} else {
+		parents = nil
+	}
+
+	id, err := l.Add(ctx, parents, payload)
+	if err != nil {
+		return EntryID{}, err
+	}
+
+	if err := l.referenceStore.Set(ctx, "HEAD", id); err != nil {
+		return EntryID{}, err
+	}
+
+	return id, nil
+}
+
+func (l *Ledger) AppendTo(ctx context.Context, parent EntryID, payload []byte) (EntryID, error) {
+	return l.Add(ctx, []EntryID{parent}, payload)
+}
+
+func (l *Ledger) Get(ctx context.Context, id EntryID) (Entry, error) {
+	return l.entryStore.Load(ctx, id)
+}
+
+func (l *Ledger) Exists(ctx context.Context, id EntryID) (bool, error) {
+	return l.entryStore.Exists(ctx, id)
+}
+
+func (l *Ledger) Verify(ctx context.Context, id EntryID) error {
+	e, err := l.entryStore.Load(ctx, id)
 	if err != nil {
 		return err
 	}
+	return l.VerifyEntry(e)
+}
 
-	err = l.referenceStore.Set(ctx, "HEAD", entry.EntryID)
+func (l *Ledger) VerifyEntry(e Entry) error {
+	want := ComputeEntryID(e.Parents, e.Timestamp, e.Payload)
+	if want != e.EntryID {
+		return ErrEntryIDMismatch
+	}
+	return nil
+}
+
+func (l *Ledger) GetRef(ctx context.Context, name string) (EntryID, bool, error) {
+	return l.referenceStore.Get(ctx, name)
+}
+
+func (l *Ledger) SetRef(ctx context.Context, name string, id EntryID) error {
+	return l.referenceStore.Set(ctx, name, id)
+}
+
+func (l *Ledger) DeleteRef(ctx context.Context, name string) error {
+	return l.referenceStore.Delete(ctx, name)
+}
+
+func (l *Ledger) ListRefs(ctx context.Context, prefix string) (map[string]EntryID, error) {
+	return l.referenceStore.List(ctx, prefix)
+}
+
+func (l *Ledger) SetRefs(ctx context.Context, refs map[string]EntryID) error {
+	return l.referenceStore.SetBatch(ctx, refs)
+}
+
+func (l *Ledger) GetRefs(ctx context.Context, names []string) (map[string]EntryID, error) {
+	return l.referenceStore.GetBatch(ctx, names)
+}
+
+func (l *Ledger) GetHead(ctx context.Context) (EntryID, bool, error) {
+	return l.referenceStore.Get(ctx, "HEAD")
+}
+
+func (l *Ledger) SetHead(ctx context.Context, id EntryID) error {
+	return l.referenceStore.Set(ctx, "HEAD", id)
+}
+
+func (l *Ledger) GetHeads(ctx context.Context, prefix string) ([]EntryID, error) {
+	m, err := l.referenceStore.List(ctx, prefix)
+	if err != nil {
+		return nil, err
+	}
+
+	seen := make(map[EntryID]struct{}, len(m))
+	for _, id := range m {
+		seen[id] = struct{}{}
+	}
+
+	out := make([]EntryID, 0, len(seen))
+	for id := range seen {
+		out = append(out, id)
+	}
+
+	sort.Slice(out, func(i, j int) bool {
+		return bytes.Compare(out[i][:], out[j][:]) < 0
+	})
+
+	return out, nil
+}
+
+func (l *Ledger) WalkAncestors(ctx context.Context, start []EntryID, fn func(Entry) bool) error {
+	frontier := dedupeIDs(start)
+	visited := make(map[EntryID]struct{}, 1024)
+
+	for len(frontier) > 0 {
+		batchIDs := make([]EntryID, 0, len(frontier))
+		for _, id := range frontier {
+			if _, ok := visited[id]; ok {
+				continue
+			}
+			visited[id] = struct{}{}
+			batchIDs = append(batchIDs, id)
+		}
+		if len(batchIDs) == 0 {
+			return nil
+		}
+
+		entries, err := l.entryStore.LoadBatch(ctx, batchIDs)
 		if err != nil {
 			return err
 		}
+		if len(entries) != len(batchIDs) {
+			return ErrBatchLoadSize
+		}
+
+		next := make([]EntryID, 0, len(batchIDs)*2)
+
+		for _, e := range entries {
+			if err := l.VerifyEntry(e); err != nil {
+				return err
+			}
+
+			if !fn(e) {
+				return nil
+			}
+
+			for _, p := range e.Parents {
+				next = append(next, p)
+			}
+		}
+
+		frontier = dedupeIDs(next)
+	}
 
 	return nil
 }
 
-func (l *Ledger) AppendTo(ctx context.Context, prevEntryID EntryID, payload []byte) error {
+func (l *Ledger) IsAncestor(ctx context.Context, ancestor, descendant EntryID) (bool, error) {
-	entryTime := time.Now()
+	if ancestor == descendant {
-	entryID := ComputeEntryID(prevEntryID, entryTime, payload)
+		return true, nil
-
-	entry := Entry{
-		EntryID:   entryID,
-		Previous:  prevEntryID,
-		Timestamp: entryTime,
-		Payload:   payload,
 	}
 
-	if ComputeEntryID(entry.Previous, entry.Timestamp, entry.Payload) != entryID {
+	found := false
-		panic("EntryID hash mismatch fuckup")
+	err := l.WalkAncestors(ctx, []EntryID{descendant}, func(e Entry) bool {
+		for _, p := range e.Parents {
+			if p == ancestor {
+				found = true
+				return false
 			}
-
+		}
-	err := l.entryStore.Store(ctx, entry)
+		return true
+	})
 	if err != nil {
-		return err
+		return false, err
+	}
+	return found, nil
 }
 
+func (l *Ledger) CommonAncestors(ctx context.Context, a, b []EntryID, limit int) ([]EntryID, error) {
+	aSet := make(map[EntryID]struct{}, 1024)
+
+	if err := l.WalkAncestors(ctx, a, func(e Entry) bool {
+		aSet[e.EntryID] = struct{}{}
+		return true
+	}); err != nil {
+		return nil, err
+	}
+
+	var out []EntryID
+	seen := make(map[EntryID]struct{}, 64)
+
+	err := l.WalkAncestors(ctx, b, func(e Entry) bool {
+		if _, ok := aSet[e.EntryID]; ok {
+			if _, dup := seen[e.EntryID]; !dup {
+				seen[e.EntryID] = struct{}{}
+				out = append(out, e.EntryID)
+				if limit > 0 && len(out) >= limit {
+					return false
+				}
+			}
+		}
+		return true
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
+func normalizeParents(parents []EntryID) []EntryID {
+	if len(parents) == 0 {
 		return nil
 	}
 
-func (l *Ledger) Get(ctx context.Context) (Entry, error) {
+	ps := make([]EntryID, len(parents))
-	headEntryID, ok, err := l.referenceStore.Get(ctx, "HEAD")
+	copy(ps, parents)
-	if err != nil {
+
-		return Entry{}, err
+	sort.Slice(ps, func(i, j int) bool {
+		return bytes.Compare(ps[i][:], ps[j][:]) < 0
+	})
+
+	out := ps[:0]
+	var last EntryID
+	for i, p := range ps {
+		if i == 0 || p != last {
+			out = append(out, p)
+			last = p
+		}
+	}
+	return out
 }
 
-	if !ok {
+func dedupeIDs(ids []EntryID) []EntryID {
-		return Entry{}, errors.New("HEAD not set")
+	if len(ids) == 0 {
+		return nil
 	}
 
-	entry, err := l.entryStore.Load(ctx, headEntryID)
+	tmp := make([]EntryID, len(ids))
-	if err != nil {
+	copy(tmp, ids)
-		return Entry{}, err
+	sort.Slice(tmp, func(i, j int) bool {
+		return bytes.Compare(tmp[i][:], tmp[j][:]) < 0
+	})
+	out := tmp[:0]
+	var last EntryID
+	for i, id := range tmp {
+		if i == 0 || id != last {
+			out = append(out, id)
+			last = id
 		}
-
-	return entry, nil
 	}
-
+	return out
-func (l *Ledger) GetFromReference(ctx context.Context, reference string) (Entry, error) {
-	referenceEntryID, ok, err := l.referenceStore.Get(ctx, reference)
-	if err != nil {
-		return Entry{}, err
-	}
-
-	if !ok {
-		return Entry{}, errors.New("HEAD not set")
-	}
-
-	entry, err := l.entryStore.Load(ctx, referenceEntryID)
-	if err != nil {
-		return Entry{}, err
-	}
-
-	return entry, nil
-}
-
-func (l *Ledger) SetHead(ctx context.Context, entryID EntryID) error {
-	return l.referenceStore.Set(ctx, "HEAD", entryID)
-}
-
-func (l *Ledger) SetReference(ctx context.Context, reference string, entryID EntryID) error {
-	return l.referenceStore.Set(ctx, reference, entryID)
-}
-
-func (l *Ledger) RemoveReference(ctx context.Context, reference string) error {
-	return l.referenceStore.Delete(ctx, reference)
 }

core/types.go

@@ -1,7 +1,9 @@
 package core
 
 import (
+	"bytes"
 	"encoding/binary"
+	"sort"
 	"time"
 
 	"lukechampine.com/blake3"
@@ -12,25 +14,35 @@ type EntryID [32]byte
 type Entry struct {
 	EntryID EntryID
 
-	Previous EntryID
+	Parents []EntryID
 
 	Timestamp time.Time
 	Payload   []byte
 }
 
-func ComputeEntryID(prev EntryID, ts time.Time, payload []byte) EntryID {
+func ComputeEntryID(parents []EntryID, ts time.Time, payload []byte) EntryID {
 	h := blake3.New(32, nil)
 
-	h.Write(prev[:])
+	ps := make([]EntryID, len(parents))
+	copy(ps, parents)
+	sort.Slice(ps, func(i, j int) bool {
+		return bytes.Compare(ps[i][:], ps[j][:]) < 0
+	})
+
+	var u32 [4]byte
+	binary.LittleEndian.PutUint32(u32[:], uint32(len(ps)))
+	h.Write(u32[:])
+
+	for _, p := range ps {
+		h.Write(p[:])
+	}
 
 	var i64 [8]byte
 	binary.LittleEndian.PutUint64(i64[:], uint64(ts.UTC().UnixNano()))
 	h.Write(i64[:])
 
-	var u32 [4]byte
 	binary.LittleEndian.PutUint32(u32[:], uint32(len(payload)))
 	h.Write(u32[:])
-
 	h.Write(payload)
 
 	var id EntryID