// Copyright 2024 The Erigon Authors
// This file is part of Erigon.
//
// Erigon is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Erigon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Erigon. If not, see <http://www.gnu.org/licenses/>.

package commitment

import (
	"bytes"
	"context"
	"encoding/binary"
	"errors"
	"fmt"
	"math/bits"
	"slices"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"unsafe"

	keccak "github.com/erigontech/fastkeccak"
	"github.com/google/btree"
	"github.com/holiman/uint256"

	"github.com/erigontech/erigon/common"
	"github.com/erigontech/erigon/common/crypto"
	"github.com/erigontech/erigon/common/dbg"
	"github.com/erigontech/erigon/common/empty"
	"github.com/erigontech/erigon/common/length"
	"github.com/erigontech/erigon/common/log/v3"
	"github.com/erigontech/erigon/common/maphash"
	"github.com/erigontech/erigon/db/etl"
	"github.com/erigontech/erigon/db/kv"
	"github.com/erigontech/erigon/diagnostics/metrics"
	"github.com/erigontech/erigon/execution/commitment/nibbles"
	"github.com/erigontech/erigon/execution/types/accounts"
)

var (
	mxTrieProcessedKeys   = metrics.GetOrCreateCounter("domain_commitment_keys")
	mxTrieBranchesUpdated = metrics.GetOrCreateCounter("domain_commitment_updates_applied")

	mxTrieStateSkipRate                 = metrics.GetOrCreateCounter("trie_state_skip_rate")
	mxTrieStateLoadRate                 = metrics.GetOrCreateCounter("trie_state_load_rate")
	mxTrieStateLevelledSkipRatesAccount = [...]metrics.Counter{
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L0",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L1",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L2",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L3",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L4",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="recent",key="account"}`),
	}
	mxTrieStateLevelledSkipRatesStorage = [...]metrics.Counter{
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L0",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L1",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L2",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L3",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="L4",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_skip_rate{level="recent",key="storage"}`),
	}
	mxTrieStateLevelledLoadRatesAccount = [...]metrics.Counter{
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L0",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L1",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L2",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L3",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L4",key="account"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="recent",key="account"}`),
	}
	mxTrieStateLevelledLoadRatesStorage = [...]metrics.Counter{
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L0",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L1",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L2",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L3",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="L4",key="storage"}`),
		metrics.GetOrCreateCounter(`trie_state_levelled_load_rate{level="recent",key="storage"}`),
	}
)

// Trie represents commitment variant.
type Trie interface {
	// RootHash produces root hash of the trie
	RootHash() (hash []byte, err error)

	// Makes trie more verbose
	SetTrace(bool)
	// Trace domain writes only (no filding etc)
	SetTraceDomain(bool)
	SetCapture(capture []string)
	GetCapture(truncate bool) []string
	EnableCsvMetrics(filePathPrefix string)
	// EnableWarmupCache enables/disables warmup cache during Process (false by default)
	EnableWarmupCache(bool)

	// Variant returns commitment trie variant
	Variant() TrieVariant

	// Reset Drops everything from the trie
	Reset()

	// Set context for state IO
	ResetContext(ctx PatriciaContext)

	// Process updates. If warmup.Enabled is true, pre-warms MDBX page cache in parallel.
	// onProgress (optional) is called periodically with commitment progress info.
	Process(ctx context.Context, updates *Updates, logPrefix string, onProgress func(*CommitProgress), warmup WarmupConfig) (rootHash []byte, err error)

	// Release returns the trie to a pool for reuse. After calling Release,
	// the caller must not use the trie.
	Release()
}

type CommitProgress struct {
	KeyIndex    uint64
	UpdateCount uint64
	Metrics     MetricValues
}

type PatriciaContext interface {
	// GetBranch load branch node and fill up the cells
	// For each cell, it sets the cell type, clears the modified flag, fills the hash,
	// and for the extension, account, and leaf type, the `l` and `k`
	Branch(prefix []byte) ([]byte, kv.Step, error)
	// store branch data
	PutBranch(prefix []byte, data []byte, prevData []byte) error
	// fetch account with given plain key
	Account(plainKey []byte) (*Update, error)
	// fetch storage with given plain key
	Storage(plainKey []byte) (*Update, error)
}

type TrieVariant string

const (
	// VariantHexPatriciaTrie used as default commitment approach
	VariantHexPatriciaTrie TrieVariant = "hex-patricia-hashed"
	// VariantBinPatriciaTrie - Experimental mode with binary key representation
	VariantBinPatriciaTrie       TrieVariant = "bin-patricia-hashed"
	VariantConcurrentHexPatricia TrieVariant = "hex-concurrent-patricia-hashed"
)

func InitializeTrieAndUpdates(mode Mode, tmpdir string, cfg TrieConfig) (Trie, *Updates) {
	switch cfg.Variant {
	case VariantConcurrentHexPatricia:
		root := NewHexPatriciaHashed(length.Addr, nil, cfg)
		trie := NewConcurrentPatriciaHashed(root, nil)
		tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash)
		// tree.SetConcurrentCommitment(true) // first run always sequential
		return trie, tree
	case VariantBinPatriciaTrie:
		//trie := NewBinPatriciaHashed(length.Addr, nil, tmpdir)
		//fn := func(key []byte) []byte { return hexToBin(key) }
		//tree := NewUpdateTree(mode, tmpdir, fn)
		//return trie, tree
		panic("VariantBinPatriciaTrie not supported")
	case VariantHexPatriciaTrie:
		fallthrough
	default:

		trie := NewHexPatriciaHashed(length.Addr, nil, cfg)
		tree := NewUpdates(mode, tmpdir, KeyToHexNibbleHash)
		return trie, tree
	}
}

type cellFields uint8

const (
	fieldExtension   cellFields = 1
	fieldAccountAddr cellFields = 2
	fieldStorageAddr cellFields = 4
	fieldHash        cellFields = 8
	fieldStateHash   cellFields = 16
)

func (p cellFields) String() string {
	var sb strings.Builder
	if p&fieldExtension != 0 {
		sb.WriteString("DownHash")
	}
	if p&fieldAccountAddr != 0 {
		sb.WriteString("+AccountPlain")
	}
	if p&fieldStorageAddr != 0 {
		sb.WriteString("+StoragePlain")
	}
	if p&fieldHash != 0 {
		sb.WriteString("+Hash")
	}
	if p&fieldStateHash != 0 {
		sb.WriteString("+LeafHash")
	}
	return sb.String()
}

// cellEncodeData contains only the fields needed for EncodeBranch.
// This is much smaller than cell (which has hashedExtension[128], Update, etc.)
// TODO: unify with cell by shrinking cell struct to eliminate this separate type
type cellEncodeData struct {
	extension   [64]byte
	accountAddr [20]byte                        // common.Address
	storageAddr [length.Addr + length.Hash]byte // 20 + 32 = 52 bytes
	hash        [32]byte                        // common.Hash
	stateHash   [32]byte                        // common.Hash

	extLen         int16
	accountAddrLen int16
	storageAddrLen int16
	hashLen        int16
	stateHashLen   int16
}

// cellEncodeDataFromCell extracts the encoding-relevant fields from a cell into a cellEncodeData.
func cellEncodeDataFromCell(c *cell) cellEncodeData {
	var d cellEncodeData
	d.extLen = c.extLen
	d.accountAddrLen = c.accountAddrLen
	d.storageAddrLen = c.storageAddrLen
	d.hashLen = c.hashLen
	d.stateHashLen = c.stateHashLen
	copy(d.extension[:], c.extension[:c.extLen])
	copy(d.accountAddr[:], c.accountAddr[:c.accountAddrLen])
	copy(d.storageAddr[:], c.storageAddr[:c.storageAddrLen])
	copy(d.hash[:], c.hash[:c.hashLen])
	copy(d.stateHash[:], c.stateHash[:c.stateHashLen])
	return d
}

// DeferredBranchUpdate holds the data needed to perform a branch update later.
// This allows collecting updates during the fold phase and running computeCellHash + EncodeBranch in parallel.
type DeferredBranchUpdate struct {
	prefix   []byte
	bitmap   uint16
	touchMap uint16
	afterMap uint16

	// Cells needed for EncodeBranch - only the fields required for encoding
	cells [16]cellEncodeData

	// Previous data from ctx.Branch (for merging)
	prev []byte
	// Result after encoding (filled by parallel workers)
	encoded BranchData
}

// Global pool for deferred branch updates.
var deferredUpdatePool = sync.Pool{
	New: func() any {
		return &DeferredBranchUpdate{}
	},
}

// Metrics for getDeferredUpdate - use atomics for thread safety
var getDeferredUpdateCount atomic.Int64

// ResetDeferredUpdateMetrics resets the getDeferredUpdate metrics.
func ResetDeferredUpdateMetrics() {
	getDeferredUpdateCount.Store(0)
}

// GetDeferredUpdateMetrics returns the count for getDeferredUpdate calls.
func GetDeferredUpdateMetrics() int64 {
	return getDeferredUpdateCount.Load()
}

// getDeferredUpdate gets a DeferredBranchUpdate from the global pool
// and copies only the fields needed for encoding.
func getDeferredUpdate(
	prefix []byte,
	bitmap, touchMap, afterMap uint16,
	cells *[16]cellEncodeData,
	prev []byte,
) *DeferredBranchUpdate {
	getDeferredUpdateCount.Add(1)
	upd := deferredUpdatePool.Get().(*DeferredBranchUpdate)

	upd.prefix = common.Copy(prefix)
	upd.bitmap = bitmap
	upd.touchMap = touchMap
	upd.afterMap = afterMap

	// Direct struct copy for each cell in bitmap
	for bitset := bitmap; bitset != 0; {
		bit := bitset & -bitset
		nibble := bits.TrailingZeros16(bit)
		upd.cells[nibble] = cells[nibble]
		bitset ^= bit
	}

	upd.prev = common.Copy(prev)
	upd.encoded = nil

	return upd
}

// putDeferredUpdate returns a DeferredBranchUpdate to the global pool.
// Clears slice references so pooled objects don't hold stale memory.
func putDeferredUpdate(upd *DeferredBranchUpdate) {
	if upd != nil {
		upd.prefix = nil
		upd.prev = nil
		upd.encoded = nil
		deferredUpdatePool.Put(upd)
	}
}

// PendingCommitmentUpdate stores deferred branch updates for a specific block.
// Used by the commitment context to defer branch update application until a later flush.
type PendingCommitmentUpdate struct {
	BlockNum uint64
	// BlockHash is the hash of the block these deferred updates were
	// generated for. It disambiguates pastChangesAccumulator lookups when
	// multiple changesets exist for the same block number (e.g. during a
	// fork-bounce reorg test where canonical and fork chains both saved a
	// block 1 changeset). Without the hash, GetChangesetByBlockNum returns
	// the first match it iterates — non-deterministic and wrong in that
	// scenario.
	BlockHash common.Hash
	TxNum     uint64
	Deferred  []*DeferredBranchUpdate
}

// Clear returns all deferred updates to the pool and nils the slice.
func (p *PendingCommitmentUpdate) Clear() {
	for _, upd := range p.Deferred {
		putDeferredUpdate(upd)
	}
	p.Deferred = nil
}

type BranchEncoder struct {
	buf       *bytes.Buffer
	bitmapBuf [binary.MaxVarintLen64]byte
	merger    *BranchMerger
	metrics   *Metrics

	// Deferred updates support
	deferUpdates       bool
	maxDeferredUpdates int // flush threshold; 0 = use DefaultMaxDeferredUpdates from config
	deferred           []*DeferredBranchUpdate
	pendingPrefixes    *maphash.NonConcurrentMap[struct{}] // tracks pending prefixes to detect duplicates
	cache              *WarmupCache
}

func NewBranchEncoder(sz uint64) *BranchEncoder {
	return &BranchEncoder{
		buf:    bytes.NewBuffer(make([]byte, sz)),
		merger: NewHexBranchMerger(sz / 2),
	}
}

// setDeferUpdates enables or disables deferred update collection.
// When enabled, CollectUpdate will store updates in a cache instead of applying them immediately.
func (be *BranchEncoder) setDeferUpdates(defer_ bool) {
	be.deferUpdates = defer_
	if defer_ {
		if be.deferred == nil {
			be.deferred = make([]*DeferredBranchUpdate, 0, 64)
		}
		if be.pendingPrefixes == nil {
			be.pendingPrefixes = maphash.NewNonConcurrentMap[struct{}]()
		}
	}
}

// DeferUpdatesEnabled returns whether deferred update collection is enabled.
func (be *BranchEncoder) DeferUpdatesEnabled() bool {
	return be.deferUpdates
}

// HasPendingPrefix returns true if the given prefix has a pending deferred update.
func (be *BranchEncoder) HasPendingPrefix(prefix []byte) bool {
	if be.pendingPrefixes == nil {
		return false
	}
	_, found := be.pendingPrefixes.Get(prefix)
	return found
}

// ClearDeferred clears the deferred updates list and returns all objects to the pool.
func (be *BranchEncoder) ClearDeferred() {
	for _, upd := range be.deferred {
		putDeferredUpdate(upd)
	}
	be.deferred = be.deferred[:0]
	if be.pendingPrefixes != nil {
		be.pendingPrefixes.Clear()
	}
	ResetDeferredUpdateMetrics()
}

// encodeDeferredUpdate encodes a branch update using the provided encoder and merger.
// Cell hashes are already computed during fold() before cells were copied.
func encodeDeferredUpdate(
	upd *DeferredBranchUpdate,
	encoder *BranchEncoder,
	merger *BranchMerger,
) error {
	update, err := encoder.EncodeBranch(upd.bitmap, upd.touchMap, upd.afterMap, &upd.cells)
	if err != nil {
		return err
	}

	if len(upd.prev) > 0 {
		if bytes.Equal(upd.prev, update) {
			upd.encoded = nil // skip unchanged
			return nil
		}
		update, err = merger.Merge(upd.prev, update)
		if err != nil {
			return err
		}
	}

	upd.encoded = common.Copy(update)
	return nil
}

// ApplyDeferredUpdates encodes branch updates concurrently and writes them.
func (be *BranchEncoder) ApplyDeferredUpdates(
	numWorkers int,
	putBranch func(prefix []byte, data []byte, prevData []byte) error,
) error {
	written, err := ApplyDeferredBranchUpdates(be.deferred, numWorkers, putBranch)
	if err != nil {
		return err
	}
	if be.metrics != nil {
		be.metrics.updateBranch.Add(uint64(written))
	}
	return nil
}

// Pools for worker encoders/mergers to avoid per-call allocations.
var (
	workerEncoderPool = sync.Pool{New: func() any { return NewBranchEncoder(1024) }}
	workerMergerPool  = sync.Pool{New: func() any { return NewHexBranchMerger(512) }}
)

// ApplyDeferredBranchUpdates encodes deferred branch updates concurrently and writes them.
// Returns the number of updates successfully written.
func ApplyDeferredBranchUpdates(
	deferred []*DeferredBranchUpdate,
	numWorkers int,
	putBranch func(prefix []byte, data []byte, prevData []byte) error,
) (int, error) {
	if len(deferred) == 0 {
		return 0, nil
	}
	if numWorkers <= 1 {
		numWorkers = 1
	}

	// Sequential fast path: avoids goroutine and channel overhead for small batches.
	if numWorkers == 1 || len(deferred) <= numWorkers {
		encoder := workerEncoderPool.Get().(*BranchEncoder)
		merger := workerMergerPool.Get().(*BranchMerger)
		defer workerEncoderPool.Put(encoder)
		defer workerMergerPool.Put(merger)

		var written int
		for _, upd := range deferred {
			if err := encodeDeferredUpdate(upd, encoder, merger); err != nil {
				return written, err
			}
			if upd.encoded == nil {
				continue
			}
			if err := putBranch(upd.prefix, upd.encoded, upd.prev); err != nil {
				return written, err
			}
			written++
		}
		mxTrieBranchesUpdated.AddInt(written)
		return written, nil
	}

	// Pipeline: workers encode in parallel, results sent to channel, main goroutine writes sequentially.
	type result struct {
		upd *DeferredBranchUpdate
		err error
	}
	// Size channels to actual batch length, not the 50K max.
	resultCh := make(chan result, len(deferred))
	workCh := make(chan *DeferredBranchUpdate, len(deferred))

	// Start workers with pooled encoders/mergers.
	var wg sync.WaitGroup
	for i := 0; i < numWorkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			encoder := workerEncoderPool.Get().(*BranchEncoder)
			merger := workerMergerPool.Get().(*BranchMerger)
			defer workerEncoderPool.Put(encoder)
			defer workerMergerPool.Put(merger)

			for upd := range workCh {
				err := encodeDeferredUpdate(upd, encoder, merger)
				resultCh <- result{upd: upd, err: err}
			}
		}()
	}

	// Close resultCh when all workers are done
	go func() {
		wg.Wait()
		close(resultCh)
	}()

	// Send work in background
	go func() {
		for _, upd := range deferred {
			workCh <- upd
		}
		close(workCh)
	}()

	// Process results as they come in - write to storage immediately
	var firstErr error
	var written int
	for res := range resultCh {
		if res.err != nil {
			if firstErr == nil {
				firstErr = res.err
			}
			continue
		}
		if res.upd.encoded == nil {
			continue // skip unchanged
		}
		if firstErr != nil {
			continue // drain channel but don't write after error
		}
		if err := putBranch(res.upd.prefix, res.upd.encoded, res.upd.prev); err != nil {
			firstErr = err
			continue
		}
		written++
	}
	mxTrieBranchesUpdated.AddInt(written)
	return written, firstErr
}

func (be *BranchEncoder) setMetrics(metrics *Metrics) {
	be.metrics = metrics
}

func (be *BranchEncoder) SetCache(cache *WarmupCache) {
	be.cache = cache
}

func (be *BranchEncoder) CollectUpdate(
	ctx PatriciaContext,
	prefix []byte,
	bitmap, touchMap, afterMap uint16,
	cells *[16]cellEncodeData,
	isNew bool,
) error {
	var prev []byte
	var foundInCache bool
	var err error

	if !isNew {
		if be.cache != nil {
			prev, foundInCache = be.cache.GetAndEvictBranch(prefix)
			if foundInCache && be.metrics != nil {
				be.metrics.cacheBranch.Add(1)
			}
		}
		if !foundInCache {
			prev, _, err = ctx.Branch(prefix)
			if err != nil {
				return err
			}
		}
	}
	if prev == nil {
		prev = []byte{}
	}

	update, err := be.EncodeBranch(bitmap, touchMap, afterMap, cells)
	if err != nil {
		return err
	}

	if len(prev) > 0 {
		if bytes.Equal(prev, update) {
			return nil // do not write the same data for prefix
		}
		update, err = be.merger.Merge(prev, update)
		if err != nil {
			return err
		}
	}

	prefixCopy := common.Copy(prefix)
	updateCopy := common.Copy(update)
	if err = ctx.PutBranch(prefixCopy, updateCopy, prev); err != nil {
		return err
	}
	if be.cache != nil {
		be.cache.PutBranch(prefixCopy, updateCopy)
	}
	if be.metrics != nil {
		be.metrics.updateBranch.Add(1)
	}
	mxTrieBranchesUpdated.Inc()
	return nil
}

// CollectDeferredUpdate stores a branch update job for later parallel processing.
// Unlike CollectUpdate, this does NOT call EncodeBranch immediately - it copies the cellEncodeData
// and defers encoding for parallel execution later.
// Cell hashes are already computed during fold() before this is called.
// Flushes pending updates if a duplicate prefix is detected or if deferred count exceeds maxDeferredUpdates.
func (be *BranchEncoder) CollectDeferredUpdate(
	ctx PatriciaContext,
	prefix []byte,
	bitmap, touchMap, afterMap uint16,
	cells *[16]cellEncodeData,
	isNew bool,
) error {
	// Flush if duplicate prefix or too many deferred updates
	limit := be.maxDeferredUpdates
	if limit == 0 {
		limit = DefaultMaxDeferredUpdates
	}
	needsFlush := len(be.deferred) >= limit
	if !needsFlush {
		_, needsFlush = be.pendingPrefixes.Get(prefix)
	}

	if needsFlush {
		if err := be.ApplyDeferredUpdates(16, ctx.PutBranch); err != nil {
			return err
		}
		be.ClearDeferred()
	}

	var (
		prev         []byte
		foundInCache bool
		err          error
	)

	if !isNew {
		if be.cache != nil {
			prev, foundInCache = be.cache.GetAndEvictBranch(prefix)
			if foundInCache && be.metrics != nil {
				be.metrics.cacheBranch.Add(1)
			}
		}
		if !foundInCache {
			prev, _, err = ctx.Branch(prefix)
		}
		if err != nil {
			return err
		}
	}
	if prev == nil {
		prev = []byte{}
	}

	// Track this prefix as pending
	be.pendingPrefixes.Set(prefix, struct{}{})

	// Get a pooled DeferredBranchUpdate and copy all fields
	upd := getDeferredUpdate(prefix, bitmap, touchMap, afterMap, cells, prev)
	be.deferred = append(be.deferred, upd)
	return nil
}

func (be *BranchEncoder) putUvarAndVal(size uint64, val []byte) error {
	n := binary.PutUvarint(be.bitmapBuf[:], size)
	if _, err := be.buf.Write(be.bitmapBuf[:n]); err != nil {
		return err
	}
	if _, err := be.buf.Write(val); err != nil {
		return err
	}
	return nil
}

// EncodeBranch encodes branch data from cellEncodeData. Pure serializer with no side effects.
// Result should be copied before next call to EncodeBranch, underlying slice is reused.
func (be *BranchEncoder) EncodeBranch(bitmap, touchMap, afterMap uint16, cells *[16]cellEncodeData) (BranchData, error) {
	be.buf.Reset()

	var encoded [4]byte
	binary.BigEndian.PutUint16(encoded[:], touchMap)
	binary.BigEndian.PutUint16(encoded[2:], afterMap)
	if _, err := be.buf.Write(encoded[:]); err != nil {
		return nil, err
	}

	for bitset := afterMap; bitset != 0; {
		bit := bitset & -bitset
		nibble := bits.TrailingZeros16(bit)
		cell := &cells[nibble]

		if bitmap&bit != 0 {
			var fields cellFields
			if cell.extLen > 0 && cell.storageAddrLen == 0 {
				fields |= fieldExtension
			}
			if cell.accountAddrLen > 0 {
				fields |= fieldAccountAddr
			}
			if cell.storageAddrLen > 0 {
				fields |= fieldStorageAddr
			}
			if cell.hashLen > 0 {
				fields |= fieldHash
			}
			if cell.stateHashLen == 32 && (cell.accountAddrLen > 0 || cell.storageAddrLen > 0) {
				fields |= fieldStateHash
			}
			if err := be.buf.WriteByte(byte(fields)); err != nil {
				return nil, err
			}
			if fields&fieldExtension != 0 {
				if err := be.putUvarAndVal(uint64(cell.extLen), cell.extension[:cell.extLen]); err != nil {
					return nil, err
				}
			}
			if fields&fieldAccountAddr != 0 {
				if err := be.putUvarAndVal(uint64(cell.accountAddrLen), cell.accountAddr[:cell.accountAddrLen]); err != nil {
					return nil, err
				}
			}
			if fields&fieldStorageAddr != 0 {
				if err := be.putUvarAndVal(uint64(cell.storageAddrLen), cell.storageAddr[:cell.storageAddrLen]); err != nil {
					return nil, err
				}
			}
			if fields&fieldHash != 0 {
				if err := be.putUvarAndVal(uint64(cell.hashLen), cell.hash[:cell.hashLen]); err != nil {
					return nil, err
				}
			}
			if fields&fieldStateHash != 0 {
				if err := be.putUvarAndVal(uint64(cell.stateHashLen), cell.stateHash[:cell.stateHashLen]); err != nil {
					return nil, err
				}
			}
		}
		bitset ^= bit
	}
	return be.buf.Bytes(), nil
}

type BranchData []byte

// ChildCount returns the number of children present in the branch.
func (branchData BranchData) ChildCount() int {
	if len(branchData) < 4 {
		return 0
	}
	return bits.OnesCount16(binary.BigEndian.Uint16(branchData[2:4]))
}

func (branchData BranchData) String() string {
	if len(branchData) == 0 {
		return ""
	}
	touchMap := binary.BigEndian.Uint16(branchData[0:])
	afterMap := binary.BigEndian.Uint16(branchData[2:])
	pos := 4
	var sb strings.Builder
	var cell cell
	fmt.Fprintf(&sb, "(%d) touchMap %016b, afterMap %016b\n", len(branchData), touchMap, afterMap)
	for bitset, j := touchMap, 0; bitset != 0; j++ {
		bit := bitset & -bitset
		nibble := bits.TrailingZeros16(bit)
		fmt.Fprintf(&sb, "   %x => ", nibble)
		if afterMap&bit == 0 {
			sb.WriteString("{DELETED}\n")
		} else {
			fields := cellFields(branchData[pos])
			pos++
			var err error
			if pos, err = cell.fillFromFields(branchData, pos, fields); err != nil {
				// This is used for test output, so ok to panic
				panic(err)
			}
			sb.WriteString("{")
			var comma string
			if cell.hashedExtLen > 0 {
				fmt.Fprintf(&sb, "hashedExtension=[%x]", cell.hashedExtension[:cell.hashedExtLen])
				comma = ","
			}
			if cell.accountAddrLen > 0 {
				fmt.Fprintf(&sb, "%saccountAddr=[%x]", comma, cell.accountAddr[:cell.accountAddrLen])
				comma = ","
			}
			if cell.storageAddrLen > 0 {
				fmt.Fprintf(&sb, "%sstorageAddr=[%x]", comma, cell.storageAddr[:cell.storageAddrLen])
				comma = ","
			}
			if cell.hashLen > 0 {
				fmt.Fprintf(&sb, "%shash=[%x]", comma, cell.hash[:cell.hashLen])
			}
			if cell.stateHashLen > 0 {
				fmt.Fprintf(&sb, "%sleafHash=[%x]", comma, cell.stateHash[:cell.stateHashLen])
			}
			sb.WriteString("}\n")
		}
		bitset ^= bit
	}
	return sb.String()
}

// if fn returns nil, the original key will be kept from branchData.
// Uses span-based lazy copy: unchanged regions of branchData are copied in bulk
// only when a key actually changes. When no keys change, returns branchData as-is
// with zero copies. When the caller passes nil or an undersized buffer, newData
// is auto-sized to len(branchData) on first use.
func (branchData BranchData) ReplacePlainKeys(newData []byte, fn func(key []byte, isStorage bool) (newKey []byte, err error)) (BranchData, error) {
	if len(branchData) < 4 {
		return branchData, nil
	}

	var numBuf [binary.MaxVarintLen64]byte
	touchMap := binary.BigEndian.Uint16(branchData[0:])
	afterMap := binary.BigEndian.Uint16(branchData[2:])
	if touchMap&afterMap == 0 {
		return branchData, nil
	}

	pos := 4
	anyChanged := false
	spanStart := 0 // start of current unchanged span in branchData
	for bitset, j := touchMap&afterMap, 0; bitset != 0; j++ {
		bit := bitset & -bitset
		fields := cellFields(branchData[pos])
		pos++
		if fields&fieldExtension != 0 {
			l, n := binary.Uvarint(branchData[pos:])
			if n == 0 {
				return nil, errors.New("replacePlainKeys buffer too small for hashedKey len")
			} else if n < 0 {
				return nil, errors.New("replacePlainKeys value overflow for hashedKey len")
			}
			pos += n
			if len(branchData) < pos+int(l) {
				return nil, fmt.Errorf("replacePlainKeys buffer too small for hashedKey: expected %d got %d", pos+int(l), len(branchData))
			}
			if l > 0 {
				pos += int(l)
			}
		}
		if fields&fieldAccountAddr != 0 {
			keyFieldStart := pos
			l, n := binary.Uvarint(branchData[pos:])
			if n == 0 {
				return nil, errors.New("replacePlainKeys buffer too small for accountAddr len")
			} else if n < 0 {
				return nil, errors.New("replacePlainKeys value overflow for accountAddr len")
			}
			pos += n
			if len(branchData) < pos+int(l) {
				return nil, fmt.Errorf("replacePlainKeys buffer too small for accountAddr: expected %d got %d", pos+int(l), len(branchData))
			}
			if l > 0 {
				pos += int(l)
			}
			newKey, err := fn(branchData[pos-int(l):pos], false)
			if err != nil {
				return nil, err
			}
			if newKey != nil {
				if !anyChanged {
					if cap(newData) < len(branchData) {
						newData = make([]byte, 0, len(branchData))
					} else {
						newData = newData[:0]
					}
					anyChanged = true
				}
				newData = append(newData, branchData[spanStart:keyFieldStart]...)
				n = binary.PutUvarint(numBuf[:], uint64(len(newKey)))
				newData = append(newData, numBuf[:n]...)
				newData = append(newData, newKey...)
				spanStart = pos
			}
		}
		if fields&fieldStorageAddr != 0 {
			keyFieldStart := pos
			l, n := binary.Uvarint(branchData[pos:])
			if n == 0 {
				return nil, errors.New("replacePlainKeys buffer too small for storageAddr len")
			} else if n < 0 {
				return nil, errors.New("replacePlainKeys value overflow for storageAddr len")
			}
			pos += n
			if len(branchData) < pos+int(l) {
				return nil, fmt.Errorf("replacePlainKeys buffer too small for storageAddr: expected %d got %d", pos+int(l), len(branchData))
			}
			if l > 0 {
				pos += int(l)
			}
			newKey, err := fn(branchData[pos-int(l):pos], true)
			if err != nil {
				return nil, err
			}
			if newKey != nil {
				if !anyChanged {
					if cap(newData) < len(branchData) {
						newData = make([]byte, 0, len(branchData))
					} else {
						newData = newData[:0]
					}
					anyChanged = true
				}
				newData = append(newData, branchData[spanStart:keyFieldStart]...)
				n = binary.PutUvarint(numBuf[:], uint64(len(newKey)))
				newData = append(newData, numBuf[:n]...)
				newData = append(newData, newKey...)
				spanStart = pos
			}
		}
		if fields&fieldHash != 0 {
			l, n := binary.Uvarint(branchData[pos:])
			if n == 0 {
				return nil, errors.New("replacePlainKeys buffer too small for hash len")
			} else if n < 0 {
				return nil, errors.New("replacePlainKeys value overflow for hash len")
			}
			pos += n
			if len(branchData) < pos+int(l) {
				return nil, fmt.Errorf("replacePlainKeys buffer too small for hash: expected %d got %d", pos+int(l), len(branchData))
			}
			if l > 0 {
				pos += int(l)
			}
		}
		if fields&fieldStateHash != 0 {
			l, n := binary.Uvarint(branchData[pos:])
			if n == 0 {
				return nil, errors.New("replacePlainKeys buffer too small for acLeaf hash len")
			} else if n < 0 {
				return nil, errors.New("replacePlainKeys value overflow for acLeafhash len")
			}
			pos += n
			if len(branchData) < pos+int(l) {
				return nil, fmt.Errorf("replacePlainKeys buffer too small for LeafHash: expected %d got %d", pos+int(l), len(branchData))
			}
			if l > 0 {
				pos += int(l)
			}
		}

		bitset ^= bit
	}

	if !anyChanged {
		return branchData, nil
	}
	// Flush remaining unchanged span
	newData = append(newData, branchData[spanStart:]...)
	return newData, nil
}

// IsComplete determines whether given branch data is complete, meaning that all information about all the children is present
// Each of 16 children of a branch node have two attributes
// touch - whether this child has been modified or deleted in this branchData (corresponding bit in touchMap is set)
// after - whether after this branchData application, the child is present in the tree or not (corresponding bit in afterMap is set)
func (branchData BranchData) IsComplete() bool {
	touchMap := binary.BigEndian.Uint16(branchData[0:])
	afterMap := binary.BigEndian.Uint16(branchData[2:])
	return ^touchMap&afterMap == 0
}

// MergeHexBranches combines two branchData, number 2 coming after (and potentially shadowing) number 1
func (branchData BranchData) MergeHexBranches(branchData2 BranchData, newData []byte) (BranchData, error) {
	if branchData2 == nil {
		return branchData, nil
	}
	if branchData == nil {
		return branchData2, nil
	}

	touchMap1 := binary.BigEndian.Uint16(branchData[0:])
	afterMap1 := binary.BigEndian.Uint16(branchData[2:])
	bitmap1 := touchMap1 & afterMap1
	pos1 := 4
	touchMap2 := binary.BigEndian.Uint16(branchData2[0:])
	afterMap2 := binary.BigEndian.Uint16(branchData2[2:])
	bitmap2 := touchMap2 & afterMap2
	pos2 := 4
	var bitmapBuf [4]byte
	binary.BigEndian.PutUint16(bitmapBuf[0:], touchMap1|touchMap2)
	binary.BigEndian.PutUint16(bitmapBuf[2:], afterMap2)
	newData = append(newData[:0], bitmapBuf[:]...)
	for bitset, j := bitmap1|bitmap2, 0; bitset != 0; j++ {
		bit := bitset & -bitset
		if bitmap2&bit != 0 {
			// Add fields from branchData2
			fields := cellFields(branchData2[pos2])
			newData = append(newData, byte(fields))
			pos2++
			for i := 0; i < bits.OnesCount8(byte(fields)); i++ {
				l, n := binary.Uvarint(branchData2[pos2:])
				if n == 0 {
					return nil, errors.New("MergeHexBranches buffer2 too small for field")
				} else if n < 0 {
					return nil, errors.New("MergeHexBranches value2 overflow for field")
				}
				newData = append(newData, branchData2[pos2:pos2+n]...)
				pos2 += n
				if len(branchData2) < pos2+int(l) {
					return nil, fmt.Errorf("MergeHexBranches buffer2 too small for %s : expected %d got %d", fields&cellFields(1<<i), pos2+int(l), len(branchData2))
				}
				if l > 0 {
					newData = append(newData, branchData2[pos2:pos2+int(l)]...)
					pos2 += int(l)
				}
			}
		}
		if bitmap1&bit != 0 {
			add := (touchMap2&bit == 0) && (afterMap2&bit != 0) // Add fields from branchData1
			fields := cellFields(branchData[pos1])
			if add {
				newData = append(newData, byte(fields))
			}
			pos1++
			for i := 0; i < bits.OnesCount8(byte(fields)); i++ {
				l, n := binary.Uvarint(branchData[pos1:])
				if n == 0 {
					return nil, errors.New("MergeHexBranches buffer1 too small for field")
				} else if n < 0 {
					return nil, errors.New("MergeHexBranches value1 overflow for field")
				}
				if add {
					newData = append(newData, branchData[pos1:pos1+n]...)
				}
				pos1 += n
				if len(branchData) < pos1+int(l) {
					return nil, fmt.Errorf("MergeHexBranches buffer1 too small for %s : expected %d got %d", fields&cellFields(1<<i), pos1+int(l), len(branchData))
				}
				if l > 0 {
					if add {
						newData = append(newData, branchData[pos1:pos1+int(l)]...)
					}
					pos1 += int(l)
				}
			}
		}
		bitset ^= bit
	}
	return newData, nil
}

func (branchData BranchData) decodeCells() (touchMap, afterMap uint16, row [16]*cell, err error) {
	touchMap = binary.BigEndian.Uint16(branchData[0:])
	afterMap = binary.BigEndian.Uint16(branchData[2:])
	pos := 4
	for bitset, j := touchMap, 0; bitset != 0; j++ {
		bit := bitset & -bitset
		nibble := bits.TrailingZeros16(bit)
		if afterMap&bit != 0 {
			fields := cellFields(branchData[pos])
			pos++
			row[nibble] = new(cell)
			if pos, err = row[nibble].fillFromFields(branchData, pos, fields); err != nil {
				err = fmt.Errorf("failed to fill cell at nibble %x: %w", nibble, err)
				return
			}
		}
		bitset ^= bit
	}
	return
}

func (branchData BranchData) Validate(branchKey []byte) error {
	if len(branchData) == 0 {
		return nil
	}
	_, afterMap, row, err := branchData.decodeCells()
	if err != nil {
		return err
	}
	if err = validateAfterMap(afterMap, row); err != nil {
		return err
	}
	if err = validatePlainKeys(branchKey, row, keccak.NewFastKeccak()); err != nil {
		return err
	}
	return nil
}

func validateAfterMap(afterMap uint16, row [16]*cell) error {
	cellsInAfterMap := bits.OnesCount16(afterMap)
	var decodedCellsCount int
	for _, c := range row {
		if c != nil {
			decodedCellsCount++
		}
	}
	if cellsInAfterMap != decodedCellsCount {
		return fmt.Errorf("cells in after map does not match branch data: %d vs %d", cellsInAfterMap, decodedCellsCount)
	}
	return nil
}

func validatePlainKeys(branchKey []byte, row [16]*cell, keccak keccak.KeccakState) error {
	uncompactedBranchKey := nibbles.CompactToHex(branchKey)
	if nibbles.HasTerm(uncompactedBranchKey) {
		uncompactedBranchKey = uncompactedBranchKey[:len(uncompactedBranchKey)-1]
	}
	if len(uncompactedBranchKey) > 128 {
		return fmt.Errorf("branch key too long: %d", len(branchKey))
	}
	var hashBuf common.Hash
	depth := int16(len(uncompactedBranchKey))
	for _, c := range row {
		if c == nil {
			continue
		}
		if c.accountAddrLen == 0 && c.storageAddrLen == 0 {
			continue
		}
		err := c.deriveHashedKeys(depth, keccak, length.Addr, hashBuf[:])
		if err != nil {
			return err
		}
		hashedExtLen := c.hashedExtLen
		hashedExt := c.hashedExtension[:hashedExtLen]
		if c.extLen > 0 && hashedExtLen >= c.extLen {
			hashedExtLen -= c.extLen
			hashedExt = hashedExt[:hashedExtLen]
		}
		branchKeyAndExtNibbles := make([]byte, len(uncompactedBranchKey)+int(hashedExtLen))
		copy(branchKeyAndExtNibbles, uncompactedBranchKey)
		copy(branchKeyAndExtNibbles[len(uncompactedBranchKey):], hashedExt)
		var plainKeyNibbles []byte
		if c.accountAddrLen > 0 {
			plainKeyNibbles = KeyToHexNibbleHash(c.accountAddr[:])
		}
		if c.storageAddrLen > 0 {
			plainKeyNibbles = KeyToHexNibbleHash(c.storageAddr[:])
			if c.accountAddrLen > 0 {
				if !bytes.Equal(c.accountAddr[:], c.storageAddr[:length.Addr]) {
					return fmt.Errorf("accountAddr mismatch with storageAddr: %s != %x", common.BytesToAddress(c.accountAddr[:]), common.BytesToHash(c.storageAddr[:length.Addr]))
				}
			}
		}
		if !bytes.Equal(plainKeyNibbles, branchKeyAndExtNibbles) {
			return fmt.Errorf("branch and hashed extension nibbles dont match plainKey nibbles: %x vs %x", plainKeyNibbles, branchKeyAndExtNibbles)
		}
	}
	return nil
}

type BranchMerger struct {
	buf []byte
	num [4]byte
}

func NewHexBranchMerger(capacity uint64) *BranchMerger {
	return &BranchMerger{buf: make([]byte, capacity)}
}

// MergeHexBranches combines two branchData, number 2 coming after (and potentially shadowing) number 1
func (m *BranchMerger) Merge(branch1 BranchData, branch2 BranchData) (BranchData, error) {
	if len(branch2) == 0 {
		return branch1, nil
	}
	if len(branch1) == 0 {
		return branch2, nil
	}

	touchMap1 := binary.BigEndian.Uint16(branch1[0:])
	afterMap1 := binary.BigEndian.Uint16(branch1[2:])
	bitmap1 := touchMap1 & afterMap1
	pos1 := 4

	touchMap2 := binary.BigEndian.Uint16(branch2[0:])
	afterMap2 := binary.BigEndian.Uint16(branch2[2:])
	bitmap2 := touchMap2 & afterMap2
	pos2 := 4

	binary.BigEndian.PutUint16(m.num[0:], touchMap1|touchMap2)
	binary.BigEndian.PutUint16(m.num[2:], afterMap2)
	dataPos := 4

	m.buf = append(m.buf[:0], m.num[:]...)

	for bitset, j := bitmap1|bitmap2, 0; bitset != 0; j++ {
		bit := bitset & -bitset
		if bitmap2&bit != 0 {
			// Add fields from branch2
			fields := cellFields(branch2[pos2])
			m.buf = append(m.buf, byte(fields))
			pos2++

			for i := 0; i < bits.OnesCount8(byte(fields)); i++ {
				l, n := binary.Uvarint(branch2[pos2:])
				if n == 0 {
					return nil, errors.New("MergeHexBranches branch2 is too small: expected node info size")
				} else if n < 0 {
					return nil, errors.New("MergeHexBranches branch2: size overflow for length")
				}

				m.buf = append(m.buf, branch2[pos2:pos2+n]...)
				pos2 += n
				dataPos += n
				if len(branch2) < pos2+int(l) {
					return nil, fmt.Errorf("MergeHexBranches branch2 is too small: expected at least %d got %d bytes", pos2+int(l), len(branch2))
				}
				if l > 0 {
					m.buf = append(m.buf, branch2[pos2:pos2+int(l)]...)
					pos2 += int(l)
					dataPos += int(l)
				}
			}
		}
		if bitmap1&bit != 0 {
			add := (touchMap2&bit == 0) && (afterMap2&bit != 0) // Add fields from branchData1
			fields := cellFields(branch1[pos1])
			if add {
				m.buf = append(m.buf, byte(fields))
			}
			pos1++
			for i := 0; i < bits.OnesCount8(byte(fields)); i++ {
				l, n := binary.Uvarint(branch1[pos1:])
				if n == 0 {
					return nil, errors.New("MergeHexBranches branch1 is too small: expected node info size")
				} else if n < 0 {
					return nil, errors.New("MergeHexBranches branch1: size overflow for length")
				}

				if add {
					m.buf = append(m.buf, branch1[pos1:pos1+n]...)
				}
				pos1 += n
				if len(branch1) < pos1+int(l) {
					return nil, fmt.Errorf("MergeHexBranches branch1 is too small: expected at least %d got %d bytes", pos1+int(l), len(branch1))
				}
				if l > 0 {
					if add {
						m.buf = append(m.buf, branch1[pos1:pos1+int(l)]...)
					}
					pos1 += int(l)
				}
			}
		}
		bitset ^= bit
	}
	return m.buf, nil
}

func ParseTrieVariant(s string) TrieVariant {
	var trieVariant TrieVariant
	switch s {
	case "bin":
		trieVariant = VariantBinPatriciaTrie
	case "hex-parallel":
		trieVariant = VariantConcurrentHexPatricia
	case "hex":
		fallthrough
	default:
		trieVariant = VariantHexPatriciaTrie
	}
	return trieVariant
}

type BranchStat struct {
	KeySize       uint64
	ValSize       uint64
	MinCellSize   uint64
	MaxCellSize   uint64
	CellCount     uint64
	APKSize       uint64
	SPKSize       uint64
	ExtSize       uint64
	HashSize      uint64
	APKCount      uint64
	SPKCount      uint64
	HashCount     uint64
	ExtCount      uint64
	TAMapsSize    uint64
	LeafHashSize  uint64
	LeafHashCount uint64
	MedianAPK     uint64
	MedianSPK     uint64
	MedianHash    uint64
	MedianExt     uint64
	MedianLH      uint64
	IsRoot        bool
}

// do not add stat of root node to other branch stat
func (bs *BranchStat) Collect(other *BranchStat) {
	if other == nil {
		return
	}

	bs.KeySize += other.KeySize
	bs.ValSize += other.ValSize
	bs.MinCellSize = min(bs.MinCellSize, other.MinCellSize)
	bs.MaxCellSize = max(bs.MaxCellSize, other.MaxCellSize)
	bs.CellCount += other.CellCount
	bs.APKSize += other.APKSize
	bs.SPKSize += other.SPKSize
	bs.ExtSize += other.ExtSize
	bs.HashSize += other.HashSize
	bs.APKCount += other.APKCount
	bs.SPKCount += other.SPKCount
	bs.HashCount += other.HashCount
	bs.ExtCount += other.ExtCount

	setMedian := func(median *uint64, otherMedian uint64) {
		if *median == 0 {
			*median = otherMedian
		} else {
			*median = (*median + otherMedian) / 2
		}
	}
	setMedian(&bs.MedianExt, other.MedianExt)
	setMedian(&bs.MedianAPK, other.MedianAPK)
	setMedian(&bs.MedianSPK, other.MedianSPK)
	setMedian(&bs.MedianHash, other.MedianHash)
	setMedian(&bs.MedianLH, other.MedianLH)
	bs.MedianHash = (bs.MedianHash + other.MedianHash) / 2
	bs.MedianAPK = (bs.MedianAPK + other.MedianAPK) / 2
	bs.MedianSPK = (bs.MedianSPK + other.MedianSPK) / 2
	bs.MedianLH = (bs.MedianLH + other.MedianLH) / 2
	bs.TAMapsSize += other.TAMapsSize
	bs.LeafHashSize += other.LeafHashSize
	bs.LeafHashCount += other.LeafHashCount
}

func DecodeBranchAndCollectStat(key, branch []byte, tv TrieVariant) *BranchStat {
	stat := &BranchStat{}
	if len(key) == 0 {
		return nil
	}

	stat.KeySize = uint64(len(key))
	stat.ValSize = uint64(len(branch))
	stat.IsRoot = true

	// if key is not "state" then we are interested in the branch data
	if !bytes.Equal(key, []byte("state")) {
		stat.IsRoot = false

		tm, am, cells, err := BranchData(branch).decodeCells()
		if err != nil {
			return nil
		}
		stat.TAMapsSize = uint64(2 + 2) // touchMap + afterMap
		stat.CellCount = uint64(bits.OnesCount16(tm & am))

		medians := make(map[string][]int16)
		for _, c := range cells {
			if c == nil {
				continue
			}
			enc := uint64(len(c.Encode()))
			stat.MinCellSize = min(stat.MinCellSize, enc)
			stat.MaxCellSize = max(stat.MaxCellSize, enc)
			switch {
			case c.accountAddrLen > 0:
				stat.APKSize += uint64(c.accountAddrLen)
				stat.APKCount++
				medians["apk"] = append(medians["apk"], c.accountAddrLen)
			case c.storageAddrLen > 0:
				stat.SPKSize += uint64(c.storageAddrLen)
				stat.SPKCount++
				medians["spk"] = append(medians["spk"], c.storageAddrLen)
			case c.hashLen > 0:
				stat.HashSize += uint64(c.hashLen)
				stat.HashCount++
				medians["hash"] = append(medians["hash"], c.hashLen)
			case c.stateHashLen > 0:
				stat.LeafHashSize += uint64(c.stateHashLen)
				stat.LeafHashCount++
				medians["lh"] = append(medians["lh"], c.stateHashLen)
			case c.extLen > 0:
				stat.ExtSize += uint64(c.extLen)
				stat.ExtCount++
				medians["ext"] = append(medians["ext"], c.extLen)
			default:
				panic("unexpected cell " + c.FullString())
			}
			if c.extLen > 0 {
				switch tv {
				case VariantBinPatriciaTrie:
					stat.ExtSize += uint64(c.extLen)
				case VariantHexPatriciaTrie, VariantConcurrentHexPatricia:
					stat.ExtSize += uint64(c.extLen)
				}
				stat.ExtCount++
			}
		}

		for k, v := range medians {
			slices.Sort(v)
			switch k {
			case "apk":
				stat.MedianAPK = uint64(v[len(v)/2])
			case "spk":
				stat.MedianSPK = uint64(v[len(v)/2])
			case "hash":
				stat.MedianHash = uint64(v[len(v)/2])
			case "ext":
				stat.MedianExt = uint64(v[len(v)/2])
			case "lh":
				stat.MedianLH = uint64(v[len(v)/2])
			}
		}
	}
	return stat
}

// Defines how to evaluate commitments
type Mode uint

const (
	ModeDisabled Mode = 0
	ModeDirect   Mode = 1
	ModeUpdate   Mode = 2
)

func (m Mode) String() string {
	switch m {
	case ModeDisabled:
		return "disabled"
	case ModeDirect:
		return "direct"
	case ModeUpdate:
		return "update"
	default:
		return "unknown"
	}
}

type Updates struct {
	hasher keyHasher
	keys   map[string]struct{} // plain keys to keep only unique keys in etl
	etl    *etl.Collector      // all-in-one collector
	// Sorted by hashedKey first, with plainKey as a tiebreaker (see
	// keyUpdateLessFn). Trie traversal must happen in hashedKey order so
	// Process's fold/unfold operates on adjacent paths; iterating by
	// plainKey produced a divergent root and was the root cause of the
	// eip1153/eip7778/eip7976/eip7825 wrong-trie-root failures. ModeDirect
	// achieves the same ordering via its etl collector keyed on hashedKey;
	// this btree must match.
	tree    *btree.BTreeG[*KeyUpdate]
	treeIdx map[string]*KeyUpdate // plainKey → btree entry for O(1) lookup in ModeUpdate
	mode    Mode
	tmpdir  string

	sortPerNibble bool // if true, use nibbles collectors instead of etl (all-in-one)
	nibbles       [16]*etl.Collector

	batchSlab []KeyUpdate // grow-only slab for HashSort batch (avoids per-key heap allocs)

	// Ring of byte arenas for HashSort key copies; a slot is reused only after its prior generation's warm items drain.
	arenas   [arenaRingSize][]byte
	curArena int
	gen      uint64
}

// arenaRingSize is how many byte arenas HashSort cycles; raising it only adds memory headroom, never affects correctness.
const arenaRingSize = 2

// arenaAlloc appends b to the byte arena and returns the sub-slice.
// The returned slice is valid until the arena is reset.
// The arena must have sufficient capacity (via arenaEnsureCap) before
// accumulating a batch; if capacity is exceeded, arenaAlloc falls back
// to an independent heap allocation to keep previously returned
// sub-slices valid.
func (t *Updates) arenaAlloc(b []byte) []byte {
	arena := t.arenas[t.curArena]
	off := len(arena)
	needed := off + len(b)
	if needed > cap(arena) {
		// Arena capacity exceeded — fall back to an independent allocation.
		// This keeps previously returned sub-slices valid while avoiding a
		// panic that would crash a production node.
		result := make([]byte, len(b))
		copy(result, b)
		return result
	}
	arena = arena[:needed]
	copy(arena[off:], b)
	t.arenas[t.curArena] = arena
	return arena[off:needed]
}

// arenaEnsureCap reserves at least c bytes in every ring buffer; call before a batch so a mid-batch grow can't reallocate and invalidate returned sub-slices.
func (t *Updates) arenaEnsureCap(c int) {
	for i := range t.arenas {
		if cap(t.arenas[i]) < c {
			t.arenas[i] = make([]byte, 0, c)
		}
	}
}

// Should be called right after updates initialisation. Otherwise could lost some data
func (t *Updates) SetConcurrentCommitment(b bool) {
	t.sortPerNibble = b
	t.initCollector()
}

// SetConcurrentCommitment returns true if updates are sorted per nibble
func (t *Updates) IsConcurrentCommitment() bool {
	return t.sortPerNibble
}

type keyHasher func(key []byte) []byte

func keyHasherNoop(key []byte) []byte { return key }

// NewEmpty creates a fresh Updates with the same mode, tmpdir, and hasher
// as the receiver. Used by SwapUpdates to replace the buffer atomically.
func (t *Updates) NewEmpty() *Updates {
	return NewUpdates(t.mode, t.tmpdir, t.hasher)
}

func NewUpdates(m Mode, tmpdir string, hasher keyHasher) *Updates {
	t := &Updates{
		hasher: hasher,
		tmpdir: tmpdir,
		mode:   m,
	}
	if t.mode == ModeDirect {
		t.keys = make(map[string]struct{})
		t.initCollector()
	} else if t.mode == ModeUpdate {
		t.tree = btree.NewG(64, keyUpdateLessFn)
		t.treeIdx = make(map[string]*KeyUpdate)
	}
	return t
}

func (t *Updates) SetMode(m Mode) {
	t.mode = m
	if t.mode == ModeDirect && t.keys == nil {
		t.keys = make(map[string]struct{})
		t.initCollector()
	} else if t.mode == ModeUpdate && t.tree == nil {
		t.tree = btree.NewG(64, keyUpdateLessFn)
		t.treeIdx = make(map[string]*KeyUpdate)
	}
	t.Reset()
}

func (t *Updates) initCollector() {
	if t.sortPerNibble {
		for i := 0; i < len(t.nibbles); i++ {
			if t.nibbles[i] != nil {
				t.nibbles[i].Close()
				t.nibbles[i] = nil
			}

			t.nibbles[i] = etl.NewCollectorWithAllocator("commitment.nibble."+strconv.Itoa(i), t.tmpdir, etl.SmallSortableBuffers, log.Root().New("update-tree")).LogLvl(log.LvlDebug)
			t.nibbles[i].SortAndFlushInBackground(true)
		}
		if t.etl != nil {
			t.etl.Close()
			t.etl = nil
		}
		return
	}

	if t.etl != nil {
		t.etl.Close()
		t.etl = nil
	}
	t.etl = etl.NewCollectorWithAllocator("commitment", t.tmpdir, etl.SmallSortableBuffers, log.Root().New("update-tree")).LogLvl(log.LvlDebug)
	t.etl.SortAndFlushInBackground(true)
}

func (t *Updates) Mode() Mode { return t.mode }

// PlainKeys returns a copy of the set of plain keys that have been touched.
// Only meaningful in ModeDirect; returns nil otherwise.
func (t *Updates) PlainKeys() map[string]struct{} {
	if t.mode != ModeDirect || t.keys == nil {
		return nil
	}
	cp := make(map[string]struct{}, len(t.keys))
	for k := range t.keys {
		cp[k] = struct{}{}
	}
	return cp
}

func (t *Updates) Size() (updates uint64) {
	switch t.mode {
	case ModeDirect:
		return uint64(len(t.keys))
	case ModeUpdate:
		return uint64(t.tree.Len())
	default:
		return 0
	}
}

// TouchPlainKey marks plainKey as updated and applies different fn for different key types
// (different behaviour for Code, Account and Storage key modifications).
func (t *Updates) TouchPlainKey(key string, val []byte, fn func(c *KeyUpdate, val []byte)) {
	switch t.mode {
	case ModeUpdate:
		if existing, ok := t.treeIdx[key]; ok {
			fn(existing, val)
		} else {
			pivot := &KeyUpdate{
				plainKey:  key,
				hashedKey: t.hasher(common.ToBytesZeroCopy(key)),
				update:    new(Update),
			}
			fn(pivot, val)
			t.tree.ReplaceOrInsert(pivot)
			t.treeIdx[key] = pivot
		}
	case ModeDirect:
		if _, ok := t.keys[key]; !ok {
			keyBytes := common.ToBytesZeroCopy(key)
			hashedKey := t.hasher(keyBytes)

			var err error
			if !t.sortPerNibble {
				err = t.etl.Collect(hashedKey, keyBytes)
			} else {
				err = t.nibbles[hashedKey[0]].Collect(hashedKey, keyBytes)
			}
			if err != nil {
				log.Warn("failed to collect updated key", "key", key, "err", err)
			}
			t.keys[key] = struct{}{}
		}
	default:
	}
}

// TouchPlainKeyDirect applies a pre-built Update to the key without
// serialization/deserialization. Used by the commitment calculator which
// receives aggregated state changes via channel instead of serialized bytes
// from DomainPut.
func (t *Updates) TouchPlainKeyDirect(key string, update *Update) {
	if dbg.TraceTouchKey {
		fmt.Printf("TOUCHDIRECT key=%x flags=%v balance=%d nonce=%d codeHash=%x\n",
			key, update.Flags, &update.Balance, update.Nonce, update.CodeHash)
	}
	switch t.mode {
	case ModeUpdate:
		if existing, ok := t.treeIdx[key]; ok {
			// Merge into existing entry
			if update.Flags&DeleteUpdate != 0 {
				existing.update.Flags = DeleteUpdate
				existing.update.CodeHash = empty.CodeHash
			} else {
				existing.update.Flags &^= DeleteUpdate
				if update.Flags&BalanceUpdate != 0 {
					existing.update.Balance.Set(&update.Balance)
					existing.update.Flags |= BalanceUpdate
				}
				if update.Flags&NonceUpdate != 0 {
					existing.update.Nonce = update.Nonce
					existing.update.Flags |= NonceUpdate
				}
				if update.Flags&CodeUpdate != 0 {
					existing.update.CodeHash = update.CodeHash
					existing.update.Flags |= CodeUpdate
				}
				if update.Flags&StorageUpdate != 0 {
					existing.update.Storage = update.Storage
					existing.update.StorageLen = update.StorageLen
					existing.update.Flags |= StorageUpdate
				}
			}
		} else {
			pivot := &KeyUpdate{
				plainKey:  key,
				hashedKey: t.hasher(common.ToBytesZeroCopy(key)),
				update:    new(Update),
			}
			*pivot.update = *update
			t.tree.ReplaceOrInsert(pivot)
			t.treeIdx[key] = pivot
		}
	case ModeDirect:
		if _, ok := t.keys[key]; !ok {
			keyBytes := common.ToBytesZeroCopy(key)
			hashedKey := t.hasher(keyBytes)

			var err error
			if !t.sortPerNibble {
				err = t.etl.Collect(hashedKey, keyBytes)
			} else {
				err = t.nibbles[hashedKey[0]].Collect(hashedKey, keyBytes)
			}
			if err != nil {
				log.Warn("failed to collect updated key", "key", key, "err", err)
			}
			t.keys[key] = struct{}{}
		}
	default:
	}
}

func (t *Updates) TouchHashedKey(hashedKey []byte) {
	switch t.mode {
	case ModeDirect:
		if len(hashedKey) == 0 {
			return
		}
		// string(hashedKey) copies the bytes, so dedupKey is safe even if the caller reuses the slice.
		// No extra copy needed before etl.Collect: see Collector.Collect — it copies k and v internally.
		dedupKey := string(hashedKey)
		if _, ok := t.keys[dedupKey]; !ok {
			var err error
			if !t.sortPerNibble {
				err = t.etl.Collect(hashedKey, []byte{})
			} else {
				err = t.nibbles[hashedKey[0]].Collect(hashedKey, []byte{})
			}
			if err != nil {
				log.Warn("failed to collect hashed key", "hashedKey", fmt.Sprintf("%x", hashedKey), "err", err)
			}
			t.keys[dedupKey] = struct{}{}
		}
	case ModeUpdate:
		pivot := &KeyUpdate{hashedKey: common.Copy(hashedKey), update: new(Update)}
		t.tree.ReplaceOrInsert(pivot)
	default:
	}
}

func (t *Updates) TouchAccount(c *KeyUpdate, val []byte) {
	if len(val) == 0 {
		c.update.Flags = DeleteUpdate
		return
	}
	if c.update.Flags&DeleteUpdate != 0 {
		c.update.Flags = 0 // also could invert with ^ but 0 is just a reset
	}

	acc := accounts.Account{}
	err := accounts.DeserialiseV3(&acc, val)
	if err != nil {
		panic(err)
	}
	if c.update.Nonce != acc.Nonce {
		c.update.Nonce = acc.Nonce
		c.update.Flags |= NonceUpdate
	}
	if !c.update.Balance.Eq(&acc.Balance) {
		c.update.Balance.Set(&acc.Balance)
		c.update.Flags |= BalanceUpdate
	}
	if acc.CodeHash.Value() != c.update.CodeHash {
		if acc.CodeHash.IsEmpty() {
			c.update.CodeHash = empty.CodeHash
		} else {
			c.update.Flags |= CodeUpdate
			c.update.CodeHash = acc.CodeHash.Value()
		}
	}
}

func (t *Updates) TouchStorage(c *KeyUpdate, val []byte) {
	c.update.StorageLen = int8(len(val))
	if len(val) == 0 {
		c.update.Flags = DeleteUpdate
	} else {
		c.update.Flags &^= DeleteUpdate
		c.update.Flags |= StorageUpdate
		copy(c.update.Storage[:], val)
	}
}

func (t *Updates) TouchCode(c *KeyUpdate, code []byte) {
	c.update.Flags |= CodeUpdate
	if len(code) == 0 {
		if c.update.Flags == 0 {
			c.update.Flags = DeleteUpdate
		}
		c.update.CodeHash = empty.CodeHash
		return
	}
	c.update.CodeHash = crypto.HashData(code)
}

func (t *Updates) Close() {
	if t.keys != nil {
		clear(t.keys)
	}
	if t.tree != nil {
		t.tree.Clear(true)
		t.tree = nil
	}
	if t.etl != nil {
		t.etl.Close()
	}
	if t.sortPerNibble {
		for i := 0; i < len(t.nibbles); i++ {
			if t.nibbles[i] != nil {
				t.nibbles[i].Close()
			}
		}
	}
}

const hashSortBatchSize = 10_000

// HashSort sorts and applies fn to each key-value pair in the order of hashed keys.
// Keys are processed in batches of 10k to control memory usage.
// If warmuper is non-nil, keys are submitted for parallel warming before processing.
// Caller is responsible for calling warmuper.Wait() after processing completes.
// IMPORTANT: fn must not retain hk or pk slices after returning; they are backed by
// reusable arena memory that is invalidated between batches.
func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, pk []byte, update *Update) error) error {
	switch t.mode {
	case ModeDirect:
		clear(t.keys)

		t.batchSlab = t.batchSlab[:0]
		if warmuper != nil {
			if err := warmuper.WaitBufferFree(t.curArena); err != nil {
				return err
			}
		}
		// Pre-size the arena so a mid-batch grow can't reallocate and invalidate live sub-slices (≤180 B/key, 192 with headroom).
		t.arenaEnsureCap(hashSortBatchSize * 192)
		t.arenas[t.curArena] = t.arenas[t.curArena][:0]
		var prevKey []byte

		err := t.etl.Load(nil, "", func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error {
			if warmuper != nil && warmuper.Cache() != nil {
				warmuper.Cache().EvictPlainKey(v)
			}
			// Copy into arena since ETL may reuse buffers
			hk := t.arenaAlloc(k)
			pk := t.arenaAlloc(v)
			t.batchSlab = append(t.batchSlab, KeyUpdate{hashedKey: hk, plainKey: unsafe.String(unsafe.SliceData(pk), len(pk))})

			// Submit to warmuper with start depth based on divergence from previous key
			if warmuper != nil {
				startDepth := 0
				if prevKey != nil {
					// Find common prefix length
					minLen := min(len(prevKey), len(hk))
					for startDepth < minLen && prevKey[startDepth] == hk[startDepth] {
						startDepth++
					}
				}
				warmuper.WarmKey(hk, startDepth, t.gen)
				prevKey = append(prevKey[:0], hk...)
			}

			// Process batch when full
			if len(t.batchSlab) >= hashSortBatchSize {
				for i := range t.batchSlab {
					select {
					case <-ctx.Done():
						return ctx.Err()
					default:
					}
					if err := fn(t.batchSlab[i].hashedKey, common.ToBytesZeroCopy(t.batchSlab[i].plainKey), nil); err != nil {
						return err
					}
				}
				t.batchSlab = t.batchSlab[:0]
				nextGen := t.gen + 1
				slot := int(nextGen % arenaRingSize)
				if warmuper != nil {
					if err := warmuper.WaitBufferFree(slot); err != nil {
						return err
					}
				}
				t.gen = nextGen
				t.arenas[slot] = t.arenas[slot][:0]
				t.curArena = slot
			}
			return nil
		}, etl.TransformArgs{Quit: ctx.Done()})
		if err != nil {
			return err
		}

		// Process remaining keys in final batch
		for i := range t.batchSlab {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
			}
			if err := fn(t.batchSlab[i].hashedKey, common.ToBytesZeroCopy(t.batchSlab[i].plainKey), nil); err != nil {
				return err
			}
		}

		t.initCollector()

	case ModeUpdate:
		t.batchSlab = t.batchSlab[:0]
		if warmuper != nil {
			if err := warmuper.WaitBufferFree(t.curArena); err != nil {
				return err
			}
		}
		t.arenaEnsureCap(hashSortBatchSize * 144)
		t.arenas[t.curArena] = t.arenas[t.curArena][:0]
		var prevKey []byte
		var processErr error

		t.tree.Ascend(func(item *KeyUpdate) bool {
			select {
			case <-ctx.Done():
				processErr = ctx.Err()
				return false
			default:
			}

			hk := t.arenaAlloc(item.hashedKey)
			t.batchSlab = append(t.batchSlab, KeyUpdate{hashedKey: hk, plainKey: item.plainKey, update: item.update})

			if warmuper != nil {
				startDepth := 0
				if prevKey != nil {
					minLen := min(len(prevKey), len(hk))
					for startDepth < minLen && prevKey[startDepth] == hk[startDepth] {
						startDepth++
					}
				}
				warmuper.WarmKey(hk, startDepth, t.gen)
				prevKey = append(prevKey[:0], hk...)
			}

			if len(t.batchSlab) >= hashSortBatchSize {
				for i := range t.batchSlab {
					select {
					case <-ctx.Done():
						processErr = ctx.Err()
						return false
					default:
					}
					if err := fn(t.batchSlab[i].hashedKey, common.ToBytesZeroCopy(t.batchSlab[i].plainKey), t.batchSlab[i].update); err != nil {
						processErr = err
						return false
					}
				}
				t.batchSlab = t.batchSlab[:0]
				nextGen := t.gen + 1
				slot := int(nextGen % arenaRingSize)
				if warmuper != nil {
					if err := warmuper.WaitBufferFree(slot); err != nil {
						processErr = err
						return false
					}
				}
				t.gen = nextGen
				t.arenas[slot] = t.arenas[slot][:0]
				t.curArena = slot
			}
			return true
		})

		if processErr != nil {
			return processErr
		}

		// Process remaining keys in final batch
		for i := range t.batchSlab {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
			}
			if err := fn(t.batchSlab[i].hashedKey, common.ToBytesZeroCopy(t.batchSlab[i].plainKey), t.batchSlab[i].update); err != nil {
				return err
			}
		}
		t.tree.Clear(true)

	default:
		return nil
	}
	return nil
}

// Reset clears all updates
func (t *Updates) Reset() {
	switch t.mode {
	case ModeDirect:
		if t.keys == nil {
			t.keys = make(map[string]struct{})
		} else {
			clear(t.keys)
		}
		t.initCollector()
	case ModeUpdate:
		t.tree.Clear(true)
		clear(t.treeIdx)
	default:
	}
	t.batchSlab = t.batchSlab[:0]
	for i := range t.arenas {
		t.arenas[i] = t.arenas[i][:0]
	}
	t.curArena = 0
	t.gen = 0
}

type KeyUpdate struct {
	plainKey  string
	hashedKey []byte
	update    *Update
}

// keyUpdateLessFn orders KeyUpdate entries by hashedKey. Process requires
// updates to arrive in hashedKey-sorted order so fold/unfold operates on
// adjacent trie paths; iterating by plainKey produces a different trie
// traversal sequence and yields a divergent root hash. ModeDirect achieves
// this via its etl collector keyed on hashedKey; ModeUpdate's btree must
// match that order.
//
// plainKey is used only as a tiebreaker (e.g. for TouchHashedKey entries
// that share their hashedKey with a "real" entry).
func keyUpdateLessFn(i, j *KeyUpdate) bool {
	if c := bytes.Compare(i.hashedKey, j.hashedKey); c != 0 {
		return c < 0
	}
	return i.plainKey < j.plainKey
}

type UpdateFlags uint8

const (
	CodeUpdate    UpdateFlags = 1
	DeleteUpdate  UpdateFlags = 2
	BalanceUpdate UpdateFlags = 4
	NonceUpdate   UpdateFlags = 8
	StorageUpdate UpdateFlags = 16
)

func (uf UpdateFlags) String() string {
	var sb strings.Builder
	if uf&DeleteUpdate != 0 {
		sb.WriteString("Delete")
	}
	if uf&BalanceUpdate != 0 {
		sb.WriteString("+Balance")
	}
	if uf&NonceUpdate != 0 {
		sb.WriteString("+Nonce")
	}
	if uf&CodeUpdate != 0 {
		sb.WriteString("+Code")
	}
	if uf&StorageUpdate != 0 {
		sb.WriteString("+Storage")
	}
	return sb.String()
}

type Update struct {
	CodeHash   common.Hash
	Storage    common.Hash
	StorageLen int8
	Flags      UpdateFlags
	Balance    uint256.Int
	Nonce      uint64
}

func (u *Update) Reset() {
	u.Flags = 0
	u.Balance.Clear()
	u.Nonce = 0
	u.StorageLen = 0
	u.CodeHash = empty.CodeHash
}

// Copy creates a deep copy of the Update.
func (u *Update) Copy() *Update {
	if u == nil {
		return nil
	}
	c := &Update{
		CodeHash:   u.CodeHash,
		Storage:    u.Storage,
		StorageLen: u.StorageLen,
		Flags:      u.Flags,
		Nonce:      u.Nonce,
	}
	c.Balance.Set(&u.Balance)
	return c
}

func (u *Update) Merge(b *Update) {
	if b.Flags == DeleteUpdate {
		u.Flags = DeleteUpdate
		return
	}
	if b.Flags&(BalanceUpdate|NonceUpdate|CodeUpdate|StorageUpdate) != 0 {
		u.Flags &^= DeleteUpdate
	}
	if b.Flags&BalanceUpdate != 0 {
		u.Flags |= BalanceUpdate
		u.Balance.Set(&b.Balance)
	}
	if b.Flags&NonceUpdate != 0 {
		u.Flags |= NonceUpdate
		u.Nonce = b.Nonce
	}
	if b.Flags&CodeUpdate != 0 {
		u.Flags |= CodeUpdate
		copy(u.CodeHash[:], b.CodeHash[:])
	}
	if b.Flags&StorageUpdate != 0 {
		u.Flags |= StorageUpdate
		copy(u.Storage[:], b.Storage[:b.StorageLen])
		u.StorageLen = b.StorageLen
	}
}

func (u *Update) Encode(buf []byte, numBuf []byte) []byte {
	buf = append(buf, byte(u.Flags))
	if u.Flags&BalanceUpdate != 0 {
		buf = append(buf, byte(u.Balance.ByteLen()))
		buf = append(buf, u.Balance.Bytes()...)
	}
	if u.Flags&NonceUpdate != 0 {
		n := binary.PutUvarint(numBuf, u.Nonce)
		buf = append(buf, numBuf[:n]...)
	}
	if u.Flags&CodeUpdate != 0 {
		buf = append(buf, u.CodeHash[:]...)
	}
	if u.Flags&StorageUpdate != 0 {
		n := binary.PutUvarint(numBuf, uint64(u.StorageLen))
		buf = append(buf, numBuf[:n]...)
		if u.StorageLen > 0 {
			buf = append(buf, u.Storage[:u.StorageLen]...)
		}
	}
	return buf
}

func (u *Update) Deleted() bool {
	return u.Flags&DeleteUpdate > 0
}

func (u *Update) Decode(buf []byte, pos int) (int, error) {
	if len(buf) < pos+1 {
		return 0, errors.New("decode Update: buffer too small for flags")
	}
	u.Reset()

	u.Flags = UpdateFlags(buf[pos])
	pos++
	if u.Flags&BalanceUpdate != 0 {
		if len(buf) < pos+1 {
			return 0, errors.New("decode Update: buffer too small for balance len")
		}
		balanceLen := int(buf[pos])
		pos++
		if len(buf) < pos+balanceLen {
			return 0, errors.New("decode Update: buffer too small for balance")
		}
		u.Balance.SetBytes(buf[pos : pos+balanceLen])
		pos += balanceLen
	}
	if u.Flags&NonceUpdate != 0 {
		var n int
		u.Nonce, n = binary.Uvarint(buf[pos:])
		if n == 0 {
			return 0, errors.New("decode Update: buffer too small for nonce")
		}
		if n < 0 {
			return 0, errors.New("decode Update: nonce overflow")
		}
		pos += n
	}
	if u.Flags&CodeUpdate != 0 {
		if len(buf) < pos+length.Hash {
			return 0, errors.New("decode Update: buffer too small for codeHash")
		}
		copy(u.CodeHash[:], buf[pos:pos+32])
		pos += length.Hash
	}
	if u.Flags&StorageUpdate != 0 {
		l, n := binary.Uvarint(buf[pos:])
		if n == 0 {
			return 0, errors.New("decode Update: buffer too small for storage len")
		}
		if n < 0 {
			return 0, errors.New("decode Update: storage pos overflow")
		}
		pos += n
		if len(buf) < pos+int(l) {
			return 0, errors.New("decode Update: buffer too small for storage")
		}
		u.StorageLen = int8(l)
		copy(u.Storage[:], buf[pos:pos+int(u.StorageLen)])
		pos += int(u.StorageLen)
	}
	return pos, nil
}

func (u *Update) String() string {
	var sb strings.Builder
	sb.WriteString(fmt.Sprintf("Flags: [%s]", u.Flags))
	if u.Deleted() {
		sb.WriteString(", DELETED")
	}
	if u.Flags&BalanceUpdate != 0 {
		sb.WriteString(fmt.Sprintf(", Balance: [%d]", &u.Balance))
	}
	if u.Flags&NonceUpdate != 0 {
		sb.WriteString(fmt.Sprintf(", Nonce: [%d]", u.Nonce))
	}
	if u.Flags&CodeUpdate != 0 {
		sb.WriteString(fmt.Sprintf(", CodeHash: [%x]", u.CodeHash))
	}
	if u.Flags&StorageUpdate != 0 {
		sb.WriteString(fmt.Sprintf(", Storage: [%x]", u.Storage[:u.StorageLen]))
	}
	return sb.String()
}
