// Copyright 2022 The Erigon Authors
// This file is part of Erigon.
//
// Erigon is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Erigon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Erigon. If not, see <http://www.gnu.org/licenses/>.

package state

import (
	"bytes"
	"context"
	"encoding/binary"
	"fmt"
	"math"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/erigontech/erigon/db/kv/prune"

	"github.com/erigontech/erigon/common/background"
	"github.com/erigontech/erigon/common/dir"
	"github.com/erigontech/erigon/common/log/v3"
	"github.com/erigontech/erigon/db/config3"
	"github.com/erigontech/erigon/db/datadir"
	"github.com/erigontech/erigon/db/kv"
	"github.com/erigontech/erigon/db/kv/dbcfg"
	"github.com/erigontech/erigon/db/kv/mdbx"
	"github.com/erigontech/erigon/db/kv/order"
	"github.com/erigontech/erigon/db/kv/stream"
	"github.com/erigontech/erigon/db/recsplit"
	"github.com/erigontech/erigon/db/recsplit/multiencseq"
	"github.com/erigontech/erigon/db/seg"
	"github.com/erigontech/erigon/db/state/statecfg"
	"github.com/erigontech/erigon/db/version"
)

func testDbAndInvertedIndex(tb testing.TB, aggStep uint64, logger log.Logger) (kv.RwDB, *InvertedIndex) {
	tb.Helper()
	dirs := datadir.New(tb.TempDir())
	keysTable := "Keys"
	indexTable := "Index"
	db := mdbx.New(dbcfg.ChainDB, logger).InMem(tb, dirs.Chaindata).WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg {
		return kv.TableCfg{
			keysTable:             kv.TableCfgItem{Flags: kv.DupSort},
			indexTable:            kv.TableCfgItem{Flags: kv.DupSort},
			kv.TblPruningProgress: kv.TableCfgItem{},
			kv.TblPruningValsProg: kv.TableCfgItem{},
		}
	}).MustOpen()
	salt := uint32(1)
	cfg := statecfg.InvIdxCfg{FilenameBase: "inv", KeysTable: keysTable, ValuesTable: indexTable, FileVersion: statecfg.IIVersionTypes{DataEF: version.V1_0_standart, AccessorEFI: version.V1_0_standart}}
	cfg.Accessors = statecfg.AccessorHashMap
	ii, err := NewInvertedIndex(cfg, aggStep, config3.DefaultStepsInFrozenFile, dirs, logger)
	require.NoError(tb, err)
	tb.Cleanup(func() {
		ii.Close()
		db.Close()

		//if runtime.GOOS == "windows" {
		//	runtime.GC()
		//	debug.FreeOSMemory()
		//}
		//
		//accFile := filepath.Join(dirs.SnapAccessors, "v1.0-inv.0-1.efi")
		//err := os.Remove(accFile)
		//tb.Log("remove accessor file after Close:", err, accFile)
		//idxFile := filepath.Join(dirs.SnapIdx, "v1.0-inv.0-1.ef")
		//err = os.Remove(idxFile)
		//tb.Log("remove idx file after Close:", err, idxFile)
	})
	ii.salt.Store(&salt)
	ii.DisableFsync()
	return db, ii
}

func TestInvertedIndexVisibleEnd(t *testing.T) {
	db, ii := testDbAndInvertedIndex(t, 16, log.New())
	tx, err := db.BeginRw(t.Context())
	require.NoError(t, err)
	defer tx.Rollback()

	iit := ii.beginForTests()
	defer iit.Close()
	require.Zero(t, iit.Progress(tx))
	require.Zero(t, iit.visibleEnd(tx))

	var txNum [8]byte
	require.NoError(t, tx.Put(ii.KeysTable, txNum[:], []byte{1}))
	require.Zero(t, iit.Progress(tx))
	require.Equal(t, uint64(1), iit.visibleEnd(tx))

	binary.BigEndian.PutUint64(txNum[:], 100)
	require.NoError(t, tx.Put(ii.KeysTable, txNum[:], []byte{1}))
	require.Equal(t, uint64(100), iit.Progress(tx))
	require.Equal(t, uint64(101), iit.visibleEnd(tx))

	iit.files = visibleFiles{{endTxNum: 200}}
	require.Equal(t, uint64(200), iit.Progress(tx))
	require.Equal(t, uint64(200), iit.visibleEnd(tx))
}

func TestInvIndexPruningCorrectness(t *testing.T) {
	t.Parallel()

	pruneLimit := uint64(10)
	pruneIters := 8

	setup := func(t *testing.T) (*InvertedIndex, kv.RwTx, *time.Ticker, [8]byte, [8]byte) {
		t.Helper()
		db, ii, _ := filledInvIndexOfSize(t, 1000, 16, 1, log.New())
		t.Cleanup(func() { ii.Close(); db.Close() })

		tx, err := db.BeginRw(t.Context())
		require.NoError(t, err)
		t.Cleanup(tx.Rollback)

		logEvery := time.NewTicker(30 * time.Second)
		t.Cleanup(logEvery.Stop)

		var from, to [8]byte
		binary.BigEndian.PutUint64(from[:], uint64(0))
		binary.BigEndian.PutUint64(to[:], uint64(pruneIters)*pruneLimit)
		return ii, tx, logEvery, from, to
	}

	t.Run("hash_prune", func(t *testing.T) {
		t.Parallel()
		ii, tx, logEvery, from, to := setup(t)

		t.Run("no_files_no_force", func(t *testing.T) {
			ic := ii.beginForTests()
			defer ic.Close()

			icc, err := tx.CursorDupSort(ii.KeysTable)
			require.NoError(t, err)
			defer icc.Close()

			count := 0
			for txn, _, err := icc.Seek(from[:]); txn != nil; txn, _, err = icc.Next() {
				require.NoError(t, err)
				if bytes.Compare(txn, to[:]) > 0 {
					break
				}
				count++
			}
			icc.Close()
			require.Equal(t, count, pruneIters*int(pruneLimit))

			// this one should not prune anything due to forced=false but no files built
			stat, err := ic.HashSeekingPrune(t.Context(), tx, 0, 10, pruneLimit, logEvery, false, nil, nil, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Zero(t, stat.PruneCountTx)
			require.Zero(t, stat.PruneCountValues)

			// this one should not prune anything as well due to given range [0,1) even it is forced
			stat, err = ic.HashSeekingPrune(t.Context(), tx, 0, 1, pruneLimit, logEvery, true, nil, nil, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Zero(t, stat.PruneCountTx)
			require.Zero(t, stat.PruneCountValues)

			ic.Close()
		})

		t.Run("retire_one_step_no_force", func(t *testing.T) {
			collation, err := ii.collate(t.Context(), 0, tx)
			require.NoError(t, err)
			sf, _ := ii.buildFiles(t.Context(), 0, collation, background.NewProgressSet())
			collation.Close()
			txFrom, txTo := firstTxNumOfStep(0, ii.stepSize), firstTxNumOfStep(1, ii.stepSize)

			// without `reCalcVisibleFiles` must be nothing to prune - because files are not visible yet.
			ic := ii.beginForTests()
			defer ic.Close()

			ii.integrateDirtyFiles(sf, txFrom, txTo)
			stat, err := ic.HashSeekingPrune(t.Context(), tx, 0, 10, pruneLimit, logEvery, false, nil, nil, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Zero(t, stat.PruneCountTx)
			require.Zero(t, stat.PruneCountValues)

			ic = ii.beginForTests()
			defer ic.Close()
			stat, err = ic.HashSeekingPrune(t.Context(), tx, 0, 10, pruneLimit, logEvery, false, nil, nil, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Equal(t, 9, int(stat.PruneCountTx))
			require.Equal(t, 9, int(stat.PruneCountValues))

			// prune only what left in step 0. Even if requested more. don't allow print more than what we have in visible files
			stat, err = ic.HashSeekingPrune(t.Context(), tx, 0, 20, pruneLimit, logEvery, false, nil, nil, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Equal(t, 6, int(stat.PruneCountTx))
			require.Equal(t, 6, int(stat.PruneCountValues))
		})

		t.Run("force", func(t *testing.T) {
			ic := ii.beginForTests()
			defer ic.Close()

			// this should prune exactly pruneLimit*pruneIter transactions
			for i := range pruneIters {
				stat, err := ic.HashSeekingPrune(t.Context(), tx, 0, 1000, pruneLimit, logEvery, true, nil, nil, prune.DefaultStorageMode)
				require.NoError(t, err)
				t.Logf("[%d] stats: %v", i, stat)
			}

			// ascending - empty
			it, err := ic.IdxRange(nil, 0, pruneIters*int(pruneLimit), order.Asc, -1, tx)
			require.NoError(t, err)
			require.False(t, it.HasNext())
			it.Close()

			// descending - empty
			it, err = ic.IdxRange(nil, pruneIters*int(pruneLimit), 0, order.Desc, -1, tx)
			require.NoError(t, err)
			require.False(t, it.HasNext())
			it.Close()

			// straight from pruned - not empty
			icc, err := tx.CursorDupSort(ii.KeysTable)
			require.NoError(t, err)
			defer icc.Close()
			txn, _, err := icc.Seek(from[:])
			require.NoError(t, err)
			println("from", binary.BigEndian.Uint64(from[:]), "txn", binary.BigEndian.Uint64(txn))

			prunedInStep0 := 16 - 1
			// we pruned by limit so next transaction after prune should be equal to `pruneIters*pruneLimit+1`
			// If we would prune by txnum then txTo prune should be available after prune is finished
			require.EqualValues(t, pruneIters*int(pruneLimit)+prunedInStep0, int(binary.BigEndian.Uint64(txn)-1))
			icc.Close()

			// check second table
			icc2, err := tx.CursorDupSort(ii.ValuesTable)
			require.NoError(t, err)
			defer icc2.Close()
			key, txn, err := icc2.First()
			t.Logf("key: %x, txn: %x", key, txn)
			require.NoError(t, err)
			// we pruned by limit so next transaction after prune should be equal to `pruneIters*pruneLimit+1`
			// If we would prune by txnum then txTo prune should be available after prune is finished
			require.EqualValues(t, pruneIters*int(pruneLimit)+prunedInStep0, int(binary.BigEndian.Uint64(txn)-1))
		})

	}) // hash_prune

	t.Run("scan_prune", func(t *testing.T) {
		t.Parallel()
		ii, tx, logEvery, from, to := setup(t)
		thr := time.Millisecond * 20
		ctx := context.WithValue(t.Context(), throttle, &thr)

		t.Run("no_files_no_force", func(t *testing.T) {
			ic := ii.beginForTests()
			defer ic.Close()

			icc, err := tx.CursorDupSort(ii.KeysTable)
			require.NoError(t, err)
			defer icc.Close()
			count := 0
			for txn, _, err := icc.Seek(from[:]); txn != nil; txn, _, err = icc.Next() {
				require.NoError(t, err)
				if bytes.Compare(txn, to[:]) > 0 {
					break
				}
				count++
			}

			require.Equal(t, count, pruneIters*int(pruneLimit))
			st := &prune.Stat{
				MinTxNum:         0,
				MaxTxNum:         0,
				PruneCountTx:     0,
				PruneCountValues: 0,
				DupsDeleted:      0,
				LastPrunedValue:  nil,
				LastPrunedKey:    nil,
				KeyProgress:      prune.Done,
				ValueProgress:    prune.Done,
				TxFrom:           0,
				TxTo:             10,
			}
			err = SavePruneValProgress(tx, ic.ii.ValuesTable, st)
			require.NoError(t, err)
			// this one should not prune anything due to forced=false but no files built
			stat, err := ic.TableScanningPrune(ctx, tx, 0, 10, pruneLimit, logEvery, false, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Zero(t, stat.PruneCountTx)
			require.Zero(t, stat.PruneCountValues)

			// this one should not prune anything as well due to given range [0,1) even it is forced
			stat, err = ic.TableScanningPrune(ctx, tx, 0, 1, pruneLimit, logEvery, true, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Zero(t, stat.PruneCountTx)
			require.Zero(t, stat.PruneCountValues)
		})

		t.Run("prune was in progress", func(t *testing.T) {
			collation, err := ii.collate(t.Context(), 0, tx)
			require.NoError(t, err)
			defer collation.Close()
			sf, _ := ii.buildFiles(t.Context(), 0, collation, background.NewProgressSet())
			defer sf.CleanupOnError()
			txFrom, txTo := firstTxNumOfStep(0, ii.stepSize), firstTxNumOfStep(1, ii.stepSize)

			// without `reCalcVisibleFiles` must be nothing to prune - because files are not visible yet.
			ic := ii.beginForTests()
			defer ic.Close()

			ii.integrateDirtyFiles(sf, txFrom, txTo)
			st := &prune.Stat{
				MinTxNum:         0,
				MaxTxNum:         0,
				PruneCountTx:     0,
				PruneCountValues: 0,
				DupsDeleted:      0,
				LastPrunedValue:  nil,
				LastPrunedKey:    nil,
				KeyProgress:      prune.Done,
				ValueProgress:    prune.InProgress,
				TxFrom:           0,
				TxTo:             10,
			}
			err = SavePruneValProgress(tx, ic.ii.ValuesTable, st)
			require.NoError(t, err)
			can := ic.CanPrune(tx, 10)
			require.True(t, can)
		})

		t.Run("prune was done", func(t *testing.T) {
			ii, tx, logEvery, _, _ := setup(t)

			collation, err := ii.collate(t.Context(), 0, tx)
			require.NoError(t, err)
			defer collation.Close()
			sf, _ := ii.buildFiles(t.Context(), 0, collation, background.NewProgressSet())
			defer sf.CleanupOnError()
			txFrom, txTo := firstTxNumOfStep(0, ii.stepSize), firstTxNumOfStep(1, ii.stepSize)

			// without `reCalcVisibleFiles` must be nothing to prune - because files are not visible yet.
			ic := ii.beginForTests()
			defer ic.Close()
			ii.integrateDirtyFiles(sf, txFrom, txTo)

			// Drain step 0 from the db so minTxNumInDB >= files.EndTxNum; otherwise
			// CanPrune's files-based clause would return true regardless of progress state.
			_, err = ic.TableScanningPrune(ctx, tx, txFrom, txTo, math.MaxUint64, logEvery, true, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(t, err)

			st := &prune.Stat{
				KeyProgress:   prune.Done,
				ValueProgress: prune.Done,
				TxFrom:        0,
				TxTo:          10,
			}
			err = SavePruneValProgress(tx, ic.ii.ValuesTable, st)
			require.NoError(t, err)
			require.False(t, ic.CanPrune(tx, 10))
		})

		t.Run("retire_one_step_no_force", func(t *testing.T) {
			ii, tx, logEvery, _, _ := setup(t)
			collation, err := ii.collate(t.Context(), 0, tx)
			require.NoError(t, err)
			sf, _ := ii.buildFiles(t.Context(), 0, collation, background.NewProgressSet())
			collation.Close()
			txFrom, txTo := firstTxNumOfStep(0, ii.stepSize), firstTxNumOfStep(1, ii.stepSize)
			ii.integrateDirtyFiles(sf, txFrom, txTo)

			ic := ii.beginForTests()
			defer ic.Close()
			stat, err := ic.TableScanningPrune(t.Context(), tx, 0, 10, pruneLimit, logEvery, false, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Equal(t, 9, int(stat.PruneCountTx))
			require.Equal(t, 9, int(stat.PruneCountValues))

			// even if requested more, don't prune beyond what visible files cover (step 0 ends at stepSize).
			stat, err = ic.TableScanningPrune(t.Context(), tx, 0, 20, pruneLimit, logEvery, false, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(t, err)
			require.Equal(t, 6, int(stat.PruneCountTx))
			require.Equal(t, 6, int(stat.PruneCountValues))
		})

		t.Run("force", func(t *testing.T) {
			ii, tx, logEvery, from, _ := setup(t)
			ic := ii.beginForTests()
			defer ic.Close()

			// Unlike HashSeekingPrune, TableScanningPrune does not honor `limit` for the
			// key range: a single forced call prunes the whole [txFrom, txTo) range.
			pruneTo := uint64(pruneIters) * pruneLimit
			stat, err := ic.TableScanningPrune(t.Context(), tx, 0, pruneTo, pruneLimit, logEvery, true, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(t, err)
			t.Logf("pruned tx=%d vals=%d", stat.PruneCountTx, stat.PruneCountValues)

			// [0, pruneTo) fully pruned
			it, err := ic.IdxRange(nil, 0, int(pruneTo), order.Asc, -1, tx)
			require.NoError(t, err)
			require.False(t, it.HasNext())
			it.Close()

			it, err = ic.IdxRange(nil, int(pruneTo), 0, order.Desc, -1, tx)
			require.NoError(t, err)
			require.False(t, it.HasNext())
			it.Close()

			// first surviving txn in the keys table == pruneTo
			icc, err := tx.CursorDupSort(ii.KeysTable)
			require.NoError(t, err)
			defer icc.Close()
			txn, _, err := icc.Seek(from[:])
			require.NoError(t, err)
			require.EqualValues(t, pruneTo, binary.BigEndian.Uint64(txn))
		})

	}) // scan_prune
}

type throttleType string

const throttle throttleType = "throttle"

func TestInvIndexCollationBuild(t *testing.T) {
	t.Parallel()

	logger := log.New()
	logEvery := time.NewTicker(30 * time.Second)
	defer logEvery.Stop()
	db, ii := testDbAndInvertedIndex(t, 16, logger)
	ctx := t.Context()
	tx, err := db.BeginRw(ctx)
	require.NoError(t, err)
	defer tx.Rollback()
	ic := ii.beginForTests()
	defer ic.Close()
	writer := ic.NewWriter()
	defer writer.close()

	err = writer.Add([]byte("key1"), 2)
	require.NoError(t, err)

	err = writer.Add([]byte("key2"), 3)
	require.NoError(t, err)

	err = writer.Add([]byte("key1"), 6)
	require.NoError(t, err)
	err = writer.Add([]byte("key3"), 6)
	require.NoError(t, err)

	err = writer.Add([]byte("key10"), 17)
	require.NoError(t, err)

	err = writer.Flush(ctx, tx)
	require.NoError(t, err)
	err = tx.Commit()
	require.NoError(t, err)

	roTx, err := db.BeginRo(ctx)
	require.NoError(t, err)
	defer roTx.Rollback()

	bs, err := ii.collate(ctx, 0, roTx)
	require.NoError(t, err)

	sf, err := ii.buildFiles(ctx, 0, bs, background.NewProgressSet())
	bs.Close()
	require.NoError(t, err)
	defer sf.CleanupOnError()

	g := sf.decomp.MakeGetter()
	g.Reset(0)
	var words []string
	var intArrs [][]uint64
	for g.HasNext() {
		w, _ := g.Next(nil)
		words = append(words, string(w))
		w, _ = g.Next(w[:0])
		ef := multiencseq.ReadMultiEncSeq(0, w)
		var ints []uint64
		it := ef.Iterator(0)
		for it.HasNext() {
			v, _ := it.Next()
			ints = append(ints, v)
		}
		intArrs = append(intArrs, ints)
	}
	require.Equal(t, []string{"key1", "key2", "key3"}, words)
	require.Equal(t, [][]uint64{{2, 6}, {3}, {6}}, intArrs)
	r := recsplit.NewIndexReader(sf.index)
	defer r.Close()
	for i := 0; i < len(words); i++ {
		offset, _ := r.TwoLayerLookup([]byte(words[i]))
		g.Reset(offset)
		w, _ := g.Next(nil)
		require.Equal(t, words[i], string(w))
	}
}

func TestInvIndexAfterPrune(t *testing.T) {
	t.Parallel()
	logEvery := time.NewTicker(30 * time.Second)
	defer logEvery.Stop()
	db, ii := testDbAndInvertedIndex(t, 16, log.New())
	ctx := t.Context()
	tx, err := db.BeginRw(ctx) //nolint:gocritic
	require.NoError(t, err)
	defer func() {
		if tx != nil {
			tx.Rollback()
		}
	}()
	ic := ii.beginForTests()
	defer ic.Close()
	writer := ic.NewWriter()
	defer writer.close()

	err = writer.Add([]byte("key1"), 2)
	require.NoError(t, err)

	err = writer.Add([]byte("key2"), 3)
	require.NoError(t, err)

	err = writer.Add([]byte("key1"), 6)
	require.NoError(t, err)
	err = writer.Add([]byte("key3"), 6)
	require.NoError(t, err)

	err = writer.Flush(ctx, tx)
	require.NoError(t, err)
	err = tx.Commit()
	require.NoError(t, err)

	roTx, err := db.BeginRo(ctx)
	require.NoError(t, err)
	defer roTx.Rollback()

	require.NoError(t, ii.collateBuildIntegrate(ctx, 0, roTx, background.NewProgressSet()))

	err = db.Update(ctx, func(tx kv.RwTx) error {
		from, to := ic.stepsRangeInDB(tx)
		require.Equal(t, "0.1", fmt.Sprintf("%.1f", from))
		require.Equal(t, "0.4", fmt.Sprintf("%.1f", to))

		ic = ii.beginForTests()
		defer ic.Close()

		_, err = ic.TableScanningPrune(ctx, tx, 0, 16, math.MaxUint64, logEvery, false, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
		require.NoError(t, err)
		return nil
	})
	require.NoError(t, err)

	require.NoError(t, err)
	tx, err = db.BeginRw(ctx)
	require.NoError(t, err)
	defer tx.Rollback()

	for _, table := range []string{ii.KeysTable, ii.ValuesTable} {
		func() {
			cur, err := tx.Cursor(table)
			require.NoError(t, err)
			defer cur.Close()
			k, _, err := cur.First()
			require.NoError(t, err)
			require.Nil(t, k, table)
		}()
	}

	from, to := ic.stepsRangeInDB(tx)
	require.Equal(t, float64(0), from) //nolint:testifylint
	require.Equal(t, float64(0), to)   //nolint:testifylint
}

// TestInvIndex_PruneRollingCursorProgress verifies the three properties of the
// rolling-cursor optimisation added to inverted_index.tableScanningPrune:
//
//  1. Early-exit: prg.TxTo >= txTo && KeyProgress==Done && ValueProgress==Done
//     → return immediately, no deletions.
//  2. Rotation reset: after Done a call with larger txTo resets cursor to First
//     and picks up newly pruneable entries.
//  3. Cursor preservation: an InProgress cursor survives a txTo advance —
//     the subsequent call resumes from LastPrunedValue rather than First.
func TestInvIndex_PruneRollingCursorProgress(t *testing.T) {
	t.Parallel()

	const (
		aggStep  = uint64(16)
		keyCount = 40 // unique keys written to the index
	)
	ctx := t.Context()
	db, ii := testDbAndInvertedIndex(t, aggStep, log.New())

	rwTx, err := db.BeginRw(ctx)
	require.NoError(t, err)
	defer rwTx.Rollback()

	// writeKeys adds keyCount entries to the index at the given txNum.
	// forced=true in TableScanningPrune bypasses the CanPrune file check.
	writeKeys := func(txNum uint64) {
		t.Helper()
		ic := ii.beginForTests()
		defer ic.Close()
		w := ic.NewWriter()
		defer w.close()
		for k := range uint64(keyCount) {
			var key [8]byte
			binary.BigEndian.PutUint64(key[:], k)
			require.NoError(t, w.Add(key[:], txNum))
		}
		require.NoError(t, w.Flush(ctx, rwTx))
	}

	logEvery := time.NewTicker(30 * time.Second)
	defer logEvery.Stop()

	// callPrune calls TableScanningPrune in table-scanning mode (txTo != MaxUint64,
	// forced=true so no file-presence check is required).
	callPrune := func(txFrom, txTo uint64) *InvertedIndexPruneStat {
		t.Helper()
		ic := ii.beginForTests()
		defer ic.Close()
		stat, err := ic.TableScanningPrune(ctx, rwTx, txFrom, txTo, math.MaxUint64, logEvery,
			true /*forced*/, nil, nil, nil, prune.DefaultStorageMode)
		require.NoError(t, err)
		return stat
	}

	// ── Write step-0 and step-1 data (txNums 1 and aggStep+1). ────────────
	writeKeys(1)           // step 0
	writeKeys(aggStep + 1) // step 1

	// ── Property 1: early-exit ─────────────────────────────────────────────
	// Full prune of step 0 (txTo = aggStep).
	stat0 := callPrune(0, aggStep)
	require.Equal(t, prune.Done, stat0.Progress)
	require.NotZero(t, stat0.PruneCountValues+stat0.PruneCountTx, "step-0 entries must be deleted")

	prg, err := GetPruneValProgress(rwTx, []byte(ii.ValuesTable))
	require.NoError(t, err)
	require.Equal(t, prune.Done, prg.ValueProgress)
	require.Equal(t, prune.Done, prg.KeyProgress)
	require.EqualValues(t, aggStep, prg.TxTo)

	// Repeat with same txTo → early-exit, no work.
	statEarly := callPrune(0, aggStep)
	require.Equal(t, prune.Done, statEarly.Progress)
	require.Zero(t, statEarly.PruneCountValues+statEarly.PruneCountTx,
		"early-exit must delete nothing")

	// ── Property 2: rotation reset when txTo advances ──────────────────────
	stat1 := callPrune(aggStep, aggStep*2)
	require.Equal(t, prune.Done, stat1.Progress)
	require.NotZero(t, stat1.PruneCountValues+stat1.PruneCountTx,
		"step-1 entries must be deleted in the new rotation")

	prg1, err := GetPruneValProgress(rwTx, []byte(ii.ValuesTable))
	require.NoError(t, err)
	require.Equal(t, prune.Done, prg1.ValueProgress)
	require.Nil(t, prg1.LastPrunedValue, "cursor must be nil after rotation completion")

	// ── Property 3: cursor preservation when txTo advances mid-rotation ────
	writeKeys(aggStep*2 + 1) // step 2

	// Inject InProgress at the midpoint key so the prune resumes from there.
	midKey := make([]byte, 8)
	binary.BigEndian.PutUint64(midKey, keyCount/2)
	txFrom2, txTo2 := aggStep*2, aggStep*3
	injected := &prune.Stat{
		ValueProgress:   prune.InProgress,
		KeyProgress:     prune.Done, // keys pass already done
		LastPrunedValue: midKey,
		TxFrom:          txFrom2,
		TxTo:            txTo2,
	}
	require.NoError(t, SavePruneValProgress(rwTx, ii.ValuesTable, injected))

	stat2 := callPrune(txFrom2, txTo2)
	require.Equal(t, prune.Done, stat2.Progress)
	// Cursor started at midKey so only the second half of entries could be pruned.
	require.LessOrEqual(t, stat2.PruneCountValues, uint64(keyCount/2),
		"cursor preserved: only entries at/after midKey should be deleted")
}

func filledInvIndex(tb testing.TB, logger log.Logger) (kv.RwDB, *InvertedIndex, uint64) {
	tb.Helper()
	return filledInvIndexOfSize(tb, uint64(1000), 16, 31, logger)
}

// Creates InvertedIndex instance and fills it with generated data.
// Txs - amount of transactions to generate
// AggStep - aggregation step for InvertedIndex
// Module - amount of keys to generate
func filledInvIndexOfSize(tb testing.TB, txs, aggStep, module uint64, logger log.Logger) (kv.RwDB, *InvertedIndex, uint64) {
	tb.Helper()
	db, ii := testDbAndInvertedIndex(tb, aggStep, logger)
	ctx, require := tb.Context(), require.New(tb)

	err := db.Update(ctx, func(tx kv.RwTx) error {
		ic := ii.beginForTests()
		defer ic.Close()
		writer := ic.NewWriter()
		defer writer.close()

		var flusher flusher

		// keys are encodings of numbers 1..31
		// each key changes value on every txNum which is multiple of the key
		for txNum := uint64(1); txNum <= txs; txNum++ {
			for keyNum := uint64(1); keyNum <= module; keyNum++ {
				if txNum%keyNum == 0 {
					var k [8]byte
					binary.BigEndian.PutUint64(k[:], keyNum)
					err := writer.Add(k[:], txNum)
					require.NoError(err)
				}
			}
			if flusher != nil {
				require.NoError(flusher.Flush(ctx, tx))
			}
			if txNum%10 == 0 {
				flusher = writer
				writer = ic.NewWriter()
			}
		}
		if flusher != nil {
			require.NoError(flusher.Flush(ctx, tx))
		}
		return writer.Flush(ctx, tx)
	})
	require.NoError(err)
	return db, ii, txs
}

func checkRanges(t *testing.T, db kv.RwDB, ii *InvertedIndex, txs uint64) {
	t.Helper()
	ctx := t.Context()
	ic := ii.beginForTests()
	defer ic.Close()

	// Check the iterator invertedIndex first without roTx
	for keyNum := uint64(1); keyNum <= uint64(31); keyNum++ {
		var k [8]byte
		binary.BigEndian.PutUint64(k[:], keyNum)
		var values []uint64
		t.Run("asc", func(t *testing.T) {
			it, err := ic.IdxRange(k[:], 0, 976, order.Asc, -1, nil)
			require.NoError(t, err)
			defer it.Close()
			for i := keyNum; i < 976; i += keyNum {
				label := fmt.Sprintf("keyNum=%d, txNum=%d", keyNum, i)
				require.True(t, it.HasNext(), label)
				n, err := it.Next()
				require.NoError(t, err)
				require.Equal(t, i, n, label)
				values = append(values, n)
			}
			require.False(t, it.HasNext())
		})

		t.Run("desc", func(t *testing.T) {
			reverseStream, err := ic.IdxRange(k[:], 976-1, 0, order.Desc, -1, nil)
			require.NoError(t, err)
			defer reverseStream.Close()
			stream.ExpectEqualU64(t, stream.ReverseArray(values), reverseStream)
		})
		t.Run("unbounded asc", func(t *testing.T) {
			forwardLimited, err := ic.IdxRange(k[:], -1, 976, order.Asc, 2, nil)
			require.NoError(t, err)
			defer forwardLimited.Close()
			stream.ExpectEqualU64(t, stream.Array(values[:2]), forwardLimited)
		})
		t.Run("unbounded desc", func(t *testing.T) {
			reverseLimited, err := ic.IdxRange(k[:], 976-1, -1, order.Desc, 2, nil)
			require.NoError(t, err)
			defer reverseLimited.Close()
			stream.ExpectEqualU64(t, stream.ReverseArray(values[len(values)-2:]), reverseLimited)
		})
		t.Run("tiny bound asc", func(t *testing.T) {
			it, err := ic.IdxRange(k[:], 100, 102, order.Asc, -1, nil)
			require.NoError(t, err)
			defer it.Close()
			expect := stream.FilterU64(stream.Array(values), func(k uint64) bool { return k >= 100 && k < 102 })
			stream.ExpectEqualU64(t, expect, it)
		})
		t.Run("tiny bound desc", func(t *testing.T) {
			it, err := ic.IdxRange(k[:], 102, 100, order.Desc, -1, nil)
			require.NoError(t, err)
			defer it.Close()
			expect := stream.FilterU64(stream.ReverseArray(values), func(k uint64) bool { return k <= 102 && k > 100 })
			stream.ExpectEqualU64(t, expect, it)
		})
	}
	// Now check invertedIndex that require access to DB
	roTx, err := db.BeginRo(ctx)
	require.NoError(t, err)
	defer roTx.Rollback()
	for keyNum := uint64(1); keyNum <= uint64(31); keyNum++ {
		var k [8]byte
		binary.BigEndian.PutUint64(k[:], keyNum)
		it, err := ic.IdxRange(k[:], 400, 1000, true, -1, roTx)
		require.NoError(t, err)
		var values []uint64
		for i := keyNum * ((400 + keyNum - 1) / keyNum); i < txs; i += keyNum {
			label := fmt.Sprintf("keyNum=%d, txNum=%d", keyNum, i)
			require.True(t, it.HasNext(), label)
			n, err := it.Next()
			require.NoError(t, err)
			require.Equal(t, i, n, label)
			values = append(values, n)
		}
		require.False(t, it.HasNext())
		it.Close()

		reverseStream, err := ic.IdxRange(k[:], 1000-1, 400-1, false, -1, roTx)
		require.NoError(t, err)
		arr := stream.ToArrU64Must(reverseStream)
		expect := stream.ToArrU64Must(stream.ReverseArray(values))
		require.Equal(t, expect, arr)
		reverseStream.Close()
	}
}

// collateBuildIntegrate collates, builds files and integrates them for the given step.
// It is a test helper that combines the common collate→buildFiles→integrateDirtyFiles pattern.
func (ii *InvertedIndex) collateBuildIntegrate(ctx context.Context, step kv.Step, tx kv.Tx, ps *background.ProgressSet) error {
	bs, err := ii.collate(ctx, step, tx)
	if err != nil {
		return err
	}
	defer bs.Close()
	sf, err := ii.buildFiles(ctx, step, bs, ps)
	if err != nil {
		return err
	}
	ii.integrateDirtyFiles(sf, step.ToTxNum(ii.stepSize), (step + 1).ToTxNum(ii.stepSize))
	return nil
}

func mergeInverted(tb testing.TB, db kv.RwDB, ii *InvertedIndex, txs uint64) {
	tb.Helper()
	logEvery := time.NewTicker(30 * time.Second)
	defer logEvery.Stop()
	ctx := tb.Context()
	// Leave the last 2 aggregation steps un-collated
	tx, err := db.BeginRw(ctx)
	require.NoError(tb, err)
	defer tx.Rollback()

	// Leave the last 2 aggregation steps un-collated
	for step := kv.Step(0); step < kv.Step(txs/ii.stepSize)-1; step++ {
		func() {
			require.NoError(tb, ii.collateBuildIntegrate(ctx, step, tx, background.NewProgressSet()))
			ic := ii.beginForTests()
			defer ic.Close()
			_, err = ic.TableScanningPrune(ctx, tx, step.ToTxNum(ii.stepSize), (step + 1).ToTxNum(ii.stepSize), math.MaxUint64, logEvery, false, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(tb, err)
			var found bool
			var startTxNum, endTxNum uint64
			maxEndTxNum := ii.dirtyFilesEndTxNumMinimax()
			maxSpan := ii.stepSize * config3.DefaultStepsInFrozenFile

			for {
				if stop := func() bool {
					ic := ii.beginForTests()
					defer ic.Close()
					mr := ic.findMergeRange(maxEndTxNum, maxSpan)
					found, startTxNum, endTxNum = mr.needMerge, mr.from, mr.to
					if !found {
						return true
					}
					outs := ic.staticFilesInRange(startTxNum, endTxNum)
					in, err := ic.mergeFiles(ctx, outs, startTxNum, endTxNum, background.NewProgressSet())
					require.NoError(tb, err)
					ii.integrateMergedDirtyFiles(in)
					return false
				}(); stop {
					break
				}
			}
		}()
	}
	err = tx.Commit()
	require.NoError(tb, err)
}

func TestInvIndexRanges(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}

	t.Parallel()

	logger := log.New()
	logEvery := time.NewTicker(30 * time.Second)
	defer logEvery.Stop()
	db, ii, txs := filledInvIndex(t, logger)
	ctx := t.Context()
	tx, err := db.BeginRw(ctx)
	require.NoError(t, err)
	defer tx.Rollback()

	// Leave the last 2 aggregation steps un-collated
	for step := kv.Step(0); step < kv.Step(txs/ii.stepSize)-1; step++ {
		func() {
			require.NoError(t, ii.collateBuildIntegrate(ctx, step, tx, background.NewProgressSet()))
			ic := ii.beginForTests()
			defer ic.Close()
			_, err = ic.TableScanningPrune(ctx, tx, step.ToTxNum(ii.stepSize), (step + 1).ToTxNum(ii.stepSize), math.MaxUint64, logEvery, false, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
			require.NoError(t, err)
		}()
	}
	err = tx.Commit()
	require.NoError(t, err)

	checkRanges(t, db, ii, txs)
}

func TestInvIndexMerge(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}

	logger := log.New()
	db, ii, txs := filledInvIndex(t, logger)

	mergeInverted(t, db, ii, txs)
	checkRanges(t, db, ii, txs)
}

func TestInvIndexScanFiles(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}

	logger, require := log.New(), require.New(t)
	db, ii, txs := filledInvIndex(t, logger)

	// Recreate InvertedIndex to scan the files
	salt := uint32(1)
	cfg := ii.InvIdxCfg

	var err error
	ii, err = NewInvertedIndex(cfg, 16, config3.DefaultStepsInFrozenFile, ii.dirs, logger)
	require.NoError(err)
	defer ii.Close()
	ii.salt.Store(&salt)

	scanDirsRes, err := scanDirs(ii.dirs)
	require.NoError(err)
	_, err = ii.openFolder(t.Context(), scanDirsRes)
	require.NoError(err)

	mergeInverted(t, db, ii, txs)
	checkRanges(t, db, ii, txs)
}

func TestScanStaticFiles(t *testing.T) {
	t.Parallel()

	ii := emptyTestInvertedIndex(t, 1)
	files := []string{
		"v1.0-accounts.0-1.ef",
		"v1.0-accounts.1-2.ef",
		"v1.0-accounts.0-4.ef",
		"v1.0-accounts.2-3.ef",
		"v1.0-accounts.3-4.ef",
		"v1.0-accounts.4-5.ef",
	}
	ii.scanDirtyFiles(files)
	require.Equal(t, 6, ii.dirtyFiles.Len())
	ic := ii.beginForTests()
	require.Equal(t, 0, len(ic.files))
	ic.Close()
}

func TestCtxFiles(t *testing.T) {
	ii := emptyTestInvertedIndex(t, 1)
	files := []string{
		"v1.0-accounts.0-1.ef", // overlap with same `endTxNum=4`
		"v1.0-accounts.1-2.ef",
		"v1.0-accounts.0-4.ef",
		"v1.0-accounts.2-3.ef",
		"v1.0-accounts.3-4.ef",
		"v1.0-accounts.4-5.ef",     // no overlap
		"v1.0-accounts.480-484.ef", // overlap with same `startTxNum=480`
		"v1.0-accounts.480-488.ef",
		"v1.0-accounts.480-496.ef",
		"v1.0-accounts.480-512.ef",
	}
	ii.scanDirtyFiles(files)
	require.Equal(t, 10, ii.dirtyFiles.Len())
	ii.dirtyFiles.Scan(func(item *FilesItem) bool {
		item.decompressor = &seg.Decompressor{}
		return true
	})

	visibleFiles := calcVisibleFiles(ii.dirtyFiles, 0, nil, false, ii.dirtyFilesEndTxNumMinimax())
	for i, item := range visibleFiles {
		if item.src.canDelete.Load() {
			require.Failf(t, "deleted file", "%d-%d", item.startTxNum, item.endTxNum)
		}
		if i == 0 {
			continue
		}
		if item.src.isProperSubsetOf(visibleFiles[i-1].src) || visibleFiles[i-1].src.isProperSubsetOf(item.src) {
			require.Failf(t, "overlaping files", "%d-%d, %d-%d", item.startTxNum, item.endTxNum, visibleFiles[i-1].startTxNum, visibleFiles[i-1].endTxNum)
		}
	}
	require.Len(t, visibleFiles, 3)

	require.Equal(t, 0, int(visibleFiles[0].startTxNum))
	require.Equal(t, 4, int(visibleFiles[0].endTxNum))

	require.Equal(t, 4, int(visibleFiles[1].startTxNum))
	require.Equal(t, 5, int(visibleFiles[1].endTxNum))

	require.Equal(t, 480, int(visibleFiles[2].startTxNum))
	require.Equal(t, 512, int(visibleFiles[2].endTxNum))
}

func TestIsSubset(t *testing.T) {
	t.Parallel()

	assert := assert.New(t)
	assert.True((&FilesItem{startTxNum: 0, endTxNum: 1}).isProperSubsetOf(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.True((&FilesItem{startTxNum: 1, endTxNum: 2}).isProperSubsetOf(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 2}).isProperSubsetOf(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 3}).isProperSubsetOf(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 2, endTxNum: 3}).isProperSubsetOf(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 1}).isProperSubsetOf(&FilesItem{startTxNum: 1, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 2}).isProperSubsetOf(&FilesItem{startTxNum: 1, endTxNum: 2}))
}

func TestIsBefore(t *testing.T) {
	t.Parallel()

	assert := assert.New(t)
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 1}).isBefore(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 1, endTxNum: 2}).isBefore(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 2}).isBefore(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 3}).isBefore(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 2, endTxNum: 3}).isBefore(&FilesItem{startTxNum: 0, endTxNum: 2}))
	assert.True((&FilesItem{startTxNum: 0, endTxNum: 1}).isBefore(&FilesItem{startTxNum: 1, endTxNum: 2}))
	assert.False((&FilesItem{startTxNum: 0, endTxNum: 2}).isBefore(&FilesItem{startTxNum: 1, endTxNum: 2}))
	assert.True((&FilesItem{startTxNum: 0, endTxNum: 1}).isBefore(&FilesItem{startTxNum: 2, endTxNum: 4}))
	assert.True((&FilesItem{startTxNum: 0, endTxNum: 2}).isBefore(&FilesItem{startTxNum: 2, endTxNum: 4}))
}

func TestInvIndex_OpenFolder(t *testing.T) {
	if testing.Short() {
		t.Skip()
	}

	t.Parallel()

	db, ii, txs := filledInvIndex(t, log.New())

	mergeInverted(t, db, ii, txs)

	ic := ii.beginForTests()
	list := ic.files
	require.NotEmpty(t, list)
	ff := list[len(list)-1]
	fn := ff.src.decompressor.FilePath()
	ic.Close()
	ii.Close()

	err := dir.RemoveFile(fn)
	require.NoError(t, err)
	err = os.WriteFile(fn, make([]byte, 33), 0644)
	require.NoError(t, err)

	scanDirsRes, err := scanDirs(ii.dirs)
	require.NoError(t, err)
	_, err = ii.openFolder(t.Context(), scanDirsRes)
	require.NoError(t, err)
	ii.Close()
}

// TestInvertedIndex_IdxRange_SkipsFileRange verifies that recentIterateRange narrows
// its DB query to exclude txNums already covered by frozen files.
// Without the fix the DB query would read prunable data in the file range,
// and after that data is pruned the results would be incomplete.
func TestInvertedIndex_IdxRange_SkipsFileRange(t *testing.T) {
	t.Parallel()
	logger := log.New()
	logEvery := time.NewTicker(30 * time.Second)
	defer logEvery.Stop()

	aggStep := uint64(16)
	db, ii := testDbAndInvertedIndex(t, aggStep, logger)
	ctx := t.Context()
	require := require.New(t)

	// Write entries for a single key across 3 steps (txNums 1..47).
	// Step 0: txNums 0-15, Step 1: 16-31, Step 2: 32-47
	var key [8]byte
	binary.BigEndian.PutUint64(key[:], 1)

	txNums := []uint64{2, 5, 10, 18, 25, 30, 35, 42}

	err := db.Update(ctx, func(tx kv.RwTx) error {
		ic := ii.beginForTests()
		defer ic.Close()
		writer := ic.NewWriter()
		defer writer.close()
		for _, txNum := range txNums {
			if err := writer.Add(key[:], txNum); err != nil {
				return err
			}
		}
		return writer.Flush(ctx, tx)
	})
	require.NoError(err)

	// Build files for steps 0 and 1 (covers txNums 0..31).
	tx, err := db.BeginRw(ctx)
	require.NoError(err)
	defer tx.Rollback()

	for step := kv.Step(0); step <= 1; step++ {
		require.NoError(ii.collateBuildIntegrate(ctx, step, tx, background.NewProgressSet()))
	}

	// Prune DB entries that are now covered by files (txNums 0..31).
	ic := ii.beginForTests()
	_, err = ic.TableScanningPrune(ctx, tx, 0, aggStep*2, math.MaxUint64, logEvery, false, nil, nil, mxPruneSizeIndex, prune.DefaultStorageMode)
	require.NoError(err)
	ic.Close()

	require.NoError(tx.Commit())

	// Now query IdxRange spanning both file and DB ranges.
	roTx, err := db.BeginRo(ctx)
	require.NoError(err)
	defer roTx.Rollback()

	ic = ii.beginForTests()
	defer ic.Close()

	// Ascending: full range [0, 48) should return all txNums.
	it, err := ic.IdxRange(key[:], 0, 48, order.Asc, -1, roTx)
	require.NoError(err)
	var got []uint64
	for it.HasNext() {
		v, err := it.Next()
		require.NoError(err)
		got = append(got, v)
	}
	it.Close()
	require.Equal(txNums, got, "ascending IdxRange after prune")

	// Descending: full range (48, 0] should return all txNums reversed.
	it, err = ic.IdxRange(key[:], 48, 0, order.Desc, -1, roTx)
	require.NoError(err)
	var gotDesc []uint64
	for it.HasNext() {
		v, err := it.Next()
		require.NoError(err)
		gotDesc = append(gotDesc, v)
	}
	it.Close()
	wantDesc := make([]uint64, len(txNums))
	copy(wantDesc, txNums)
	for i, j := 0, len(wantDesc)-1; i < j; i, j = i+1, j-1 {
		wantDesc[i], wantDesc[j] = wantDesc[j], wantDesc[i]
	}
	require.Equal(wantDesc, gotDesc, "descending IdxRange after prune")
}

// TestInvertedIndex_IdxRange_IgnoresDBInFileRange proves the DB is not consulted
// for txNums within the file range. It inserts a rogue txNum directly into the DB's
// ValuesTable inside the frozen range; IdxRange must NOT return it.
func TestInvertedIndex_IdxRange_IgnoresDBInFileRange(t *testing.T) {
	t.Parallel()
	logger := log.New()

	aggStep := uint64(16)
	db, ii := testDbAndInvertedIndex(t, aggStep, logger)
	ctx := t.Context()
	require := require.New(t)

	var key [8]byte
	binary.BigEndian.PutUint64(key[:], 1)

	txNums := []uint64{2, 5, 10, 18, 25, 30, 35, 42}

	err := db.Update(ctx, func(tx kv.RwTx) error {
		ic := ii.beginForTests()
		defer ic.Close()
		writer := ic.NewWriter()
		defer writer.close()
		for _, txNum := range txNums {
			if err := writer.Add(key[:], txNum); err != nil {
				return err
			}
		}
		return writer.Flush(ctx, tx)
	})
	require.NoError(err)

	// Build files for steps 0 and 1 (covers txNums 0..31).
	tx, err := db.BeginRw(ctx)
	require.NoError(err)
	defer tx.Rollback()

	for step := kv.Step(0); step <= 1; step++ {
		require.NoError(ii.collateBuildIntegrate(ctx, step, tx, background.NewProgressSet()))
	}

	// Inject a rogue txNum (7) into the DB's ValuesTable within the frozen range.
	// If the DB is queried for the file range, this would appear in results.
	var rogueTxNum [8]byte
	binary.BigEndian.PutUint64(rogueTxNum[:], 7)
	require.NoError(tx.Put(ii.ValuesTable, key[:], rogueTxNum[:]))
	require.NoError(tx.Commit())

	roTx, err := db.BeginRo(ctx)
	require.NoError(err)
	defer roTx.Rollback()

	ic := ii.beginForTests()
	defer ic.Close()

	// Ascending: rogue txNum 7 must not appear.
	it, err := ic.IdxRange(key[:], 0, 48, order.Asc, -1, roTx)
	require.NoError(err)
	var got []uint64
	for it.HasNext() {
		v, err := it.Next()
		require.NoError(err)
		got = append(got, v)
	}
	it.Close()
	require.Equal(txNums, got, "ascending: rogue DB entry in file range must be invisible")

	// Descending: rogue txNum 7 must not appear.
	it, err = ic.IdxRange(key[:], 48, 0, order.Desc, -1, roTx)
	require.NoError(err)
	var gotDesc []uint64
	for it.HasNext() {
		v, err := it.Next()
		require.NoError(err)
		gotDesc = append(gotDesc, v)
	}
	it.Close()
	wantDesc := make([]uint64, len(txNums))
	copy(wantDesc, txNums)
	for i, j := 0, len(wantDesc)-1; i < j; i, j = i+1, j-1 {
		wantDesc[i], wantDesc[j] = wantDesc[j], wantDesc[i]
	}
	require.Equal(wantDesc, gotDesc, "descending: rogue DB entry in file range must be invisible")
}
