mirror of
https://github.com/filecoin-project/lotus.git
synced 2025-08-24 01:08:42 +08:00

* chain index complete for msgs and txns
* dont need observer changes for now
* changes
* fix tests
* fix tests
* use th right context
* index empty tipsets correctly
* implement automated backfilling
* add event indexing and remove all old indices
* fix test
* revert deployment test changes
* revert test changes and better error handling for eth tx index lookups
* fix sql statments naming convention
* address review for Index GC
* more changes as per review
* changes as per review
* fix config
* mark events as reverted during reconciliation
* better reconciliation; pens down and code complete; also reconcile events
* fix tests
* improve config and docs
* improve docs and error handling
* improve read logic
* improve docs
* better logging and handle ennable event storage
* improve logs and index init proc
* better logging
* fix bugs based on calibnet testing
* create sqliite Indices
* gc should be based on epochs
* fix event query
* foreign keys should be enabled on the DB
* reverted tipsets should be removed as part of GC
* release read lock
* make it easy to backfill an empty index using reconciliation
* better docs for reconciliation
* fix conflicts with master
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix go mod
* fix formatting
* revert config changes
* address changes in observer
* remove top level chainindex package
* changes as per review
* changes as per review
* changes as per review
* handle index with reverted tipsets during reconciliation
* changes as per review
* fix type of max reconcile epoch
* changes to reconciliation as per review
* log ipld error
* better logging of progress
* disable chain indexer hydrate from snapshot based on config
* always populate index
* make config easy to reason about
* fix config
* fix messaging
* revert config changes
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* changes as per review
* make error messages homogenous
* fix indentation
* changes as per review
* feat: recompute tipset to generate missing events if event indexing is enabled (#12463)
* auto repair events
* make jen
* fix leaky abstraction
* better docs for gc retention epoch
* imrpove DB handling (#12485)
* fix conflict
* fix lite node config for indexer
* exclude reverted events from eth get logs if client queries by epoch
* Simply addressing for event lookups in the index.
simply addressing for event lookups
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix tests
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* feat: migration("re-indexing"), backfilling and diasgnostics tooling for the `ChainIndexer` (#12450)
* fix conflicts with chain indexer
* feat: chain indexer todos [skip changelog] (#12462)
* feat: finish todos of validation api
* feat: add indexed data verification with chain store
* feat: address comments and finish TODO
* fix: build issue
* address comments
* fix: ci issue
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* changes to Index Validation API based on Rodds first review
* build chain indexer API
* improve error handling
* feat: lotus-shed tooling for chain indexer (#12474)
* feat: add lotus-shed command for backfilling chain indexer
* feat: add lotus-shed command for inspecting the chain indexer
* feat: use single lotus-shed command to inspect and backfill
* fix: remove the unused queries
* small changes
* add change log
* backfilling improvements and fixes
* finish chain index validation and backfill tooling
* user documentation for the
* validate from epoch
* Apply suggestions from code review
Suggestions from Steve's read of the user doc.
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* changes to user doc as per review
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* changes to user doc as per review
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* changes as per review
* feat: add event entries count in validation API (#12506)
* feat: add event entry count in validation API
* address comments
* use sqllite defaults (#12504)
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* write chain index to a different dir
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* fix conflicts
* UX improvements to backfilling
* feat: tests for the chain indexer (#12521)
* ddl tests
* tests for the chain indexer
* finish unit tests for chain indexer
* fix formatting
* cleanup reverted tipsets to avoid db bloat
* fix logging
* test for filter by address
* test gc cascade delete
* fix db locked error during backfilling
* fix var name
* increase db locked timeout
* fix db locked issue
* reduce db lock timeout
* no lock in gc
* reconcile does not need lock
* improved error handling
* Update chain-indexing-overview-for-rpc-providers.md
Doc updates based on @jennijuju feedack.
* Update chain-indexing-overview-for-rpc-providers.MD
Fixes after reviewing 33c1ca1831
* better metrics for backfilling
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* tests for changes to event addressing
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* changes as per review -> round 1
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* log tipset key cid
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix docs
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix tests
* fix tests
* make jen
* fix conflicts
---------
Co-authored-by: Aryan Tikarya <aryan.tikarya@dojima.network>
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* fix lint
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* remove reverted flag from RPC
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix testing of events and dummy chain store
* remove lotus shed commands for old Indices
* change type of event counts to uint64
* only recompute events if theyre not found
* short-circuit empty events path for older tipsets
* chain indexer must be enabled if ETH RPC is enabled
* change name of message_id column to id in tipset_message table
* only expose SetRecomputeTipSetStateFunc
* dont block on head indexing for reading messages
* document why we're only checking for missing events for a single tipset
* document when we query for reverted events
* simplify event collection
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix test
* change event_id to id in the event table
* change head indexed timeout
* remove deprecated config options
* fail ETH RPC calls if ChainIndexer is disabled
* fix docs
* remove the tipset key cid func from lotus shed
* address review comments
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* chore(events): remove unnecessary DisableRealTimeFilterAPI (#12610)
* feat(cli): add --quiet to chainindex validate-backfill + cleanups (#12611)
* fix tests
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* error type for disabled chainindexer
* fix(chainindex): recompute tipset when we find no receipts
* fix(chainindexer): backfilling should halt when chain state data is missing and not backfill parents (#12619)
* fix backfilling UX
* Update chain/index/api.go
Co-authored-by: Rod Vagg <rod@vagg.org>
* address review
---------
Co-authored-by: Rod Vagg <rod@vagg.org>
* reduce log noise
* make jen
* make jen
* docs: finishing chain-indexer-overview-for-operators.md (#12600)
* Followup to PR #12450 for doc updates
This is being used to resolve the unresolved items in https://github.com/filecoin-project/lotus/pull/12450 since that PR is unwieldly at this point.
* Incorporated some items and added TODOs based on unresolved items from https://github.com/filecoin-project/lotus/pull/12450
* Incorporating more feedback
* Pointing to issue to learn about benefits
* Formatting fixes
* Apply most of the suggestions from @rvagg code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Incorporating feedback from https://github.com/filecoin-project/lotus/pull/12600#discussion_r1802519453
* Addressing https://github.com/filecoin-project/lotus/pull/12600#discussion_r1802540042 and more
* Moved chain-indexer docs to documentation
Renamed
Added ToC
We can move to lotus-docs later
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
* Added upgrade path when importing chain state from a snapshot.
* Typo fixes
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
* chore(doc): "regular checks" section for chainindexer docs (#12612)
* Apply suggestions from @rvagg code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Incorporating @aarshkshah1992 feedback
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
---------
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
* remove go mod replace
* remove unnecessary changes from CHANGELOG
* fix test
* compare events AMT root (#12632)
* fix(chainindex): retry transaction if database connection is lost (#12657)
* retry database lost connection
* log context cancellation
* address review
* fix gateway itest: no chainindexer for lite nodes
* fix changelog
---------
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Aryan Tikarya <aryan.tikarya@dojima.network>
Co-authored-by: Steve Loeppky <biglep@filoz.org>
124 lines
3.2 KiB
Go
124 lines
3.2 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
pseudo "math/rand"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/filecoin-project/go-address"
|
|
"github.com/filecoin-project/go-state-types/abi"
|
|
|
|
"github.com/filecoin-project/lotus/chain/types"
|
|
)
|
|
|
|
func TestGC(t *testing.T) {
|
|
ctx := context.Background()
|
|
seed := time.Now().UnixNano()
|
|
t.Logf("seed: %d", seed)
|
|
rng := pseudo.New(pseudo.NewSource(seed))
|
|
headHeight := abi.ChainEpoch(60)
|
|
si, _, cs := setupWithHeadIndexed(t, headHeight, rng)
|
|
t.Cleanup(func() { _ = si.Close() })
|
|
|
|
si.gcRetentionEpochs = 20
|
|
|
|
ev1 := fakeEvent(
|
|
abi.ActorID(1),
|
|
[]kv{
|
|
{k: "type", v: []byte("approval")},
|
|
{k: "signer", v: []byte("addr1")},
|
|
},
|
|
[]kv{
|
|
{k: "amount", v: []byte("2988181")},
|
|
},
|
|
)
|
|
|
|
ev2 := fakeEvent(
|
|
abi.ActorID(2),
|
|
[]kv{
|
|
{k: "type", v: []byte("approval")},
|
|
{k: "signer", v: []byte("addr2")},
|
|
},
|
|
[]kv{
|
|
{k: "amount", v: []byte("2988181")},
|
|
},
|
|
)
|
|
|
|
events := []types.Event{*ev1, *ev2}
|
|
|
|
fm := fakeMessage(address.TestAddress, address.TestAddress)
|
|
em1 := executedMessage{
|
|
msg: fm,
|
|
evs: events,
|
|
}
|
|
|
|
si.SetActorToDelegatedAddresFunc(func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool) {
|
|
idAddr, err := address.NewIDAddress(uint64(emitter))
|
|
if err != nil {
|
|
return address.Undef, false
|
|
}
|
|
|
|
return idAddr, true
|
|
})
|
|
|
|
si.setExecutedMessagesLoaderFunc(func(ctx context.Context, cs ChainStore, msgTs, rctTs *types.TipSet) ([]executedMessage, error) {
|
|
if msgTs.Height() == 1 {
|
|
return []executedMessage{em1}, nil
|
|
}
|
|
return nil, nil
|
|
})
|
|
|
|
// Create a fake tipset at height 1
|
|
fakeTipSet1 := fakeTipSet(t, rng, 1, nil)
|
|
fakeTipSet2 := fakeTipSet(t, rng, 10, nil)
|
|
fakeTipSet3 := fakeTipSet(t, rng, 50, nil)
|
|
|
|
// Set the dummy chainstore to return this tipset for height 1
|
|
cs.SetTipsetByHeightAndKey(1, fakeTipSet1.Key(), fakeTipSet1) // empty DB
|
|
cs.SetTipsetByHeightAndKey(10, fakeTipSet2.Key(), fakeTipSet2) // empty DB
|
|
cs.SetTipsetByHeightAndKey(50, fakeTipSet3.Key(), fakeTipSet3) // empty DB
|
|
cs.SetTipSetByCid(t, fakeTipSet1)
|
|
cs.SetTipSetByCid(t, fakeTipSet2)
|
|
cs.SetTipSetByCid(t, fakeTipSet3)
|
|
|
|
cs.SetMessagesForTipset(fakeTipSet1, []types.ChainMsg{fm})
|
|
|
|
// index tipset and events
|
|
require.NoError(t, si.Apply(ctx, fakeTipSet1, fakeTipSet2))
|
|
require.NoError(t, si.Apply(ctx, fakeTipSet2, fakeTipSet3))
|
|
|
|
// getLogs works for height 1
|
|
filter := &EventFilter{
|
|
MinHeight: 1,
|
|
MaxHeight: 1,
|
|
}
|
|
ces, err := si.GetEventsForFilter(ctx, filter)
|
|
require.NoError(t, err)
|
|
require.Len(t, ces, 2)
|
|
|
|
si.gc(ctx)
|
|
|
|
// getLogs does not work for height 1
|
|
_, err = si.GetEventsForFilter(ctx, filter)
|
|
require.Error(t, err)
|
|
|
|
// Verify that the tipset at height 1 is removed
|
|
var count int
|
|
err = si.db.QueryRow("SELECT COUNT(*) FROM tipset_message WHERE height = 1").Scan(&count)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 0, count)
|
|
|
|
// Verify that the tipset at height 10 is not removed
|
|
err = si.db.QueryRow("SELECT COUNT(*) FROM tipset_message WHERE height = 10").Scan(&count)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 0, count)
|
|
|
|
// Verify that the tipset at height 50 is not removed
|
|
err = si.db.QueryRow("SELECT COUNT(*) FROM tipset_message WHERE height = 50").Scan(&count)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, count)
|
|
}
|