mirror of
https://github.com/filecoin-project/lotus.git
synced 2025-08-24 01:08:42 +08:00

* chain index complete for msgs and txns
* dont need observer changes for now
* changes
* fix tests
* fix tests
* use th right context
* index empty tipsets correctly
* implement automated backfilling
* add event indexing and remove all old indices
* fix test
* revert deployment test changes
* revert test changes and better error handling for eth tx index lookups
* fix sql statments naming convention
* address review for Index GC
* more changes as per review
* changes as per review
* fix config
* mark events as reverted during reconciliation
* better reconciliation; pens down and code complete; also reconcile events
* fix tests
* improve config and docs
* improve docs and error handling
* improve read logic
* improve docs
* better logging and handle ennable event storage
* improve logs and index init proc
* better logging
* fix bugs based on calibnet testing
* create sqliite Indices
* gc should be based on epochs
* fix event query
* foreign keys should be enabled on the DB
* reverted tipsets should be removed as part of GC
* release read lock
* make it easy to backfill an empty index using reconciliation
* better docs for reconciliation
* fix conflicts with master
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix go mod
* fix formatting
* revert config changes
* address changes in observer
* remove top level chainindex package
* changes as per review
* changes as per review
* changes as per review
* handle index with reverted tipsets during reconciliation
* changes as per review
* fix type of max reconcile epoch
* changes to reconciliation as per review
* log ipld error
* better logging of progress
* disable chain indexer hydrate from snapshot based on config
* always populate index
* make config easy to reason about
* fix config
* fix messaging
* revert config changes
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* changes as per review
* make error messages homogenous
* fix indentation
* changes as per review
* feat: recompute tipset to generate missing events if event indexing is enabled (#12463)
* auto repair events
* make jen
* fix leaky abstraction
* better docs for gc retention epoch
* imrpove DB handling (#12485)
* fix conflict
* fix lite node config for indexer
* exclude reverted events from eth get logs if client queries by epoch
* Simply addressing for event lookups in the index.
simply addressing for event lookups
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix tests
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* feat: migration("re-indexing"), backfilling and diasgnostics tooling for the `ChainIndexer` (#12450)
* fix conflicts with chain indexer
* feat: chain indexer todos [skip changelog] (#12462)
* feat: finish todos of validation api
* feat: add indexed data verification with chain store
* feat: address comments and finish TODO
* fix: build issue
* address comments
* fix: ci issue
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* changes to Index Validation API based on Rodds first review
* build chain indexer API
* improve error handling
* feat: lotus-shed tooling for chain indexer (#12474)
* feat: add lotus-shed command for backfilling chain indexer
* feat: add lotus-shed command for inspecting the chain indexer
* feat: use single lotus-shed command to inspect and backfill
* fix: remove the unused queries
* small changes
* add change log
* backfilling improvements and fixes
* finish chain index validation and backfill tooling
* user documentation for the
* validate from epoch
* Apply suggestions from code review
Suggestions from Steve's read of the user doc.
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* changes to user doc as per review
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* changes to user doc as per review
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* changes as per review
* feat: add event entries count in validation API (#12506)
* feat: add event entry count in validation API
* address comments
* use sqllite defaults (#12504)
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* write chain index to a different dir
* Apply suggestions from code review
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* fix conflicts
* UX improvements to backfilling
* feat: tests for the chain indexer (#12521)
* ddl tests
* tests for the chain indexer
* finish unit tests for chain indexer
* fix formatting
* cleanup reverted tipsets to avoid db bloat
* fix logging
* test for filter by address
* test gc cascade delete
* fix db locked error during backfilling
* fix var name
* increase db locked timeout
* fix db locked issue
* reduce db lock timeout
* no lock in gc
* reconcile does not need lock
* improved error handling
* Update chain-indexing-overview-for-rpc-providers.md
Doc updates based on @jennijuju feedack.
* Update chain-indexing-overview-for-rpc-providers.MD
Fixes after reviewing 33c1ca1831
* better metrics for backfilling
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update chain/index/chain-indexing-overview-for-rpc-providers.MD
Co-authored-by: Rod Vagg <rod@vagg.org>
* tests for changes to event addressing
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* changes as per review -> round 1
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* log tipset key cid
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix docs
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix tests
* fix tests
* make jen
* fix conflicts
---------
Co-authored-by: Aryan Tikarya <aryan.tikarya@dojima.network>
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Steve Loeppky <biglep@filoz.org>
* fix lint
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* remove reverted flag from RPC
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix testing of events and dummy chain store
* remove lotus shed commands for old Indices
* change type of event counts to uint64
* only recompute events if theyre not found
* short-circuit empty events path for older tipsets
* chain indexer must be enabled if ETH RPC is enabled
* change name of message_id column to id in tipset_message table
* only expose SetRecomputeTipSetStateFunc
* dont block on head indexing for reading messages
* document why we're only checking for missing events for a single tipset
* document when we query for reverted events
* simplify event collection
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* fix test
* change event_id to id in the event table
* change head indexed timeout
* remove deprecated config options
* fail ETH RPC calls if ChainIndexer is disabled
* fix docs
* remove the tipset key cid func from lotus shed
* address review comments
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* chore(events): remove unnecessary DisableRealTimeFilterAPI (#12610)
* feat(cli): add --quiet to chainindex validate-backfill + cleanups (#12611)
* fix tests
* Apply suggestions from code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* error type for disabled chainindexer
* fix(chainindex): recompute tipset when we find no receipts
* fix(chainindexer): backfilling should halt when chain state data is missing and not backfill parents (#12619)
* fix backfilling UX
* Update chain/index/api.go
Co-authored-by: Rod Vagg <rod@vagg.org>
* address review
---------
Co-authored-by: Rod Vagg <rod@vagg.org>
* reduce log noise
* make jen
* make jen
* docs: finishing chain-indexer-overview-for-operators.md (#12600)
* Followup to PR #12450 for doc updates
This is being used to resolve the unresolved items in https://github.com/filecoin-project/lotus/pull/12450 since that PR is unwieldly at this point.
* Incorporated some items and added TODOs based on unresolved items from https://github.com/filecoin-project/lotus/pull/12450
* Incorporating more feedback
* Pointing to issue to learn about benefits
* Formatting fixes
* Apply most of the suggestions from @rvagg code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Incorporating feedback from https://github.com/filecoin-project/lotus/pull/12600#discussion_r1802519453
* Addressing https://github.com/filecoin-project/lotus/pull/12600#discussion_r1802540042 and more
* Moved chain-indexer docs to documentation
Renamed
Added ToC
We can move to lotus-docs later
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
* Added upgrade path when importing chain state from a snapshot.
* Typo fixes
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
* chore(doc): "regular checks" section for chainindexer docs (#12612)
* Apply suggestions from @rvagg code review
Co-authored-by: Rod Vagg <rod@vagg.org>
* Incorporating @aarshkshah1992 feedback
* Update documentation/en/chain-indexer-overview-for-operators.md
Co-authored-by: Rod Vagg <rod@vagg.org>
---------
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Aarsh Shah <aarshkshah1992@gmail.com>
* remove go mod replace
* remove unnecessary changes from CHANGELOG
* fix test
* compare events AMT root (#12632)
* fix(chainindex): retry transaction if database connection is lost (#12657)
* retry database lost connection
* log context cancellation
* address review
* fix gateway itest: no chainindexer for lite nodes
* fix changelog
---------
Co-authored-by: Rod Vagg <rod@vagg.org>
Co-authored-by: Aryan Tikarya <aryan.tikarya@dojima.network>
Co-authored-by: Steve Loeppky <biglep@filoz.org>
188 lines
5.4 KiB
Go
188 lines
5.4 KiB
Go
package modules
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/ipfs/boxo/bitswap"
|
|
"github.com/ipfs/boxo/bitswap/network"
|
|
"github.com/ipfs/boxo/blockservice"
|
|
"github.com/libp2p/go-libp2p/core/host"
|
|
"github.com/libp2p/go-libp2p/core/routing"
|
|
"go.uber.org/fx"
|
|
"golang.org/x/xerrors"
|
|
|
|
"github.com/filecoin-project/lotus/blockstore"
|
|
"github.com/filecoin-project/lotus/blockstore/splitstore"
|
|
"github.com/filecoin-project/lotus/build/buildconstants"
|
|
"github.com/filecoin-project/lotus/chain"
|
|
"github.com/filecoin-project/lotus/chain/beacon"
|
|
"github.com/filecoin-project/lotus/chain/consensus"
|
|
"github.com/filecoin-project/lotus/chain/consensus/filcns"
|
|
"github.com/filecoin-project/lotus/chain/exchange"
|
|
"github.com/filecoin-project/lotus/chain/gen/slashfilter"
|
|
"github.com/filecoin-project/lotus/chain/messagepool"
|
|
"github.com/filecoin-project/lotus/chain/stmgr"
|
|
"github.com/filecoin-project/lotus/chain/store"
|
|
"github.com/filecoin-project/lotus/chain/vm"
|
|
"github.com/filecoin-project/lotus/journal"
|
|
"github.com/filecoin-project/lotus/node/modules/dtypes"
|
|
"github.com/filecoin-project/lotus/node/modules/helpers"
|
|
)
|
|
|
|
// ChainBitswap uses a blockstore that bypasses all caches.
|
|
func ChainBitswap(lc fx.Lifecycle, mctx helpers.MetricsCtx, host host.Host, rt routing.Routing, bs dtypes.ExposedBlockstore) dtypes.ChainBitswap {
|
|
// prefix protocol for chain bitswap
|
|
// (so bitswap uses /chain/ipfs/bitswap/1.0.0 internally for chain sync stuff)
|
|
bitswapNetwork := network.NewFromIpfsHost(host, rt, network.Prefix("/chain"))
|
|
bitswapOptions := []bitswap.Option{bitswap.ProvideEnabled(false)}
|
|
|
|
// Write all incoming bitswap blocks into a temporary blockstore for two
|
|
// block times. If they validate, they'll be persisted later.
|
|
cache := blockstore.NewTimedCacheBlockstore(2 * time.Duration(buildconstants.BlockDelaySecs) * time.Second)
|
|
lc.Append(fx.Hook{OnStop: cache.Stop, OnStart: cache.Start})
|
|
|
|
bitswapBs := blockstore.NewTieredBstore(bs, cache)
|
|
|
|
// Use just exch.Close(), closing the context is not needed
|
|
exch := bitswap.New(mctx, bitswapNetwork, bitswapBs, bitswapOptions...)
|
|
lc.Append(fx.Hook{
|
|
OnStop: func(ctx context.Context) error {
|
|
return exch.Close()
|
|
},
|
|
})
|
|
|
|
return exch
|
|
}
|
|
|
|
func ChainBlockService(bs dtypes.ExposedBlockstore, rem dtypes.ChainBitswap) dtypes.ChainBlockService {
|
|
return blockservice.New(bs, rem)
|
|
}
|
|
|
|
func MessagePool(lc fx.Lifecycle, mctx helpers.MetricsCtx, us stmgr.UpgradeSchedule, mpp messagepool.Provider, ds dtypes.MetadataDS, nn dtypes.NetworkName, j journal.Journal, protector dtypes.GCReferenceProtector) (*messagepool.MessagePool, error) {
|
|
mp, err := messagepool.New(helpers.LifecycleCtx(mctx, lc), mpp, ds, us, nn, j)
|
|
if err != nil {
|
|
return nil, xerrors.Errorf("constructing mpool: %w", err)
|
|
}
|
|
lc.Append(fx.Hook{
|
|
OnStop: func(_ context.Context) error {
|
|
return mp.Close()
|
|
},
|
|
})
|
|
protector.AddProtector(mp.ForEachPendingMessage)
|
|
return mp, nil
|
|
}
|
|
|
|
func ChainStore(lc fx.Lifecycle,
|
|
mctx helpers.MetricsCtx,
|
|
cbs dtypes.ChainBlockstore,
|
|
sbs dtypes.StateBlockstore,
|
|
ds dtypes.MetadataDS,
|
|
basebs dtypes.BaseBlockstore,
|
|
weight store.WeightFunc,
|
|
us stmgr.UpgradeSchedule,
|
|
j journal.Journal) (*store.ChainStore, error) {
|
|
|
|
chain := store.NewChainStore(cbs, sbs, ds, weight, j)
|
|
|
|
if err := chain.Load(helpers.LifecycleCtx(mctx, lc)); err != nil {
|
|
return nil, xerrors.Errorf("loading chain state from disk: %w", err)
|
|
}
|
|
|
|
var startHook func(context.Context) error
|
|
if ss, ok := basebs.(*splitstore.SplitStore); ok {
|
|
startHook = func(_ context.Context) error {
|
|
err := ss.Start(chain, us)
|
|
if err != nil {
|
|
err = xerrors.Errorf("error starting splitstore: %w", err)
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
lc.Append(fx.Hook{
|
|
OnStart: startHook,
|
|
OnStop: func(_ context.Context) error {
|
|
return chain.Close()
|
|
},
|
|
})
|
|
|
|
return chain, nil
|
|
}
|
|
|
|
func NetworkName(mctx helpers.MetricsCtx,
|
|
lc fx.Lifecycle,
|
|
cs *store.ChainStore,
|
|
tsexec stmgr.Executor,
|
|
syscalls vm.SyscallBuilder,
|
|
us stmgr.UpgradeSchedule,
|
|
_ dtypes.AfterGenesisSet) (dtypes.NetworkName, error) {
|
|
if !buildconstants.Devnet {
|
|
return "testnetnet", nil
|
|
}
|
|
|
|
ctx := helpers.LifecycleCtx(mctx, lc)
|
|
|
|
sm, err := stmgr.NewStateManager(cs, tsexec, syscalls, us, nil, nil, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
netName, err := stmgr.GetNetworkName(ctx, sm, cs.GetHeaviestTipSet().ParentState())
|
|
return netName, err
|
|
}
|
|
|
|
type SyncerParams struct {
|
|
fx.In
|
|
|
|
Lifecycle fx.Lifecycle
|
|
MetadataDS dtypes.MetadataDS
|
|
StateManager *stmgr.StateManager
|
|
ChainXchg exchange.Client
|
|
SyncMgrCtor chain.SyncManagerCtor
|
|
Host host.Host
|
|
Beacon beacon.Schedule
|
|
Gent chain.Genesis
|
|
Consensus consensus.Consensus
|
|
}
|
|
|
|
func NewSyncer(params SyncerParams) (*chain.Syncer, error) {
|
|
var (
|
|
lc = params.Lifecycle
|
|
ds = params.MetadataDS
|
|
sm = params.StateManager
|
|
ex = params.ChainXchg
|
|
smCtor = params.SyncMgrCtor
|
|
h = params.Host
|
|
b = params.Beacon
|
|
)
|
|
syncer, err := chain.NewSyncer(ds, sm, ex, smCtor, h.ConnManager(), h.ID(), b, params.Gent, params.Consensus)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
lc.Append(fx.Hook{
|
|
OnStart: func(_ context.Context) error {
|
|
syncer.Start()
|
|
return nil
|
|
},
|
|
OnStop: func(_ context.Context) error {
|
|
syncer.Stop()
|
|
return nil
|
|
},
|
|
})
|
|
return syncer, nil
|
|
}
|
|
|
|
func NewSlashFilter(ds dtypes.MetadataDS) *slashfilter.SlashFilter {
|
|
return slashfilter.New(ds)
|
|
}
|
|
|
|
func UpgradeSchedule() stmgr.UpgradeSchedule {
|
|
return filcns.DefaultUpgradeSchedule()
|
|
}
|
|
|
|
func EnableStoringEvents(cs *store.ChainStore) {
|
|
cs.StoreEvents(true)
|
|
}
|