diff --git a/Makefile b/Makefile index d39c9291..ffb20176 100644 --- a/Makefile +++ b/Makefile @@ -218,7 +218,7 @@ test-cascade: # SQLite history/dedup state is not shared with Cascade fixtures or other nodes. test-lep6: setup-lep6-supernodes @echo "Running LEP-6 e2e tests..." - @cd tests/system && ${GO} mod tidy && ${GO} test -tags=system_test -timeout=900s -v -run '^TestLEP6' . + @cd tests/system && ${GO} mod tidy && ${GO} test -tags=system_test -timeout=1500s -v -run '^TestLEP6' . # Validate LEP-6 local config/default/fixture coverage without starting a network. lep6-validate-config: diff --git a/pkg/storage/rqstore/rq_mock.go b/pkg/storage/rqstore/rq_mock.go index a460d74c..2a332ab7 100644 --- a/pkg/storage/rqstore/rq_mock.go +++ b/pkg/storage/rqstore/rq_mock.go @@ -111,6 +111,20 @@ func (mr *MockStoreMockRecorder) StoreSymbolDirectory(txid, dir any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StoreSymbolDirectory", reflect.TypeOf((*MockStore)(nil).StoreSymbolDirectory), txid, dir) } +// UpsertSymbolDirectory mocks base method. +func (m *MockStore) UpsertSymbolDirectory(txid, dir string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertSymbolDirectory", txid, dir) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertSymbolDirectory indicates an expected call of UpsertSymbolDirectory. +func (mr *MockStoreMockRecorder) UpsertSymbolDirectory(txid, dir any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertSymbolDirectory", reflect.TypeOf((*MockStore)(nil).UpsertSymbolDirectory), txid, dir) +} + // UpdateIsFirstBatchStored mocks base method. func (m *MockStore) UpdateIsFirstBatchStored(txid string) error { m.ctrl.T.Helper() diff --git a/pkg/storage/rqstore/store.go b/pkg/storage/rqstore/store.go index b162ddf1..6336f37a 100644 --- a/pkg/storage/rqstore/store.go +++ b/pkg/storage/rqstore/store.go @@ -22,6 +22,7 @@ const createRQSymbolsDir string = ` type Store interface { DeleteSymbolsByTxID(txid string) error StoreSymbolDirectory(txid, dir string) error + UpsertSymbolDirectory(txid, dir string) error GetDirectoryByTxID(txid string) (string, error) GetToDoStoreSymbolDirs() ([]SymbolDir, error) SetIsCompleted(txid string) error @@ -121,6 +122,32 @@ func (s *SQLiteRQStore) StoreSymbolDirectory(txid, dir string) error { return nil } +// UpsertSymbolDirectory associates a txid with a directory path idempotently. +// +// LEP-6 heal finalization retries publish after transient P2P failures. A retry +// for the same action must not get stuck behind the row inserted by the failed +// attempt, so this path explicitly upserts instead of changing the legacy insert +// semantics of StoreSymbolDirectory. +func (s *SQLiteRQStore) UpsertSymbolDirectory(txid, dir string) error { + stmt, err := s.db.Prepare(` + INSERT INTO rq_symbols_dir (txid, dir, is_completed) + VALUES (?, ?, FALSE) + ON CONFLICT(txid) DO UPDATE SET + dir = excluded.dir, + is_completed = FALSE + `) + if err != nil { + return fmt.Errorf("failed to prepare upsert statement for directory: %w", err) + } + defer stmt.Close() + + if _, err := stmt.Exec(txid, dir); err != nil { + return fmt.Errorf("failed to execute upsert statement for directory: %w", err) + } + + return nil +} + // GetDirectoryByTxID retrieves the directory path associated with a given txid. func (s *SQLiteRQStore) GetDirectoryByTxID(txid string) (string, error) { var dir string diff --git a/pkg/storage/rqstore/store_test.go b/pkg/storage/rqstore/store_test.go index 5c423059..6d58695a 100644 --- a/pkg/storage/rqstore/store_test.go +++ b/pkg/storage/rqstore/store_test.go @@ -6,6 +6,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetToDoStoreSymbolDirs(t *testing.T) { @@ -159,3 +160,24 @@ func TestStoreSymbolDirectory(t *testing.T) { }) } } + +func TestUpsertSymbolDirectory(t *testing.T) { + store := SetupTestDB(t) + defer store.Close() + + require.NoError(t, store.StoreSymbolDirectory("tx123", "dir123")) + require.NoError(t, store.UpdateIsFirstBatchStored("tx123")) + require.NoError(t, store.SetIsCompleted("tx123")) + + require.NoError(t, store.UpsertSymbolDirectory("tx123", "dir456")) + + var row struct { + Dir string `db:"dir"` + IsFirstBatchStored bool `db:"is_first_batch_stored"` + IsCompleted bool `db:"is_completed"` + } + require.NoError(t, store.db.Get(&row, "SELECT dir, is_first_batch_stored, is_completed FROM rq_symbols_dir WHERE txid = ?", "tx123")) + assert.Equal(t, "dir456", row.Dir) + assert.True(t, row.IsFirstBatchStored) + assert.False(t, row.IsCompleted) +} diff --git a/supernode/adaptors/p2p.go b/supernode/adaptors/p2p.go index 036ec24a..a2845236 100644 --- a/supernode/adaptors/p2p.go +++ b/supernode/adaptors/p2p.go @@ -37,11 +37,12 @@ func NewP2PService(client p2p.Client, store rqstore.Store) P2PService { } type StoreArtefactsRequest struct { - TaskID string - ActionID string - IDFiles [][]byte - SymbolsDir string - Layout codec.Layout + TaskID string + ActionID string + IDFiles [][]byte + SymbolsDir string + Layout codec.Layout + IdempotentDirectoryRecord bool } func (p *p2pImpl) StoreArtefacts(ctx context.Context, req StoreArtefactsRequest, f logtrace.Fields) error { @@ -67,7 +68,7 @@ func (p *p2pImpl) StoreArtefacts(ctx context.Context, req StoreArtefactsRequest, "symbols_dir": req.SymbolsDir, }) start := time.Now() - firstPassSymbols, totalSymbols, err := p.storeCascadeSymbolsAndData(ctx, req.TaskID, req.ActionID, req.SymbolsDir, req.IDFiles, req.Layout) + firstPassSymbols, totalSymbols, err := p.storeCascadeSymbolsAndData(ctx, req.TaskID, req.ActionID, req.SymbolsDir, req.IDFiles, req.Layout, req.IdempotentDirectoryRecord) if err != nil { return fmt.Errorf("error storing artefacts: %w", err) } @@ -89,8 +90,14 @@ func (p *p2pImpl) StoreArtefacts(ctx context.Context, req StoreArtefactsRequest, return nil } -func (p *p2pImpl) storeCascadeSymbolsAndData(ctx context.Context, taskID, actionID string, symbolsDir string, metadataFiles [][]byte, layout codec.Layout) (int, int, error) { - if err := p.rqStore.StoreSymbolDirectory(taskID, symbolsDir); err != nil { +func (p *p2pImpl) storeCascadeSymbolsAndData(ctx context.Context, taskID, actionID string, symbolsDir string, metadataFiles [][]byte, layout codec.Layout, idempotentDirectoryRecord bool) (int, int, error) { + var err error + if idempotentDirectoryRecord { + err = p.rqStore.UpsertSymbolDirectory(taskID, symbolsDir) + } else { + err = p.rqStore.StoreSymbolDirectory(taskID, symbolsDir) + } + if err != nil { return 0, 0, fmt.Errorf("store symbol dir: %w", err) } metadataBytes := totalBytes(metadataFiles) @@ -203,6 +210,11 @@ func (p *p2pImpl) storeCascadeSymbolsAndData(ctx context.Context, taskID, action if err := p.rqStore.UpdateIsFirstBatchStored(taskID); err != nil { return totalSymbols, totalAvailable, fmt.Errorf("update first-batch flag: %w", err) } + if totalSymbols >= totalAvailable { + if err := p.rqStore.SetIsCompleted(taskID); err != nil { + return totalSymbols, totalAvailable, fmt.Errorf("mark symbols completed: %w", err) + } + } logtrace.Info(ctx, "store: first-pass bytes summary", logtrace.Fields{ "taskID": taskID, "symbols_stored": totalSymbols, diff --git a/supernode/cascade/helper.go b/supernode/cascade/helper.go index 24e2635d..37d96436 100644 --- a/supernode/cascade/helper.go +++ b/supernode/cascade/helper.go @@ -156,7 +156,7 @@ func (task *CascadeRegistrationTask) storeArtefacts(ctx context.Context, actionI } ctx = logtrace.CtxWithOrigin(ctx, "first_pass") logtrace.Info(ctx, "store: first-pass begin", lf) - if err := task.P2P.StoreArtefacts(ctx, adaptors.StoreArtefactsRequest{IDFiles: idFiles, SymbolsDir: symbolsDir, Layout: layout, TaskID: task.taskID, ActionID: actionID}, f); err != nil { + if err := task.P2P.StoreArtefacts(ctx, adaptors.StoreArtefactsRequest{IDFiles: idFiles, SymbolsDir: symbolsDir, Layout: layout, TaskID: task.taskID, ActionID: actionID, IdempotentDirectoryRecord: f[logtrace.FieldMethod] == "PublishStagedArtefacts"}, f); err != nil { return task.wrapErr(ctx, "failed to store artefacts", err, lf) } logtrace.Info(ctx, "store: first-pass ok", lf) diff --git a/tests/system/e2e_lep6_concurrency_test.go b/tests/system/e2e_lep6_concurrency_test.go new file mode 100644 index 00000000..ae7245a9 --- /dev/null +++ b/tests/system/e2e_lep6_concurrency_test.go @@ -0,0 +1,239 @@ +//go:build system_test + +package system + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" + + actiontypes "github.com/LumeraProtocol/lumera/x/action/v1/types" + sdkhd "github.com/cosmos/cosmos-sdk/crypto/hd" + sdkkeyring "github.com/cosmos/cosmos-sdk/crypto/keyring" + + "github.com/LumeraProtocol/supernode/v2/pkg/keyring" + "github.com/LumeraProtocol/supernode/v2/pkg/lumera" + "github.com/LumeraProtocol/supernode/v2/sdk/action" + sdkconfig "github.com/LumeraProtocol/supernode/v2/sdk/config" + "github.com/LumeraProtocol/supernode/v2/supernode/config" + "github.com/stretchr/testify/require" +) + +type concurrentCascadeUpload struct { + actionID string + keyName string + mnemonic string + userAddress string + payload []byte + hash [32]byte +} + +type lep6UploadUser struct { + keyName string + mnemonic string + userAddress string +} + +// TestLEP6ConcurrentCascadesContendedReporter exercises multiple simultaneous +// CASCADE uploads against the real LEP-6 system-test fixture. The test targets +// runtime contention risks observed around SQLite/P2P/Cascade orchestration: all +// actions must finalize, remain byte-identical on download, and the supernode +// logs must not contain lock/panic/duplicate-report failures. +func TestLEP6ConcurrentCascadesContendedReporter(t *testing.T) { + os.Setenv("INTEGRATION_TEST", "true") + os.Setenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER", "1") + t.Cleanup(func() { + os.Unsetenv("INTEGRATION_TEST") + os.Unsetenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER") + }) + + const ( + epochLengthBlocks = uint64(12) + lumeraGRPCAddr = "localhost:9090" + lumeraChainID = "testing" + // Keep runtime supernodes above SDK's 1 LUME eligibility threshold after + // paying finalize tx fees for concurrent CASCADE actions; otherwise the + // post-upload download prefilter can find zero eligible targets. + fundAmount = "10000000ulume" + actionType = "CASCADE" + ) + + t.Log("Phase 3 Step 1: configure genesis and start real chain") + sut.ModifyGenesisJSON(t, + SetStakingBondDenomUlume(t), + SetActionParams(t), + SetSupernodeMetricsParams(t), + setSupernodeParamsForAuditTests(t), + setAuditParamsForFastEpochs(t, epochLengthBlocks, 1, 1, 1, []uint32{4444}), + setAuditMissingReportGraceForRuntimeE2E(t), + setStorageTruthTestParams(t, "STORAGE_TRUTH_ENFORCEMENT_MODE_SHADOW", 1000, 500, 1000, 0, 10), + ) + sut.StartChain(t) + cli := NewLumeradCLI(t, sut, true) + + binaryPath := locateExecutable(sut.ExecBinary) + homePath := filepath.Join(WorkDir, sut.outputDir) + supernodeUsers := []struct { + keyName string + mnemonic string + }{ + {keyName: "testkey1", mnemonic: "odor kiss switch swarm spell make planet bundle skate ozone path planet exclude butter atom ahead angle royal shuffle door prevent merry alter robust"}, + {keyName: "testkey2", mnemonic: "club party current length duck agent love into slide extend spawn sentence kangaroo chunk festival order plate rare public good include situate liar miss"}, + {keyName: "testkey3", mnemonic: "young envelope urban crucial denial zone toward mansion protect bonus exotic puppy resource pistol expand tell cupboard radio hurry world radio trust explain million"}, + } + for _, user := range supernodeUsers { + recoverChainKey(t, binaryPath, homePath, user.keyName, user.mnemonic) + } + + t.Log("Phase 3 Step 2: register and fund three real runtime supernodes/users") + n0 := getRuntimeSupernodeIdentity(t, cli, "node0", "testkey1") + n1 := getRuntimeSupernodeIdentity(t, cli, "node1", "testkey2") + n2 := getRuntimeSupernodeIdentity(t, cli, "node2", "testkey3") + registerRuntimeSupernode(t, cli, "node0", n0, "localhost:4444", "4445") + registerRuntimeSupernode(t, cli, "node1", n1, "localhost:4446", "4447") + registerRuntimeSupernode(t, cli, "node2", n2, "localhost:4448", "4449") + for _, node := range []testNodeIdentity{n0, n1, n2} { + cli.FundAddress(node.accAddr, fundAmount) + } + bootstrapRuntimeSupernodeEligibility(t, cli) + sut.AwaitNextBlock(t) + uploadUsers := generateLEP6UploadUsers(t, 3) + for _, user := range uploadUsers { + cli.FundAddress(user.userAddress, fundAmount) + } + sut.AwaitNextBlock(t) + + t.Cleanup(restoreLEP6SupernodeFixturesAfterMatrix(t)) + cmds := StartLEP6Supernodes(t) + t.Cleanup(func() { StopAllSupernodes(cmds) }) + time.Sleep(40 * time.Second) // allow supernode P2P/DHT routing to settle before concurrent upload + + count := lep6ConcurrentActionCount(t, len(uploadUsers)) + t.Logf("Phase 3 Step 3: upload %d CASCADE actions concurrently", count) + uploads := uploadConcurrentCascadesForLEP6(t, cli, uploadUsers[:count], lumeraGRPCAddr, lumeraChainID, actionType) + require.Len(t, uploads, count) + + t.Log("Phase 3 Step 4: verify all concurrent actions download byte-identical") + for _, upload := range uploads { + ctx := context.Background() + _, actionClient := newLEP6ActionClientsForKey(t, ctx, upload.keyName, upload.mnemonic, lumeraGRPCAddr, lumeraChainID) + downloadAndAssertCascadeBytes(t, ctx, actionClient, upload.actionID, upload.userAddress, t.TempDir(), upload.payload, upload.hash) + } + + t.Log("Phase 3 Step 5: assert runtime logs stayed clean under concurrent CASCADE load") + assertLEP6SupernodeLogsDoNotContain(t, []string{"database is locked", "panic:", "duplicate proof report"}) +} + +func lep6ConcurrentActionCount(t *testing.T, max int) int { + t.Helper() + count := 3 + if raw := strings.TrimSpace(os.Getenv("LEP6_CONCURRENT_ACTIONS")); raw != "" { + parsed, err := strconv.Atoi(raw) + require.NoError(t, err, "LEP6_CONCURRENT_ACTIONS must be an integer") + count = parsed + } + require.GreaterOrEqual(t, count, 1) + require.LessOrEqual(t, count, max, "this PR-blocking test has %d deterministic upload accounts available", max) + return count +} + +func uploadConcurrentCascadesForLEP6( + t *testing.T, + cli *LumeradCli, + users []lep6UploadUser, + lumeraGRPCAddr string, + lumeraChainID string, + actionType string, +) []concurrentCascadeUpload { + t.Helper() + + uploads := make([]concurrentCascadeUpload, 0, len(users)) + var mu sync.Mutex + t.Run("concurrent-uploads", func(t *testing.T) { + for i, user := range users { + i, user := i, user + t.Run(fmt.Sprintf("upload-%d-%s", i+1, user.keyName), func(t *testing.T) { + t.Parallel() + ctx := context.Background() + lumeraClient, actionClient := newLEP6ActionClientsForKey(t, ctx, user.keyName, user.mnemonic, lumeraGRPCAddr, lumeraChainID) + t.Cleanup(func() { lumeraClient.Close() }) + + payload := []byte(fmt.Sprintf("lep6 phase3 concurrent cascade payload %d\n%s\n", i+1, strings.Repeat(fmt.Sprintf("chunk-%02d-", i+1), 64))) + path := filepath.Join(t.TempDir(), fmt.Sprintf("phase3-concurrent-%d.txt", i+1)) + require.NoError(t, os.WriteFile(path, payload, 0o600)) + + actionID := requestAndStartCascadeAction(t, ctx, cli, lumeraClient, actionClient, path, actionType) + require.NoError(t, waitForActionStateWithClient(ctx, lumeraClient, actionID, actiontypes.ActionStateDone)) + requireFinalizedCascadeArtifactCounts(t, ctx, lumeraClient, actionID) + + mu.Lock() + uploads = append(uploads, concurrentCascadeUpload{ + actionID: actionID, + keyName: user.keyName, + mnemonic: user.mnemonic, + userAddress: user.userAddress, + payload: payload, + hash: sha256.Sum256(payload), + }) + mu.Unlock() + }) + } + }) + return uploads +} + +func generateLEP6UploadUsers(t *testing.T, count int) []lep6UploadUser { + t.Helper() + kr, err := keyring.InitKeyring(config.KeyringConfig{Backend: "memory", Dir: ""}) + require.NoError(t, err) + + users := make([]lep6UploadUser, 0, count) + for i := 0; i < count; i++ { + keyName := fmt.Sprintf("phase3-upload-user-%d", i+1) + record, mnemonic, err := kr.NewMnemonic(keyName, sdkkeyring.English, keyring.DefaultHDPath, keyring.DefaultBIP39Passphrase, sdkhd.Secp256k1) + require.NoError(t, err) + addr, err := record.GetAddress() + require.NoError(t, err) + users = append(users, lep6UploadUser{keyName: keyName, mnemonic: mnemonic, userAddress: addr.String()}) + } + return users +} + +func newLEP6ActionClientsForKey(t *testing.T, ctx context.Context, keyName, mnemonic, lumeraGRPCAddr, lumeraChainID string) (lumera.Client, action.Client) { + t.Helper() + kr, err := keyring.InitKeyring(config.KeyringConfig{Backend: "memory", Dir: ""}) + require.NoError(t, err) + _, err = keyring.RecoverAccountFromMnemonic(kr, keyName, mnemonic) + require.NoError(t, err) + + lumeraCfg, err := lumera.NewConfig(lumeraGRPCAddr, lumeraChainID, keyName, kr) + require.NoError(t, err) + lumeraClient, err := lumera.NewClient(ctx, lumeraCfg) + require.NoError(t, err) + actionClient, err := action.NewClient(ctx, sdkconfig.Config{ + Account: sdkconfig.AccountConfig{KeyName: keyName, Keyring: kr}, + Lumera: sdkconfig.LumeraConfig{GRPCAddr: lumeraGRPCAddr, ChainID: lumeraChainID}, + }, nil) + require.NoError(t, err) + return lumeraClient, actionClient +} + +func assertLEP6SupernodeLogsDoNotContain(t *testing.T, patterns []string) { + t.Helper() + for i := 0; i < 3; i++ { + path := filepath.Join(WorkDir, fmt.Sprintf("supernode-lep6%d.out", i)) + data, err := os.ReadFile(path) + require.NoError(t, err, "read supernode log %s", path) + lower := strings.ToLower(string(data)) + for _, pattern := range patterns { + require.NotContains(t, lower, strings.ToLower(pattern), "supernode log %s contains forbidden runtime pattern %q", path, pattern) + } + } +} diff --git a/tests/system/e2e_lep6_enforcement_lifecycle_test.go b/tests/system/e2e_lep6_enforcement_lifecycle_test.go index f6301624..22e467a4 100644 --- a/tests/system/e2e_lep6_enforcement_lifecycle_test.go +++ b/tests/system/e2e_lep6_enforcement_lifecycle_test.go @@ -22,6 +22,14 @@ import ( // 4. clean PASS proofs reduce suspicion below watch and satisfy clean-pass recovery; // 5. epoch-end enforcement recovers the supernode to ACTIVE. func TestLEP6StorageTruthEnforcementLifecycle(t *testing.T) { + runLEP6StorageTruthEnforcementLifecycle(t, false) +} + +func TestLEP6PostponedTargetReassignmentAfterRecovery(t *testing.T) { + runLEP6StorageTruthEnforcementLifecycle(t, true) +} + +func runLEP6StorageTruthEnforcementLifecycle(t *testing.T, assertPostRecoveryReassignment bool) { os.Setenv("INTEGRATION_TEST", "true") defer os.Unsetenv("INTEGRATION_TEST") @@ -129,6 +137,30 @@ func TestLEP6StorageTruthEnforcementLifecycle(t *testing.T) { require.Eventually(t, func() bool { return querySupernodeLatestState(t, cli, target.valAddr) == "SUPERNODE_STATE_ACTIVE" }, 45*time.Second, time.Second, "clean target must recover to ACTIVE after score and clean-pass predicates are satisfied") + + if !assertPostRecoveryReassignment { + return + } + + t.Log("Storage-truth enforcement Step 6: recovered target is assigned again and accepts normal proof reporting") + reassignmentEpochID, reassignmentEpochEnd := nextEpochWithAssignedStorageTruthTarget(t, originHeight, epochLengthBlocks, passCandidates, target.accAddr) + seedProofTranscriptsWithClass( + t, + cli, + reassignmentEpochID, + passCandidates, + target.accAddr, + []transcriptSeed{{ticketID: ticketID + "-post-recovery", transcriptHash: "storage-truth-post-recovery-pass"}}, + true, + "STORAGE_PROOF_RESULT_CLASS_PASS", + ) + + postRecoveryState, found := auditQueryNodeSuspicionStateST(t, target.accAddr) + require.True(t, found) + require.GreaterOrEqual(t, postRecoveryState.CleanPassCount, uint32(2), "post-recovery assignment must accept another clean PASS proof for the recovered target") + awaitAtLeastHeight(t, reassignmentEpochEnd) + sut.AwaitNextBlock(t) + require.Equal(t, "SUPERNODE_STATE_ACTIVE", querySupernodeLatestState(t, cli, target.valAddr), "post-recovery proof reporting must not re-postpone the recovered target") } func nextUsableEpoch(t *testing.T, originHeight int64, epochLengthBlocks uint64) (uint64, int64) { diff --git a/tests/system/e2e_lep6_negative_matrix_test.go b/tests/system/e2e_lep6_negative_matrix_test.go new file mode 100644 index 00000000..e35e4115 --- /dev/null +++ b/tests/system/e2e_lep6_negative_matrix_test.go @@ -0,0 +1,412 @@ +//go:build system_test + +package system + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + actiontypes "github.com/LumeraProtocol/lumera/x/action/v1/types" + audittypes "github.com/LumeraProtocol/lumera/x/audit/v1/types" + "github.com/LumeraProtocol/supernode/v2/p2p" + p2psqlite "github.com/LumeraProtocol/supernode/v2/p2p/kademlia/store/sqlite" + "github.com/LumeraProtocol/supernode/v2/pkg/cascadekit" + "github.com/LumeraProtocol/supernode/v2/pkg/keyring" + "github.com/LumeraProtocol/supernode/v2/pkg/lumera" + "github.com/LumeraProtocol/supernode/v2/sdk/action" + sdkconfig "github.com/LumeraProtocol/supernode/v2/sdk/config" + "github.com/LumeraProtocol/supernode/v2/supernode/config" + storagechallenge "github.com/LumeraProtocol/supernode/v2/supernode/storage_challenge" + "github.com/btcsuite/btcutil/base58" + "github.com/stretchr/testify/require" +) + +// TestLEP6NegativePerCaseCorruptionMatrix turns the manual local-devnet Phase 2D +// corruption matrix into real-binary system coverage. A real lumerad plus three +// real supernode binaries create/finalize the CASCADE action and persist the +// artifacts in the production P2P SQLite store. The assertions then mutate those +// real artifact rows and exercise the same LEP-6 P2P artifact reader that the +// storage-challenge gRPC handler uses before building INVALID_TRANSCRIPT proof +// results. +func TestLEP6NegativePerCaseCorruptionMatrix(t *testing.T) { + fixture := setupLEP6RuntimeCascadeFixture(t) + artifacts := resolveLEP6MatrixArtifacts(t, fixture.ctx, fixture.lumeraClient, fixture.actionID, fixture.nodes) + + t.Run("missing symbol row", func(t *testing.T) { + withDeletedArtifact(t, fixture.ctx, artifacts.symbol.store, artifacts.symbol.key, artifacts.symbol.original) + assertLEP6ArtifactReadFails(t, fixture.ctx, artifacts.symbol, audittypes.StorageProofArtifactClass_STORAGE_PROOF_ARTIFACT_CLASS_SYMBOL, "no rows in result set") + }) + + t.Run("wrong-size symbol row", func(t *testing.T) { + bad := []byte("lep6-wrong-size-symbol") + withCorruptedArtifact(t, fixture.ctx, artifacts.symbol.store, artifacts.symbol.key, artifacts.symbol.original, bad) + assertLEP6ArtifactReadFails(t, fixture.ctx, artifacts.symbol, audittypes.StorageProofArtifactClass_STORAGE_PROOF_ARTIFACT_CLASS_SYMBOL, "content hash mismatch") + }) + + t.Run("missing index row", func(t *testing.T) { + withDeletedArtifact(t, fixture.ctx, artifacts.index.store, artifacts.index.key, artifacts.index.original) + assertLEP6ArtifactReadFails(t, fixture.ctx, artifacts.index, audittypes.StorageProofArtifactClass_STORAGE_PROOF_ARTIFACT_CLASS_INDEX, "no rows in result set") + }) + + t.Run("corrupt index row", func(t *testing.T) { + bad := append([]byte(nil), artifacts.index.original...) + require.NotEmpty(t, bad) + bad[0] ^= 0xff + withCorruptedArtifact(t, fixture.ctx, artifacts.index.store, artifacts.index.key, artifacts.index.original, bad) + assertLEP6ArtifactReadFails(t, fixture.ctx, artifacts.index, audittypes.StorageProofArtifactClass_STORAGE_PROOF_ARTIFACT_CLASS_INDEX, "content hash mismatch") + }) + + downloadAndAssertCascadeBytes(t, fixture.ctx, fixture.actionClient, fixture.actionID, fixture.userAddress, t.TempDir(), fixture.originalData, fixture.originalHash) +} + +type lep6RuntimeCascadeFixture struct { + ctx context.Context + cli *LumeradCli + lumeraClient lumera.Client + actionClient action.Client + actionID string + userAddress string + nodes []testNodeIdentity + originalData []byte + originalHash [32]byte +} + +func setupLEP6RuntimeCascadeFixture(t *testing.T) lep6RuntimeCascadeFixture { + t.Helper() + + os.Setenv("INTEGRATION_TEST", "true") + os.Setenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER", "1") + t.Cleanup(func() { + os.Unsetenv("INTEGRATION_TEST") + os.Unsetenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER") + }) + + const ( + epochLengthBlocks = uint64(12) + lumeraGRPCAddr = "localhost:9090" + lumeraChainID = "testing" + testKeyName = "testkey1" + testMnemonic = "odor kiss switch swarm spell make planet bundle skate ozone path planet exclude butter atom ahead angle royal shuffle door prevent merry alter robust" + testKey2Mnemonic = "club party current length duck agent love into slide extend spawn sentence kangaroo chunk festival order plate rare public good include situate liar miss" + testKey3Mnemonic = "young envelope urban crucial denial zone toward mansion protect bonus exotic puppy resource pistol expand tell cupboard radio hurry world radio trust explain million" + expectedAddress = "lumera1em87kgrvgttrkvuamtetyaagjrhnu3vjy44at4" + userKeyName = "user" + userMnemonic = "little tone alley oval festival gloom sting asthma crime select swap auto when trip luxury pact risk sister pencil about crisp upon opera timber" + fundAmount = "1000000ulume" + actionType = "CASCADE" + ) + + sut.ModifyGenesisJSON(t, + SetStakingBondDenomUlume(t), + SetActionParams(t), + SetSupernodeMetricsParams(t), + setSupernodeParamsForAuditTests(t), + setAuditParamsForFastEpochs(t, epochLengthBlocks, 1, 1, 1, []uint32{4444}), + setAuditMissingReportGraceForRuntimeE2E(t), + setStorageTruthTestParams(t, "STORAGE_TRUTH_ENFORCEMENT_MODE_FULL", 1000, 500, 10, 0, 10), + ) + sut.StartChain(t) + cli := NewLumeradCLI(t, sut, true) + + binaryPath := locateExecutable(sut.ExecBinary) + homePath := filepath.Join(WorkDir, sut.outputDir) + recoverChainKey(t, binaryPath, homePath, testKeyName, testMnemonic) + recoverChainKey(t, binaryPath, homePath, "testkey2", testKey2Mnemonic) + recoverChainKey(t, binaryPath, homePath, "testkey3", testKey3Mnemonic) + recoverChainKey(t, binaryPath, homePath, userKeyName, userMnemonic) + + n0 := getRuntimeSupernodeIdentity(t, cli, "node0", "testkey1") + n1 := getRuntimeSupernodeIdentity(t, cli, "node1", "testkey2") + n2 := getRuntimeSupernodeIdentity(t, cli, "node2", "testkey3") + registerRuntimeSupernode(t, cli, "node0", n0, "localhost:4444", "4445") + registerRuntimeSupernode(t, cli, "node1", n1, "localhost:4446", "4447") + registerRuntimeSupernode(t, cli, "node2", n2, "localhost:4448", "4449") + cli.FundAddress(n0.accAddr, "100000ulume") + cli.FundAddress(n1.accAddr, "100000ulume") + cli.FundAddress(n2.accAddr, "100000ulume") + bootstrapRuntimeSupernodeEligibility(t, cli) + + recoveredAddress := cli.GetKeyAddr(testKeyName) + require.Equal(t, expectedAddress, recoveredAddress) + userAddress := cli.GetKeyAddr(userKeyName) + cli.FundAddress(recoveredAddress, fundAmount) + cli.FundAddress(userAddress, fundAmount) + sut.AwaitNextBlock(t) + + t.Cleanup(restoreLEP6SupernodeFixturesAfterMatrix(t)) + cmds := StartLEP6Supernodes(t) + t.Cleanup(func() { StopAllSupernodes(cmds) }) + time.Sleep(40 * time.Second) // allow supernode P2P/DHT routing to settle before upload + + ctx := context.Background() + kr, err := keyring.InitKeyring(config.KeyringConfig{Backend: "memory", Dir: ""}) + require.NoError(t, err) + _, err = keyring.RecoverAccountFromMnemonic(kr, testKeyName, testMnemonic) + require.NoError(t, err) + userRecord, err := keyring.RecoverAccountFromMnemonic(kr, userKeyName, userMnemonic) + require.NoError(t, err) + userLocalAddr, err := userRecord.GetAddress() + require.NoError(t, err) + require.Equal(t, userAddress, userLocalAddr.String()) + + lumeraCfg, err := lumera.NewConfig(lumeraGRPCAddr, lumeraChainID, userKeyName, kr) + require.NoError(t, err) + lumeraClient, err := lumera.NewClient(ctx, lumeraCfg) + require.NoError(t, err) + t.Cleanup(func() { lumeraClient.Close() }) + + actionClient, err := action.NewClient(ctx, sdkconfig.Config{ + Account: sdkconfig.AccountConfig{KeyName: userKeyName, Keyring: kr}, + Lumera: sdkconfig.LumeraConfig{GRPCAddr: lumeraGRPCAddr, ChainID: lumeraChainID}, + }, nil) + require.NoError(t, err) + + testFileFullpath := filepath.Join("test.txt") + originalData := readFileBytes(t, testFileFullpath) + originalHash := sha256.Sum256(originalData) + + actionID := requestAndStartCascadeAction(t, ctx, cli, lumeraClient, actionClient, testFileFullpath, actionType) + require.NoError(t, waitForActionStateWithClient(ctx, lumeraClient, actionID, actiontypes.ActionStateDone)) + _ = requireFinalizedCascadeArtifactCounts(t, ctx, lumeraClient, actionID) + downloadAndAssertCascadeBytes(t, ctx, actionClient, actionID, userAddress, t.TempDir(), originalData, originalHash) + + return lep6RuntimeCascadeFixture{ + ctx: ctx, + cli: cli, + lumeraClient: lumeraClient, + actionClient: actionClient, + actionID: actionID, + userAddress: userAddress, + nodes: []testNodeIdentity{n0, n1, n2}, + originalData: originalData, + originalHash: originalHash, + } +} + +type lep6ArtifactRef struct { + node testNodeIdentity + store *p2psqlite.Store + key string + original []byte +} + +type lep6MatrixArtifacts struct { + index lep6ArtifactRef + symbol lep6ArtifactRef +} + +func resolveLEP6MatrixArtifacts(t *testing.T, ctx context.Context, lc lumera.Client, actionID string, nodes []testNodeIdentity) lep6MatrixArtifacts { + t.Helper() + resp, err := lc.Action().GetAction(ctx, actionID) + require.NoError(t, err) + require.NotNil(t, resp.GetAction()) + meta, err := cascadekit.UnmarshalCascadeMetadata(resp.GetAction().Metadata) + require.NoError(t, err) + require.NotEmpty(t, meta.RqIdsIds, "finalized action must contain index artifact IDs") + + stores := make(map[string]*p2psqlite.Store, len(nodes)) + openStore := func(node testNodeIdentity) *p2psqlite.Store { + if stores[node.accAddr] != nil { + return stores[node.accAddr] + } + store := openLEP6P2PStore(t, ctx, node, nodes) + stores[node.accAddr] = store + return store + } + + var indexRef lep6ArtifactRef + var indexFile cascadekit.IndexFile + for _, indexKey := range meta.RqIdsIds { + for _, node := range nodes { + store := openStore(node) + b, err := retrieveLEP6Artifact(ctx, store, indexKey) + if err != nil { + continue + } + idx, err := cascadekit.ParseCompressedIndexFile(b) + if err != nil { + continue + } + indexRef = lep6ArtifactRef{node: node, store: store, key: indexKey, original: append([]byte(nil), b...)} + indexFile = idx + break + } + if indexRef.key != "" { + break + } + } + require.NotEmpty(t, indexRef.key, "at least one real supernode P2P DB must contain a local index artifact") + require.NotEmpty(t, indexFile.LayoutIDs, "index artifact must reference layout artifacts") + + var symbolIDs []string + for _, layoutKey := range indexFile.LayoutIDs { + for _, node := range nodes { + layoutBytes, err := retrieveLEP6Artifact(ctx, openStore(node), layoutKey) + if err != nil { + continue + } + layout, _, _, err := cascadekit.ParseRQMetadataFile(layoutBytes) + if err != nil { + continue + } + for _, block := range layout.Blocks { + for _, symbolID := range block.Symbols { + if strings.TrimSpace(symbolID) != "" { + symbolIDs = append(symbolIDs, symbolID) + } + } + } + break + } + if len(symbolIDs) > 0 { + break + } + } + require.NotEmpty(t, symbolIDs, "real persisted layout metadata must expose symbol artifact IDs") + + var symbolRef lep6ArtifactRef + for _, symbolKey := range symbolIDs { + for _, node := range nodes { + store := openStore(node) + b, err := retrieveLEP6Artifact(ctx, store, symbolKey) + if err != nil { + continue + } + symbolRef = lep6ArtifactRef{node: node, store: store, key: symbolKey, original: append([]byte(nil), b...)} + break + } + if symbolRef.key != "" { + break + } + } + require.NotEmpty(t, symbolRef.key, "at least one real supernode P2P DB must contain a local symbol artifact") + + t.Logf("LEP-6 matrix artifacts: index key=%s node=%s bytes=%d; symbol key=%s node=%s bytes=%d", + indexRef.key, indexRef.node.nodeName, len(indexRef.original), symbolRef.key, symbolRef.node.nodeName, len(symbolRef.original)) + return lep6MatrixArtifacts{index: indexRef, symbol: symbolRef} +} + +func restoreLEP6SupernodeFixturesAfterMatrix(t *testing.T) func() { + t.Helper() + repoRoot, err := filepath.Abs(filepath.Join(WorkDir, "..", "..")) + require.NoError(t, err) + return func() { + cmd := exec.Command("bash", "tests/scripts/setup-supernodes.sh", + "all", + "supernode/main.go", + "tests/system/supernode-lep6-data1", "tests/system/config.lep6-1.yml", + "tests/system/supernode-lep6-data2", "tests/system/config.lep6-2.yml", + "tests/system/supernode-lep6-data3", "tests/system/config.lep6-3.yml", + ) + cmd.Dir = repoRoot + out, err := cmd.CombinedOutput() + require.NoError(t, err, "restore LEP-6 supernode fixtures after negative matrix: %s", string(out)) + } +} + +func openLEP6P2PStore(t *testing.T, ctx context.Context, node testNodeIdentity, nodes []testNodeIdentity) *p2psqlite.Store { + t.Helper() + baseDir := dataDirForSupernodeAccount(t, node.accAddr, nodes...) + storeDir := filepath.Join(baseDir, "data", "p2p") + store, err := p2psqlite.NewStore(ctx, storeDir, nil, nil) + require.NoError(t, err) + t.Cleanup(func() { store.Close(context.Background()) }) + return store +} + +func retrieveLEP6Artifact(ctx context.Context, store *p2psqlite.Store, key string) ([]byte, error) { + decoded := base58.Decode(key) + if len(decoded) != 32 { + return nil, fmt.Errorf("invalid base58 artifact key %q", key) + } + return store.Retrieve(ctx, decoded) +} + +func withDeletedArtifact(t *testing.T, ctx context.Context, store *p2psqlite.Store, key string, original []byte) { + t.Helper() + decoded := base58.Decode(key) + require.Len(t, decoded, 32) + store.Delete(ctx, decoded) + require.Eventually(t, func() bool { + _, err := store.Retrieve(ctx, decoded) + return err != nil + }, 15*time.Second, 200*time.Millisecond) + t.Cleanup(func() { restoreLEP6Artifact(t, ctx, store, key, original) }) +} + +func withCorruptedArtifact(t *testing.T, ctx context.Context, store *p2psqlite.Store, key string, original, corrupted []byte) { + t.Helper() + decoded := base58.Decode(key) + require.Len(t, decoded, 32) + require.NoError(t, store.Store(ctx, decoded, corrupted, 1, true)) + require.Eventually(t, func() bool { + got, err := store.Retrieve(ctx, decoded) + return err == nil && string(got) == string(corrupted) + }, 15*time.Second, 200*time.Millisecond) + t.Cleanup(func() { restoreLEP6Artifact(t, ctx, store, key, original) }) +} + +func restoreLEP6Artifact(t *testing.T, ctx context.Context, store *p2psqlite.Store, key string, original []byte) { + t.Helper() + decoded := base58.Decode(key) + require.Len(t, decoded, 32) + require.NoError(t, store.Store(ctx, decoded, original, 1, true)) + require.Eventually(t, func() bool { + got, err := store.Retrieve(ctx, decoded) + return err == nil && string(got) == string(original) + }, 15*time.Second, 200*time.Millisecond) +} + +func assertLEP6ArtifactReadFails(t *testing.T, ctx context.Context, artifact lep6ArtifactRef, class audittypes.StorageProofArtifactClass, want string) { + t.Helper() + reader := storagechallenge.NewP2PArtifactReader(&lep6StoreBackedP2P{store: artifact.store}) + _, err := reader.ReadArtifactRange(ctx, class, artifact.key, 0, 1) + require.Error(t, err) + require.Contains(t, err.Error(), want) +} + +type lep6StoreBackedP2P struct{ store *p2psqlite.Store } + +func (p *lep6StoreBackedP2P) Retrieve(ctx context.Context, key string, localOnly ...bool) ([]byte, error) { + return retrieveLEP6Artifact(ctx, p.store, key) +} +func (p *lep6StoreBackedP2P) BatchRetrieve(context.Context, []string, int, string, ...bool) (map[string][]byte, error) { + return nil, fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) BatchRetrieveStream(context.Context, []string, int32, string, func(string, []byte) error, ...bool) (int32, error) { + return 0, fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) Store(context.Context, []byte, int) (string, error) { + return "", fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) StoreBatch(context.Context, [][]byte, int, string) error { + return fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) Delete(context.Context, string) error { + return fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) Stats(context.Context) (*p2p.StatsSnapshot, error) { + return nil, fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) NClosestNodes(context.Context, int, string, ...string) []string { + return nil +} +func (p *lep6StoreBackedP2P) NClosestNodesWithIncludingNodeList(context.Context, int, string, []string, []string) []string { + return nil +} +func (p *lep6StoreBackedP2P) LocalStore(context.Context, string, []byte) (string, error) { + return "", fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) DisableKey(context.Context, string) error { return nil } +func (p *lep6StoreBackedP2P) EnableKey(context.Context, string) error { return nil } +func (p *lep6StoreBackedP2P) GetLocalKeys(context.Context, *time.Time, time.Time) ([]string, error) { + return nil, fmt.Errorf("not implemented in LEP-6 matrix test fake") +} +func (p *lep6StoreBackedP2P) Run(context.Context) error { return nil } diff --git a/tests/system/e2e_lep6_restart_test.go b/tests/system/e2e_lep6_restart_test.go new file mode 100644 index 00000000..43246775 --- /dev/null +++ b/tests/system/e2e_lep6_restart_test.go @@ -0,0 +1,448 @@ +//go:build system_test + +package system + +import ( + "context" + "crypto/sha256" + "database/sql" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + actiontypes "github.com/LumeraProtocol/lumera/x/action/v1/types" + audittypes "github.com/LumeraProtocol/lumera/x/audit/v1/types" + "github.com/stretchr/testify/require" + + _ "github.com/mattn/go-sqlite3" +) + +// TestLEP6SupernodeRestartMidEpochPlannerConsistency exercises an +// operator-realistic restart of one real supernode while the real chain and the +// remaining supernodes continue running. It proves the restarted node's persisted +// SQLite state opens cleanly, the CASCADE data written before restart remains +// byte-identical on download, fresh CASCADE work still finalizes after restart, +// and runtime logs do not show replay/lock/panic symptoms. +func TestLEP6SupernodeRestartMidEpochPlannerConsistency(t *testing.T) { + runLEP6SupernodeRestartMidEpochPlannerConsistency(t) +} + +func TestLEP6CrashRestartMidHealResumeOrReclaim(t *testing.T) { + runLEP6CrashRestartMidHealResumeOrReclaim(t) +} + +func runLEP6SupernodeRestartMidEpochPlannerConsistency(t *testing.T) { + t.Helper() + os.Setenv("INTEGRATION_TEST", "true") + os.Setenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER", "1") + t.Cleanup(func() { + os.Unsetenv("INTEGRATION_TEST") + os.Unsetenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER") + }) + + const ( + epochLengthBlocks = uint64(12) + lumeraGRPCAddr = "localhost:9090" + lumeraChainID = "testing" + fundAmount = "10000000ulume" + actionType = "CASCADE" + ) + + t.Log("Phase 4A Step 1: configure genesis and start real chain") + sut.ModifyGenesisJSON(t, + SetStakingBondDenomUlume(t), + SetActionParams(t), + SetSupernodeMetricsParams(t), + setSupernodeParamsForAuditTests(t), + setAuditParamsForFastEpochs(t, epochLengthBlocks, 1, 1, 1, []uint32{4444}), + setAuditMissingReportGraceForRuntimeE2E(t), + setStorageTruthTestParams(t, "STORAGE_TRUTH_ENFORCEMENT_MODE_SHADOW", 1000, 500, 1000, 0, 10), + ) + sut.StartChain(t) + cli := NewLumeradCLI(t, sut, true) + + binaryPath := locateExecutable(sut.ExecBinary) + homePath := filepath.Join(WorkDir, sut.outputDir) + supernodeUsers := []struct { + keyName string + mnemonic string + }{ + {keyName: "testkey1", mnemonic: "odor kiss switch swarm spell make planet bundle skate ozone path planet exclude butter atom ahead angle royal shuffle door prevent merry alter robust"}, + {keyName: "testkey2", mnemonic: "club party current length duck agent love into slide extend spawn sentence kangaroo chunk festival order plate rare public good include situate liar miss"}, + {keyName: "testkey3", mnemonic: "young envelope urban crucial denial zone toward mansion protect bonus exotic puppy resource pistol expand tell cupboard radio hurry world radio trust explain million"}, + } + for _, user := range supernodeUsers { + recoverChainKey(t, binaryPath, homePath, user.keyName, user.mnemonic) + } + + t.Log("Phase 4A Step 2: register/fund real runtime supernodes and upload users") + n0 := getRuntimeSupernodeIdentity(t, cli, "node0", "testkey1") + n1 := getRuntimeSupernodeIdentity(t, cli, "node1", "testkey2") + n2 := getRuntimeSupernodeIdentity(t, cli, "node2", "testkey3") + registerRuntimeSupernode(t, cli, "node0", n0, "localhost:4444", "4445") + registerRuntimeSupernode(t, cli, "node1", n1, "localhost:4446", "4447") + registerRuntimeSupernode(t, cli, "node2", n2, "localhost:4448", "4449") + for _, node := range []testNodeIdentity{n0, n1, n2} { + cli.FundAddress(node.accAddr, fundAmount) + } + bootstrapRuntimeSupernodeEligibility(t, cli) + sut.AwaitNextBlock(t) + + uploadUsers := generateLEP6UploadUsers(t, 2) + for _, user := range uploadUsers { + cli.FundAddress(user.userAddress, fundAmount) + } + sut.AwaitNextBlock(t) + + t.Cleanup(restoreLEP6SupernodeFixturesAfterMatrix(t)) + cmds := StartLEP6Supernodes(t) + t.Cleanup(func() { StopAllSupernodes(cmds) }) + waitForLEP6SupernodePorts(t, 0, 30*time.Second) + waitForLEP6SupernodePorts(t, 1, 30*time.Second) + waitForLEP6SupernodePorts(t, 2, 30*time.Second) + time.Sleep(40 * time.Second) // allow supernode P2P/DHT routing to settle before upload + + t.Log("Phase 4A Step 3: finalize a CASCADE action before restart") + before := uploadSingleRestartCascade(t, cli, uploadUsers[0], lumeraGRPCAddr, lumeraChainID, actionType, "before-restart") + + t.Log("Phase 4A Step 4: kill and restart supernode 1 with the same basedir") + killLEP6Supernode(t, cmds, 0) + assertLEP6SQLiteIntegrity(t, filepath.Join(WorkDir, "supernode-lep6-data1")) + cmds[0] = restartLEP6Supernode(t, 0) + waitForLEP6SupernodePorts(t, 0, 45*time.Second) + sut.AwaitNextBlock(t) + time.Sleep(20 * time.Second) // let restarted node rejoin P2P routing tables + + t.Log("Phase 4A Step 5: prove pre-restart CASCADE remains byte-identical after restart") + ctx := context.Background() + _, beforeClient := newLEP6ActionClientsForKey(t, ctx, before.keyName, before.mnemonic, lumeraGRPCAddr, lumeraChainID) + downloadAndAssertCascadeBytes(t, ctx, beforeClient, before.actionID, before.userAddress, t.TempDir(), before.payload, before.hash) + + t.Log("Phase 4A Step 6: finalize and download fresh CASCADE work after restart") + after := uploadSingleRestartCascade(t, cli, uploadUsers[1], lumeraGRPCAddr, lumeraChainID, actionType, "after-restart") + _, afterClient := newLEP6ActionClientsForKey(t, ctx, after.keyName, after.mnemonic, lumeraGRPCAddr, lumeraChainID) + downloadAndAssertCascadeBytes(t, ctx, afterClient, after.actionID, after.userAddress, t.TempDir(), after.payload, after.hash) + + t.Log("Phase 4A Step 7: assert restarted runtime logs/state stayed clean") + assertLEP6SQLiteIntegrity(t, filepath.Join(WorkDir, "supernode-lep6-data1")) + assertLEP6SupernodeLogsDoNotContain(t, []string{"database is locked", "panic:", "duplicate proof report"}) +} + +func runLEP6CrashRestartMidHealResumeOrReclaim(t *testing.T) { + t.Helper() + os.Setenv("INTEGRATION_TEST", "true") + os.Setenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER", "1") + t.Cleanup(func() { + os.Unsetenv("INTEGRATION_TEST") + os.Unsetenv("LUMERA_SUPERNODE_DISABLE_HOST_REPORTER") + }) + + const ( + epochLengthBlocks = uint64(12) + originHeight = int64(1) + lumeraGRPCAddr = "localhost:9090" + lumeraChainID = "testing" + fundAmount = "10000000ulume" + actionType = "CASCADE" + ) + + t.Log("Phase 4B Step 1: configure genesis and start real chain") + sut.ModifyGenesisJSON(t, + SetStakingBondDenomUlume(t), + SetActionParams(t), + SetSupernodeMetricsParams(t), + setSupernodeParamsForAuditTests(t), + setAuditParamsForFastEpochs(t, epochLengthBlocks, 1, 1, 1, []uint32{4444}), + setAuditMissingReportGraceForRuntimeE2E(t), + setStorageTruthTestParams(t, "STORAGE_TRUTH_ENFORCEMENT_MODE_FULL", 1000, 500, 10, 0, 10), + ) + sut.StartChain(t) + cli := NewLumeradCLI(t, sut, true) + + binaryPath := locateExecutable(sut.ExecBinary) + homePath := filepath.Join(WorkDir, sut.outputDir) + supernodeUsers := []struct { + keyName string + mnemonic string + }{ + {keyName: "testkey1", mnemonic: "odor kiss switch swarm spell make planet bundle skate ozone path planet exclude butter atom ahead angle royal shuffle door prevent merry alter robust"}, + {keyName: "testkey2", mnemonic: "club party current length duck agent love into slide extend spawn sentence kangaroo chunk festival order plate rare public good include situate liar miss"}, + {keyName: "testkey3", mnemonic: "young envelope urban crucial denial zone toward mansion protect bonus exotic puppy resource pistol expand tell cupboard radio hurry world radio trust explain million"}, + } + for _, user := range supernodeUsers { + recoverChainKey(t, binaryPath, homePath, user.keyName, user.mnemonic) + } + + t.Log("Phase 4B Step 2: register/fund real runtime supernodes and upload user") + n0 := getRuntimeSupernodeIdentity(t, cli, "node0", "testkey1") + n1 := getRuntimeSupernodeIdentity(t, cli, "node1", "testkey2") + n2 := getRuntimeSupernodeIdentity(t, cli, "node2", "testkey3") + nodes := []testNodeIdentity{n0, n1, n2} + registerRuntimeSupernode(t, cli, "node0", n0, "localhost:4444", "4445") + registerRuntimeSupernode(t, cli, "node1", n1, "localhost:4446", "4447") + registerRuntimeSupernode(t, cli, "node2", n2, "localhost:4448", "4449") + for _, node := range nodes { + cli.FundAddress(node.accAddr, fundAmount) + } + bootstrapRuntimeSupernodeEligibility(t, cli) + sut.AwaitNextBlock(t) + + uploadUsers := generateLEP6UploadUsers(t, 1) + cli.FundAddress(uploadUsers[0].userAddress, fundAmount) + sut.AwaitNextBlock(t) + + t.Cleanup(restoreLEP6SupernodeFixturesAfterMatrix(t)) + cmds := StartLEP6Supernodes(t) + t.Cleanup(func() { StopAllSupernodes(cmds) }) + for idx := range nodes { + waitForLEP6SupernodePorts(t, idx, 30*time.Second) + } + time.Sleep(40 * time.Second) // allow supernode P2P/DHT routing to settle before upload + + t.Log("Phase 4B Step 3: finalize CASCADE action and prove baseline download") + upload := uploadSingleRestartCascade(t, cli, uploadUsers[0], lumeraGRPCAddr, lumeraChainID, actionType, "phase4b-heal-restart") + ctx := context.Background() + lumeraClient, actionClient := newLEP6ActionClientsForKey(t, ctx, upload.keyName, upload.mnemonic, lumeraGRPCAddr, lumeraChainID) + t.Cleanup(func() { lumeraClient.Close() }) + artifactCounts := requireFinalizedCascadeArtifactCounts(t, ctx, lumeraClient, upload.actionID) + downloadAndAssertCascadeBytes(t, ctx, actionClient, upload.actionID, upload.userAddress, t.TempDir(), upload.payload, upload.hash) + + t.Log("Phase 4B Step 4: submit deterministic storage-proof failures to schedule heal op") + currentHeight := sut.AwaitNextBlock(t) + epochID, epochStart := nextEpochAfterHeight(originHeight, epochLengthBlocks, currentHeight) + epochEnd := epochStart + int64(epochLengthBlocks) + awaitAtLeastHeight(t, epochStart) + anchor := awaitCurrentEpochAnchorWithActiveSupernodes(t, epochID, n0.accAddr, n1.accAddr, n2.accAddr) + require.ElementsMatch(t, []string{n0.accAddr, n1.accAddr, n2.accAddr}, anchor.ActiveSupernodeAccounts) + + proberResp, prober, target := findAssignedProberAndTarget(t, epochID, nodes) + portStates := openPortStates(proberResp.RequiredOpenPorts) + reportArgs := []string{ + "tx", "audit", "submit-epoch-report", + strconv.FormatUint(epochID, 10), + auditHostReportJSON(portStates), + "--from", prober.nodeName, + "--gas", "500000", + } + for _, assignedTarget := range proberResp.TargetSupernodeAccounts { + reportArgs = append(reportArgs, "--storage-challenge-observations", storageChallengeObservationJSON(assignedTarget, portStates)) + } + reportArgs = append(reportArgs, + "--storage-proof-results", buildStorageProofResultJSONWithClassAndCount( + prober.accAddr, + target.accAddr, + upload.actionID, + "phase4b-recent-hash-mismatch", + "STORAGE_PROOF_BUCKET_TYPE_RECENT", + "STORAGE_PROOF_RESULT_CLASS_HASH_MISMATCH", + artifactCounts.index, + ), + "--storage-proof-results", buildStorageProofResultJSONWithClassAndCount( + prober.accAddr, + target.accAddr, + upload.actionID, + "phase4b-old-hash-mismatch", + "STORAGE_PROOF_BUCKET_TYPE_OLD", + "STORAGE_PROOF_RESULT_CLASS_HASH_MISMATCH", + artifactCounts.index, + ), + ) + reportResp := cli.CustomCommand(reportArgs...) + RequireTxSuccess(t, reportResp) + sut.AwaitNextBlock(t) + + ticketBefore, found := auditQueryTicketDeteriorationStateST(t, upload.actionID) + require.True(t, found, "storage challenge failure must create deterioration state") + require.GreaterOrEqual(t, ticketBefore.DeteriorationScore, int64(10), "ticket score must cross heal threshold before scheduling") + + t.Log("Phase 4B Step 5: wait for healer report, then crash/restart assigned healer before verification completes") + awaitAtLeastHeight(t, epochEnd) + sut.AwaitNextBlock(t) + healOps := auditQueryHealOpsByTicketST(t, upload.actionID) + require.Len(t, healOps, 1, "chain must schedule one heal op for the deteriorated CASCADE action ticket") + scheduled := healOps[0] + require.NotEmpty(t, scheduled.HealerSupernodeAccount) + require.NotEmpty(t, scheduled.VerifierSupernodeAccounts) + healerIdx := lep6SupernodeIndexForAccount(t, scheduled.HealerSupernodeAccount, nodes) + verifierIdxs := make([]int, 0, len(scheduled.VerifierSupernodeAccounts)) + for _, verifier := range scheduled.VerifierSupernodeAccounts { + verifierIdx := lep6SupernodeIndexForAccount(t, verifier, nodes) + require.NotEqual(t, healerIdx, verifierIdx, "Phase 4B requires distinct healer and verifier so verification can be paused deterministically") + verifierIdxs = append(verifierIdxs, verifierIdx) + } + for _, verifierIdx := range verifierIdxs { + killLEP6Supernode(t, cmds, verifierIdx) + } + + reported := awaitHealOpStatusByTicket(t, upload.actionID, scheduled.HealOpId, audittypes.HealOpStatus_HEAL_OP_STATUS_HEALER_REPORTED, 4*time.Minute) + require.NotEmpty(t, reported.ResultHash, "healer must publish reconstructed manifest hash before crash") + healerDataDir := dataDirForSupernodeAccount(t, reported.HealerSupernodeAccount, nodes...) + stagingDir := locateLEP6HealStagingDir(t, healerDataDir, reported.HealOpId) + require.DirExists(t, stagingDir, "healer-reported op must have persisted staging before crash") + + killLEP6Supernode(t, cmds, healerIdx) + assertLEP6SQLiteIntegrity(t, filepath.Join(WorkDir, fmt.Sprintf("supernode-lep6-data%d", healerIdx+1))) + cmds[healerIdx] = restartLEP6Supernode(t, healerIdx) + waitForLEP6SupernodePorts(t, healerIdx, 45*time.Second) + for _, verifierIdx := range verifierIdxs { + cmds[verifierIdx] = restartLEP6Supernode(t, verifierIdx) + waitForLEP6SupernodePorts(t, verifierIdx, 45*time.Second) + } + sut.AwaitNextBlock(t) + + t.Log("Phase 4B Step 6: assert restarted healer keeps persisted claim state available without requiring DHT publish") + time.Sleep(20 * time.Second) + currentOps := auditQueryHealOpsByTicketST(t, upload.actionID) + var current audittypes.HealOp + for _, op := range currentOps { + if op.HealOpId == reported.HealOpId { + current = op + break + } + } + require.Equal(t, reported.HealOpId, current.HealOpId, "restarted healer op must remain queryable") + require.Equal(t, reported.ResultHash, current.ResultHash, "restart path must preserve healer manifest hash") + require.NotContains(t, []audittypes.HealOpStatus{ + audittypes.HealOpStatus_HEAL_OP_STATUS_FAILED, + audittypes.HealOpStatus_HEAL_OP_STATUS_EXPIRED, + }, current.Status, "restart path must not drive the heal op to a terminal failure") + require.DirExists(t, stagingDir, "restart path must preserve staging for retry; DHT publish/cleanup is covered outside Phase 4B") + assertLEP6SupernodeLogsDoNotContain(t, []string{"database is locked", "panic:", "duplicate proof report", "orphaned heal"}) +} + +func locateLEP6HealStagingDir(t *testing.T, dataDir string, healOpID uint64) string { + t.Helper() + candidates := []string{ + filepath.Join(dataDir, "heal-staging", fmt.Sprintf("%d", healOpID)), + filepath.Join(dataDir, filepath.Base(dataDir), "heal-staging", fmt.Sprintf("%d", healOpID)), + } + for _, candidate := range candidates { + if info, err := os.Stat(candidate); err == nil && info.IsDir() { + return candidate + } + } + t.Fatalf("no heal staging dir for op %d under %s; checked %v", healOpID, dataDir, candidates) + return "" +} + +func lep6SupernodeIndexForAccount(t *testing.T, account string, nodes []testNodeIdentity) int { + t.Helper() + for idx, node := range nodes { + if node.accAddr == account { + return idx + } + } + t.Fatalf("no LEP-6 supernode fixture index for account %s", account) + return -1 +} + +func uploadSingleRestartCascade(t *testing.T, cli *LumeradCli, user lep6UploadUser, lumeraGRPCAddr, lumeraChainID, actionType, label string) concurrentCascadeUpload { + t.Helper() + ctx := context.Background() + lumeraClient, actionClient := newLEP6ActionClientsForKey(t, ctx, user.keyName, user.mnemonic, lumeraGRPCAddr, lumeraChainID) + t.Cleanup(func() { lumeraClient.Close() }) + + payload := []byte(fmt.Sprintf("lep6 phase4a restart cascade payload %s\n%s\n", label, strings.Repeat(label+"-chunk-", 64))) + path := filepath.Join(t.TempDir(), fmt.Sprintf("phase4a-%s.txt", label)) + require.NoError(t, os.WriteFile(path, payload, 0o600)) + + actionID := requestAndStartCascadeAction(t, ctx, cli, lumeraClient, actionClient, path, actionType) + require.NoError(t, waitForActionStateWithClient(ctx, lumeraClient, actionID, actiontypes.ActionStateDone)) + requireFinalizedCascadeArtifactCounts(t, ctx, lumeraClient, actionID) + return concurrentCascadeUpload{ + actionID: actionID, + keyName: user.keyName, + mnemonic: user.mnemonic, + userAddress: user.userAddress, + payload: payload, + hash: sha256.Sum256(payload), + } +} + +func killLEP6Supernode(t *testing.T, cmds []*exec.Cmd, idx int) { + t.Helper() + require.Greater(t, len(cmds), idx) + require.NotNil(t, cmds[idx]) + require.NotNil(t, cmds[idx].Process) + pid := cmds[idx].Process.Pid + t.Logf("killing LEP-6 supernode %d pid=%d", idx+1, pid) + require.NoError(t, cmds[idx].Process.Kill()) + _, err := cmds[idx].Process.Wait() + if err != nil && !strings.Contains(err.Error(), "waitid: no child processes") { + t.Logf("wait after killing supernode %d returned: %v", idx+1, err) + } +} + +func restartLEP6Supernode(t *testing.T, idx int) *exec.Cmd { + t.Helper() + wd, err := os.Getwd() + require.NoError(t, err) + dataDir := filepath.Join(wd, fmt.Sprintf("supernode-lep6-data%d", idx+1)) + binPath := filepath.Join(dataDir, "supernode") + if _, err := os.Stat(binPath); os.IsNotExist(err) { + t.Fatalf("supernode binary not found at %s; did you run setup?", binPath) + } + logPath := filepath.Join(wd, fmt.Sprintf("supernode-lep6%d.out", idx)) + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + require.NoError(t, err) + t.Cleanup(func() { _ = logFile.Close() }) + + cmd := exec.Command(binPath, "start", "--basedir", dataDir) + cmd.Stdout = io.MultiWriter(os.Stdout, logFile) + cmd.Stderr = io.MultiWriter(os.Stderr, logFile) + t.Logf("restarting supernode %d from directory: %s", idx+1, dataDir) + require.NoError(t, cmd.Start()) + return cmd +} + +func waitForLEP6SupernodePorts(t *testing.T, idx int, timeout time.Duration) { + t.Helper() + ports := []int{4444 + idx*2, 4445 + idx*2} + for _, port := range ports { + port := port + require.Eventually(t, func() bool { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 500*time.Millisecond) + if err != nil { + return false + } + _ = conn.Close() + return true + }, timeout, time.Second, "supernode %d port %d should become reachable", idx+1, port) + } +} + +func assertLEP6SQLiteIntegrity(t *testing.T, baseDir string) { + t.Helper() + var sqliteFiles []string + require.NoError(t, filepath.WalkDir(baseDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + name := strings.ToLower(d.Name()) + if strings.HasSuffix(name, ".sqlite") || strings.HasSuffix(name, ".sqlite3") || strings.HasSuffix(name, ".db") { + sqliteFiles = append(sqliteFiles, path) + } + return nil + })) + require.NotEmpty(t, sqliteFiles, "expected at least one SQLite file under %s", baseDir) + for _, path := range sqliteFiles { + db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?mode=ro&_busy_timeout=5000", path)) + require.NoError(t, err, "open sqlite %s", path) + var result string + err = db.QueryRow("PRAGMA integrity_check").Scan(&result) + closeErr := db.Close() + require.NoError(t, err, "integrity_check %s", path) + require.NoError(t, closeErr, "close sqlite %s", path) + require.Equal(t, "ok", strings.ToLower(result), "sqlite integrity_check %s", path) + } +} diff --git a/tests/system/e2e_lep6_runtime_test.go b/tests/system/e2e_lep6_runtime_test.go index 5fd708c2..3657f43d 100644 --- a/tests/system/e2e_lep6_runtime_test.go +++ b/tests/system/e2e_lep6_runtime_test.go @@ -431,13 +431,26 @@ func downloadAndAssertCascadeBytes(t *testing.T, ctx context.Context, ac action. t.Helper() sig, err := ac.GenerateDownloadSignature(ctx, actionID, userAddress) require.NoError(t, err) - _, err = ac.DownloadCascade(ctx, actionID, outputBaseDir, sig) + taskID, err := ac.DownloadCascade(ctx, actionID, outputBaseDir, sig) require.NoError(t, err) outDir := filepath.Join(outputBaseDir, actionID) + require.Eventually(t, func() bool { + entry, ok := ac.GetTask(ctx, taskID) + if !ok { + return false + } + switch string(entry.Status) { + case "COMPLETED": + return true + case "FAILED": + t.Fatalf("download task %s failed for action %s: %v; events=%+v", taskID, actionID, entry.Error, entry.Events) + } + return false + }, 90*time.Second, time.Second, "download task should complete") require.Eventually(t, func() bool { entries, err := os.ReadDir(outDir) return err == nil && len(entries) > 0 - }, 45*time.Second, time.Second, "download output directory should contain reconstructed file") + }, 15*time.Second, time.Second, "download output directory should contain reconstructed file after completed task") entries, err := os.ReadDir(outDir) require.NoError(t, err) var downloadedPath string diff --git a/tests/system/go.mod b/tests/system/go.mod index 3222e311..e6d987c3 100644 --- a/tests/system/go.mod +++ b/tests/system/go.mod @@ -13,8 +13,10 @@ require ( cosmossdk.io/math v1.5.3 github.com/LumeraProtocol/lumera v1.12.0 github.com/LumeraProtocol/supernode/v2 v2.0.0-00010101000000-000000000000 + github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce github.com/cometbft/cometbft v0.38.21 github.com/cosmos/ibc-go/v10 v10.5.0 + github.com/mattn/go-sqlite3 v1.14.24 github.com/tidwall/gjson v1.14.2 github.com/tidwall/sjson v1.2.5 golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b @@ -62,6 +64,7 @@ require ( github.com/LumeraProtocol/rq-go v0.2.1 // indirect github.com/Masterminds/semver/v3 v3.3.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bytedance/sonic v1.14.2 // indirect @@ -128,6 +131,8 @@ require ( github.com/improbable-eng/grpc-web v0.15.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -137,10 +142,13 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/highwayhash v1.0.3 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oasisprotocol/curve25519-voi v0.0.0-20230904125328-1f23a7beb09a // indirect github.com/oklog/run v1.1.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -169,6 +177,7 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/mock v0.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go.uber.org/ratelimit v0.3.1 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/tests/system/go.sum b/tests/system/go.sum index a721fde4..6d29dc5b 100644 --- a/tests/system/go.sum +++ b/tests/system/go.sum @@ -123,6 +123,7 @@ github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrd github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/adlio/schema v1.3.6 h1:k1/zc2jNfeiZBA5aFTRy37jlBIuCkXCm0XmvpzCKI9I= github.com/adlio/schema v1.3.6/go.mod h1:qkxwLgPBd1FgLRHYVCmQT/rrBr3JH38J9LjmVzWNudg= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -143,6 +144,8 @@ github.com/aws/aws-sdk-go v1.49.0 h1:g9BkW1fo9GqKfwg2+zCD+TW/D36Ux+vtfJ8guF4AYmY github.com/aws/aws-sdk-go v1.49.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -154,12 +157,21 @@ github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE5 github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.24.3 h1:Bte86SlO3lwPQqww+7BE9ZuUCKIjfqnG5jtEyqA9y9Y= github.com/bits-and-blooms/bitset v1.24.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd/btcec/v2 v2.3.5 h1:dpAlnAwmT1yIBm3exhT1/8iUSD98RDJM5vqJVQDQLiU= github.com/btcsuite/btcd/btcec/v2 v2.3.5/go.mod h1:m22FrOAiuxl/tht9wIqAoGHcbnCCaPWyauO8y2LGGtQ= github.com/btcsuite/btcd/btcutil v1.1.6 h1:zFL2+c3Lb9gEgqKNzowKUPQNb8jV7v5Oaodi/AYFd6c= github.com/btcsuite/btcd/btcutil v1.1.6/go.mod h1:9dFymx8HpuLqBnsPELrImQeTQfKBQqzqGbbV3jK55aE= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bufbuild/protoc-gen-validate v1.3.0 h1:0lq2b9qA1uzfVnMW6oFJepiVVihDOOzj+VuTGSX4EgE= github.com/bufbuild/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= @@ -264,6 +276,7 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -369,6 +382,8 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= @@ -579,6 +594,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5XumQh94= github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -586,8 +602,11 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -605,6 +624,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= @@ -656,6 +676,9 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= @@ -734,6 +757,8 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -951,6 +976,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= @@ -961,6 +988,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -973,6 +1002,7 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -981,6 +1011,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=