From ce4a5a774f281b08fcc103ca5c43777db9cd94a5 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:29:12 -0300 Subject: [PATCH 01/11] feat(machine): expose complete terminal state proofs --- pkg/emulator/emulator.go | 2 + pkg/emulator/machine.go | 12 +- pkg/emulator/remote.go | 2 + pkg/emulator/types.go | 16 +- pkg/machine/backend.go | 15 +- pkg/machine/backend_test.go | 34 ++- pkg/machine/doc.go | 10 +- pkg/machine/implementation.go | 211 ++++++++++--- pkg/machine/implementation_test.go | 467 +++++++++++++++++------------ pkg/machine/libcartesi.go | 42 ++- pkg/machine/libcartesi_test.go | 104 ++++++- pkg/machine/machine.go | 88 ++++-- pkg/machine/machine_test.go | 78 +++-- pkg/machine/util_test.go | 4 +- 14 files changed, 755 insertions(+), 330 deletions(-) diff --git a/pkg/emulator/emulator.go b/pkg/emulator/emulator.go index 3761aabdd..fe0c6aeee 100644 --- a/pkg/emulator/emulator.go +++ b/pkg/emulator/emulator.go @@ -4,6 +4,8 @@ // This package is a binding to the emulator's C API. // Refer to the machine-c files in the emulator's repository for documentation // (mainly machine-c-api.h and jsonrpc-machine-c-api.h). +// +//nolint:gocritic // CGo output-parameter wrappers trigger dupSubExpr false positives. package emulator // #cgo LDFLAGS: -lcartesi -lcartesi_jsonrpc diff --git a/pkg/emulator/machine.go b/pkg/emulator/machine.go index 5c3cb09df..dc8f27803 100644 --- a/pkg/emulator/machine.go +++ b/pkg/emulator/machine.go @@ -4,6 +4,8 @@ // This package is a binding to the emulator's C API. // Refer to the machine-c files in the emulator's repository for documentation // (mainly machine-c-api.h and jsonrpc-machine-c-api.h). +// +//nolint:gocritic // CGo output-parameter wrappers trigger dupSubExpr false positives. package emulator // #include @@ -143,13 +145,19 @@ func (m *Machine) GetInitialConfig() (string, error) { } // get_proof -func (m *Machine) GetProof(address uint64, log2size int32) (string, error) { +func (m *Machine) GetProof(address uint64, log2TargetSize, log2RootSize int32) (string, error) { var proof *C.char var err error var res string m.callCAPI(func() { - err = newError(C.cm_get_proof(m.ptr, C.uint64_t(address), C.int32_t(log2size), C.int32_t(HashTreeLog2RootSize), &proof)) + err = newError(C.cm_get_proof( + m.ptr, + C.uint64_t(address), + C.int32_t(log2TargetSize), + C.int32_t(log2RootSize), + &proof, + )) if err != nil || proof == nil { return } diff --git a/pkg/emulator/remote.go b/pkg/emulator/remote.go index 551c9e5ff..4911a6bed 100644 --- a/pkg/emulator/remote.go +++ b/pkg/emulator/remote.go @@ -4,6 +4,8 @@ // This package is a binding to the emulator's C API. // Refer to the machine-c files in the emulator's repository for documentation // (mainly machine-c-api.h and jsonrpc-machine-c-api.h). +// +//nolint:gocritic // CGo output-parameter wrappers trigger dupSubExpr false positives. package emulator // #include diff --git a/pkg/emulator/types.go b/pkg/emulator/types.go index 97a5bf361..eb405fe09 100644 --- a/pkg/emulator/types.go +++ b/pkg/emulator/types.go @@ -4,6 +4,8 @@ // This package is a binding to the emulator's C API. // Refer to the machine-c files in the emulator's repository for documentation // (mainly machine-c-api.h and jsonrpc-machine-c-api.h). +// +//nolint:revive // Exported ALL_CAPS names intentionally mirror the emulator C API. package emulator // #include @@ -28,8 +30,8 @@ const ( ErrCodeRangeError ErrorCode = C.CM_ERROR_RANGE_ERROR ErrCodeOverflowError ErrorCode = C.CM_ERROR_OVERFLOW_ERROR ErrCodeUnderflowError ErrorCode = C.CM_ERROR_UNDERFLOW_ERROR - //ErrCodeRegexError ErrorCode = C.CM_ERROR_REGEX_ERROR - //ErrCodeSystemError ErrorCode = C.CM_ERROR_SYSTEM_ERROR + // ErrCodeRegexError ErrorCode = C.CM_ERROR_REGEX_ERROR + // ErrCodeSystemError ErrorCode = C.CM_ERROR_SYSTEM_ERROR ErrCodeBadTypeid ErrorCode = C.CM_ERROR_BAD_TYPEID ErrCodeBadCast ErrorCode = C.CM_ERROR_BAD_CAST ErrCodeBadAnyCast ErrorCode = C.CM_ERROR_BAD_ANY_CAST @@ -101,6 +103,10 @@ func (reason BreakReason) String() (s string) { s = "yielded softly" case BreakReasonReachedTargetMcycle: s = "reached target mcycle" + case BreakReasonConsoleOutput: + s = "console output" + case BreakReasonConsoleInput: + s = "console input" case BreakReasonMcycleOverflow: s = "mcycle overflow" default: @@ -125,6 +131,11 @@ type ( ) const ( + HtifDeviceShift uint64 = C.CM_HTIF_DEV_SHIFT + HtifCommandShift uint64 = C.CM_HTIF_CMD_SHIFT + HtifReasonShift uint64 = C.CM_HTIF_REASON_SHIFT + HtifDeviceYield uint64 = C.CM_HTIF_DEV_YIELD + // type YieldAutomatic CmioYieldCommand = C.CM_HTIF_YIELD_CMD_AUTOMATIC YieldManual CmioYieldCommand = C.CM_HTIF_YIELD_CMD_MANUAL @@ -358,5 +369,6 @@ func NewMachineRuntimeConfig() *MachineRuntimeConfig { } const ( + HashTreeLog2WordSize uint32 = C.CM_HASH_TREE_LOG2_WORD_SIZE HashTreeLog2RootSize uint32 = C.CM_HASH_TREE_LOG2_ROOT_SIZE ) diff --git a/pkg/machine/backend.go b/pkg/machine/backend.go index 7ccd99ef7..c340b734f 100644 --- a/pkg/machine/backend.go +++ b/pkg/machine/backend.go @@ -29,6 +29,18 @@ type HashCollectorState struct { ConsoleIOError string } +// MemoryProof is the complete Merkle proof returned by the emulator for a +// memory range. Keeping the root and target metadata allows callers to verify +// that a proof was produced for the requested machine state and address. +type MemoryProof struct { + Log2RootSize int32 + Log2TargetSize int32 + RootHash Hash + Siblings []Hash + TargetAddress uint64 + TargetHash Hash +} + // This Backend interface covers the methods used from the emulator / remote machine server. // It is to abstract the emulator package and allow for easier testing and mocking in unit tests. type Backend interface { @@ -49,9 +61,10 @@ type Backend interface { ReceiveCmioRequest(timeout time.Duration) (cmd uint8, reason uint16, data []byte, err error) WriteMemory(address uint64, data []byte, timeout time.Duration) error + ReadMemory(address uint64, length uint64, timeout time.Duration) ([]byte, error) GetRootHash(timeout time.Duration) (Hash, error) - GetProof(address uint64, log2size int32, timeout time.Duration) ([]Hash, error) + GetProof(address uint64, log2TargetSize, log2RootSize int32, timeout time.Duration) (MemoryProof, error) Delete() ForkServer(timeout time.Duration) (Backend, string, uint32, error) diff --git a/pkg/machine/backend_test.go b/pkg/machine/backend_test.go index 50f11ea69..57d54dbac 100644 --- a/pkg/machine/backend_test.go +++ b/pkg/machine/backend_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/mock" ) +const testMachineAddress = "127.0.0.1:12345" + // MockBackend is a testify mock implementation of the Backend interface type MockBackend struct { mock.Mock @@ -42,10 +44,11 @@ func (m *MockBackend) ReadMCycle(timeout time.Duration) (uint64, error) { func (m *MockBackend) SendCmioResponse(reason uint16, data []byte, revertRootHash *Hash, timeout time.Duration) error { args := mock.Arguments{} - if requestType(reason) == AdvanceStateRequest { + switch reason { + case uint16(AdvanceStateRequest): // match hash by value on advance args = m.Called(reason, data, *revertRootHash, timeout) - } else if requestType(reason) == InspectStateRequest { + case uint16(InspectStateRequest): // nil on inspect args = m.Called(reason, data, nil, timeout) } @@ -67,6 +70,14 @@ func (m *MockBackend) WriteMemory(address uint64, data []byte, timeout time.Dura return args.Error(0) } +func (m *MockBackend) ReadMemory(address uint64, length uint64, timeout time.Duration) ([]byte, error) { + args := m.Called(address, length, timeout) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]byte), args.Error(1) +} + func (m *MockBackend) Delete() { m.Called() } @@ -91,14 +102,23 @@ func (m *MockBackend) CmioRxBufferSize() uint64 { return args.Get(0).(uint64) } -func (m *MockBackend) RunAndCollectRootHashes(mcycleEnd uint64, state *HashCollectorState, timeout time.Duration) (reason BreakReason, err error) { +func (m *MockBackend) RunAndCollectRootHashes( + mcycleEnd uint64, + state *HashCollectorState, + timeout time.Duration, +) (reason BreakReason, err error) { args := m.Called(mcycleEnd, state, timeout) return args.Get(0).(BreakReason), args.Error(1) } -func (m *MockBackend) GetProof(address uint64, log2size int32, timeout time.Duration) ([]Hash, error) { - args := m.Called(address, log2size, timeout) - return args.Get(0).([]Hash), args.Error(1) +func (m *MockBackend) GetProof( + address uint64, + log2TargetSize, + log2RootSize int32, + timeout time.Duration, +) (MemoryProof, error) { + args := m.Called(address, log2TargetSize, log2RootSize, timeout) + return args.Get(0).(MemoryProof), args.Error(1) } // Helper functions for setting up common mock scenarios @@ -185,7 +205,7 @@ func NewMockBackend() *MockBackend { // MockBackendFactory creates a backend factory that returns the provided mock func MockBackendFactory(backend *MockBackend) BackendFactory { return func(_ string, _ time.Duration) (Backend, string, uint32, error) { - return backend, "127.0.0.1:12345", 12345, nil + return backend, testMachineAddress, 12345, nil } } diff --git a/pkg/machine/doc.go b/pkg/machine/doc.go index cf70cfba7..0909911a6 100644 --- a/pkg/machine/doc.go +++ b/pkg/machine/doc.go @@ -6,10 +6,12 @@ // // A request that reaches a deterministic guest completion returns a response // value with CompletionStatusAccepted, CompletionStatusRejected, -// CompletionStatusException, or CompletionStatusHalted. Anything that prevents -// completion—including deadlines, local resource limits, backend failures, and -// cycle exhaustion—returns an error. Advance then returns no response; Inspect -// may return partial reports with CompletionStatusUnknown. In short: terminal +// CompletionStatusException, CompletionStatusHalted, +// CompletionStatusOverflow, or CompletionStatusUnexpectedYield. Anything that +// prevents completion—including deadlines, local resource limits, backend +// failures, and configured cycle exhaustion—returns an error. Advance then +// returns no response; Inspect may return partial reports with +// CompletionStatusUnknown. In short: terminal // guest outcomes travel as values; incomplete execution travels as an error. // Callers decide how a completed outcome affects canonical application state. // diff --git a/pkg/machine/implementation.go b/pkg/machine/implementation.go index bceae6a61..1fddb5e5d 100644 --- a/pkg/machine/implementation.go +++ b/pkg/machine/implementation.go @@ -5,6 +5,7 @@ package machine import ( "context" + "encoding/binary" "errors" "fmt" "log/slog" @@ -13,6 +14,7 @@ import ( "time" "github.com/cartesi/rollups-node/internal/model" + "github.com/ethereum/go-ethereum/crypto" ) // RequestType represents the type of request to send to the machine @@ -88,8 +90,26 @@ const ( const maxOutputs = 65536 // 2^16 const maxReports = 65536 // 2^16 -const TxBufferAddress uint64 = 0x60800000 -const HashLog2Size = 5 // 32 bytes +const ( + // These addresses and hash-tree sizes are defined by the Cartesi Machine + // emulator. + iflagsYAddress uint64 = 0x308 + htifTohostAddress uint64 = 0x330 + TxBufferAddress uint64 = 0x60800000 + HashLog2Size = 5 // 32-byte data block + machineMemoryLog2Size int32 = 64 + memoryProofSiblingCount = int(machineMemoryLog2Size) - HashLog2Size +) + +const ( + // These values encode the HTIF fields proved by the accepted-state check. + htifDeviceYield uint64 = 2 + htifCommandManual uint64 = 1 + htifReasonInputAccepted uint64 = 1 + htifDeviceShift = 56 + htifCommandShift = 48 + htifReasonShift = 32 +) // machineImpl implements the Machine interface by wrapping an emulator.RemoteMachine type machineImpl struct { @@ -141,51 +161,154 @@ func (m *machineImpl) Hash(ctx context.Context) (Hash, error) { return hash, nil } -// OutputsHash returns the outputs hash stored in the cmio tx buffer -func (m *machineImpl) OutputsHash(ctx context.Context) (Hash, error) { - result, err := m.readManualYieldResult(ctx) - if err != nil { - err = fmt.Errorf("could not read the outputs hash: %w", err) - return Hash{}, err +func (m *machineImpl) StateProof(ctx context.Context) (*StateProof, error) { + if err := checkContext(ctx); err != nil { + return nil, err } - - switch result.status { - case CompletionStatusAccepted: - // Intentionally empty. - case CompletionStatusRejected: - return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrRejected) - case CompletionStatusException: - return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrException) - case CompletionStatusHalted: - return Hash{}, fmt.Errorf("could not read the outputs hash: %w", ErrHalted) - case CompletionStatusUnknown: - return Hash{}, fmt.Errorf( - "could not read the outputs hash with completion status %d: %w", - result.status, + machineHash, err := m.backend.GetRootHash(m.params.LoadDeadline) + if err != nil { + return nil, errors.Join( ErrMachineInternal, + fmt.Errorf("could not get the machine root for its validity proof: %w", err), ) } - if length := len(result.data); length != HashSize { - err = fmt.Errorf("invalid outputs hash: %w (it has %d bytes)", ErrHashLength, length) - return Hash{}, err + iflagsYProof, err := m.readLeafProof(ctx, machineHash, iflagsYAddress) + if err != nil { + return nil, fmt.Errorf("could not prove iflags_Y: %w", err) + } + htifTohostProof, err := m.readLeafProof(ctx, machineHash, htifTohostAddress) + if err != nil { + return nil, fmt.Errorf("could not prove HTIF tohost: %w", err) } + txBufferProof, err := m.readLeafProof(ctx, machineHash, TxBufferAddress) + if err != nil { + return nil, fmt.Errorf("could not prove the CMIO TX buffer: %w", err) + } + + return &StateProof{ + MachineHash: machineHash, + IflagsYProof: iflagsYProof, + HtifTohostProof: htifTohostProof, + TxBufferProof: txBufferProof, + }, nil +} - var outputsHash Hash - copy(outputsHash[:], result.data) - return outputsHash, nil +// ValidateAcceptedState checks the state semantics required when an epoch +// is published through the released v3 contracts. StateProof itself remains +// generic so the exact post-run proof can also be persisted for terminal +// outcomes. +func ValidateAcceptedState(proof *StateProof) error { + if proof == nil { + return fmt.Errorf("state proof is nil: %w", ErrInvalidMachineProof) + } + if readMachineWord(proof.IflagsYProof.DataBlock, iflagsYAddress) == 0 { + return fmt.Errorf("iflags_Y is zero: %w", ErrInvalidMachineProof) + } + tohost := readMachineWord(proof.HtifTohostProof.DataBlock, htifTohostAddress) + if !isAcceptedManualYield(tohost) { + return fmt.Errorf("HTIF tohost does not signal an accepted manual yield: %w", ErrInvalidMachineProof) + } + return nil } -func (m *machineImpl) OutputsHashProof(ctx context.Context) ([]Hash, error) { +func (m *machineImpl) readLeafProof(ctx context.Context, machineHash Hash, wordAddress uint64) (LeafProof, error) { if err := checkContext(ctx); err != nil { - return nil, err + return LeafProof{}, err + } + dataBlockAddress := wordAddress &^ ((uint64(1) << HashLog2Size) - 1) + proof, err := m.backend.GetProof( + dataBlockAddress, + int32(HashLog2Size), + machineMemoryLog2Size, + m.params.LoadDeadline, + ) + if err != nil { + return LeafProof{}, errors.Join(ErrMachineInternal, fmt.Errorf("could not get memory proof: %w", err)) } - siblings, err := m.backend.GetProof(TxBufferAddress, HashLog2Size, m.params.LoadDeadline) + data, err := m.backend.ReadMemory( + dataBlockAddress, + uint64(1)<> HashLog2Size + for _, sibling := range proof.Siblings { + if index&1 == 0 { + root = Hash(crypto.Keccak256Hash(root[:], sibling[:])) + } else { + root = Hash(crypto.Keccak256Hash(sibling[:], root[:])) + } + index >>= 1 + } + if root != machineHash { + return LeafProof{}, fmt.Errorf("proof siblings do not reconstruct the current machine root: %w", ErrInvalidMachineProof) + } + + return LeafProof{ + DataBlock: dataBlock, + Siblings: append([]Hash(nil), proof.Siblings...), + }, nil +} + +func readMachineWord(dataBlock Hash, address uint64) uint64 { + offset := int(address & ((uint64(1) << HashLog2Size) - 1)) + return binary.LittleEndian.Uint64(dataBlock[offset : offset+8]) +} + +func isAcceptedManualYield(tohost uint64) bool { + return tohost>>htifDeviceShift == htifDeviceYield && + (tohost>>htifCommandShift)&0xff == htifCommandManual && + (tohost>>htifReasonShift)&0xffff == htifReasonInputAccepted } // Advance sends an input to the machine and processes it @@ -214,10 +337,11 @@ func (m *machineImpl) Advance(ctx context.Context, input []byte, checkpointHash if length := len(result.completion.data); length != HashSize { return nil, fmt.Errorf("%w (it has %d bytes)", ErrHashLength, length) } - copy(resp.OutputsHash[:], result.completion.data) - } else if resp.Status == CompletionStatusException { + } + if resp.Status == CompletionStatusException { resp.ExceptionData = append([]byte{}, result.completion.data...) } + return resp, nil } @@ -345,8 +469,7 @@ func (m *machineImpl) readManualYieldResult(ctx context.Context) (completionResu case ManualYieldReasonException: return completionResult{status: CompletionStatusException, data: data}, nil default: - err = fmt.Errorf("invalid manual yield reason: %d: %w", yieldReason, ErrMachineInternal) - return completionResult{}, err + return completionResult{status: CompletionStatusUnexpectedYield}, nil } } @@ -423,6 +546,9 @@ func (m *machineImpl) process( case errors.Is(err, ErrHalted): result.completion.status = CompletionStatusHalted return result, nil + case errors.Is(err, ErrMcycleOverflow): + result.completion.status = CompletionStatusOverflow + return result, nil default: return result, err } @@ -522,7 +648,7 @@ func (m *machineImpl) run( case Halted: return finish(ErrHalted, terminalHashAppended) case McycleOverflow: - return fail(executionLimitError(reqType, bounds, currentCycle, ErrMcycleOverflow)) + return finish(ErrMcycleOverflow, terminalHashAppended) case ReachedTargetMcycle, YieldedSoftly: continue case Failed: @@ -867,11 +993,12 @@ func checkContext(ctx context.Context) error { return nil } err := ctx.Err() - if errors.Is(err, context.DeadlineExceeded) { + switch { + case errors.Is(err, context.DeadlineExceeded): return ErrDeadlineExceeded - } else if errors.Is(err, context.Canceled) { + case errors.Is(err, context.Canceled): return ErrCanceled - } else { + default: return err } } diff --git a/pkg/machine/implementation_test.go b/pkg/machine/implementation_test.go index 39df3c38a..b112b7e9b 100644 --- a/pkg/machine/implementation_test.go +++ b/pkg/machine/implementation_test.go @@ -5,6 +5,7 @@ package machine import ( "context" + "encoding/binary" "errors" "fmt" "io" @@ -13,6 +14,7 @@ import ( "time" "github.com/cartesi/rollups-node/internal/model" + "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -49,7 +51,7 @@ func (s *ImplementationSuite) TestFork() { machine := &machineImpl{ backend: mockBackend, - address: "127.0.0.1:12345", + address: testMachineAddress, logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, @@ -69,7 +71,7 @@ func (s *ImplementationSuite) TestFork() { machine2 := &machineImpl{ backend: mockBackend2, - address: "127.0.0.1:12345", + address: testMachineAddress, logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, @@ -135,139 +137,205 @@ func (s *ImplementationSuite) TestHash() { require.ErrorIs(err, ErrCanceled) } -// Test OutputsHash method -func (s *ImplementationSuite) TestOutputsHash() { - require := s.Require() - ctx := context.Background() +func (s *ImplementationSuite) TestStateProof() { + blocks := acceptedStateTestBlocks(1, acceptedTohostTestValue()) + machine, expected := s.machineWithMemoryProofs(blocks) - // Test successful outputs hash retrieval - mockBackend := NewMockBackend() - expectedHash := randomFakeHash() - mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(ManualYieldReasonAccepted), expectedHash[:], nil) + actual, err := machine.StateProof(context.Background()) + s.Require().NoError(err) + s.Require().Equal(expected, actual) + machine.backend.(*MockBackend).AssertExpectations(s.T()) +} - machine := &machineImpl{ - backend: mockBackend, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - }, +func (s *ImplementationSuite) TestValidateAcceptedState() { + for _, test := range []struct { + name string + iflags uint64 + tohost uint64 + valid bool + }{ + {name: "accepted", iflags: 1, tohost: acceptedTohostTestValue(), valid: true}, + {name: "iflags_Y is zero", iflags: 0, tohost: acceptedTohostTestValue()}, + {name: "wrong HTIF device", iflags: 1, tohost: uint64(3)<<56 | uint64(1)<<48 | uint64(1)<<32}, + {name: "wrong HTIF command", iflags: 1, tohost: uint64(2)<<56 | uint64(2)<<48 | uint64(1)<<32}, + {name: "tohost is rejected", iflags: 1, tohost: uint64(2)<<56 | uint64(1)<<48 | uint64(2)<<32}, + } { + s.Run(test.name, func() { + blocks := acceptedStateTestBlocks(test.iflags, test.tohost) + machine, _ := s.machineWithMemoryProofs(blocks) + + proof, err := machine.StateProof(context.Background()) + s.Require().NoError(err) + err = ValidateAcceptedState(proof) + if test.valid { + s.Require().NoError(err) + } else { + s.Require().ErrorIs(err, ErrInvalidMachineProof) + } + machine.backend.(*MockBackend).AssertExpectations(s.T()) + }) } + s.Require().ErrorIs(ValidateAcceptedState(nil), ErrInvalidMachineProof) +} - hash, err := machine.OutputsHash(ctx) - require.NoError(err) - require.Equal(expectedHash, hash) - mockBackend.AssertExpectations(s.T()) +func (s *ImplementationSuite) TestStateProofHonorsCanceledContext() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + machine := &machineImpl{backend: NewMockBackend()} - // Test outputs hash with rejected request - mockBackend2 := NewMockBackend() - mockBackend2.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(ManualYieldReasonRejected), make([]byte, 32), nil) - machine2 := &machineImpl{ - backend: mockBackend2, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - }, - } - _, err = machine2.OutputsHash(ctx) - require.ErrorIs(err, ErrRejected) - mockBackend2.AssertExpectations(s.T()) + proof, err := machine.StateProof(ctx) + s.Require().Nil(proof) + s.Require().ErrorIs(err, ErrCanceled) +} - // Exception remains distinguishable through the public error taxonomy. - mockBackendException := NewMockBackend() - mockBackendException.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(ManualYieldReasonException), []byte("exception"), nil) - machineException := &machineImpl{ - backend: mockBackendException, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - }, - } - _, err = machineException.OutputsHash(ctx) - require.ErrorIs(err, ErrException) - mockBackendException.AssertExpectations(s.T()) +func (s *ImplementationSuite) TestVerifyMemoryProofRejectsMalformedProof() { + address := TxBufferAddress + block := randomFakeHash() + root, proofs := buildTestMemoryProofs(map[uint64]Hash{address: block}) + data := append([]byte(nil), block[:]...) - // Test outputs hash with invalid length - mockBackend3 := NewMockBackend() - mockBackend3.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(ManualYieldReasonAccepted), make([]byte, 16), nil) // Invalid length - machine3 := &machineImpl{ - backend: mockBackend3, - logger: s.logger, - params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, - }, + for _, test := range []struct { + name string + mutate func(*MemoryProof, *[]byte) + }{ + {name: "root size", mutate: func(p *MemoryProof, _ *[]byte) { p.Log2RootSize-- }}, + {name: "target size", mutate: func(p *MemoryProof, _ *[]byte) { p.Log2TargetSize-- }}, + {name: "target address", mutate: func(p *MemoryProof, _ *[]byte) { p.TargetAddress += 32 }}, + {name: "reported root", mutate: func(p *MemoryProof, _ *[]byte) { p.RootHash[0] ^= 0xff }}, + {name: "sibling count", mutate: func(p *MemoryProof, _ *[]byte) { p.Siblings = p.Siblings[:len(p.Siblings)-1] }}, + {name: "data length", mutate: func(_ *MemoryProof, data *[]byte) { *data = (*data)[:len(*data)-1] }}, + {name: "target hash", mutate: func(p *MemoryProof, _ *[]byte) { p.TargetHash[0] ^= 0xff }}, + {name: "sibling path", mutate: func(p *MemoryProof, _ *[]byte) { p.Siblings[0][0] ^= 0xff }}, + } { + s.Run(test.name, func() { + proof := proofs[address] + proof.Siblings = append([]Hash(nil), proof.Siblings...) + proofData := append([]byte(nil), data...) + test.mutate(&proof, &proofData) + + leaf, err := verifyMemoryProof(root, address, proof, proofData) + s.Require().Equal(LeafProof{}, leaf) + s.Require().ErrorIs(err, ErrInvalidMachineProof) + }) } - _, err = machine3.OutputsHash(ctx) - require.Error(err) - require.ErrorIs(err, ErrHashLength) - mockBackend3.AssertExpectations(s.T()) +} - // Test outputs hash with backend error - mockBackend4 := NewMockBackend() - mockBackend4.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( - uint8(0), uint16(0), []byte{}, errors.New("receive failed")) - machine4 := &machineImpl{ - backend: mockBackend4, +func (s *ImplementationSuite) machineWithMemoryProofs(blocks map[uint64]Hash) (*machineImpl, *StateProof) { + root, proofs := buildTestMemoryProofs(blocks) + backend := NewMockBackend() + backend.On("GetRootHash", mock.AnythingOfType("time.Duration")).Return(root, nil).Once() + for _, address := range []uint64{ + iflagsYAddress &^ 31, + htifTohostAddress &^ 31, + TxBufferAddress, + } { + block := blocks[address] + backend.On( + "GetProof", + address, + int32(HashLog2Size), + machineMemoryLog2Size, + mock.AnythingOfType("time.Duration"), + ). + Return(proofs[address], nil).Once() + backend.On("ReadMemory", address, uint64(32), mock.AnythingOfType("time.Duration")). + Return(append([]byte(nil), block[:]...), nil).Once() + } + machine := &machineImpl{ + backend: backend, logger: s.logger, params: model.ExecutionParameters{ - FastDeadline: time.Second * 5, + LoadDeadline: 5 * time.Second, }, } - _, err = machine4.OutputsHash(ctx) - require.Error(err) - require.Contains(err.Error(), "could not read the outputs hash") - mockBackend4.AssertExpectations(s.T()) + expected := &StateProof{ + MachineHash: root, + IflagsYProof: LeafProof{DataBlock: blocks[iflagsYAddress&^31], Siblings: proofs[iflagsYAddress&^31].Siblings}, + HtifTohostProof: LeafProof{DataBlock: blocks[htifTohostAddress&^31], Siblings: proofs[htifTohostAddress&^31].Siblings}, + TxBufferProof: LeafProof{DataBlock: blocks[TxBufferAddress], Siblings: proofs[TxBufferAddress].Siblings}, + } + return machine, expected } -// Test OutputsHashProof method -func (s *ImplementationSuite) TestOutputsHashProof() { - require := s.Require() - ctx := context.Background() - - // Test successful outputs hash proof retrieval - mockBackend := NewMockBackend() - expectedProof := []Hash{randomFakeHash(), randomFakeHash(), randomFakeHash()} - mockBackend.On("GetProof", TxBufferAddress, int32(HashLog2Size), mock.AnythingOfType("time.Duration")). - Return(expectedProof, nil) - - machine := &machineImpl{ - backend: mockBackend, - logger: s.logger, - params: model.ExecutionParameters{ - LoadDeadline: time.Second * 5, - }, +func acceptedStateTestBlocks(iflags, tohost uint64) map[uint64]Hash { + var iflagsBlock Hash + binary.LittleEndian.PutUint64(iflagsBlock[iflagsYAddress&31:], iflags) + var tohostBlock Hash + binary.LittleEndian.PutUint64(tohostBlock[htifTohostAddress&31:], tohost) + return map[uint64]Hash{ + iflagsYAddress &^ 31: iflagsBlock, + htifTohostAddress &^ 31: tohostBlock, + TxBufferAddress: randomFakeHash(), } +} - proof, err := machine.OutputsHashProof(ctx) - require.NoError(err) - require.Equal(expectedProof, proof) - mockBackend.AssertExpectations(s.T()) +func acceptedTohostTestValue() uint64 { + return uint64(2)<<56 | uint64(1)<<48 | uint64(1)<<32 +} - // Test outputs hash proof with backend error - mockBackend2 := NewMockBackend() - mockBackend2.On("GetProof", TxBufferAddress, int32(HashLog2Size), mock.AnythingOfType("time.Duration")). - Return([]Hash(nil), errors.New("proof failed")) - machine2 := &machineImpl{ - backend: mockBackend2, - logger: s.logger, - params: model.ExecutionParameters{ - LoadDeadline: time.Second * 5, - }, - } - _, err = machine2.OutputsHashProof(ctx) - require.Error(err) - require.ErrorIs(err, ErrMachineInternal) - require.Contains(err.Error(), "could not get outputs hash machine proof") - mockBackend2.AssertExpectations(s.T()) +func buildTestMemoryProofs(blocks map[uint64]Hash) (Hash, map[uint64]MemoryProof) { + const depth = memoryProofSiblingCount + defaultHashes := make([]Hash, depth+1) + defaultHashes[0] = Hash(crypto.Keccak256Hash(make([]byte, 32))) + for level := 1; level <= depth; level++ { + defaultHashes[level] = Hash(crypto.Keccak256Hash( + defaultHashes[level-1][:], + defaultHashes[level-1][:], + )) + } + + current := make(map[uint64]Hash, len(blocks)) + indexes := make(map[uint64]uint64, len(blocks)) + siblings := make(map[uint64][]Hash, len(blocks)) + for address, block := range blocks { + index := address >> HashLog2Size + indexes[address] = index + current[index] = Hash(crypto.Keccak256Hash(block[:])) + siblings[address] = make([]Hash, 0, depth) + } + + for level := 0; level < depth; level++ { + for address, originalIndex := range indexes { + index := originalIndex >> level + sibling, ok := current[index^1] + if !ok { + sibling = defaultHashes[level] + } + siblings[address] = append(siblings[address], sibling) + } - // Test outputs hash proof with canceled context - canceledCtx, cancel := context.WithCancel(ctx) - cancel() - _, err = machine.OutputsHashProof(canceledCtx) - require.ErrorIs(err, ErrCanceled) + parents := make(map[uint64]struct{}, len(current)) + for index := range current { + parents[index>>1] = struct{}{} + } + next := make(map[uint64]Hash, len(parents)) + for parent := range parents { + left, ok := current[parent<<1] + if !ok { + left = defaultHashes[level] + } + right, ok := current[parent<<1|1] + if !ok { + right = defaultHashes[level] + } + next[parent] = Hash(crypto.Keccak256Hash(left[:], right[:])) + } + current = next + } + + root := current[0] + proofs := make(map[uint64]MemoryProof, len(blocks)) + for address, block := range blocks { + proofs[address] = MemoryProof{ + Log2RootSize: machineMemoryLog2Size, + Log2TargetSize: int32(HashLog2Size), + RootHash: root, + Siblings: siblings[address], + TargetAddress: address, + TargetHash: Hash(crypto.Keccak256Hash(block[:])), + } + } + return root, proofs } // Test Advance method @@ -298,7 +366,6 @@ func (s *ImplementationSuite) TestAdvance() { require.Equal(CompletionStatusAccepted, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) - require.NotEqual(Hash{}, resp.OutputsHash) mockBackend.AssertExpectations(s.T()) // Test advance with rejection @@ -320,7 +387,6 @@ func (s *ImplementationSuite) TestAdvance() { require.Equal(CompletionStatusRejected, resp.Status) require.Empty(resp.Outputs) require.Empty(resp.Reports) - require.Equal(Hash{}, resp.OutputsHash) mockBackend2.AssertExpectations(s.T()) // Test advance with exception @@ -341,7 +407,6 @@ func (s *ImplementationSuite) TestAdvance() { require.NotNil(resp) require.Equal(CompletionStatusException, resp.Status) require.Equal([]byte("exception data"), resp.ExceptionData) - require.Equal(Hash{}, resp.OutputsHash) mockBackend3.AssertExpectations(s.T()) // Halting is also a completed deterministic status and retains @@ -442,8 +507,8 @@ func (s *ImplementationSuite) TestAdvance() { mockBackend4.AssertExpectations(s.T()) // A configured target beyond uint64 saturates at MaxUint64, just like the - // emulator's imcyclemax. If the machine reaches that endpoint, the machine - // overflow—not the configured-cap sentinel—explains why execution stopped. + // emulator's imcyclemax. The emulator's overflow break reason completes the + // run even when the saturated local target is the same cycle. mockBackendOverflow := NewMockBackend() mockBackendOverflow.On("CmioRxBufferSize").Return(uint64(1024)) mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). @@ -466,18 +531,18 @@ func (s *ImplementationSuite) TestAdvance() { }, } resp, err = machineOverflow.Advance(ctx, input, expectedHash, false) - require.ErrorIs(err, ErrReachedLimitMcycle) - require.ErrorIs(err, ErrMcycleOverflow) - require.Contains(err.Error(), "requested_span=1000") - require.Contains(err.Error(), "target_span=999") - require.Contains(err.Error(), "executed_cycles=999") - require.Nil(resp) + require.NoError(err) + require.NotNil(resp) + require.Equal(CompletionStatusOverflow, resp.Status) mockBackendOverflow.AssertExpectations(s.T()) // Test advance with invalid hash length mockBackend5 := NewMockBackend() mockBackend5.On("CmioRxBufferSize").Return(uint64(1024)) - mockBackend5.On("SendCmioResponse", uint16(AdvanceStateRequest), mock.Anything, expectedHash, mock.AnythingOfType("time.Duration")).Return(nil) + mockBackend5.On( + "SendCmioResponse", uint16(AdvanceStateRequest), mock.Anything, + expectedHash, mock.AnythingOfType("time.Duration"), + ).Return(nil) mockBackend5.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) mockBackend5.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) mockBackend5.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -682,10 +747,13 @@ func (b *statefulAdvanceBackend) ReceiveCmioRequest(time.Duration) (uint8, uint1 } } func (b *statefulAdvanceBackend) WriteMemory(uint64, []byte, time.Duration) error { return nil } -func (b *statefulAdvanceBackend) GetRootHash(time.Duration) (Hash, error) { return Hash{}, nil } -func (b *statefulAdvanceBackend) GetProof(uint64, int32, time.Duration) ([]Hash, error) { +func (b *statefulAdvanceBackend) ReadMemory(uint64, uint64, time.Duration) ([]byte, error) { return nil, nil } +func (b *statefulAdvanceBackend) GetRootHash(time.Duration) (Hash, error) { return Hash{}, nil } +func (b *statefulAdvanceBackend) GetProof(uint64, int32, int32, time.Duration) (MemoryProof, error) { + return MemoryProof{}, nil +} func (b *statefulAdvanceBackend) Delete() {} func (b *statefulAdvanceBackend) ForkServer(time.Duration) (Backend, string, uint32, error) { return nil, "", 0, errors.New("not implemented") @@ -734,7 +802,6 @@ func (s *ImplementationSuite) TestInterruptedAdvanceReturnsNilAndCanBeRetried() s.Require().NoError(err) s.Require().NotNil(result) s.Require().Equal(CompletionStatusAccepted, result.Status) - s.Require().Equal(expectedOutputsHash, result.OutputsHash) backend.AssertExpectations(s.T()) } @@ -782,16 +849,16 @@ func (s *ImplementationSuite) TestRunUsesHardExecutionSpanWhenMaximumIsZero() { params: tt.params, } - _, err := machine.run( + result, err := machine.run( context.Background(), tt.reqType, false, executionBounds{ start: startCycle, limit: startCycle + executionCycleSpan, span: executionCycleSpan, }, ) - s.Require().ErrorIs(err, ErrReachedLimitMcycle) s.Require().ErrorIs(err, ErrMcycleOverflow) - s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", executionCycleSpan)) + s.Empty(result.outputs) + s.Empty(result.reports) mockBackend.AssertExpectations(s.T()) }) } @@ -1062,7 +1129,7 @@ func (s *ImplementationSuite) TestAdvanceConfiguredCycleExhaustionReturnsNoResul backend.AssertExpectations(s.T()) } -func (s *ImplementationSuite) TestAdvanceFixedSpanExhaustionPreservesMachineOverflow() { +func (s *ImplementationSuite) TestAdvanceMachineOverflowCompletesRun() { const start = uint64(30) backend := NewMockBackend() backend.On("CmioRxBufferSize").Return(uint64(1024)) @@ -1084,14 +1151,13 @@ func (s *ImplementationSuite) TestAdvanceFixedSpanExhaustionPreservesMachineOver }} response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) - s.Require().Nil(response) - s.Require().ErrorIs(err, ErrReachedLimitMcycle) - s.Require().ErrorIs(err, ErrMcycleOverflow) - s.Contains(err.Error(), "advance execution reached fixed (machine imcyclemax) cycle limit") + s.Require().NoError(err) + s.Require().NotNil(response) + s.Equal(CompletionStatusOverflow, response.Status) backend.AssertExpectations(s.T()) } -func (s *ImplementationSuite) TestAdvanceConfiguredLimitTiePreservesMachineOverflowPrecedence() { +func (s *ImplementationSuite) TestAdvanceConfiguredLimitTieCompletesWithMachineOverflow() { const start = uint64(30) configuredSpan := model.MaxExecutionCycleSpan limit := start + configuredSpan @@ -1113,16 +1179,13 @@ func (s *ImplementationSuite) TestAdvanceConfiguredLimitTiePreservesMachineOverf }} response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) - s.Require().Nil(response) - s.Require().ErrorIs(err, ErrReachedLimitMcycle) - s.Require().ErrorIs(err, ErrMcycleOverflow) - s.Contains(err.Error(), "advance execution stopped at machine imcyclemax coincident with configured target") - s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", configuredSpan)) - s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", configuredSpan)) + s.Require().NoError(err) + s.Require().NotNil(response) + s.Equal(CompletionStatusOverflow, response.Status) backend.AssertExpectations(s.T()) } -func (s *ImplementationSuite) TestAdvanceStartingAtMachineMaximumPreservesOverflowOrigin() { +func (s *ImplementationSuite) TestAdvanceStartingAtMachineMaximumCompletesWithOverflow() { for _, test := range []struct { name string configuredMax uint64 @@ -1152,16 +1215,9 @@ func (s *ImplementationSuite) TestAdvanceStartingAtMachineMaximumPreservesOverfl }} response, err := machine.Advance(context.Background(), []byte("input"), Hash{}, false) - s.Require().Nil(response) - s.Require().ErrorIs(err, ErrReachedLimitMcycle) - s.Require().ErrorIs(err, ErrMcycleOverflow) - expectedRequestedSpan := test.configuredMax - if expectedRequestedSpan == 0 { - expectedRequestedSpan = model.MaxExecutionCycleSpan - } - s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", expectedRequestedSpan)) - s.Contains(err.Error(), "target_span=0") - s.Contains(err.Error(), "executed_cycles=0") + s.Require().NoError(err) + s.Require().NotNil(response) + s.Equal(CompletionStatusOverflow, response.Status) backend.AssertExpectations(s.T()) }) } @@ -1192,35 +1248,31 @@ func (s *ImplementationSuite) TestRunRejectsNonOverflowReasonAtMachineMaximum() } } -func (s *ImplementationSuite) TestInspectInheritedMcycleOverflowDoesNotClaimLocalLimitExhaustion() { +func (s *ImplementationSuite) TestInspectInheritedMcycleOverflowCompletesRun() { const start = uint64(100) for _, test := range []struct { - name string - configuredMax uint64 - requestedSpan uint64 - executedCycles uint64 - stopDescription string + name string + configuredMax uint64 + requestedSpan uint64 + executedCycles uint64 }{ { - name: "before configured limit", - configuredMax: 50, - requestedSpan: 50, - executedCycles: 7, - stopDescription: "stopped before configured cycle limit", + name: "before configured limit", + configuredMax: 50, + requestedSpan: 50, + executedCycles: 7, }, { - name: "at configured limit", - configuredMax: 7, - requestedSpan: 7, - executedCycles: 7, - stopDescription: "stopped at machine imcyclemax coincident with configured target", + name: "at configured limit", + configuredMax: 7, + requestedSpan: 7, + executedCycles: 7, }, { - name: "before zero default fixed limit", - requestedSpan: model.MaxExecutionCycleSpan, - executedCycles: 7, - stopDescription: "stopped before fixed cycle limit", + name: "before zero default fixed limit", + requestedSpan: model.MaxExecutionCycleSpan, + executedCycles: 7, }, } { s.Run(test.name, func() { @@ -1246,13 +1298,9 @@ func (s *ImplementationSuite) TestInspectInheritedMcycleOverflowDoesNotClaimLoca response, err := machine.Inspect(context.Background(), []byte("query")) s.Require().NotNil(response) - s.Equal(CompletionStatusUnknown, response.Status) + s.Equal(CompletionStatusOverflow, response.Status) s.Empty(response.Reports) - s.Require().ErrorIs(err, ErrReachedLimitMcycle) - s.Require().ErrorIs(err, ErrMcycleOverflow) - s.Contains(err.Error(), test.stopDescription) - s.Contains(err.Error(), fmt.Sprintf("requested_span=%d", test.requestedSpan)) - s.Contains(err.Error(), fmt.Sprintf("executed_cycles=%d", test.executedCycles)) + s.Require().NoError(err) backend.AssertExpectations(s.T()) }) } @@ -1514,8 +1562,8 @@ func (s *ImplementationSuite) TestInspect() { require.ErrorIs(err, ErrPayloadLengthLimitExceeded) mockBackend4.AssertExpectations(s.T()) - // Inspect uses the same saturating target as advance. Reaching MaxUint64 is - // reported as a machine overflow and never as a completed input result. + // Inspect uses the same saturating target as advance. The emulator's overflow + // break reason completes the run even though this inspect fork is disposable. mockBackendOverflow := NewMockBackend() mockBackendOverflow.On("CmioRxBufferSize").Return(uint64(1024)) mockBackendOverflow.On("ReadMCycle", mock.AnythingOfType("time.Duration")). @@ -1539,13 +1587,9 @@ func (s *ImplementationSuite) TestInspect() { } response, err = machineOverflow.Inspect(ctx, query) require.NotNil(response) - require.Equal(CompletionStatusUnknown, response.Status) + require.Equal(CompletionStatusOverflow, response.Status) require.Empty(response.Reports) - require.ErrorIs(err, ErrReachedLimitMcycle) - require.ErrorIs(err, ErrMcycleOverflow) - require.Contains(err.Error(), "requested_span=1000") - require.Contains(err.Error(), "target_span=999") - require.Contains(err.Error(), "executed_cycles=999") + require.NoError(err) mockBackendOverflow.AssertExpectations(s.T()) } @@ -1576,6 +1620,12 @@ func (s *ImplementationSuite) TestAdvanceCanonicalizesTerminalBoundaryHash() { status: CompletionStatusException, terminalAtBoundary: true, }, + { + name: "unexpected yield at first boundary", + yieldReason: manualYieldReason(0x7f), + status: CompletionStatusUnexpectedYield, + terminalAtBoundary: true, + }, { name: "off-boundary terminal keeps prior boundary", yieldReason: ManualYieldReasonAccepted, @@ -1720,7 +1770,7 @@ func (s *ImplementationSuite) TestAdvanceCanonicalizesTerminalBoundaryHash() { } } -func (s *ImplementationSuite) TestRunDiscardsOverflowCollection() { +func (s *ImplementationSuite) TestRunCanonicalizesOverflowCollection() { const fixedEndpoint uint64 = model.MaxExecutionCycleSpan const startCycle uint64 = fixedEndpoint - mcycleComputationHashChunkSize terminalSample := randomFakeHash() @@ -1754,7 +1804,7 @@ func (s *ImplementationSuite) TestRunDiscardsOverflowCollection() { s.Empty(result.outputs) s.Empty(result.reports) s.Empty(result.periodicStateHashes) - s.Zero(result.paddingRepetitions) + s.Equal(uint64(InputEntryCapacity), result.paddingRepetitions) backend.AssertExpectations(s.T()) } @@ -1894,7 +1944,7 @@ func (s *ImplementationSuite) TestClose() { mockBackend.On("Delete").Return() machine := &machineImpl{ backend: mockBackend, - address: "127.0.0.1:12345", + address: testMachineAddress, logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, @@ -1916,7 +1966,7 @@ func (s *ImplementationSuite) TestClose() { mockBackend2.On("Delete").Return() machine2 := &machineImpl{ backend: mockBackend2, - address: "127.0.0.1:12345", + address: testMachineAddress, logger: s.logger, params: model.ExecutionParameters{ FastDeadline: time.Second * 5, @@ -1936,11 +1986,11 @@ func (s *ImplementationSuite) TestAddress() { require := s.Require() machine := &machineImpl{ - address: "127.0.0.1:12345", + address: testMachineAddress, } address := machine.Address() - require.Equal("127.0.0.1:12345", address) + require.Equal(testMachineAddress, address) } // Test helper methods @@ -2029,6 +2079,22 @@ func (s *ImplementationSuite) TestHelperMethods() { require.NotNil(manualResult.data) mockBackend5.AssertExpectations(s.T()) + mockUnexpected := NewMockBackend() + mockUnexpected.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(0x7f), []byte("unexpected"), nil) + machineUnexpected := &machineImpl{ + backend: mockUnexpected, + logger: s.logger, + params: model.ExecutionParameters{ + FastDeadline: time.Second * 5, + }, + } + manualResult, err = machineUnexpected.readManualYieldResult(ctx) + require.NoError(err) + require.Equal(CompletionStatusUnexpectedYield, manualResult.status) + require.Nil(manualResult.data) + mockUnexpected.AssertExpectations(s.T()) + // Test readMCycle mockBackend6 := NewMockBackend() mockBackend6.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(12345), nil) @@ -2328,7 +2394,10 @@ func (s *ImplementationSuite) TestProcess() { // Test successful process mockBackend := NewMockBackend() mockBackend.On("CmioRxBufferSize").Return(uint64(1024)) - mockBackend.On("SendCmioResponse", mock.AnythingOfType("uint16"), mock.Anything, expectedHash, mock.AnythingOfType("time.Duration")).Return(nil) + mockBackend.On( + "SendCmioResponse", mock.AnythingOfType("uint16"), mock.Anything, + expectedHash, mock.AnythingOfType("time.Duration"), + ).Return(nil) mockBackend.On("ReadMCycle", mock.AnythingOfType("time.Duration")).Return(uint64(0), nil) mockBackend.On("Run", mock.AnythingOfType("uint64"), mock.AnythingOfType("time.Duration")).Return(YieldedManually, nil) mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( @@ -2544,7 +2613,7 @@ func (s *ImplementationSuite) TestCheckContext() { require.ErrorIs(err, ErrDeadlineExceeded) // Test nil context (should not panic) - err = checkContext(nil) // nolint + err = checkContext(nil) //nolint:staticcheck // The nil-context contract is intentional. require.NoError(err) // nil context is valid in Go } diff --git a/pkg/machine/libcartesi.go b/pkg/machine/libcartesi.go index 8044f013c..fa7c32536 100644 --- a/pkg/machine/libcartesi.go +++ b/pkg/machine/libcartesi.go @@ -20,11 +20,12 @@ type RemoteMachineInterface interface { Load(dir string, runtimeConfig string) error Run(mcycleEnd uint64) (emulator.BreakReason, error) GetRootHash() (emulator.Hash, error) - GetProof(address uint64, log2size int32) (string, error) + GetProof(address uint64, log2TargetSize, log2RootSize int32) (string, error) ReadReg(reg emulator.RegID) (uint64, error) SendCmioResponse(reason uint16, data []byte, revertRootHash *emulator.Hash) error ReceiveCmioRequest() (uint8, uint16, []byte, error) WriteMemory(address uint64, data []byte) error + ReadMemory(address uint64, length uint64) ([]byte, error) Store(directory string) error Delete() ForkServer() (*emulator.RemoteMachine, string, uint32, error) @@ -38,7 +39,7 @@ type RemoteMachineInterface interface { ) ([]byte, error) } -type proofJson struct { +type proofJSON struct { Log2RootSize int32 `json:"log2_root_size"` Log2TargetSize int32 `json:"log2_target_size"` RootHash Hash `json:"root_hash"` @@ -68,7 +69,7 @@ func decodeB64To32(dst *Hash, s string) error { return nil } -func (p *proofJson) UnmarshalJSON(data []byte) error { +func (p *proofJSON) UnmarshalJSON(data []byte) error { var aux struct { Log2RootSize int32 `json:"log2_root_size"` Log2TargetSize int32 `json:"log2_target_size"` @@ -189,20 +190,32 @@ func (e *LibCartesiBackend) GetRootHash(timeout time.Duration) (Hash, error) { return e.inner.GetRootHash() } -func (e *LibCartesiBackend) GetProof(address uint64, log2size int32, timeout time.Duration) ([]Hash, error) { +func (e *LibCartesiBackend) GetProof( + address uint64, + log2TargetSize, + log2RootSize int32, + timeout time.Duration, +) (MemoryProof, error) { if err := e.inner.SetTimeout(timeout.Milliseconds()); err != nil { - return nil, fmt.Errorf("failed to set operation timeout: %w", err) + return MemoryProof{}, fmt.Errorf("failed to set operation timeout: %w", err) } - jsonMessage, err := e.inner.GetProof(address, log2size) + jsonMessage, err := e.inner.GetProof(address, log2TargetSize, log2RootSize) if err != nil { - return nil, fmt.Errorf("failed to get proof: %w", err) + return MemoryProof{}, fmt.Errorf("failed to get proof: %w", err) } - proof := &proofJson{} + proof := &proofJSON{} err = json.Unmarshal([]byte(jsonMessage), proof) if err != nil { - return nil, fmt.Errorf("failed to unmarshal proof JSON: %w", err) - } - return proof.Siblings, nil + return MemoryProof{}, fmt.Errorf("failed to unmarshal proof JSON: %w", err) + } + return MemoryProof{ + Log2RootSize: proof.Log2RootSize, + Log2TargetSize: proof.Log2TargetSize, + RootHash: proof.RootHash, + Siblings: proof.Siblings, + TargetAddress: proof.TargetAddress, + TargetHash: proof.TargetHash, + }, nil } func (e *LibCartesiBackend) IsAtManualYield(timeout time.Duration) (bool, error) { @@ -255,6 +268,13 @@ func (e *LibCartesiBackend) WriteMemory(address uint64, data []byte, timeout tim return e.inner.WriteMemory(address, data) } +func (e *LibCartesiBackend) ReadMemory(address uint64, length uint64, timeout time.Duration) ([]byte, error) { + if err := e.inner.SetTimeout(timeout.Milliseconds()); err != nil { + return nil, fmt.Errorf("failed to set operation timeout: %w", err) + } + return e.inner.ReadMemory(address, length) +} + func (e *LibCartesiBackend) Delete() { e.inner.Delete() } diff --git a/pkg/machine/libcartesi_test.go b/pkg/machine/libcartesi_test.go index aafd798f1..bb412fbf8 100644 --- a/pkg/machine/libcartesi_test.go +++ b/pkg/machine/libcartesi_test.go @@ -30,6 +30,28 @@ func TestValidateEmulatorComputationHashLimits(t *testing.T) { require.ErrorContains(t, err, "CM_ROLLUP_LOG2_MAX_MCYCLES_PER_ADVANCE_STATE=47") } +func TestStateProofConstantsMatchEmulator(t *testing.T) { + require.EqualValues(t, emulator.CmioTxBufferStart, TxBufferAddress) + require.EqualValues(t, emulator.HashTreeLog2WordSize, HashLog2Size) + require.EqualValues(t, emulator.HashTreeLog2RootSize, machineMemoryLog2Size) + require.EqualValues(t, emulator.HtifDeviceYield, htifDeviceYield) + require.EqualValues(t, emulator.YieldManual, htifCommandManual) + require.EqualValues(t, emulator.ManualYieldReasonAccepted, htifReasonInputAccepted) + require.EqualValues(t, emulator.HtifDeviceShift, htifDeviceShift) + require.EqualValues(t, emulator.HtifCommandShift, htifCommandShift) + require.EqualValues(t, emulator.HtifReasonShift, htifReasonShift) + + // cm_get_reg_address accepts a nil local machine and returns the static + // address defined by the linked emulator. + var localMachine emulator.Machine + actualIflagsYAddress, err := localMachine.GetRegAddress(emulator.REG_IFLAGS_Y) + require.NoError(t, err) + require.Equal(t, iflagsYAddress, actualIflagsYAddress) + actualHtifTohostAddress, err := localMachine.GetRegAddress(emulator.REG_HTIF_TOHOST) + require.NoError(t, err) + require.Equal(t, htifTohostAddress, actualHtifTohostAddress) +} + type LibCartesiSuite struct { suite.Suite mockRemoteMachine *MockRemoteMachine @@ -375,6 +397,76 @@ func (s *LibCartesiSuite) TestGetRootHash() { s.mockRemoteMachine.AssertExpectations(s.T()) } +func (s *LibCartesiSuite) TestGetProofPreservesEmulatorMetadata() { + require := s.Require() + expected := MemoryProof{ + Log2RootSize: 64, + Log2TargetSize: 5, + RootHash: randomFakeHash(), + Siblings: []Hash{randomFakeHash(), randomFakeHash()}, + TargetAddress: 0x60800000, + TargetHash: randomFakeHash(), + } + siblingHashes := make([]string, len(expected.Siblings)) + for i := range expected.Siblings { + siblingHashes[i] = base64.StdEncoding.EncodeToString(expected.Siblings[i][:]) + } + encoded, err := json.Marshal(map[string]any{ + "log2_root_size": expected.Log2RootSize, + "log2_target_size": expected.Log2TargetSize, + "root_hash": base64.StdEncoding.EncodeToString(expected.RootHash[:]), + "sibling_hashes": siblingHashes, + "target_address": expected.TargetAddress, + "target_hash": base64.StdEncoding.EncodeToString(expected.TargetHash[:]), + }) + require.NoError(err) + + s.mockRemoteMachine.On("SetTimeout", int64(5000)).Return(nil) + s.mockRemoteMachine.On("GetProof", expected.TargetAddress, expected.Log2TargetSize, expected.Log2RootSize). + Return(string(encoded), nil) + + actual, err := s.backend.GetProof( + expected.TargetAddress, + expected.Log2TargetSize, + expected.Log2RootSize, + 5*time.Second, + ) + require.NoError(err) + require.Equal(expected, actual) + s.mockRemoteMachine.AssertExpectations(s.T()) +} + +func (s *LibCartesiSuite) TestReadMemory() { + require := s.Require() + expected := []byte{0xde, 0xad, 0xbe, 0xef} + s.mockRemoteMachine.On("SetTimeout", int64(5000)).Return(nil) + s.mockRemoteMachine.On("ReadMemory", uint64(0x300), uint64(len(expected))). + Return(expected, nil) + + actual, err := s.backend.ReadMemory(0x300, uint64(len(expected)), 5*time.Second) + require.NoError(err) + require.Equal(expected, actual) + s.mockRemoteMachine.AssertExpectations(s.T()) + + s.mockRemoteMachine = new(MockRemoteMachine) + s.backend = &LibCartesiBackend{inner: s.mockRemoteMachine} + s.mockRemoteMachine.On("SetTimeout", int64(5000)).Return(errors.New("timeout error")) + actual, err = s.backend.ReadMemory(0x300, uint64(len(expected)), 5*time.Second) + require.Nil(actual) + require.ErrorContains(err, "failed to set operation timeout") + s.mockRemoteMachine.AssertExpectations(s.T()) + + s.mockRemoteMachine = new(MockRemoteMachine) + s.backend = &LibCartesiBackend{inner: s.mockRemoteMachine} + s.mockRemoteMachine.On("SetTimeout", int64(5000)).Return(nil) + s.mockRemoteMachine.On("ReadMemory", uint64(0x300), uint64(len(expected))). + Return([]byte(nil), errors.New("read error")) + actual, err = s.backend.ReadMemory(0x300, uint64(len(expected)), 5*time.Second) + require.Nil(actual) + require.ErrorContains(err, "read error") + s.mockRemoteMachine.AssertExpectations(s.T()) +} + func (s *LibCartesiSuite) TestIsAtManualYield() { require := s.Require() @@ -684,8 +776,8 @@ func (m *MockRemoteMachine) GetRootHash() (emulator.Hash, error) { return args.Get(0).(Hash), args.Error(1) } -func (m *MockRemoteMachine) GetProof(address uint64, log2size int32) (string, error) { - args := m.Called(address, log2size) +func (m *MockRemoteMachine) GetProof(address uint64, log2TargetSize, log2RootSize int32) (string, error) { + args := m.Called(address, log2TargetSize, log2RootSize) return args.Get(0).(string), args.Error(1) } @@ -714,6 +806,14 @@ func (m *MockRemoteMachine) WriteMemory(address uint64, data []byte) error { return args.Error(0) } +func (m *MockRemoteMachine) ReadMemory(address uint64, length uint64) ([]byte, error) { + args := m.Called(address, length) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]byte), args.Error(1) +} + func (m *MockRemoteMachine) Delete() { m.Called() } diff --git a/pkg/machine/machine.go b/pkg/machine/machine.go index 6dc582c8b..793d78473 100644 --- a/pkg/machine/machine.go +++ b/pkg/machine/machine.go @@ -26,10 +26,9 @@ type ( ) // CompletionStatus identifies how a guest-machine request completed. Advance -// and Inspect have the same completion outcomes; their callers decide whether -// and how those outcomes affect canonical state. If execution does not -// complete, the operation returns CompletionStatusUnknown together with an -// error instead. +// and Inspect share the same outcomes; their callers decide whether and how a +// completed outcome affects canonical state. If execution does not complete, +// the operation returns CompletionStatusUnknown with an error. type CompletionStatus uint8 const ( @@ -40,6 +39,8 @@ const ( CompletionStatusRejected CompletionStatusException CompletionStatusHalted + CompletionStatusOverflow + CompletionStatusUnexpectedYield ) // IsCompleted reports whether the status is a completed guest-machine outcome. @@ -48,7 +49,9 @@ func (s CompletionStatus) IsCompleted() bool { case CompletionStatusAccepted, CompletionStatusRejected, CompletionStatusException, - CompletionStatusHalted: + CompletionStatusHalted, + CompletionStatusOverflow, + CompletionStatusUnexpectedYield: return true case CompletionStatusUnknown: return false @@ -57,6 +60,24 @@ func (s CompletionStatus) IsCompleted() bool { } } +// LeafProof proves a 32-byte data block at a known machine-memory address. +// Its shape matches the proof consumed by the released v3 contracts. +type LeafProof struct { + DataBlock Hash + Siblings []Hash +} + +// StateProof binds the three state leaves used by the released v3 contracts to +// one machine root. The proof is intentionally outcome-neutral: accepted and +// terminal post-run states have the same Merkle shape, while callers that need +// an accepted post-epoch state must additionally call ValidateAcceptedState. +type StateProof struct { + MachineHash Hash + IflagsYProof LeafProof + HtifTohostProof LeafProof + TxBufferProof LeafProof +} + // AdvanceResponse contains the result of a completed advance operation. type AdvanceResponse struct { Status CompletionStatus @@ -68,7 +89,6 @@ type AdvanceResponse struct { ExceptionData []byte PeriodicStateHashes []Hash PaddingRepetitions uint64 - OutputsHash Hash } // InspectResponse contains the result of an inspect operation. On incomplete @@ -91,12 +111,14 @@ var ( ErrNotAtManualYield = errors.New("not at manual yield") ErrException = errors.New("last request yielded an exception") ErrRejected = errors.New("last request yielded as rejected") + ErrUnexpectedYield = errors.New("last request yielded with an unsupported reason") ErrHalted = errors.New("machine halted") ErrOutputsLimitExceeded = errors.New("outputs limit exceeded") ErrReportsLimitExceeded = errors.New("reports limit exceeded") ErrPayloadLengthLimitExceeded = errors.New("payload length limit exceeded") ErrHashLength = errors.New("hash does not have the exactly number of bytes") ErrReachedLimitMcycle = errors.New("machine reached limit mcycle") + ErrInvalidMachineProof = errors.New("invalid machine validity proof") // ErrMcycleOverflow preserves the emulator-reported fact that the machine // itself reached imcyclemax, rather than a node target. Canonical overflow @@ -122,23 +144,22 @@ type Machine interface { Fork(ctx context.Context) (Machine, error) // Hash returns the machine's merkle tree root hash. Hash(ctx context.Context) (Hash, error) - // OutputsHash returns the outputs merkle root hash stored in the cmio tx buffer. - OutputsHash(ctx context.Context) (Hash, error) - // OutputsHashProof returns the proof that the outputs merkle root hash is stored in the cmio tx buffer. - OutputsHashProof(ctx context.Context) ([]Hash, error) + // StateProof returns a complete, locally verified proof of the current + // machine root and the three state leaves used by the rollups contracts. + StateProof(ctx context.Context) (*StateProof, error) // Advance sends an input to the machine. // The checkpointHash is the machine's root hash before processing the input, // sent along with the request so the machine can revert to it if needed. // A non-nil response and nil error mean the machine completed with one of - // the four completed CompletionStatus values. Any incomplete execution—input + // the six completed CompletionStatus values. Any incomplete execution—input // validation, an operational limit, deadline/cancellation, or infrastructure // failure—returns a nil response and a non-nil error. CompletionStatusUnknown // is never returned by a successful call. Advance(ctx context.Context, input []byte, checkpointHash Hash, computeHashes bool) (*AdvanceResponse, error) // Inspect sends a query to the machine. A nil error means the guest completed - // with one of the four non-unknown CompletionStatus values. On incomplete + // with one of the six non-unknown CompletionStatus values. On incomplete // execution, the response preserves reports emitted before the failure and // the error identifies why inspection could not complete. Inspect(ctx context.Context, query []byte) (*InspectResponse, error) @@ -155,7 +176,7 @@ type Machine interface { } // MachineConfig contains configuration for a machine instance -type MachineConfig struct { +type MachineConfig struct { //nolint:revive // Keep the established public API name. Address string // Address to connect to the machine backend Path string // Path to the machine's directory ExecutionParameters model.ExecutionParameters // Execution parameters for the machine @@ -173,13 +194,13 @@ func DefaultConfig(path string) *MachineConfig { AdvanceMaxCycles: 0, InspectIncCycles: 1 << 22, //nolint:mnd InspectMaxCycles: 0, - AdvanceIncDeadline: time.Second * 10, // nolint: mnd - AdvanceMaxDeadline: time.Second * 180, // nolint: mnd - InspectIncDeadline: time.Second * 10, // nolint: mnd - InspectMaxDeadline: time.Second * 180, // nolint: mnd - LoadDeadline: time.Second * 300, // nolint: mnd - StoreDeadline: time.Second * 180, // nolint: mnd - FastDeadline: time.Second * 5, // nolint: mnd + AdvanceIncDeadline: time.Second * 10, //nolint:mnd + AdvanceMaxDeadline: time.Second * 180, //nolint:mnd + InspectIncDeadline: time.Second * 10, //nolint:mnd + InspectMaxDeadline: time.Second * 180, //nolint:mnd + LoadDeadline: time.Second * 300, //nolint:mnd + StoreDeadline: time.Second * 180, //nolint:mnd + FastDeadline: time.Second * 5, //nolint:mnd }, BackendFactoryFn: DefaultBackendFactory, // Use the default backend factory } @@ -263,14 +284,31 @@ func Load(ctx context.Context, logger *slog.Logger, config *MachineConfig) (Mach machine.Close() return nil, err } - if manualResult.status == CompletionStatusException { + switch manualResult.status { + case CompletionStatusAccepted: + return machine, nil + case CompletionStatusException: machine.Close() return nil, ErrException - } - if manualResult.status != CompletionStatusAccepted { + case CompletionStatusRejected: machine.Close() return nil, ErrRejected + case CompletionStatusUnexpectedYield: + machine.Close() + return nil, ErrUnexpectedYield + case CompletionStatusUnknown, CompletionStatusHalted, CompletionStatusOverflow: + machine.Close() + return nil, fmt.Errorf( + "invalid initial completion status %d: %w", + manualResult.status, + ErrMachineInternal, + ) + default: + machine.Close() + return nil, fmt.Errorf( + "unsupported initial completion status %d: %w", + manualResult.status, + ErrMachineInternal, + ) } - - return machine, nil } diff --git a/pkg/machine/machine_test.go b/pkg/machine/machine_test.go index 7f002a1af..a8469ee8e 100644 --- a/pkg/machine/machine_test.go +++ b/pkg/machine/machine_test.go @@ -157,6 +157,25 @@ func (s *MachineSuite) TestLoad() { require.ErrorIs(err, ErrRejected) mockBackend.AssertExpectations(s.T()) + // Test with an unsupported manual yield reason. + mockBackend = NewMockBackend() + mockBackend.On("NewMachineRuntimeConfig").Return(`{"concurrency":{"update_merkle_tree":1}}`, nil) + mockBackend.On("Load", + mock.AnythingOfType("string"), + mock.AnythingOfType("string"), + mock.AnythingOfType("time.Duration"), + ).Return(nil).Once() + mockBackend.On("IsAtManualYield", mock.AnythingOfType("time.Duration")).Return(true, nil).Once() + mockBackend.On("ReceiveCmioRequest", mock.AnythingOfType("time.Duration")).Return( + uint8(0), uint16(9), make([]byte, HashSize), nil).Once() + mockBackend.SetupForCleanup() + config = DefaultConfig("some/path") + config.BackendFactoryFn = MockBackendFactory(mockBackend) + machine, err = Load(ctx, s.logger, config) + require.Nil(machine) + require.ErrorIs(err, ErrUnexpectedYield) + mockBackend.AssertExpectations(s.T()) + // Test successful load mockBackend = NewMockBackend() mockBackend.SetupForLoad() @@ -166,7 +185,7 @@ func (s *MachineSuite) TestLoad() { machine, err = Load(ctx, s.logger, config) require.NoError(err) require.NotNil(machine) - require.Equal("127.0.0.1:12345", machine.Address()) + require.Equal(testMachineAddress, machine.Address()) // Clean up err = machine.Close() @@ -231,6 +250,8 @@ func (s *MachineSuite) TestCompletionStatusIsCompleted() { CompletionStatusRejected, CompletionStatusException, CompletionStatusHalted, + CompletionStatusOverflow, + CompletionStatusUnexpectedYield, } { s.Require().True(status.IsCompleted()) } @@ -245,13 +266,14 @@ func (s *MachineSuite) TestMachineInterface() { // Create a mock machine mockMachine := &MockMachine{ - AddressReturn: "127.0.0.1:12345", - HashReturn: Hash{1, 2, 3, 4, 5}, - OutputsHashReturn: Hash{6, 7, 8, 9, 10}, + AddressReturn: testMachineAddress, + HashReturn: Hash{1, 2, 3, 4, 5}, + StateProofReturn: &StateProof{ + MachineHash: Hash{1, 2, 3, 4, 5}, + }, CompletionStatusReturn: CompletionStatusAccepted, AdvanceOutputsReturn: []Output{[]byte("output1"), []byte("output2")}, AdvanceReportsReturn: []Report{[]byte("report1")}, - AdvanceHashReturn: Hash{11, 12, 13, 14, 15}, InspectResponseReturn: &InspectResponse{ Status: CompletionStatusAccepted, Reports: []Report{[]byte("inspect report")}, @@ -263,17 +285,17 @@ func (s *MachineSuite) TestMachineInterface() { // Test Address address := machine.Address() - require.Equal("127.0.0.1:12345", address) + require.Equal(testMachineAddress, address) // Test Hash hash, err := machine.Hash(ctx) require.NoError(err) require.Equal(Hash{1, 2, 3, 4, 5}, hash) - // Test OutputsHash - outputsHash, err := machine.OutputsHash(ctx) + // Test state proof + acceptedState, err := machine.StateProof(ctx) require.NoError(err) - require.Equal(Hash{6, 7, 8, 9, 10}, outputsHash) + require.Equal(Hash{1, 2, 3, 4, 5}, acceptedState.MachineHash) // Test Advance advanceResp, err := machine.Advance(ctx, []byte("input"), Hash{}, false) @@ -284,7 +306,6 @@ func (s *MachineSuite) TestMachineInterface() { require.Equal([]byte("output2"), advanceResp.Outputs[1]) require.Len(advanceResp.Reports, 1) require.Equal([]byte("report1"), advanceResp.Reports[0]) - require.Equal(Hash{11, 12, 13, 14, 15}, advanceResp.OutputsHash) // Test Inspect inspectResponse, err := machine.Inspect(ctx, []byte("query")) @@ -314,13 +335,13 @@ func (s *MachineSuite) TestMachineInterfaceErrors() { // Create a mock machine that returns errors mockMachine := &MockMachine{ - ForkError: errors.New("fork error"), - HashError: errors.New("hash error"), - OutputsHashError: errors.New("outputs hash error"), - AdvanceError: errors.New("advance error"), - InspectError: errors.New("inspect error"), - StoreError: errors.New("store error"), - CloseError: errors.New("close error"), + ForkError: errors.New("fork error"), + HashError: errors.New("hash error"), + StateProofError: errors.New("state proof error"), + AdvanceError: errors.New("advance error"), + InspectError: errors.New("inspect error"), + StoreError: errors.New("store error"), + CloseError: errors.New("close error"), } var machine Machine = mockMachine @@ -335,10 +356,10 @@ func (s *MachineSuite) TestMachineInterfaceErrors() { require.Error(err) require.Contains(err.Error(), "hash error") - // Test OutputsHash error - _, err = machine.OutputsHash(ctx) + // Test state proof error + _, err = machine.StateProof(ctx) require.Error(err) - require.Contains(err.Error(), "outputs hash error") + require.Contains(err.Error(), "state proof error") // Test Advance error _, err = machine.Advance(ctx, []byte("input"), Hash{}, false) @@ -369,18 +390,14 @@ type MockMachine struct { HashReturn Hash HashError error - OutputsHashReturn Hash - OutputsHashError error - - OutputsHashProofReturn []Hash - OutputsHashProofError error + StateProofReturn *StateProof + StateProofError error CompletionStatusReturn CompletionStatus AdvanceOutputsReturn []Output AdvanceReportsReturn []Report AdvanceHashesReturn []Hash AdvanceRemainingReturn uint64 - AdvanceHashReturn Hash AdvanceError error InspectResponseReturn *InspectResponse @@ -401,12 +418,8 @@ func (m *MockMachine) Hash(_ context.Context) (Hash, error) { return m.HashReturn, m.HashError } -func (m *MockMachine) OutputsHash(_ context.Context) (Hash, error) { - return m.OutputsHashReturn, m.OutputsHashError -} - -func (m *MockMachine) OutputsHashProof(_ context.Context) ([]Hash, error) { - return m.OutputsHashProofReturn, m.OutputsHashProofError +func (m *MockMachine) StateProof(_ context.Context) (*StateProof, error) { + return m.StateProofReturn, m.StateProofError } func (m *MockMachine) Advance(_ context.Context, _ []byte, _ Hash, _ bool) (*AdvanceResponse, error) { @@ -419,7 +432,6 @@ func (m *MockMachine) Advance(_ context.Context, _ []byte, _ Hash, _ bool) (*Adv Reports: m.AdvanceReportsReturn, PeriodicStateHashes: m.AdvanceHashesReturn, PaddingRepetitions: m.AdvanceRemainingReturn, - OutputsHash: m.AdvanceHashReturn, }, nil } diff --git a/pkg/machine/util_test.go b/pkg/machine/util_test.go index a4956d2cb..d622c79ea 100644 --- a/pkg/machine/util_test.go +++ b/pkg/machine/util_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/suite" ) -const FAST_DEADLINE = 2 * time.Second +const FAST_DEADLINE = 2 * time.Second //nolint:revive // Keep the established test helper name. func TestUtil(t *testing.T) { suite.Run(t, new(UtilSuite)) @@ -61,7 +61,7 @@ func (s *UtilSuite) TestStopServer() { require := s.Require() // Test with nil logger - err := StopServer("127.0.0.1:12345", nil, FAST_DEADLINE) + err := StopServer(testMachineAddress, nil, FAST_DEADLINE) require.Error(err) require.Contains(err.Error(), "logger must not be nil") From 8d7e228e99d3cfce7a3e92672c39dad817809218 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:30:29 -0300 Subject: [PATCH 02/11] feat(repository): persist complete terminal state proofs --- internal/claimer/accept.go | 4 +- internal/claimer/accept_test.go | 18 +- internal/claimer/blockchain.go | 20 +- internal/claimer/divergence.go | 29 +- internal/claimer/divergence_test.go | 49 ++ internal/claimer/fixtures_test.go | 10 +- internal/claimer/inflight.go | 8 +- internal/claimer/matchers.go | 14 +- internal/claimer/mocks_test.go | 9 +- internal/claimer/repository.go | 2 +- internal/claimer/reverts.go | 24 +- internal/claimer/stage.go | 4 +- internal/claimer/stage_test.go | 19 +- internal/claimer/submit.go | 18 +- internal/claimer/submit_test.go | 12 +- internal/model/execution_parameters_test.go | 59 +++ internal/model/models.go | 191 ++++++-- internal/model/models_json_test.go | 54 ++- internal/prt/prt.go | 12 +- internal/prt/validation_test.go | 4 +- internal/repository/postgres/application.go | 9 +- internal/repository/postgres/bulk.go | 214 +++++++-- internal/repository/postgres/claimer.go | 124 +++-- .../public/enum/applicationstatus.go | 24 +- .../public/enum/inputcompletionstatus.go | 24 +- .../db/rollupsdb/public/table/epoch.go | 28 +- .../db/rollupsdb/public/table/input.go | 10 +- internal/repository/postgres/epoch.go | 171 ++++--- internal/repository/postgres/input.go | 16 +- .../postgres/input_exception_data_test.go | 18 +- .../repository/postgres/postgres_repo_test.go | 70 ++- internal/repository/postgres/replay.go | 6 +- .../repository/postgres/replay_source_test.go | 21 +- .../000001_create_initial_schema.up.sql | 150 ++++-- internal/repository/repository.go | 38 +- .../repotest/application_test_cases.go | 178 +++++++- internal/repository/repotest/builders.go | 81 +++- .../repository/repotest/bulk_test_cases.go | 432 +++++++++++++----- .../repository/repotest/claimer_test_cases.go | 48 +- .../repository/repotest/epoch_test_cases.go | 316 +++++++------ .../repository/repotest/input_test_cases.go | 2 +- .../repository/repotest/output_test_cases.go | 15 +- .../repository/repotest/report_test_cases.go | 5 +- internal/repository/repotest/repotest.go | 5 +- .../repotest/state_hash_test_cases.go | 10 +- test/integration/divergent_claim_test.go | 6 +- .../echo_authority_staging_test.go | 2 +- test/integration/echo_quorum_test.go | 36 +- test/integration/same_block_inputs_test.go | 6 +- test/integration/withdrawal_lifecycle_test.go | 10 +- 50 files changed, 1871 insertions(+), 764 deletions(-) diff --git a/internal/claimer/accept.go b/internal/claimer/accept.go index 780ec0f25..ab622b3e6 100644 --- a/internal/claimer/accept.go +++ b/internal/claimer/accept.go @@ -153,7 +153,7 @@ func (s *Service) processAcceptedClaimEvent( } s.Logger.Debug("Updating claim status to accepted", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, ) txHash := currEvent.Raw.TxHash @@ -297,7 +297,7 @@ func (s *Service) handlePreAcceptClaimStatus( s.dropAcceptAttempt(acceptAttemptKey{currEpoch.ApplicationID, currEpoch.Index}) s.Logger.Info("Claim accepted (front-run; observed via getClaim)", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, ) return claimProgressed(1), true diff --git a/internal/claimer/accept_test.go b/internal/claimer/accept_test.go index c4d7bc12c..5db1d6fc8 100644 --- a/internal/claimer/accept_test.go +++ b/internal/claimer/accept_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/pkg/contracts/iconsensus" "github.com/ethereum/go-ethereum/common" @@ -128,7 +129,7 @@ func TestAcceptClaimWithAntecessorMismatch(t *testing.T) { prevEvent := &iconsensus.IConsensusClaimAccepted{ LastProcessedBlockNumber: new(big.Int).SetUint64(prevEpoch.LastBlock + 1), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *prevEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *prevEpoch.TxBufferDataBlock, MachineMerkleRoot: testMachineHash(prevEpoch), } var currEvent *iconsensus.IConsensusClaimAccepted = nil @@ -614,7 +615,7 @@ func TestAcceptanceDivergence_QuorumStagedDoesNotRejectEpoch(t *testing.T) { divergent := &iconsensus.IConsensusClaimAccepted{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: common.HexToHash("0xbad"), } @@ -646,7 +647,7 @@ func TestAcceptanceDivergence_QuorumComputedRejectsEpoch(t *testing.T) { divergent := &iconsensus.IConsensusClaimAccepted{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: common.HexToHash("0xbad"), } @@ -657,7 +658,10 @@ func TestAcceptanceDivergence_QuorumComputedRejectsEpoch(t *testing.T) { r.On("RejectEpochAndSetApplicationDiverged", mock.Anything, app.ID, currEpoch.Index, mock.MatchedBy(func(reason string) bool { return strings.Contains(reason, "quorum_divergence_at_acceptance") })). - Return(nil).Once() + Return(repository.RejectEpochAndDivergeResult{ + EpochRejected: true, + ApplicationDiverged: true, + }, nil).Once() _, errs := m.submitClaimsAndUpdateDatabase(makeEpochMap(), makeEpochMap(currEpoch), makeApplicationMap(app), endBlock) assert.Equal(t, 1, len(errs)) @@ -677,7 +681,7 @@ func TestAcceptanceDivergence_AuthorityComputedSetsDivergedWithoutRejectingEpoch divergent := &iconsensus.IConsensusClaimAccepted{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: common.HexToHash("0xbad"), } @@ -709,7 +713,7 @@ func TestAcceptanceDivergence_AuthorityDoesNotRejectEpoch(t *testing.T) { divergent := &iconsensus.IConsensusClaimAccepted{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: common.HexToHash("0xbad"), } @@ -746,7 +750,7 @@ func TestAcceptanceDivergenceReaderMode_Quorum(t *testing.T) { divergent := &iconsensus.IConsensusClaimAccepted{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: common.HexToHash("0xbad"), } diff --git a/internal/claimer/blockchain.go b/internal/claimer/blockchain.go index 29c59b19b..b49cf22e3 100644 --- a/internal/claimer/blockchain.go +++ b/internal/claimer/blockchain.go @@ -125,22 +125,22 @@ func (cb *claimerBlockchain) submitClaimToBlockchain( if cb.txOptsFactory == nil { return txHash, fmt.Errorf("txOptsFactory is required for claim submission") } - if epoch.OutputsMerkleRoot == nil { + if epoch.TxBufferDataBlock == nil { return txHash, fmt.Errorf( - "epoch %d (%d) has no outputs_merkle_root; refusing to submit claim", + "epoch %d (%d) has no tx_buffer_data_block to supply as the contract outputs Merkle root; refusing to submit claim", epoch.Index, epoch.VirtualIndex) } - // The DB trigger checks outputs_merkle_proof when an epoch moves to + // The DB trigger checks tx_buffer_proof when an epoch moves to // CLAIM_COMPUTED. It does not stop a later UPDATE from clearing the proof. // Submitting without a proof would revert on chain, so fail here with a // clear local error. - if epoch.OutputsMerkleProof == nil { + if epoch.TxBufferProof == nil { return txHash, fmt.Errorf( - "epoch %d (%d) has no outputs_merkle_proof; refusing to submit claim", + "epoch %d (%d) has no tx_buffer_proof to supply as the contract outputs Merkle proof; refusing to submit claim", epoch.Index, epoch.VirtualIndex) } - proof := make([][32]byte, len(epoch.OutputsMerkleProof)) - for i, h := range epoch.OutputsMerkleProof { + proof := make([][32]byte, len(epoch.TxBufferProof)) + for i, h := range epoch.TxBufferProof { proof[i] = h } txOpts, err := cb.txOptsFactory.NewTransactOpts(ctx) @@ -149,18 +149,18 @@ func (cb *claimerBlockchain) submitClaimToBlockchain( } lastBlockNumber := new(big.Int).SetUint64(epoch.LastBlock) tx, err := ic.SubmitClaim(txOpts, application.IApplicationAddress, - lastBlockNumber, *epoch.OutputsMerkleRoot, proof) + lastBlockNumber, *epoch.TxBufferDataBlock, proof) if err != nil { cb.logger.Warn("submitClaimToBlockchain:failed", "appContractAddress", application.IApplicationAddress, - "claimHash", *epoch.OutputsMerkleRoot, + "claimHash", *epoch.TxBufferDataBlock, "last_block", epoch.LastBlock, "error", err) } else { txHash = tx.Hash() cb.logger.Debug("submitClaimToBlockchain:success", "appContractAddress", application.IApplicationAddress, - "claimHash", *epoch.OutputsMerkleRoot, + "claimHash", *epoch.TxBufferDataBlock, "last_block", epoch.LastBlock, "TxHash", txHash) } diff --git a/internal/claimer/divergence.go b/internal/claimer/divergence.go index 77a7befb3..23de515da 100644 --- a/internal/claimer/divergence.go +++ b/internal/claimer/divergence.go @@ -115,8 +115,8 @@ func (s *Service) markSubmittedDivergence( site string, ) error { ourOutputsMerkleRoot := common.Hash{} - if epoch.OutputsMerkleRoot != nil { - ourOutputsMerkleRoot = *epoch.OutputsMerkleRoot + if epoch.TxBufferDataBlock != nil { + ourOutputsMerkleRoot = *epoch.TxBufferDataBlock } ourMachineMerkleRoot := common.Hash{} if epoch.MachineHash != nil { @@ -146,8 +146,8 @@ func (s *Service) markAcceptedDivergence( site string, ) error { ourOutputsMerkleRoot := common.Hash{} - if epoch.OutputsMerkleRoot != nil { - ourOutputsMerkleRoot = *epoch.OutputsMerkleRoot + if epoch.TxBufferDataBlock != nil { + ourOutputsMerkleRoot = *epoch.TxBufferDataBlock } ourMachineMerkleRoot := common.Hash{} if epoch.MachineHash != nil { @@ -192,12 +192,13 @@ func (s *Service) rejectEpochAndSetApplicationDiverged( epoch *model.Epoch, reason string, ) error { - s.Logger.Error("marking application as diverged (terminal)", + s.Logger.Error("claim divergence detected", "application", app.Name, "address", app.IApplicationAddress.String(), + "epoch_index", epoch.Index, "reason", reason) - err := s.repository.RejectEpochAndSetApplicationDiverged( + result, err := s.repository.RejectEpochAndSetApplicationDiverged( s.Context, app.ID, epoch.Index, reason) reasonErr := errors.New(reason) if err != nil { @@ -209,9 +210,13 @@ func (s *Service) rejectEpochAndSetApplicationDiverged( return errors.Join(reasonErr, err) } - app.Status = model.ApplicationStatus_Diverged - app.Reason = &reason - epoch.Status = model.EpochStatus_ClaimRejected + if result.ApplicationDiverged { + app.Status = model.ApplicationStatus_Diverged + app.Reason = &reason + } + if result.EpochRejected { + epoch.Status = model.EpochStatus_ClaimRejected + } return reasonErr } @@ -246,13 +251,13 @@ func (s *Service) verifyClaimOutputsMatch( claim iconsensus.IConsensusClaim, site string, ) error { - if epoch.OutputsMerkleRoot == nil { + if epoch.TxBufferDataBlock == nil { // Other paths mark this as a local data problem. Here we only compare // outputs when the local value exists. return nil } chainStagedOutputs := common.BytesToHash(claim.StagedOutputsMerkleRoot[:]) - if chainStagedOutputs == *epoch.OutputsMerkleRoot { + if chainStagedOutputs == *epoch.TxBufferDataBlock { return nil } status := fmt.Sprintf("status %d", claim.Status) @@ -270,7 +275,7 @@ func (s *Service) verifyClaimOutputsMatch( "machineMerkleRoot; manual remediation required.", site, status, chainStagedOutputs.Hex(), - epoch.OutputsMerkleRoot.Hex(), + epoch.TxBufferDataBlock.Hex(), epoch.Index, epoch.LastBlock) } diff --git a/internal/claimer/divergence_test.go b/internal/claimer/divergence_test.go index 3a4890358..d5ccf84e5 100644 --- a/internal/claimer/divergence_test.go +++ b/internal/claimer/divergence_test.go @@ -8,11 +8,13 @@ import ( "testing" "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) func TestVerifyClaimOutputsMismatch(t *testing.T) { @@ -41,5 +43,52 @@ func TestVerifyClaimOutputsMismatch(t *testing.T) { assert.Equal(t, 0, len(m.acceptsInFlight)) } +func TestRejectEpochAndSetApplicationDiverged_MirrorsAppliedWrites(t *testing.T) { + t.Run("preserves terminal application mirror", func(t *testing.T) { + service, repo, _ := newServiceMock(t) + defer repo.AssertExpectations(t) + + app := makeApplication() + app.Status = model.ApplicationStatus_MachineHalted + originalReason := "machine halted" + app.Reason = &originalReason + epoch := makeComputedEpoch(app, 3) + + repo.On("RejectEpochAndSetApplicationDiverged", + mock.Anything, app.ID, epoch.Index, mock.Anything). + Return(repository.RejectEpochAndDivergeResult{EpochRejected: true}, nil).Once() + + err := service.rejectEpochAndSetApplicationDiverged(app, epoch, "later claim disagreement") + + require.Error(t, err) + assert.Equal(t, model.ApplicationStatus_MachineHalted, app.Status) + require.NotNil(t, app.Reason) + assert.Equal(t, originalReason, *app.Reason) + assert.Equal(t, model.EpochStatus_ClaimRejected, epoch.Status) + }) + + t.Run("preserves epoch mirror when rejection did not apply", func(t *testing.T) { + service, repo, _ := newServiceMock(t) + defer repo.AssertExpectations(t) + + app := makeApplication() + epoch := makeComputedEpoch(app, 3) + epoch.Status = model.EpochStatus_Closed + reason := "claim disagreement" + + repo.On("RejectEpochAndSetApplicationDiverged", + mock.Anything, app.ID, epoch.Index, reason). + Return(repository.RejectEpochAndDivergeResult{ApplicationDiverged: true}, nil).Once() + + err := service.rejectEpochAndSetApplicationDiverged(app, epoch, reason) + + require.Error(t, err) + assert.Equal(t, model.ApplicationStatus_Diverged, app.Status) + require.NotNil(t, app.Reason) + assert.Equal(t, reason, *app.Reason) + assert.Equal(t, model.EpochStatus_Closed, epoch.Status) + }) +} + // TestCleanupOrphanedInFlight — entries whose app is no longer in any work // map (e.g. transitioned to FAILED/DIVERGED/CORRUPTED/DISABLED mid-flight) must be diff --git a/internal/claimer/fixtures_test.go b/internal/claimer/fixtures_test.go index 9b102991f..249375626 100644 --- a/internal/claimer/fixtures_test.go +++ b/internal/claimer/fixtures_test.go @@ -97,7 +97,7 @@ func makeEpoch(id int64, status model.EpochStatus, i uint64) *model.Epoch { WithBlocks(i*10, i*10+9). WithStatus(status). WithClaimTransactionHash(txHash). - WithOutputsMerkleRoot(outputsMerkleRoot). + WithTxBufferDataBlock(outputsMerkleRoot). WithMachineHash(machineHash). Build() if status == model.EpochStatus_ClaimStaged { @@ -157,7 +157,7 @@ func makeSubmittedEventWithTxHash( return &iconsensus.IConsensusClaimSubmitted{ LastProcessedBlockNumber: new(big.Int).SetUint64(epoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *epoch.OutputsMerkleRoot, + OutputsMerkleRoot: *epoch.TxBufferDataBlock, MachineMerkleRoot: testMachineHash(epoch), Raw: types.Log{ TxHash: txHash, @@ -191,7 +191,7 @@ func makeClaimStagedLog(app *model.Application, epoch *model.Epoch) types.Log { } data, err := event.Inputs.NonIndexed().Pack( new(big.Int).SetUint64(epoch.LastBlock), - *epoch.OutputsMerkleRoot, + *epoch.TxBufferDataBlock, testMachineHash(epoch), ) if err != nil { @@ -212,7 +212,7 @@ func makeStagedEvent(app *model.Application, epoch *model.Epoch) *iconsensus.ICo return &iconsensus.IConsensusClaimStaged{ LastProcessedBlockNumber: new(big.Int).SetUint64(epoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *epoch.OutputsMerkleRoot, + OutputsMerkleRoot: *epoch.TxBufferDataBlock, MachineMerkleRoot: testMachineHash(epoch), Raw: types.Log{ BlockNumber: epoch.LastBlock + 5, @@ -224,7 +224,7 @@ func makeAcceptedEvent(app *model.Application, epoch *model.Epoch) *iconsensus.I return &iconsensus.IConsensusClaimAccepted{ LastProcessedBlockNumber: new(big.Int).SetUint64(epoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *epoch.OutputsMerkleRoot, + OutputsMerkleRoot: *epoch.TxBufferDataBlock, MachineMerkleRoot: testMachineHash(epoch), Raw: types.Log{ TxHash: common.HexToHash(epoch.ClaimTransactionHash.Hex()), diff --git a/internal/claimer/inflight.go b/internal/claimer/inflight.go index b118919e3..8d9f3a1c6 100644 --- a/internal/claimer/inflight.go +++ b/internal/claimer/inflight.go @@ -145,7 +145,7 @@ func (s *Service) handleConfirmedSubmitInFlight( s.Logger.Info("Claim submitted (and staged in same tx)", "app", appAddress, "receipt_block_number", receipt.BlockNumber, - "outputs_merkle_root", hashToHex(computedEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(computedEpoch.TxBufferDataBlock), "last_block", computedEpoch.LastBlock, "tx", txHash) s.dropClaimInFlight(appID) @@ -199,7 +199,7 @@ func (s *Service) handleConfirmedSubmitInFlight( s.Logger.Info("Claim submitted", "app", appAddress, "receipt_block_number", receipt.BlockNumber, - "outputs_merkle_root", hashToHex(computedEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(computedEpoch.TxBufferDataBlock), "last_block", computedEpoch.LastBlock, "tx", txHash) s.dropClaimInFlight(appID) @@ -322,7 +322,7 @@ func (s *Service) handleRevertedAcceptInFlight( s.dropAcceptAttempt(acceptAttemptKey{stagedEpoch.ApplicationID, stagedEpoch.Index}) s.Logger.Info("Claim accepted by front-runner (own accept tx reverted; reconciled via getClaim)", "app", appAddress, "tx", txHash, - "outputs_merkle_root", hashToHex(stagedEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(stagedEpoch.TxBufferDataBlock), "last_block", stagedEpoch.LastBlock) return claimWorkCompleted(1) case claimStatusStaged: @@ -372,7 +372,7 @@ func (s *Service) handleConfirmedAcceptInFlight( s.Logger.Info("Claim accepted (own tx)", "app", appAddress, "receipt_block_number", receipt.BlockNumber, - "outputs_merkle_root", hashToHex(stagedEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(stagedEpoch.TxBufferDataBlock), "last_block", stagedEpoch.LastBlock, "tx", txHash) s.dropAcceptInFlight(appID) diff --git a/internal/claimer/matchers.go b/internal/claimer/matchers.go index ab5f3ca34..84b1ea163 100644 --- a/internal/claimer/matchers.go +++ b/internal/claimer/matchers.go @@ -20,7 +20,7 @@ func checkEpochConstraint(epoch *model.Epoch) error { epoch.Status == model.EpochStatus_ClaimAccepted || epoch.Status == model.EpochStatus_ClaimComputed if mustHaveOutputsMerkleRoot { - if epoch.OutputsMerkleRoot == nil { + if epoch.TxBufferDataBlock == nil { return fmt.Errorf("unexpected epoch state. missing outputs_merkle_root.") } } @@ -75,11 +75,11 @@ func claimSubmittedEventMatches(application *model.Application, epoch *model.Epo if application == nil || epoch == nil || event == nil { return false, false } - if epoch.OutputsMerkleRoot == nil || epoch.MachineHash == nil { + if epoch.TxBufferDataBlock == nil || epoch.MachineHash == nil { return false, false } return application.IApplicationAddress == event.AppContract && - *epoch.OutputsMerkleRoot == event.OutputsMerkleRoot && + *epoch.TxBufferDataBlock == event.OutputsMerkleRoot && *epoch.MachineHash == event.MachineMerkleRoot && epoch.LastBlock == event.LastProcessedBlockNumber.Uint64(), true } @@ -88,11 +88,11 @@ func claimAcceptedEventMatches(application *model.Application, epoch *model.Epoc if application == nil || epoch == nil || event == nil { return false, false } - if epoch.OutputsMerkleRoot == nil || epoch.MachineHash == nil { + if epoch.TxBufferDataBlock == nil || epoch.MachineHash == nil { return false, false } return application.IApplicationAddress == event.AppContract && - *epoch.OutputsMerkleRoot == event.OutputsMerkleRoot && + *epoch.TxBufferDataBlock == event.OutputsMerkleRoot && *epoch.MachineHash == event.MachineMerkleRoot && epoch.LastBlock == event.LastProcessedBlockNumber.Uint64(), true } @@ -128,11 +128,11 @@ func claimStagedEventMatches(application *model.Application, epoch *model.Epoch, if application == nil || epoch == nil || event == nil { return false, false } - if epoch.OutputsMerkleRoot == nil || epoch.MachineHash == nil { + if epoch.TxBufferDataBlock == nil || epoch.MachineHash == nil { return false, false } return application.IApplicationAddress == event.AppContract && - *epoch.OutputsMerkleRoot == event.OutputsMerkleRoot && + *epoch.TxBufferDataBlock == event.OutputsMerkleRoot && *epoch.MachineHash == event.MachineMerkleRoot && epoch.LastBlock == event.LastProcessedBlockNumber.Uint64(), true } diff --git a/internal/claimer/mocks_test.go b/internal/claimer/mocks_test.go index 965258284..2212aad6a 100644 --- a/internal/claimer/mocks_test.go +++ b/internal/claimer/mocks_test.go @@ -96,9 +96,10 @@ func (m *claimerRepositoryMock) RejectEpochAndSetApplicationDiverged( appID int64, index uint64, reason string, -) error { +) (repository.RejectEpochAndDivergeResult, error) { args := m.Called(ctx, appID, index, reason) - return args.Error(0) + result, _ := args.Get(0).(repository.RejectEpochAndDivergeResult) + return result, args.Error(1) } func (m *claimerRepositoryMock) HasUnreconciledClaimsBeforeBlock( @@ -408,8 +409,8 @@ func expectPreSubmitPath(b *claimerBlockchainMock, app *model.Application, epoch func makeClaimStatus(status uint8, epoch *model.Epoch, stagedAtBlock uint64) iconsensus.IConsensusClaim { claim := iconsensus.IConsensusClaim{Status: status} - if epoch.OutputsMerkleRoot != nil { - claim.StagedOutputsMerkleRoot = *epoch.OutputsMerkleRoot + if epoch.TxBufferDataBlock != nil { + claim.StagedOutputsMerkleRoot = *epoch.TxBufferDataBlock } if stagedAtBlock != 0 { claim.StagingBlockNumber = new(big.Int).SetUint64(stagedAtBlock) diff --git a/internal/claimer/repository.go b/internal/claimer/repository.go index f848ad7ee..2920bef9d 100644 --- a/internal/claimer/repository.go +++ b/internal/claimer/repository.go @@ -85,7 +85,7 @@ type iclaimerRepository interface { applicationID int64, index uint64, reason string, - ) error + ) (repository.RejectEpochAndDivergeResult, error) UpdateApplicationStatus( ctx context.Context, diff --git a/internal/claimer/reverts.go b/internal/claimer/reverts.go index 3659c5e51..0ba1732cf 100644 --- a/internal/claimer/reverts.go +++ b/internal/claimer/reverts.go @@ -112,7 +112,7 @@ func (s *Service) handleSubmitClaimRevert( "submitClaim broadcast rejected with 'nonce too low'; "+ "deferring to the next tick's getClaim reconciliation", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return submitClaimRetryLater, nil @@ -135,14 +135,14 @@ func (s *Service) handleSubmitClaimRevert( s.Logger.Warn( "submitClaim reverted with NotFirstClaim on Quorum; waiting for event reconciliation", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return submitClaimRetryLater, nil } s.Logger.Info("Claim already on-chain, waiting for event sync", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return submitClaimAlreadyOnChain, nil @@ -155,7 +155,7 @@ func (s *Service) handleSubmitClaimRevert( s.Logger.Warn("submitClaim reverted with ApplicationForeclosed; "+ "awaiting Foreclosure observer to record the foreclosure marker", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return submitClaimRetryLater, nil @@ -164,7 +164,7 @@ func (s *Service) handleSubmitClaimRevert( stateErr := s.setApplicationCorrupted( s.Context, app, "submitClaim reverted with InvalidOutputsMerkleRootProofSize for "+ - "epoch %d (%d), last_block %d — outputs_merkle_proof in DB is "+ + "epoch %d (%d), last_block %d — tx_buffer_proof in DB is "+ "the wrong length for the machine memory tree.", epoch.Index, epoch.VirtualIndex, epoch.LastBlock, ) @@ -190,7 +190,7 @@ func (s *Service) handleSubmitClaimRevert( stateErr := s.setApplicationCorrupted( s.Context, app, "submitClaim reverted with InvalidNodeIndex for "+ - "epoch %d (%d), last_block %d — outputs_merkle_proof in DB does "+ + "epoch %d (%d), last_block %d — tx_buffer_proof in DB does "+ "not form a valid replacement proof for the machine memory tree.", epoch.Index, epoch.VirtualIndex, epoch.LastBlock, ) @@ -306,7 +306,7 @@ func (s *Service) classifySharedConsensusRevert( logArgs := []any{ "app", app.IApplicationAddress, "epoch_index", epoch.Index, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, } if lastProcessed, upperBound, ok := decodeNotPastBlockBounds(err); ok { @@ -358,7 +358,7 @@ func (s *Service) handleAcceptClaimRevert( "acceptClaim broadcast rejected with 'nonce too low'; "+ "deferring to the next tick's getClaim reconciliation", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return acceptClaimRetryLater, nil @@ -377,7 +377,7 @@ func (s *Service) handleAcceptClaimRevert( case claimStatusAccepted: s.Logger.Info("Claim was accepted by a front-runner; reconciling", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return acceptClaimReconciledAccepted, nil @@ -387,7 +387,7 @@ func (s *Service) handleAcceptClaimRevert( "This can happen under reorgs when reading non-final blocks; "+ "retry on the next tick.", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return acceptClaimRetryLater, nil @@ -422,7 +422,7 @@ func (s *Service) handleAcceptClaimRevert( s.Logger.Warn("acceptClaim reverted with ClaimStagingPeriodNotOverYet; "+ "local arithmetic disagrees with chain. Will retry next tick.", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return acceptClaimRetryLater, nil @@ -431,7 +431,7 @@ func (s *Service) handleAcceptClaimRevert( s.Logger.Warn("acceptClaim reverted with ApplicationForeclosed; "+ "awaiting Foreclosure observer to record the foreclosure marker", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return acceptClaimRetryLater, nil diff --git a/internal/claimer/stage.go b/internal/claimer/stage.go index 72336df21..48161a898 100644 --- a/internal/claimer/stage.go +++ b/internal/claimer/stage.go @@ -111,7 +111,7 @@ func (s *Service) tryStageFromReceipt( s.Logger.Info("Claim staged (fast path)", "app", app.IApplicationAddress, "epoch_index", epoch.Index, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, "staged_at_block", log.BlockNumber, "tx", receipt.TxHash) @@ -201,7 +201,7 @@ func (s *Service) processSubmittedClaim( } s.Logger.Info("Claim staged", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, "staged_at_block", currEvent.Raw.BlockNumber, ) diff --git a/internal/claimer/stage_test.go b/internal/claimer/stage_test.go index c5ce626a4..739389a6c 100644 --- a/internal/claimer/stage_test.go +++ b/internal/claimer/stage_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/pkg/contracts/iconsensus" "github.com/ethereum/go-ethereum/common" @@ -37,7 +38,7 @@ func TestStagingFastPathDivergence(t *testing.T) { divergent := makeStagedEvent(app, currEpoch) differentMachineMerkleRoot := common.HexToHash("0xdeadbeef") divergent.MachineMerkleRoot = differentMachineMerkleRoot - stagedLog := buildClaimStagedLog(app, currEpoch, *currEpoch.OutputsMerkleRoot, differentMachineMerkleRoot) + stagedLog := buildClaimStagedLog(app, currEpoch, *currEpoch.TxBufferDataBlock, differentMachineMerkleRoot) receiptBlock := currEpoch.LastBlock + 1 b.On("pollTransaction", mock.Anything, txHash, endBlock). @@ -176,7 +177,7 @@ func TestStagingDivergence_Quorum(t *testing.T) { divergent := &iconsensus.IConsensusClaimStaged{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: differentMachineMerkleRoot, } @@ -187,7 +188,10 @@ func TestStagingDivergence_Quorum(t *testing.T) { r.On("RejectEpochAndSetApplicationDiverged", mock.Anything, app.ID, currEpoch.Index, mock.MatchedBy(func(reason string) bool { return strings.Contains(reason, "quorum_divergence_at_staging") })). - Return(nil).Once() + Return(repository.RejectEpochAndDivergeResult{ + EpochRejected: true, + ApplicationDiverged: true, + }, nil).Once() _, errs := m.stageClaimsAndUpdateDatabase(makeEpochMap(), makeEpochMap(currEpoch), makeApplicationMap(app), endBlock) assert.Equal(t, 1, len(errs)) @@ -207,7 +211,7 @@ func TestStagingDivergence_AuthorityDoesNotRejectEpoch(t *testing.T) { divergent := &iconsensus.IConsensusClaimStaged{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: common.HexToHash("0xfeed"), } @@ -267,7 +271,7 @@ func TestStagingDivergenceReaderMode_Quorum(t *testing.T) { divergent := &iconsensus.IConsensusClaimStaged{ LastProcessedBlockNumber: new(big.Int).SetUint64(currEpoch.LastBlock), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *currEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *currEpoch.TxBufferDataBlock, MachineMerkleRoot: differentMachineMerkleRoot, } @@ -278,7 +282,10 @@ func TestStagingDivergenceReaderMode_Quorum(t *testing.T) { r.On("RejectEpochAndSetApplicationDiverged", mock.Anything, app.ID, currEpoch.Index, mock.MatchedBy(func(reason string) bool { return strings.Contains(reason, "quorum_divergence_at_staging") })). - Return(nil).Once() + Return(repository.RejectEpochAndDivergeResult{ + EpochRejected: true, + ApplicationDiverged: true, + }, nil).Once() _, errs := m.stageClaimsAndUpdateDatabase(makeEpochMap(), makeEpochMap(currEpoch), makeApplicationMap(app), endBlock) assert.Equal(t, 1, len(errs), "divergence detection must fire in reader mode") diff --git a/internal/claimer/submit.go b/internal/claimer/submit.go index 335b286b3..088382477 100644 --- a/internal/claimer/submit.go +++ b/internal/claimer/submit.go @@ -139,8 +139,8 @@ func (s *Service) shouldIgnoreQuorumSubmittedMismatch( } ourOutputsMerkleRoot := common.Hash{} - if epoch.OutputsMerkleRoot != nil { - ourOutputsMerkleRoot = *epoch.OutputsMerkleRoot + if epoch.TxBufferDataBlock != nil { + ourOutputsMerkleRoot = *epoch.TxBufferDataBlock } outputsMatch := common.Hash(event.OutputsMerkleRoot) == ourOutputsMerkleRoot @@ -179,7 +179,7 @@ func (s *Service) shouldRecordMatchingClaimSubmitted( "app", app.IApplicationAddress, "event_submitter", event.Submitter, "our_submitter", submitter, - "outputs_merkle_root", hashToHex(epoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(epoch.TxBufferDataBlock), "last_block", epoch.LastBlock, ) return false @@ -260,7 +260,7 @@ func (s *Service) processComputedClaim( if prevEpoch != nil && prevEpoch.Status != model.EpochStatus_ClaimAccepted { s.Logger.Debug("Waiting previous claim to be accepted before submitting new one. Previous:", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(prevEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(prevEpoch.TxBufferDataBlock), "last_block", prevEpoch.LastBlock, ) return claimNoProgress() @@ -391,7 +391,7 @@ func (s *Service) recordSubmittedEvent( ) claimStepResult { s.Logger.Debug("Updating claim status to submitted", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, ) txHash := currEvent.Raw.TxHash @@ -408,7 +408,7 @@ func (s *Service) recordSubmittedEvent( s.Logger.Info("Claim previously submitted", "app", app.IApplicationAddress, "event_block_number", currEvent.Raw.BlockNumber, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, ) return claimProgressed(1) @@ -422,7 +422,7 @@ func (s *Service) broadcastComputedClaim( ) claimStepResult { s.Logger.Debug("Submitting claim to blockchain", "app", app.IApplicationAddress, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, ) txCtx, cancel := context.WithTimeout(s.Context, s.submissionTimeout) @@ -486,7 +486,7 @@ func (s *Service) reconcileBeforeSubmit( s.Logger.Info("Claim already accepted on chain (reconciled pre-submit)", "app", app.IApplicationAddress, "epoch_index", currEpoch.Index, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, ) return true, nil @@ -499,7 +499,7 @@ func (s *Service) reconcileBeforeSubmit( s.Logger.Info("Claim already staged on chain (reconciled pre-submit)", "app", app.IApplicationAddress, "epoch_index", currEpoch.Index, - "outputs_merkle_root", hashToHex(currEpoch.OutputsMerkleRoot), + "outputs_merkle_root", hashToHex(currEpoch.TxBufferDataBlock), "last_block", currEpoch.LastBlock, "staged_at_block", stagingBlock, ) diff --git a/internal/claimer/submit_test.go b/internal/claimer/submit_test.go index 37cc84613..1ec6bc61f 100644 --- a/internal/claimer/submit_test.go +++ b/internal/claimer/submit_test.go @@ -603,7 +603,7 @@ func TestQuorumSubmittedEventsIgnoresForeignAdversarialProofAndSubmitsLocalClaim adversarialEvent := makeSubmittedEventWithRoots( app, currEpoch, - *currEpoch.OutputsMerkleRoot, + *currEpoch.TxBufferDataBlock, common.HexToHash("0xf003"), ) adversarialEvent.Submitter = common.HexToAddress("0x0000000000000000000000000000000000000003") @@ -636,7 +636,7 @@ func TestQuorumSubmittedEventsOwnMismatchSetsDiverged(t *testing.T) { adversarialEvent := makeSubmittedEventWithRoots( app, currEpoch, - *currEpoch.OutputsMerkleRoot, + *currEpoch.TxBufferDataBlock, common.HexToHash("0xf003"), ) adversarialEvent.Submitter = b.submitterAddress @@ -671,7 +671,7 @@ func TestQuorumReaderModeIgnoresNonMatchingSubmittedEvent(t *testing.T) { foreignEvent := makeSubmittedEventWithRoots( app, currEpoch, - *currEpoch.OutputsMerkleRoot, + *currEpoch.TxBufferDataBlock, common.HexToHash("0xf003"), ) foreignEvent.Submitter = common.HexToAddress("0x0000000000000000000000000000000000000002") @@ -703,7 +703,7 @@ func TestSubmitClaimWithAntecessorMismatch(t *testing.T) { prevEvent := &iconsensus.IConsensusClaimSubmitted{ LastProcessedBlockNumber: new(big.Int).SetUint64(prevEpoch.LastBlock + 1), AppContract: app.IApplicationAddress, - OutputsMerkleRoot: *prevEpoch.OutputsMerkleRoot, + OutputsMerkleRoot: *prevEpoch.TxBufferDataBlock, MachineMerkleRoot: testMachineHash(prevEpoch), } var currEvent *iconsensus.IConsensusClaimSubmitted = nil @@ -765,7 +765,7 @@ func TestQuorumPreviousSubmittedEventsIgnoresForeignMismatchAndSubmitsCurrentCla foreignPrevEvent := makeSubmittedEventWithRoots( app, prevEpoch, - *prevEpoch.OutputsMerkleRoot, + *prevEpoch.TxBufferDataBlock, common.HexToHash("0xf003"), ) foreignPrevEvent.Submitter = common.HexToAddress("0x0000000000000000000000000000000000000002") @@ -801,7 +801,7 @@ func TestQuorumPreviousSubmittedEventsOwnMismatchSetsDiverged(t *testing.T) { wrongPrevEvent := makeSubmittedEventWithRoots( app, prevEpoch, - *prevEpoch.OutputsMerkleRoot, + *prevEpoch.TxBufferDataBlock, common.HexToHash("0xf003"), ) wrongPrevEvent.Submitter = b.submitterAddress diff --git a/internal/model/execution_parameters_test.go b/internal/model/execution_parameters_test.go index 7a22256e3..c0622bf9e 100644 --- a/internal/model/execution_parameters_test.go +++ b/internal/model/execution_parameters_test.go @@ -132,6 +132,8 @@ func TestInputCompletionStatusContract(t *testing.T) { InputCompletionStatus_Rejected, InputCompletionStatus_Exception, InputCompletionStatus_MachineHalted, + InputCompletionStatus_Overflow, + InputCompletionStatus_UnexpectedYield, } require.Equal(t, expected, InputCompletionStatusAllValues) @@ -145,6 +147,13 @@ func TestInputCompletionStatusContract(t *testing.T) { require.NoError(t, fromBytes.Scan([]byte(value.String()))) require.Equal(t, value, fromBytes) require.Equal(t, value != InputCompletionStatus_None, value.IsCompleted()) + require.Equal(t, + value == InputCompletionStatus_Exception || + value == InputCompletionStatus_MachineHalted || + value == InputCompletionStatus_Overflow || + value == InputCompletionStatus_UnexpectedYield, + value.IsTerminal(), + ) }) } @@ -161,6 +170,56 @@ func TestInputCompletionStatusContract(t *testing.T) { var status InputCompletionStatus require.Error(t, status.Scan(value)) require.False(t, InputCompletionStatus(value).IsCompleted()) + require.False(t, InputCompletionStatus(value).IsTerminal()) }) } } + +func TestApplicationStatusContract(t *testing.T) { + expected := []ApplicationStatus{ + ApplicationStatus_OK, + ApplicationStatus_Failed, + ApplicationStatus_Diverged, + ApplicationStatus_Corrupted, + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } + require.Equal(t, expected, ApplicationStatusAllValues) + + for _, value := range expected { + t.Run(value.String(), func(t *testing.T) { + var scanned ApplicationStatus + require.NoError(t, scanned.Scan(value.String())) + require.Equal(t, value, scanned) + require.Equal(t, + value != ApplicationStatus_OK && value != ApplicationStatus_Failed, + value.IsTerminal(), + ) + }) + } +} + +func TestTerminalApplicationStatus(t *testing.T) { + expected := map[InputCompletionStatus]ApplicationStatus{ + InputCompletionStatus_Exception: ApplicationStatus_GuestException, + InputCompletionStatus_MachineHalted: ApplicationStatus_MachineHalted, + InputCompletionStatus_Overflow: ApplicationStatus_McycleOverflow, + InputCompletionStatus_UnexpectedYield: ApplicationStatus_UnexpectedYield, + } + for inputStatus, applicationStatus := range expected { + got, terminal := inputStatus.TerminalApplicationStatus() + require.True(t, terminal) + require.Equal(t, applicationStatus, got) + } + for _, inputStatus := range []InputCompletionStatus{ + InputCompletionStatus_None, + InputCompletionStatus_Accepted, + InputCompletionStatus_Rejected, + } { + got, terminal := inputStatus.TerminalApplicationStatus() + require.False(t, terminal) + require.Empty(t, got) + } +} diff --git a/internal/model/models.go b/internal/model/models.go index 432650f62..ebe9ed03a 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -330,27 +330,44 @@ func (a *Application) IsDaveConsensus() bool { return a.ConsensusType == Consensus_PRT } -// ApplicationStatus is the node's processing-integrity health for an -// application. It is independent of lifecycle (foreclosure lives in the -// foreclose_block column) and of operator intent (the enabled flag). +// ApplicationStatus records why the node may no longer execute an application +// or produce claims for it. It combines recoverable operational failures, +// integrity failures, and deterministic terminal machine outcomes. It is +// independent of lifecycle (foreclosure lives in the foreclose_block column) +// and of operator intent (the enabled flag). // // Transitions (enforced by DB trigger): // -// OK ⇄ FAILED (FAILED is recoverable by the operator) -// OK, FAILED → DIVERGED (terminal) -// OK, FAILED → CORRUPTED (terminal) +// OK ⇄ FAILED (FAILED is recoverable) +// OK, FAILED → DIVERGED, CORRUPTED (integrity terminal) +// OK → GUEST_EXCEPTION, MACHINE_HALTED (execution terminal) +// MCYCLE_OVERFLOW, UNEXPECTED_YIELD +// execution terminal → CORRUPTED (integrity escalation) // -// DIVERGED means the node's computed claim disagrees with what the chain -// accepted; CORRUPTED means local state is missing or inconsistent. Both are -// terminal and carry a reason. Foreclosure is orthogonal and may coexist with -// any health value. +// FAILED is recoverable and suspends execution, but enabled applications in +// every status continue L1 observation. Consequently, later evidence may +// supersede FAILED with DIVERGED or CORRUPTED. DIVERGED means the node's +// computed claim disagrees with what the chain accepted; CORRUPTED means local +// state or its relationship with L1 history is missing or inconsistent. +// +// GUEST_EXCEPTION, MACHINE_HALTED, MCYCLE_OVERFLOW, and UNEXPECTED_YIELD record +// deterministic machine outcomes that stop execution and must not be retried. +// Later L1 observation may supersede one with CORRUPTED when it establishes +// that local history is untrustworthy. The input retains its original +// completion status and terminal state proof. Foreclosure is orthogonal and +// may coexist with any application status. type ApplicationStatus string +//nolint:revive // Public enum names preserve the generated/API naming convention. const ( - ApplicationStatus_OK ApplicationStatus = "OK" // healthy; eligible for work when enabled and not foreclosed - ApplicationStatus_Failed ApplicationStatus = "FAILED" // recoverable failure (e.g., OOM, process crash) - ApplicationStatus_Diverged ApplicationStatus = "DIVERGED" // computed claim disagrees with the chain (terminal) - ApplicationStatus_Corrupted ApplicationStatus = "CORRUPTED" // local state missing or inconsistent (terminal) + ApplicationStatus_OK ApplicationStatus = "OK" // healthy; eligible for work when enabled and not foreclosed + ApplicationStatus_Failed ApplicationStatus = "FAILED" // recoverable failure (e.g., OOM, process crash) + ApplicationStatus_Diverged ApplicationStatus = "DIVERGED" // computed claim disagrees with the chain (terminal) + ApplicationStatus_Corrupted ApplicationStatus = "CORRUPTED" // local state missing or inconsistent (terminal) + ApplicationStatus_GuestException ApplicationStatus = "GUEST_EXCEPTION" + ApplicationStatus_MachineHalted ApplicationStatus = "MACHINE_HALTED" + ApplicationStatus_McycleOverflow ApplicationStatus = "MCYCLE_OVERFLOW" + ApplicationStatus_UnexpectedYield ApplicationStatus = "UNEXPECTED_YIELD" ) var ApplicationStatusAllValues = []ApplicationStatus{ @@ -358,6 +375,26 @@ var ApplicationStatusAllValues = []ApplicationStatus{ ApplicationStatus_Failed, ApplicationStatus_Diverged, ApplicationStatus_Corrupted, + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, +} + +func (e ApplicationStatus) IsTerminal() bool { + switch e { + case ApplicationStatus_Diverged, + ApplicationStatus_Corrupted, + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield: + return true + case ApplicationStatus_OK, ApplicationStatus_Failed: + return false + default: + return false + } } func (e *ApplicationStatus) Scan(value any) error { @@ -380,6 +417,14 @@ func (e *ApplicationStatus) Scan(value any) error { *e = ApplicationStatus_Diverged case "CORRUPTED": *e = ApplicationStatus_Corrupted + case "GUEST_EXCEPTION": + *e = ApplicationStatus_GuestException + case "MACHINE_HALTED": + *e = ApplicationStatus_MachineHalted + case "MCYCLE_OVERFLOW": + *e = ApplicationStatus_McycleOverflow + case "UNEXPECTED_YIELD": + *e = ApplicationStatus_UnexpectedYield default: return errors.New("invalid value '" + enumValue + "' for ApplicationStatus enum") } @@ -806,8 +851,12 @@ type Epoch struct { InputIndexLowerBound uint64 `json:"input_index_lower_bound"` InputIndexUpperBound uint64 `json:"input_index_upper_bound"` MachineHash *common.Hash `json:"machine_hash"` - OutputsMerkleRoot *common.Hash `json:"outputs_merkle_root"` - OutputsMerkleProof []common.Hash `json:"outputs_merkle_proof,omitempty"` + TxBufferDataBlock *common.Hash `json:"tx_buffer_data_block"` + TxBufferProof []common.Hash `json:"tx_buffer_proof,omitempty"` + IflagsYDataBlock *common.Hash `json:"iflags_y_data_block"` + IflagsYProof []common.Hash `json:"iflags_y_proof,omitempty"` + HtifTohostDataBlock *common.Hash `json:"htif_tohost_data_block"` + HtifTohostProof []common.Hash `json:"htif_tohost_proof,omitempty"` ClaimTransactionHash *common.Hash `json:"claim_transaction_hash"` Commitment *common.Hash `json:"commitment"` CommitmentProof []common.Hash `json:"commitment_proof,omitempty"` @@ -988,7 +1037,7 @@ type Input struct { Status InputCompletionStatus `json:"status"` ExceptionData []byte `json:"-"` MachineHash *common.Hash `json:"machine_hash"` - OutputsHash *common.Hash `json:"outputs_hash"` + TxBufferDataBlock *common.Hash `json:"tx_buffer_data_block"` TransactionHash common.Hash `json:"transaction_hash"` LogIndex uint64 `json:"log_index"` SnapshotURI *string `json:"-"` @@ -1080,12 +1129,15 @@ func (i *Input) UnmarshalJSON(in []byte) error { type InputCompletionStatus string +//nolint:revive // Public enum names preserve the generated/API naming convention. const ( - InputCompletionStatus_None InputCompletionStatus = "NONE" - InputCompletionStatus_Accepted InputCompletionStatus = "ACCEPTED" - InputCompletionStatus_Rejected InputCompletionStatus = "REJECTED" - InputCompletionStatus_Exception InputCompletionStatus = "EXCEPTION" - InputCompletionStatus_MachineHalted InputCompletionStatus = "MACHINE_HALTED" + InputCompletionStatus_None InputCompletionStatus = "NONE" + InputCompletionStatus_Accepted InputCompletionStatus = "ACCEPTED" + InputCompletionStatus_Rejected InputCompletionStatus = "REJECTED" + InputCompletionStatus_Exception InputCompletionStatus = "EXCEPTION" + InputCompletionStatus_MachineHalted InputCompletionStatus = "MACHINE_HALTED" + InputCompletionStatus_Overflow InputCompletionStatus = "OVERFLOW" + InputCompletionStatus_UnexpectedYield InputCompletionStatus = "UNEXPECTED_YIELD" ) var InputCompletionStatusAllValues = []InputCompletionStatus{ @@ -1094,6 +1146,8 @@ var InputCompletionStatusAllValues = []InputCompletionStatus{ InputCompletionStatus_Rejected, InputCompletionStatus_Exception, InputCompletionStatus_MachineHalted, + InputCompletionStatus_Overflow, + InputCompletionStatus_UnexpectedYield, } // IsCompleted reports whether the status is a deterministic completed result @@ -1103,7 +1157,9 @@ func (e InputCompletionStatus) IsCompleted() bool { case InputCompletionStatus_Accepted, InputCompletionStatus_Rejected, InputCompletionStatus_Exception, - InputCompletionStatus_MachineHalted: + InputCompletionStatus_MachineHalted, + InputCompletionStatus_Overflow, + InputCompletionStatus_UnexpectedYield: return true case InputCompletionStatus_None: return false @@ -1112,6 +1168,35 @@ func (e InputCompletionStatus) IsCompleted() bool { } } +// IsTerminal reports whether a completed input leaves the canonical machine +// unable to process another advance. Rejection is completed but not terminal. +func (e InputCompletionStatus) IsTerminal() bool { + _, terminal := e.TerminalApplicationStatus() + return terminal +} + +// TerminalApplicationStatus maps an execution terminal to the durable status +// exposed for the application. The boolean is false for pending, accepted, and +// rejected inputs. +func (e InputCompletionStatus) TerminalApplicationStatus() (ApplicationStatus, bool) { + switch e { + case InputCompletionStatus_Exception: + return ApplicationStatus_GuestException, true + case InputCompletionStatus_MachineHalted: + return ApplicationStatus_MachineHalted, true + case InputCompletionStatus_Overflow: + return ApplicationStatus_McycleOverflow, true + case InputCompletionStatus_UnexpectedYield: + return ApplicationStatus_UnexpectedYield, true + case InputCompletionStatus_None, + InputCompletionStatus_Accepted, + InputCompletionStatus_Rejected: + return "", false + default: + return "", false + } +} + func (e *InputCompletionStatus) Scan(value any) error { var enumValue string switch val := value.(type) { @@ -1134,6 +1219,10 @@ func (e *InputCompletionStatus) Scan(value any) error { *e = InputCompletionStatus_Exception case "MACHINE_HALTED": *e = InputCompletionStatus_MachineHalted + case "OVERFLOW": + *e = InputCompletionStatus_Overflow + case "UNEXPECTED_YIELD": + *e = InputCompletionStatus_UnexpectedYield default: return errors.New("invalid value '" + enumValue + "' for InputCompletionStatus enum") } @@ -1369,14 +1458,44 @@ type NodeConfig[T any] struct { UpdatedAt time.Time } -type OutputsProof struct { - OutputsHash common.Hash - OutputsHashProof [][32]byte - MachineHash common.Hash +type StateProof struct { + TxBufferDataBlock common.Hash + TxBufferProof [][32]byte + MachineHash common.Hash + IflagsYDataBlock common.Hash + IflagsYProof [][32]byte + HtifTohostDataBlock common.Hash + HtifTohostProof [][32]byte +} + +// StateProofSiblingCount is the height of the canonical machine +// memory tree above a 32-byte data block (64 - 5). +const StateProofSiblingCount = 59 + +// IsComplete reports whether all three state leaves have the canonical sibling +// depth. Leaf contents and roots are verified by pkg/machine. +func (p *StateProof) IsComplete() bool { + return p != nil && + len(p.TxBufferProof) == StateProofSiblingCount && + len(p.IflagsYProof) == StateProofSiblingCount && + len(p.HtifTohostProof) == StateProofSiblingCount +} + +// HasCompleteStateProof reports whether an epoch contains every persisted +// component of the machine state proof. +func (e *Epoch) HasCompleteStateProof() bool { + return e != nil && + e.MachineHash != nil && + e.TxBufferDataBlock != nil && + e.IflagsYDataBlock != nil && + e.HtifTohostDataBlock != nil && + len(e.TxBufferProof) == StateProofSiblingCount && + len(e.IflagsYProof) == StateProofSiblingCount && + len(e.HtifTohostProof) == StateProofSiblingCount } type AdvanceResult struct { - OutputsProof + StateProof EpochIndex uint64 InputIndex uint64 Status InputCompletionStatus @@ -1400,14 +1519,14 @@ type ReplaySummary struct { // machine execution. It is deliberately narrower than Input: L1 metadata, // timestamps, and snapshot location do not participate in the comparison. type ReplayInput struct { - ApplicationID int64 - EpochIndex uint64 - InputIndex uint64 - RawData []byte - Status InputCompletionStatus - ExceptionData []byte - MachineHash *common.Hash - OutputsHash *common.Hash + ApplicationID int64 + EpochIndex uint64 + InputIndex uint64 + RawData []byte + Status InputCompletionStatus + ExceptionData []byte + MachineHash *common.Hash + TxBufferDataBlock *common.Hash } // ReplayStateHash is one persisted row of a PRT input hash collection. Keeping diff --git a/internal/model/models_json_test.go b/internal/model/models_json_test.go index 929d23dae..4c5c9ff91 100644 --- a/internal/model/models_json_test.go +++ b/internal/model/models_json_test.go @@ -14,6 +14,8 @@ import ( func TestEpochJSONRoundtrip(t *testing.T) { root := common.HexToHash("0xabcd") + iflagsY := common.HexToHash("0x1234") + htifTohost := common.HexToHash("0x5678") original := Epoch{ ApplicationID: 1, Index: 42, @@ -23,13 +25,22 @@ func TestEpochJSONRoundtrip(t *testing.T) { InputIndexUpperBound: 10, VirtualIndex: 5, Status: EpochStatus_ClaimAccepted, - OutputsMerkleRoot: &root, + TxBufferDataBlock: &root, + TxBufferProof: []common.Hash{common.HexToHash("0xaaaa")}, + IflagsYDataBlock: &iflagsY, + IflagsYProof: []common.Hash{common.HexToHash("0x1111")}, + HtifTohostDataBlock: &htifTohost, + HtifTohostProof: []common.Hash{common.HexToHash("0x2222")}, CreatedAt: time.Now().Truncate(time.Microsecond).UTC(), UpdatedAt: time.Now().Truncate(time.Microsecond).UTC(), } data, err := json.Marshal(&original) require.NoError(t, err) + require.Contains(t, string(data), `"tx_buffer_data_block"`) + require.Contains(t, string(data), `"tx_buffer_proof"`) + require.NotContains(t, string(data), `"outputs_merkle_root"`) + require.NotContains(t, string(data), `"outputs_merkle_proof"`) var decoded Epoch err = json.Unmarshal(data, &decoded) @@ -44,11 +55,46 @@ func TestEpochJSONRoundtrip(t *testing.T) { require.Equal(t, original.InputIndexUpperBound, decoded.InputIndexUpperBound) require.Equal(t, original.VirtualIndex, decoded.VirtualIndex) require.Equal(t, original.Status, decoded.Status) - require.Equal(t, original.OutputsMerkleRoot, decoded.OutputsMerkleRoot) + require.Equal(t, original.TxBufferDataBlock, decoded.TxBufferDataBlock) + require.Equal(t, original.TxBufferProof, decoded.TxBufferProof) + require.Equal(t, original.IflagsYDataBlock, decoded.IflagsYDataBlock) + require.Equal(t, original.IflagsYProof, decoded.IflagsYProof) + require.Equal(t, original.HtifTohostDataBlock, decoded.HtifTohostDataBlock) + require.Equal(t, original.HtifTohostProof, decoded.HtifTohostProof) +} + +func TestStateProofCompleteness(t *testing.T) { + siblings := make([][32]byte, StateProofSiblingCount) + proof := &StateProof{ + TxBufferProof: append([][32]byte(nil), siblings...), + IflagsYProof: append([][32]byte(nil), siblings...), + HtifTohostProof: append([][32]byte(nil), siblings...), + } + require.True(t, proof.IsComplete()) + + proof.HtifTohostProof = proof.HtifTohostProof[:len(proof.HtifTohostProof)-1] + require.False(t, proof.IsComplete()) + require.False(t, (*StateProof)(nil).IsComplete()) + + hash := common.Hash{1} + epoch := &Epoch{ + MachineHash: &hash, + TxBufferDataBlock: &hash, + TxBufferProof: make([]common.Hash, StateProofSiblingCount), + IflagsYDataBlock: &hash, + IflagsYProof: make([]common.Hash, StateProofSiblingCount), + HtifTohostDataBlock: &hash, + HtifTohostProof: make([]common.Hash, StateProofSiblingCount), + } + require.True(t, epoch.HasCompleteStateProof()) + epoch.IflagsYDataBlock = nil + require.False(t, epoch.HasCompleteStateProof()) + require.False(t, (*Epoch)(nil).HasCompleteStateProof()) } func TestInputJSONRoundtrip(t *testing.T) { machineHash := common.HexToHash("0x1234") + txBufferDataBlock := common.HexToHash("0xabcd") original := Input{ EpochApplicationID: 1, EpochIndex: 3, @@ -58,6 +104,7 @@ func TestInputJSONRoundtrip(t *testing.T) { Status: InputCompletionStatus_Exception, ExceptionData: []byte{0xff, 0x00, 0x80}, MachineHash: &machineHash, + TxBufferDataBlock: &txBufferDataBlock, TransactionHash: common.HexToHash("0x5678"), LogIndex: 11, CreatedAt: time.Now().Truncate(time.Microsecond).UTC(), @@ -70,6 +117,8 @@ func TestInputJSONRoundtrip(t *testing.T) { // LogIndex must be hex-encoded like the other uint64 fields. require.Contains(t, string(data), `"log_index":"0xb"`) require.Contains(t, string(data), `"exception_data":"0xff0080"`) + require.Contains(t, string(data), `"tx_buffer_data_block"`) + require.NotContains(t, string(data), `"outputs_hash"`) var decoded Input err = json.Unmarshal(data, &decoded) @@ -84,6 +133,7 @@ func TestInputJSONRoundtrip(t *testing.T) { require.Equal(t, original.Status, decoded.Status) require.Equal(t, original.ExceptionData, decoded.ExceptionData) require.Equal(t, original.MachineHash, decoded.MachineHash) + require.Equal(t, original.TxBufferDataBlock, decoded.TxBufferDataBlock) require.Equal(t, original.TransactionHash, decoded.TransactionHash) require.Equal(t, original.LogIndex, decoded.LogIndex) } diff --git a/internal/prt/prt.go b/internal/prt/prt.go index f64c77fe1..bf2dd5c66 100644 --- a/internal/prt/prt.go +++ b/internal/prt/prt.go @@ -405,7 +405,7 @@ func (s *Service) checkEpochs(ctx context.Context, app *Application, mostRecentB for _, epoch := range epochs { if epoch.TournamentAddress == nil || epoch.Commitment == nil || - epoch.MachineHash == nil || epoch.OutputsMerkleRoot == nil { + epoch.MachineHash == nil || epoch.TxBufferDataBlock == nil { return s.setApplicationCorrupted(ctx, app, "epoch %d has missing required fields for ClaimComputed status", epoch.Index) } @@ -453,9 +453,9 @@ func (s *Service) checkEpochs(ctx context.Context, app *Application, mostRecentB return s.setApplicationDiverged(ctx, app, "Epoch %d has inconsistent machine hash between off-chain (%s) and on-chain (%s)", epoch.Index, epoch.MachineHash.String(), hexutil.Encode(event.InitialMachineStateHash[:])) } - if *epoch.OutputsMerkleRoot != event.OutputsMerkleRoot { + if *epoch.TxBufferDataBlock != event.OutputsMerkleRoot { return s.setApplicationDiverged(ctx, app, "Epoch %d has inconsistent claim hash between off-chain (%s) and on-chain (%s)", - epoch.Index, epoch.OutputsMerkleRoot.String(), hexutil.Encode(event.OutputsMerkleRoot[:])) + epoch.Index, epoch.TxBufferDataBlock.String(), hexutil.Encode(event.OutputsMerkleRoot[:])) } err = s.fetchTournamentData(ctx, app, epoch, RootLevel, nil, nil, *epoch.TournamentAddress, mostRecentBlock) @@ -695,7 +695,7 @@ func (s *Service) trySettle(ctx context.Context, app *Application, mostRecentBlo return nil // nothing to do } - if epoch.OutputsMerkleRoot == nil || epoch.OutputsMerkleProof == nil { + if epoch.TxBufferDataBlock == nil || epoch.TxBufferProof == nil { return s.setApplicationCorrupted(ctx, app, "epoch %d has missing required fields for settlement", epoch.Index) } @@ -716,7 +716,7 @@ func (s *Service) trySettle(ctx context.Context, app *Application, mostRecentBlo } s.Logger.Info("Sending Settle transaction", "application", app.Name, "epoch_index", epoch.Index, - "outputs_merkle_root", epoch.OutputsMerkleRoot.String()) + "outputs_merkle_root", epoch.TxBufferDataBlock.String()) if s.txOptsFactory == nil { return fmt.Errorf("txOpts is required for settlement") @@ -728,7 +728,7 @@ func (s *Service) trySettle(ctx context.Context, app *Application, mostRecentBlo return fmt.Errorf("creating transaction options for settlement: %w", err) } tx, err := consensus.Settle(txOpts, result.EpochNumber, - *epoch.OutputsMerkleRoot, hashSliceToByteSlice(epoch.OutputsMerkleProof)) + *epoch.TxBufferDataBlock, hashSliceToByteSlice(epoch.TxBufferProof)) if err != nil { return s.handleSettleRevert(ctx, app, result.EpochNumber.Uint64(), err) } diff --git a/internal/prt/validation_test.go b/internal/prt/validation_test.go index 2abefc57a..d04c60e44 100644 --- a/internal/prt/validation_test.go +++ b/internal/prt/validation_test.go @@ -84,14 +84,14 @@ func newValidationService(t *testing.T) (*Service, *model.Application) { epoch := repotest.NewEpochBuilder(app.ID). WithStatus(model.EpochStatus_ClaimComputed). WithMachineHash(common.HexToHash("0x6")). - WithOutputsMerkleRoot(common.HexToHash("0x8")). + WithTxBufferDataBlock(common.HexToHash("0x8")). Build() tournamentAddress := common.HexToAddress("0x4") commitment := common.HexToHash("0x5") epoch.TournamentAddress = &tournamentAddress epoch.Commitment = &commitment epoch.CommitmentProof = []common.Hash{common.HexToHash("0x7")} - epoch.OutputsMerkleProof = []common.Hash{} + epoch.TxBufferProof = []common.Hash{} repo := &prtRepositoryMock{} repo.On("GetEpoch", mock.Anything, app.IApplicationAddress.Hex(), uint64(0)). diff --git a/internal/repository/postgres/application.go b/internal/repository/postgres/application.go index c372c9db9..da6d39bad 100644 --- a/internal/repository/postgres/application.go +++ b/internal/repository/postgres/application.go @@ -722,7 +722,10 @@ func (r *PostgresRepository) UpdateEventLastCheckBlock( SET( uint64Expr(blockNumber), ). - WHERE(table.Application.ID.IN(ids...)) + WHERE( + table.Application.ID.IN(ids...). + AND(column.LT(uint64Expr(blockNumber))), + ) sqlStr, args := updateStmt.Sql() _, err = r.db.Exec(ctx, sqlStr, args...) @@ -743,7 +746,7 @@ func (r *PostgresRepository) GetLastSnapshot(ctx context.Context, nameOrAddress table.Input.Status, table.Input.ExceptionData, table.Input.MachineHash, - table.Input.OutputsHash, + table.Input.TxBufferDataBlock, table.Input.TransactionHash, table.Input.LogIndex, table.Input.SnapshotURI, @@ -777,7 +780,7 @@ func (r *PostgresRepository) GetLastSnapshot(ctx context.Context, nameOrAddress &inp.Status, &inp.ExceptionData, &inp.MachineHash, - &inp.OutputsHash, + &inp.TxBufferDataBlock, &inp.TransactionHash, &inp.LogIndex, &inp.SnapshotURI, diff --git a/internal/repository/postgres/bulk.go b/internal/repository/postgres/bulk.go index cf57379a8..ea207894c 100644 --- a/internal/repository/postgres/bulk.go +++ b/internal/repository/postgres/bulk.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "math" - "unsafe" "github.com/ethereum/go-ethereum/common" "github.com/go-jet/jet/v2/postgres" @@ -19,13 +18,7 @@ import ( "github.com/cartesi/rollups-node/internal/repository/postgres/db/rollupsdb/public/table" ) -// byteSliceToHashSlice converts [][32]byte to []common.Hash without copying. -// This is safe because common.Hash is defined as [32]byte, so the memory layout is identical. -func byteSliceToHashSlice(b [][32]byte) []common.Hash { - return *(*[]common.Hash)(unsafe.Pointer(&b)) -} - -func encodeSiblings(siblings []common.Hash) [][]byte { +func encodeSiblings[T ~[32]byte](siblings []T) [][]byte { arr := make([][]byte, len(siblings)) for i, h := range siblings { arr[i] = make([]byte, len(h)) @@ -317,10 +310,11 @@ func updateInput( ctx context.Context, tx pgx.Tx, appID int64, + epochIndex uint64, inputIndex uint64, status model.InputCompletionStatus, exceptionData []byte, - outputsHash common.Hash, + txBufferDataBlock common.Hash, machineHash common.Hash, ) error { @@ -329,16 +323,17 @@ func updateInput( table.Input.Status, table.Input.ExceptionData, table.Input.MachineHash, - table.Input.OutputsHash, + table.Input.TxBufferDataBlock, ). SET( status, exceptionData, machineHash[:], - outputsHash[:], + txBufferDataBlock[:], ). WHERE( table.Input.EpochApplicationID.EQ(postgres.Int64(appID)). + AND(table.Input.EpochIndex.EQ(uint64Expr(epochIndex))). AND(table.Input.Index.EQ(uint64Expr(inputIndex))). AND(table.Input.Status.EQ(postgres.NewEnumValue(model.InputCompletionStatus_None.String()))), ) @@ -354,26 +349,32 @@ func updateInput( return nil } -func updateEpochOutputsMerkleProof( +func updateEpochState( ctx context.Context, tx pgx.Tx, appID int64, epochIndex uint64, - outputsHash common.Hash, - outputsHashProof []common.Hash, - machineHash common.Hash, + proof *model.StateProof, ) error { updStmt := table.Epoch. UPDATE( - table.Epoch.OutputsMerkleRoot, - table.Epoch.OutputsMerkleProof, + table.Epoch.TxBufferDataBlock, + table.Epoch.TxBufferProof, table.Epoch.MachineHash, + table.Epoch.IflagsYDataBlock, + table.Epoch.IflagsYProof, + table.Epoch.HtifTohostDataBlock, + table.Epoch.HtifTohostProof, ). SET( - outputsHash[:], - encodeSiblings(outputsHashProof), - machineHash[:], + proof.TxBufferDataBlock[:], + encodeSiblings(proof.TxBufferProof), + proof.MachineHash[:], + proof.IflagsYDataBlock[:], + encodeSiblings(proof.IflagsYProof), + proof.HtifTohostDataBlock[:], + encodeSiblings(proof.HtifTohostProof), ). WHERE( table.Epoch.ApplicationID.EQ(postgres.Int64(appID)). @@ -391,12 +392,129 @@ func updateEpochOutputsMerkleProof( return nil } +// lockAdvanceRows acquires and validates StoreAdvanceResult's lock set in the +// shared Application -> child-row order. The application row is the aggregate +// lock that serializes multi-row work for one application; the input row lock +// then keeps the exact input's epoch and completion status stable until the +// result transaction finishes. +// +// These are PostgreSQL transaction-scoped row locks, not Go mutexes or session +// advisory locks. Commit or rollback releases them. If the node connection is +// lost, PostgreSQL aborts its transaction when it detects the disconnect; if +// PostgreSQL itself restarts, crash recovery discards the uncommitted +// transaction. A network failure can delay release until the server detects +// the dead connection (or a configured timeout terminates it), but the lock has +// no lifetime independent of that transaction and server backend. +func lockAdvanceRows( + ctx context.Context, + tx pgx.Tx, + appID int64, + epochIndex uint64, + inputIndex uint64, +) error { + // SELECT FOR NO KEY UPDATE reads the cursor and status while taking a row + // lock that conflicts with concurrent updates of this application. It does + // not modify the row and does not block ordinary SELECTs. Because each + // participating multi-row writer takes this row before an input or epoch, + // two writers cannot form an Application <-> child-row deadlock cycle. + appStmt := table.Application. + SELECT(table.Application.ProcessedInputs, table.Application.Status). + WHERE(table.Application.ID.EQ(postgres.Int64(appID))). + FOR(postgres.NO_KEY_UPDATE()) + appSQL, appArgs := appStmt.Sql() + var processedInputs uint64 + var applicationStatus model.ApplicationStatus + if err := tx.QueryRow(ctx, appSQL, appArgs...).Scan( + &processedInputs, &applicationStatus, + ); errors.Is(err, pgx.ErrNoRows) { + return repository.ErrNotFound + } else if err != nil { + return err + } + if applicationStatus != model.ApplicationStatus_OK { + classification := repository.ErrApplicationNotRunnable + if applicationStatus.IsTerminal() { + classification = errors.Join(classification, repository.ErrAdvanceAfterTerminal) + } + return fmt.Errorf( + "%w: application %d has status %q", + classification, + appID, + applicationStatus, + ) + } + if processedInputs != inputIndex { + return fmt.Errorf( + "%w: application %d expects input %d, got %d", + repository.ErrAdvanceCursorMismatch, + appID, + processedInputs, + inputIndex, + ) + } + + // SELECT FOR UPDATE is stronger because this exact input will be updated. + // Taking it after the aggregate lock makes the preflight check durable: no + // concurrent writer can change its epoch or completion status between this + // validation and the result's output/proof/application writes. + inputStmt := table.Input. + SELECT(table.Input.EpochIndex, table.Input.Status). + WHERE( + table.Input.EpochApplicationID.EQ(postgres.Int64(appID)). + AND(table.Input.Index.EQ(uint64Expr(inputIndex))), + ). + FOR(postgres.UPDATE()) + inputSQL, inputArgs := inputStmt.Sql() + var storedEpochIndex uint64 + var storedStatus model.InputCompletionStatus + if err := tx.QueryRow(ctx, inputSQL, inputArgs...).Scan(&storedEpochIndex, &storedStatus); errors.Is(err, pgx.ErrNoRows) { + return repository.ErrNotFound + } else if err != nil { + return err + } + if storedEpochIndex != epochIndex || storedStatus != model.InputCompletionStatus_None { + return fmt.Errorf( + "%w: input %d has epoch %d and status %q, result has epoch %d", + repository.ErrAdvanceCursorMismatch, + inputIndex, + storedEpochIndex, + storedStatus, + epochIndex, + ) + } + + return nil +} + func updateApp( ctx context.Context, tx pgx.Tx, appID int64, inputIndex uint64, + completionStatus model.InputCompletionStatus, ) error { + where := table.Application.ID.EQ(postgres.Int64(appID)). + AND(table.Application.ProcessedInputs.EQ(uint64Expr(inputIndex))). + AND(table.Application.Status.EQ( + postgres.NewEnumValue(model.ApplicationStatus_OK.String()), + )) + + if applicationStatus, terminal := completionStatus.TerminalApplicationStatus(); terminal { + reason := fmt.Sprintf("input %d completed with %s", inputIndex, completionStatus) + updStmt := table.Application. + UPDATE( + table.Application.ProcessedInputs, + table.Application.Status, + table.Application.Reason, + ). + SET( + uint64Expr(inputIndex+1), + postgres.NewEnumValue(applicationStatus.String()), + reason, + ). + WHERE(where) + return executeApplicationAdvanceUpdate(ctx, tx, updStmt) + } updStmt := table.Application. UPDATE( @@ -405,17 +523,22 @@ func updateApp( SET( uint64Expr(inputIndex + 1), ). - WHERE( - table.Application.ID.EQ(postgres.Int64(appID)), - ) + WHERE(where) + return executeApplicationAdvanceUpdate(ctx, tx, updStmt) +} +func executeApplicationAdvanceUpdate( + ctx context.Context, + tx pgx.Tx, + updStmt postgres.UpdateStatement, +) error { sqlStr, args := updStmt.Sql() cmd, err := tx.Exec(ctx, sqlStr, args...) if err != nil { return err } if cmd.RowsAffected() == 0 { - return repository.ErrNotFound + return repository.ErrApplicationNotRunnable } return nil } @@ -434,12 +557,30 @@ func (r *PostgresRepository) StoreAdvanceResult( if err := validateAdvanceExceptionData(res.Status, res.ExceptionData); err != nil { return err } - + if res.InputIndex == math.MaxUint64 { + return errors.New("cannot store an advance result at the maximum input index") + } + if res.Status != model.InputCompletionStatus_Accepted && + (len(res.Outputs) != 0 || len(res.Reports) != 0) { + return fmt.Errorf("advance result with status %q must not contain outputs or reports", res.Status) + } + if !res.IsComplete() { + return repository.ErrInvalidStateProof + } tx, err := r.db.Begin(ctx) if err != nil { return err } defer tx.Rollback(ctx) //nolint:errcheck + // Acquire and validate the transaction-scoped Application -> Input lock set + // before inserting any effects. Holding both rows through Commit keeps the + // application cursor/status and the pending input used by this result + // unchanged while outputs, reports, proofs, and terminal status are written. + // Cancellation makes the waiting query return; the deferred rollback then + // releases any lock that this transaction had already acquired. + if err := lockAdvanceRows(ctx, tx, appID, res.EpochIndex, res.InputIndex); err != nil { + return err + } if res.Status == model.InputCompletionStatus_Accepted { err = insertOutputs(ctx, tx, appID, res.InputIndex, res.Outputs) @@ -464,19 +605,34 @@ func (r *PostgresRepository) StoreAdvanceResult( } err = updateInput( - ctx, tx, appID, res.InputIndex, res.Status, res.ExceptionData, res.OutputsHash, res.MachineHash, + ctx, tx, appID, res.EpochIndex, res.InputIndex, res.Status, res.ExceptionData, + res.TxBufferDataBlock, res.MachineHash, ) if err != nil { return err } - err = updateEpochOutputsMerkleProof(ctx, tx, appID, res.EpochIndex, res.OutputsHash, - byteSliceToHashSlice(res.OutputsHashProof), res.MachineHash) + switch res.Status { + case model.InputCompletionStatus_Accepted: + err = updateEpochState(ctx, tx, appID, res.EpochIndex, &res.StateProof) + case model.InputCompletionStatus_Rejected: + // A rejection consumes the input but leaves the pre-input epoch state + // and proof tuple unchanged. + case model.InputCompletionStatus_Exception, + model.InputCompletionStatus_MachineHalted, + model.InputCompletionStatus_Overflow, + model.InputCompletionStatus_UnexpectedYield: + err = updateEpochState(ctx, tx, appID, res.EpochIndex, &res.StateProof) + case model.InputCompletionStatus_None: + return fmt.Errorf("unsupported noncompleted advance status %q", res.Status) + default: + return fmt.Errorf("unsupported completed advance status %q", res.Status) + } if err != nil { return err } - err = updateApp(ctx, tx, appID, res.InputIndex) + err = updateApp(ctx, tx, appID, res.InputIndex, res.Status) if err != nil { return err } diff --git a/internal/repository/postgres/claimer.go b/internal/repository/postgres/claimer.go index 971d6aff2..faebc34f1 100644 --- a/internal/repository/postgres/claimer.go +++ b/internal/repository/postgres/claimer.go @@ -5,6 +5,7 @@ package postgres import ( "context" + "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -50,8 +51,8 @@ func (r *PostgresRepository) selectOldestClaimPerApp( table.Epoch.FirstBlock, table.Epoch.LastBlock, table.Epoch.MachineHash, - table.Epoch.OutputsMerkleRoot, - table.Epoch.OutputsMerkleProof, + table.Epoch.TxBufferDataBlock, + table.Epoch.TxBufferProof, table.Epoch.ClaimTransactionHash, table.Epoch.Status, table.Epoch.StagedAtBlock, @@ -124,8 +125,8 @@ func (r *PostgresRepository) selectOldestClaimPerApp( &epoch.FirstBlock, &epoch.LastBlock, &epoch.MachineHash, - &epoch.OutputsMerkleRoot, - &epoch.OutputsMerkleProof, + &epoch.TxBufferDataBlock, + &epoch.TxBufferProof, &epoch.ClaimTransactionHash, &epoch.Status, &epoch.StagedAtBlock, @@ -199,8 +200,8 @@ func (r *PostgresRepository) selectNewestClaimBarrierPerApp( table.Epoch.FirstBlock, table.Epoch.LastBlock, table.Epoch.MachineHash, - table.Epoch.OutputsMerkleRoot, - table.Epoch.OutputsMerkleProof, + table.Epoch.TxBufferDataBlock, + table.Epoch.TxBufferProof, table.Epoch.ClaimTransactionHash, table.Epoch.Status, table.Epoch.StagedAtBlock, @@ -243,8 +244,8 @@ func (r *PostgresRepository) selectNewestClaimBarrierPerApp( &epoch.FirstBlock, &epoch.LastBlock, &epoch.MachineHash, - &epoch.OutputsMerkleRoot, - &epoch.OutputsMerkleProof, + &epoch.TxBufferDataBlock, + &epoch.TxBufferProof, &epoch.ClaimTransactionHash, &epoch.Status, &epoch.StagedAtBlock, @@ -478,25 +479,51 @@ func (r *PostgresRepository) UpdateEpochWithForeclosedClaim( } // RejectEpochAndSetApplicationDiverged atomically records that the local claim -// lost the applicable consensus/dispute process and halts the application as -// DIVERGED. Quorum rejection is only a normal outcome before the local claim -// has staged; once CLAIM_STAGED is recorded, a different staged or accepted -// claim for the same epoch would violate the contract's single-staged claim -// invariant. The epoch reject and the application halt share one transaction, -// and the halt runs even when the epoch reject matched no row, so a detected -// divergence can never leave the application runnable. +// lost the applicable consensus/dispute process. Quorum rejection is only a +// normal outcome before the local claim has staged; once CLAIM_STAGED is +// recorded, a different staged or accepted claim for the same epoch would +// violate the contract's single-staged claim invariant. +// +// Unlike an executed-output byte mismatch, losing consensus does not prove the +// node's stored history is corrupt. An existing terminal machine outcome is +// therefore preserved while the epoch rejection is still attempted. For a +// non-terminal application, the DIVERGED halt runs even when the epoch reject +// matched no row. The returned flags identify the writes that committed. func (r *PostgresRepository) RejectEpochAndSetApplicationDiverged( ctx context.Context, applicationID int64, index uint64, reason string, -) error { +) (repository.RejectEpochAndDivergeResult, error) { + var result repository.RejectEpochAndDivergeResult tx, err := r.db.Begin(ctx) if err != nil { - return fmt.Errorf("beginning transaction for rejected claim update: %w", err) + return result, fmt.Errorf("beginning transaction for rejected claim update: %w", err) } defer tx.Rollback(ctx) //nolint:errcheck + // Take the shared aggregate lock before updating the epoch. SELECT FOR NO KEY + // UPDATE reads the current status without changing the application row, but + // its transaction-scoped row lock conflicts with concurrent application + // updates. Matching StoreAdvanceResult and CreateEpochsAndInputs' Application + // -> child-row order prevents an Epoch -> Application deadlock cycle. + // + // Commit or rollback releases the lock. PostgreSQL also aborts the + // transaction after detecting a lost node connection, and crash recovery + // discards it after a database restart. Detection can be delayed by a network + // failure, but the lock is not persisted independently of the transaction. + lockAppStmt := table.Application. + SELECT(table.Application.Status). + WHERE(table.Application.ID.EQ(postgres.Int64(applicationID))). + FOR(postgres.NO_KEY_UPDATE()) + lockAppSQL, lockAppArgs := lockAppStmt.Sql() + var applicationStatus model.ApplicationStatus + if err := tx.QueryRow(ctx, lockAppSQL, lockAppArgs...).Scan(&applicationStatus); errors.Is(err, pgx.ErrNoRows) { + return result, repository.ErrNotFound + } else if err != nil { + return result, fmt.Errorf("locking application for rejected claim update (app=%d, index=%d): %w", applicationID, index, err) + } + rejectStmt := table.Epoch. UPDATE(table.Epoch.Status). SET(postgres.NewEnumValue(model.EpochStatus_ClaimRejected.String())). @@ -510,34 +537,49 @@ func (r *PostgresRepository) RejectEpochAndSetApplicationDiverged( ) // The epoch reject is best-effort: an epoch outside CLAIM_COMPUTED/ - // CLAIM_SUBMITTED is left untouched. The application halt below does not - // depend on it — a detected divergence always stops the application. + // CLAIM_SUBMITTED is left untouched. For a non-terminal application, the + // DIVERGED transition below is independent of whether this update matched. sqlStr, args := rejectStmt.Sql() - if _, err := tx.Exec(ctx, sqlStr, args...); err != nil { - return fmt.Errorf("executing rejected claim update (app=%d, index=%d): %w", applicationID, index, err) - } - - appStmt := table.Application. - UPDATE( - table.Application.Status, - table.Application.Reason, - ). - SET( - model.ApplicationStatus_Diverged, - &reason, - ). - WHERE(table.Application.ID.EQ(postgres.Int64(applicationID))) - - sqlStr, args = appStmt.Sql() - cmd, err := tx.Exec(ctx, sqlStr, args...) + rejectCmd, err := tx.Exec(ctx, sqlStr, args...) if err != nil { - return fmt.Errorf("executing diverged application update (app=%d, index=%d): %w", applicationID, index, err) - } - if cmd.RowsAffected() == 0 { - return repository.ErrNotFound + return result, fmt.Errorf("executing rejected claim update (app=%d, index=%d): %w", applicationID, index, err) + } + result.EpochRejected = rejectCmd.RowsAffected() == 1 + + // Preserve an already-terminal cause. The best-effort epoch rejection still + // commits, but a later claim observation must not overwrite a stronger or + // earlier durable terminal status. + if !applicationStatus.IsTerminal() { + appStmt := table.Application. + UPDATE( + table.Application.Status, + table.Application.Reason, + ). + SET( + model.ApplicationStatus_Diverged, + &reason, + ). + WHERE(table.Application.ID.EQ(postgres.Int64(applicationID))) + + sqlStr, args = appStmt.Sql() + cmd, err := tx.Exec(ctx, sqlStr, args...) + if err != nil { + return repository.RejectEpochAndDivergeResult{}, fmt.Errorf( + "executing diverged application update (app=%d, index=%d): %w", + applicationID, index, err) + } + if cmd.RowsAffected() == 0 { + return repository.RejectEpochAndDivergeResult{}, repository.ErrNotFound + } + result.ApplicationDiverged = true } - return tx.Commit(ctx) + if err := tx.Commit(ctx); err != nil { + return repository.RejectEpochAndDivergeResult{}, fmt.Errorf( + "committing rejected claim update (app=%d, index=%d): %w", + applicationID, index, err) + } + return result, nil } // UpdateEpochToStaged transitions an epoch from CLAIM_SUBMITTED to diff --git a/internal/repository/postgres/db/rollupsdb/public/enum/applicationstatus.go b/internal/repository/postgres/db/rollupsdb/public/enum/applicationstatus.go index 5b4a349d0..4ff7b119c 100644 --- a/internal/repository/postgres/db/rollupsdb/public/enum/applicationstatus.go +++ b/internal/repository/postgres/db/rollupsdb/public/enum/applicationstatus.go @@ -10,13 +10,21 @@ package enum import "github.com/go-jet/jet/v2/postgres" var ApplicationStatus = &struct { - Ok postgres.StringExpression - Failed postgres.StringExpression - Diverged postgres.StringExpression - Corrupted postgres.StringExpression + Ok postgres.StringExpression + Failed postgres.StringExpression + Diverged postgres.StringExpression + Corrupted postgres.StringExpression + GuestException postgres.StringExpression + MachineHalted postgres.StringExpression + McycleOverflow postgres.StringExpression + UnexpectedYield postgres.StringExpression }{ - Ok: postgres.NewEnumValue("OK"), - Failed: postgres.NewEnumValue("FAILED"), - Diverged: postgres.NewEnumValue("DIVERGED"), - Corrupted: postgres.NewEnumValue("CORRUPTED"), + Ok: postgres.NewEnumValue("OK"), + Failed: postgres.NewEnumValue("FAILED"), + Diverged: postgres.NewEnumValue("DIVERGED"), + Corrupted: postgres.NewEnumValue("CORRUPTED"), + GuestException: postgres.NewEnumValue("GUEST_EXCEPTION"), + MachineHalted: postgres.NewEnumValue("MACHINE_HALTED"), + McycleOverflow: postgres.NewEnumValue("MCYCLE_OVERFLOW"), + UnexpectedYield: postgres.NewEnumValue("UNEXPECTED_YIELD"), } diff --git a/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go b/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go index ce3410837..dba7ab965 100644 --- a/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go +++ b/internal/repository/postgres/db/rollupsdb/public/enum/inputcompletionstatus.go @@ -10,15 +10,19 @@ package enum import "github.com/go-jet/jet/v2/postgres" var InputCompletionStatus = &struct { - None postgres.StringExpression - Accepted postgres.StringExpression - Rejected postgres.StringExpression - Exception postgres.StringExpression - MachineHalted postgres.StringExpression + None postgres.StringExpression + Accepted postgres.StringExpression + Rejected postgres.StringExpression + Exception postgres.StringExpression + MachineHalted postgres.StringExpression + Overflow postgres.StringExpression + UnexpectedYield postgres.StringExpression }{ - None: postgres.NewEnumValue("NONE"), - Accepted: postgres.NewEnumValue("ACCEPTED"), - Rejected: postgres.NewEnumValue("REJECTED"), - Exception: postgres.NewEnumValue("EXCEPTION"), - MachineHalted: postgres.NewEnumValue("MACHINE_HALTED"), + None: postgres.NewEnumValue("NONE"), + Accepted: postgres.NewEnumValue("ACCEPTED"), + Rejected: postgres.NewEnumValue("REJECTED"), + Exception: postgres.NewEnumValue("EXCEPTION"), + MachineHalted: postgres.NewEnumValue("MACHINE_HALTED"), + Overflow: postgres.NewEnumValue("OVERFLOW"), + UnexpectedYield: postgres.NewEnumValue("UNEXPECTED_YIELD"), } diff --git a/internal/repository/postgres/db/rollupsdb/public/table/epoch.go b/internal/repository/postgres/db/rollupsdb/public/table/epoch.go index 2823afa5e..01eecb3c0 100644 --- a/internal/repository/postgres/db/rollupsdb/public/table/epoch.go +++ b/internal/repository/postgres/db/rollupsdb/public/table/epoch.go @@ -24,8 +24,12 @@ type epochTable struct { InputIndexLowerBound postgres.ColumnFloat InputIndexUpperBound postgres.ColumnFloat MachineHash postgres.ColumnBytea - OutputsMerkleRoot postgres.ColumnBytea - OutputsMerkleProof postgres.ColumnByteaArray + TxBufferDataBlock postgres.ColumnBytea + TxBufferProof postgres.ColumnByteaArray + IflagsYDataBlock postgres.ColumnBytea + IflagsYProof postgres.ColumnByteaArray + HtifTohostDataBlock postgres.ColumnBytea + HtifTohostProof postgres.ColumnByteaArray Commitment postgres.ColumnBytea CommitmentProof postgres.ColumnByteaArray TournamentAddress postgres.ColumnBytea @@ -83,8 +87,12 @@ func newEpochTableImpl(schemaName, tableName, alias string) epochTable { InputIndexLowerBoundColumn = postgres.FloatColumn("input_index_lower_bound") InputIndexUpperBoundColumn = postgres.FloatColumn("input_index_upper_bound") MachineHashColumn = postgres.ByteaColumn("machine_hash") - OutputsMerkleRootColumn = postgres.ByteaColumn("outputs_merkle_root") - OutputsMerkleProofColumn = postgres.ByteaArrayColumn("outputs_merkle_proof") + TxBufferDataBlockColumn = postgres.ByteaColumn("tx_buffer_data_block") + TxBufferProofColumn = postgres.ByteaArrayColumn("tx_buffer_proof") + IflagsYDataBlockColumn = postgres.ByteaColumn("iflags_y_data_block") + IflagsYProofColumn = postgres.ByteaArrayColumn("iflags_y_proof") + HtifTohostDataBlockColumn = postgres.ByteaColumn("htif_tohost_data_block") + HtifTohostProofColumn = postgres.ByteaArrayColumn("htif_tohost_proof") CommitmentColumn = postgres.ByteaColumn("commitment") CommitmentProofColumn = postgres.ByteaArrayColumn("commitment_proof") TournamentAddressColumn = postgres.ByteaColumn("tournament_address") @@ -94,8 +102,8 @@ func newEpochTableImpl(schemaName, tableName, alias string) epochTable { VirtualIndexColumn = postgres.FloatColumn("virtual_index") CreatedAtColumn = postgres.TimestampzColumn("created_at") UpdatedAtColumn = postgres.TimestampzColumn("updated_at") - allColumns = postgres.ColumnList{ApplicationIDColumn, IndexColumn, FirstBlockColumn, LastBlockColumn, InputIndexLowerBoundColumn, InputIndexUpperBoundColumn, MachineHashColumn, OutputsMerkleRootColumn, OutputsMerkleProofColumn, CommitmentColumn, CommitmentProofColumn, TournamentAddressColumn, ClaimTransactionHashColumn, StatusColumn, StagedAtBlockColumn, VirtualIndexColumn, CreatedAtColumn, UpdatedAtColumn} - mutableColumns = postgres.ColumnList{FirstBlockColumn, LastBlockColumn, InputIndexLowerBoundColumn, InputIndexUpperBoundColumn, MachineHashColumn, OutputsMerkleRootColumn, OutputsMerkleProofColumn, CommitmentColumn, CommitmentProofColumn, TournamentAddressColumn, ClaimTransactionHashColumn, StatusColumn, StagedAtBlockColumn, VirtualIndexColumn, CreatedAtColumn, UpdatedAtColumn} + allColumns = postgres.ColumnList{ApplicationIDColumn, IndexColumn, FirstBlockColumn, LastBlockColumn, InputIndexLowerBoundColumn, InputIndexUpperBoundColumn, MachineHashColumn, TxBufferDataBlockColumn, TxBufferProofColumn, IflagsYDataBlockColumn, IflagsYProofColumn, HtifTohostDataBlockColumn, HtifTohostProofColumn, CommitmentColumn, CommitmentProofColumn, TournamentAddressColumn, ClaimTransactionHashColumn, StatusColumn, StagedAtBlockColumn, VirtualIndexColumn, CreatedAtColumn, UpdatedAtColumn} + mutableColumns = postgres.ColumnList{FirstBlockColumn, LastBlockColumn, InputIndexLowerBoundColumn, InputIndexUpperBoundColumn, MachineHashColumn, TxBufferDataBlockColumn, TxBufferProofColumn, IflagsYDataBlockColumn, IflagsYProofColumn, HtifTohostDataBlockColumn, HtifTohostProofColumn, CommitmentColumn, CommitmentProofColumn, TournamentAddressColumn, ClaimTransactionHashColumn, StatusColumn, StagedAtBlockColumn, VirtualIndexColumn, CreatedAtColumn, UpdatedAtColumn} defaultColumns = postgres.ColumnList{CreatedAtColumn, UpdatedAtColumn} ) @@ -110,8 +118,12 @@ func newEpochTableImpl(schemaName, tableName, alias string) epochTable { InputIndexLowerBound: InputIndexLowerBoundColumn, InputIndexUpperBound: InputIndexUpperBoundColumn, MachineHash: MachineHashColumn, - OutputsMerkleRoot: OutputsMerkleRootColumn, - OutputsMerkleProof: OutputsMerkleProofColumn, + TxBufferDataBlock: TxBufferDataBlockColumn, + TxBufferProof: TxBufferProofColumn, + IflagsYDataBlock: IflagsYDataBlockColumn, + IflagsYProof: IflagsYProofColumn, + HtifTohostDataBlock: HtifTohostDataBlockColumn, + HtifTohostProof: HtifTohostProofColumn, Commitment: CommitmentColumn, CommitmentProof: CommitmentProofColumn, TournamentAddress: TournamentAddressColumn, diff --git a/internal/repository/postgres/db/rollupsdb/public/table/input.go b/internal/repository/postgres/db/rollupsdb/public/table/input.go index 11c33e20c..62dfa9526 100644 --- a/internal/repository/postgres/db/rollupsdb/public/table/input.go +++ b/internal/repository/postgres/db/rollupsdb/public/table/input.go @@ -25,7 +25,7 @@ type inputTable struct { Status postgres.ColumnString ExceptionData postgres.ColumnBytea MachineHash postgres.ColumnBytea - OutputsHash postgres.ColumnBytea + TxBufferDataBlock postgres.ColumnBytea TransactionHash postgres.ColumnBytea LogIndex postgres.ColumnFloat SnapshotURI postgres.ColumnString @@ -80,14 +80,14 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { StatusColumn = postgres.StringColumn("status") ExceptionDataColumn = postgres.ByteaColumn("exception_data") MachineHashColumn = postgres.ByteaColumn("machine_hash") - OutputsHashColumn = postgres.ByteaColumn("outputs_hash") + TxBufferDataBlockColumn = postgres.ByteaColumn("tx_buffer_data_block") TransactionHashColumn = postgres.ByteaColumn("transaction_hash") LogIndexColumn = postgres.FloatColumn("log_index") SnapshotURIColumn = postgres.StringColumn("snapshot_uri") CreatedAtColumn = postgres.TimestampzColumn("created_at") UpdatedAtColumn = postgres.TimestampzColumn("updated_at") - allColumns = postgres.ColumnList{EpochApplicationIDColumn, EpochIndexColumn, IndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} - mutableColumns = postgres.ColumnList{EpochIndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, OutputsHashColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} + allColumns = postgres.ColumnList{EpochApplicationIDColumn, EpochIndexColumn, IndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, TxBufferDataBlockColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} + mutableColumns = postgres.ColumnList{EpochIndexColumn, BlockNumberColumn, RawDataColumn, StatusColumn, ExceptionDataColumn, MachineHashColumn, TxBufferDataBlockColumn, TransactionHashColumn, LogIndexColumn, SnapshotURIColumn, CreatedAtColumn, UpdatedAtColumn} defaultColumns = postgres.ColumnList{CreatedAtColumn, UpdatedAtColumn} ) @@ -103,7 +103,7 @@ func newInputTableImpl(schemaName, tableName, alias string) inputTable { Status: StatusColumn, ExceptionData: ExceptionDataColumn, MachineHash: MachineHashColumn, - OutputsHash: OutputsHashColumn, + TxBufferDataBlock: TxBufferDataBlockColumn, TransactionHash: TransactionHashColumn, LogIndex: LogIndexColumn, SnapshotURI: SnapshotURIColumn, diff --git a/internal/repository/postgres/epoch.go b/internal/repository/postgres/epoch.go index 5a9b8986e..43159e328 100644 --- a/internal/repository/postgres/epoch.go +++ b/internal/repository/postgres/epoch.go @@ -99,6 +99,31 @@ func (r *PostgresRepository) CreateEpochsAndInputs( } defer tx.Rollback(ctx) //nolint:errcheck + // Participate in the same Application -> child-row order as + // StoreAdvanceResult and RejectEpochAndSetApplicationDiverged. SELECT FOR NO + // KEY UPDATE does not change the application row; it takes a + // transaction-scoped row lock that conflicts with application updates. L1 + // ingestion holds it while upserting epochs and inputs and advancing the + // input-scan cursor, so another multi-row writer for this application waits + // before touching a child row instead of forming a child -> Application + // deadlock cycle. + // + // Commit or rollback releases the lock. A lost node connection aborts the + // transaction once PostgreSQL detects it (which a network failure can + // delay), and a PostgreSQL restart discards the uncommitted transaction + // during crash recovery. The lock is not persisted independently. + appLockStmt := table.Application. + SELECT(table.Application.ID). + WHERE(whereClause). + FOR(postgres.NO_KEY_UPDATE()) + appLockSQL, appLockArgs := appLockStmt.Sql() + var appID int64 + if err := tx.QueryRow(ctx, appLockSQL, appLockArgs...).Scan(&appID); errors.Is(err, pgx.ErrNoRows) { + return repository.ErrNotFound + } else if err != nil { + return err + } + epochs := orderEpochs(epochInputsMap) for _, epoch := range epochs { inputs := epochInputsMap[epoch] @@ -261,8 +286,12 @@ func (r *PostgresRepository) GetEpoch( table.Epoch.InputIndexLowerBound, table.Epoch.InputIndexUpperBound, table.Epoch.MachineHash, - table.Epoch.OutputsMerkleRoot, - table.Epoch.OutputsMerkleProof, + table.Epoch.TxBufferDataBlock, + table.Epoch.TxBufferProof, + table.Epoch.IflagsYDataBlock, + table.Epoch.IflagsYProof, + table.Epoch.HtifTohostDataBlock, + table.Epoch.HtifTohostProof, table.Epoch.Commitment, table.Epoch.CommitmentProof, table.Epoch.ClaimTransactionHash, @@ -296,8 +325,12 @@ func (r *PostgresRepository) GetEpoch( &ep.InputIndexLowerBound, &ep.InputIndexUpperBound, &ep.MachineHash, - &ep.OutputsMerkleRoot, - &ep.OutputsMerkleProof, + &ep.TxBufferDataBlock, + &ep.TxBufferProof, + &ep.IflagsYDataBlock, + &ep.IflagsYProof, + &ep.HtifTohostDataBlock, + &ep.HtifTohostProof, &ep.Commitment, &ep.CommitmentProof, &ep.ClaimTransactionHash, @@ -532,8 +565,12 @@ func (r *PostgresRepository) GetLastNonOpenEpoch( table.Epoch.InputIndexLowerBound, table.Epoch.InputIndexUpperBound, table.Epoch.MachineHash, - table.Epoch.OutputsMerkleRoot, - table.Epoch.OutputsMerkleProof, + table.Epoch.TxBufferDataBlock, + table.Epoch.TxBufferProof, + table.Epoch.IflagsYDataBlock, + table.Epoch.IflagsYProof, + table.Epoch.HtifTohostDataBlock, + table.Epoch.HtifTohostProof, table.Epoch.Commitment, table.Epoch.CommitmentProof, table.Epoch.ClaimTransactionHash, @@ -569,8 +606,12 @@ func (r *PostgresRepository) GetLastNonOpenEpoch( &ep.InputIndexLowerBound, &ep.InputIndexUpperBound, &ep.MachineHash, - &ep.OutputsMerkleRoot, - &ep.OutputsMerkleProof, + &ep.TxBufferDataBlock, + &ep.TxBufferProof, + &ep.IflagsYDataBlock, + &ep.IflagsYProof, + &ep.HtifTohostDataBlock, + &ep.HtifTohostProof, &ep.Commitment, &ep.CommitmentProof, &ep.ClaimTransactionHash, @@ -607,8 +648,12 @@ func (r *PostgresRepository) GetEpochByVirtualIndex( table.Epoch.InputIndexLowerBound, table.Epoch.InputIndexUpperBound, table.Epoch.MachineHash, - table.Epoch.OutputsMerkleRoot, - table.Epoch.OutputsMerkleProof, + table.Epoch.TxBufferDataBlock, + table.Epoch.TxBufferProof, + table.Epoch.IflagsYDataBlock, + table.Epoch.IflagsYProof, + table.Epoch.HtifTohostDataBlock, + table.Epoch.HtifTohostProof, table.Epoch.Commitment, table.Epoch.CommitmentProof, table.Epoch.ClaimTransactionHash, @@ -642,8 +687,12 @@ func (r *PostgresRepository) GetEpochByVirtualIndex( &ep.InputIndexLowerBound, &ep.InputIndexUpperBound, &ep.MachineHash, - &ep.OutputsMerkleRoot, - &ep.OutputsMerkleProof, + &ep.TxBufferDataBlock, + &ep.TxBufferProof, + &ep.IflagsYDataBlock, + &ep.IflagsYProof, + &ep.HtifTohostDataBlock, + &ep.HtifTohostProof, &ep.Commitment, &ep.CommitmentProof, &ep.ClaimTransactionHash, @@ -698,22 +747,6 @@ func (r *PostgresRepository) UpdateEpochClaimTransactionHash( return nil } -func (r *PostgresRepository) UpdateEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64, proof *model.OutputsProof) error { - tx, err := r.db.Begin(ctx) - if err != nil { - return err - } - defer tx.Rollback(ctx) //nolint:errcheck - - err = updateEpochOutputsMerkleProof(ctx, tx, appID, epochIndex, - proof.OutputsHash, byteSliceToHashSlice(proof.OutputsHashProof), proof.MachineHash) - if err != nil { - return err - } - - return tx.Commit(ctx) -} - func (r *PostgresRepository) UpdateEpochStatus( ctx context.Context, nameOrAddress string, @@ -753,7 +786,11 @@ func (r *PostgresRepository) UpdateEpochInputsProcessed( ctx context.Context, nameOrAddress string, epochIndex uint64, + proof *model.StateProof, ) error { + if !proof.IsComplete() { + return repository.ErrInvalidStateProof + } whereClause := getWhereClauseFromNameOrAddress(nameOrAddress) @@ -794,13 +831,33 @@ func (r *PostgresRepository) UpdateEpochInputsProcessed( inputsCondition := hasNoInputs.OR(allInputsPresentAndProcessed) - // Update statement to set epoch status to InputsProcessed - updateStmt := table.Epoch.UPDATE(table.Epoch.Status). - SET(enum.EpochStatus.InputsProcessed). + // Publish the final state proof and status atomically. Readers can never + // observe INPUTS_PROCESSED without all three canonical proof leaves. + updateStmt := table.Epoch.UPDATE( + table.Epoch.Status, + table.Epoch.TxBufferDataBlock, + table.Epoch.TxBufferProof, + table.Epoch.MachineHash, + table.Epoch.IflagsYDataBlock, + table.Epoch.IflagsYProof, + table.Epoch.HtifTohostDataBlock, + table.Epoch.HtifTohostProof, + ). + SET( + enum.EpochStatus.InputsProcessed, + proof.TxBufferDataBlock[:], + encodeSiblings(proof.TxBufferProof), + proof.MachineHash[:], + proof.IflagsYDataBlock[:], + encodeSiblings(proof.IflagsYProof), + proof.HtifTohostDataBlock[:], + encodeSiblings(proof.HtifTohostProof), + ). FROM(table.Application). WHERE(postgres.AND( table.Epoch.Status.EQ(postgres.NewEnumValue(model.EpochStatus_Closed.String())), table.Epoch.ApplicationID.EQ(table.Application.ID), + table.Application.Status.EQ(enum.ApplicationStatus.Ok), table.Epoch.Index.EQ(uint64Expr(epochIndex)), whereClause, prevCondition, @@ -815,7 +872,7 @@ func (r *PostgresRepository) UpdateEpochInputsProcessed( err := r.db.QueryRow(ctx, sqlStr, args...).Scan(&index) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - return nil + return repository.ErrNoUpdate } return err } @@ -885,7 +942,9 @@ func (r *PostgresRepository) ListEpochs( table.Epoch.InputIndexLowerBound, table.Epoch.InputIndexUpperBound, table.Epoch.MachineHash, - table.Epoch.OutputsMerkleRoot, + table.Epoch.TxBufferDataBlock, + table.Epoch.IflagsYDataBlock, + table.Epoch.HtifTohostDataBlock, table.Epoch.Commitment, table.Epoch.ClaimTransactionHash, table.Epoch.TournamentAddress, @@ -930,7 +989,9 @@ func (r *PostgresRepository) ListEpochs( &ep.InputIndexLowerBound, &ep.InputIndexUpperBound, &ep.MachineHash, - &ep.OutputsMerkleRoot, + &ep.TxBufferDataBlock, + &ep.IflagsYDataBlock, + &ep.HtifTohostDataBlock, &ep.Commitment, &ep.ClaimTransactionHash, &ep.TournamentAddress, @@ -950,45 +1011,3 @@ func (r *PostgresRepository) ListEpochs( } return epochs, total, nil } - -func (r *PostgresRepository) RepeatPreviousEpochOutputsProof( - ctx context.Context, - appID int64, - epochIndex uint64, -) error { - if epochIndex == 0 { - return fmt.Errorf("cannot repeat outputs proof for epoch 0") - } - - e1 := table.Epoch.AS("e1") - e2 := table.Epoch.AS("e2") - updStmt := e1. - UPDATE( - e1.OutputsMerkleRoot, - e1.OutputsMerkleProof, - e1.MachineHash, - ). - SET( - e2.OutputsMerkleRoot, - e2.OutputsMerkleProof, - e2.MachineHash, - ). - FROM(e2). - WHERE(postgres.AND( - e1.ApplicationID.EQ(postgres.Int64(appID)), - e1.Index.EQ(uint64Expr(epochIndex)), - e2.ApplicationID.EQ(postgres.Int64(appID)), - e2.Index.EQ(uint64Expr(epochIndex-1)), - )) - - sqlStr, args := updStmt.Sql() - - cmd, err := r.db.Exec(ctx, sqlStr, args...) - if err != nil { - return err - } - if cmd.RowsAffected() == 0 { - return repository.ErrNotFound - } - return nil -} diff --git a/internal/repository/postgres/input.go b/internal/repository/postgres/input.go index 67049469e..b330e7ae7 100644 --- a/internal/repository/postgres/input.go +++ b/internal/repository/postgres/input.go @@ -33,7 +33,7 @@ func (r *PostgresRepository) GetInput( table.Input.Status, table.Input.ExceptionData, table.Input.MachineHash, - table.Input.OutputsHash, + table.Input.TxBufferDataBlock, table.Input.TransactionHash, table.Input.LogIndex, table.Input.SnapshotURI, @@ -64,7 +64,7 @@ func (r *PostgresRepository) GetInput( &inp.Status, &inp.ExceptionData, &inp.MachineHash, - &inp.OutputsHash, + &inp.TxBufferDataBlock, &inp.TransactionHash, &inp.LogIndex, &inp.SnapshotURI, @@ -98,7 +98,7 @@ func (r *PostgresRepository) GetLastInput( table.Input.Status, table.Input.ExceptionData, table.Input.MachineHash, - table.Input.OutputsHash, + table.Input.TxBufferDataBlock, table.Input.TransactionHash, table.Input.LogIndex, table.Input.SnapshotURI, @@ -131,7 +131,7 @@ func (r *PostgresRepository) GetLastInput( &inp.Status, &inp.ExceptionData, &inp.MachineHash, - &inp.OutputsHash, + &inp.TxBufferDataBlock, &inp.TransactionHash, &inp.LogIndex, &inp.SnapshotURI, @@ -164,7 +164,7 @@ func (r *PostgresRepository) GetLastProcessedInput( table.Input.Status, table.Input.ExceptionData, table.Input.MachineHash, - table.Input.OutputsHash, + table.Input.TxBufferDataBlock, table.Input.TransactionHash, table.Input.LogIndex, table.Input.SnapshotURI, @@ -197,7 +197,7 @@ func (r *PostgresRepository) GetLastProcessedInput( &inp.Status, &inp.ExceptionData, &inp.MachineHash, - &inp.OutputsHash, + &inp.TxBufferDataBlock, &inp.TransactionHash, &inp.LogIndex, &inp.SnapshotURI, @@ -281,7 +281,7 @@ func (r *PostgresRepository) ListInputs( table.Input.Status, table.Input.ExceptionData, table.Input.MachineHash, - table.Input.OutputsHash, + table.Input.TxBufferDataBlock, table.Input.TransactionHash, table.Input.LogIndex, table.Input.SnapshotURI, @@ -322,7 +322,7 @@ func (r *PostgresRepository) ListInputs( &in.Status, &in.ExceptionData, &in.MachineHash, - &in.OutputsHash, + &in.TxBufferDataBlock, &in.TransactionHash, &in.LogIndex, &in.SnapshotURI, diff --git a/internal/repository/postgres/input_exception_data_test.go b/internal/repository/postgres/input_exception_data_test.go index 602b83543..8030f1d59 100644 --- a/internal/repository/postgres/input_exception_data_test.go +++ b/internal/repository/postgres/input_exception_data_test.go @@ -33,33 +33,33 @@ func TestPostgresInputExceptionDataContract(t *testing.T) { seed := repotest.Seed(ctx, t, repo) machineHash := repotest.UniqueHash() - outputsHash := repotest.UniqueHash() + txBufferDataBlock := repotest.UniqueHash() _, err = conn.Exec(ctx, ` UPDATE input - SET status = 'EXCEPTION', machine_hash = $2, outputs_hash = $3 + SET status = 'EXCEPTION', machine_hash = $2, tx_buffer_data_block = $3 WHERE epoch_application_id = $1 AND index = 0`, - seed.App.ID, machineHash[:], outputsHash[:], + seed.App.ID, machineHash[:], txBufferDataBlock[:], ) requirePostgresConstraint(t, err, "input_exception_data_check") _, err = conn.Exec(ctx, ` UPDATE input - SET status = 'ACCEPTED', exception_data = '\x01', machine_hash = $2, outputs_hash = $3 + SET status = 'ACCEPTED', exception_data = '\x01', machine_hash = $2, tx_buffer_data_block = $3 WHERE epoch_application_id = $1 AND index = 0`, - seed.App.ID, machineHash[:], outputsHash[:], + seed.App.ID, machineHash[:], txBufferDataBlock[:], ) requirePostgresConstraint(t, err, "input_exception_data_check") + proof := repotest.DummyStateProof() + proof.MachineHash = machineHash + proof.TxBufferDataBlock = txBufferDataBlock require.NoError(t, repo.StoreAdvanceResult(ctx, seed.App.ID, &model.AdvanceResult{ EpochIndex: seed.Epoch.Index, InputIndex: seed.Input.Index, Status: model.InputCompletionStatus_Exception, ExceptionData: []byte{}, - OutputsProof: model.OutputsProof{ - MachineHash: machineHash, - OutputsHash: outputsHash, - }, + StateProof: *proof, })) completed, err := repo.GetInput(ctx, seed.App.IApplicationAddress.String(), seed.Input.Index) require.NoError(t, err) diff --git a/internal/repository/postgres/postgres_repo_test.go b/internal/repository/postgres/postgres_repo_test.go index 0fd18784c..a182beae6 100644 --- a/internal/repository/postgres/postgres_repo_test.go +++ b/internal/repository/postgres/postgres_repo_test.go @@ -73,7 +73,35 @@ func TestPostgresSchemaExecutionOutcomeContract(t *testing.T) { require.NoError(t, err) labels, err := pgx.CollectRows(rows, pgx.RowTo[string]) require.NoError(t, err) - require.Equal(t, []string{"NONE", "ACCEPTED", "REJECTED", "EXCEPTION", "MACHINE_HALTED"}, labels) + require.Equal(t, []string{ + "NONE", + "ACCEPTED", + "REJECTED", + "EXCEPTION", + "MACHINE_HALTED", + "OVERFLOW", + "UNEXPECTED_YIELD", + }, labels) + + rows, err = conn.Query(ctx, ` + SELECT enumlabel + FROM pg_enum + JOIN pg_type ON pg_type.oid = pg_enum.enumtypid + WHERE pg_type.typname = 'ApplicationStatus' + ORDER BY enumsortorder`) + require.NoError(t, err) + labels, err = pgx.CollectRows(rows, pgx.RowTo[string]) + require.NoError(t, err) + require.Equal(t, []string{ + "OK", + "FAILED", + "DIVERGED", + "CORRUPTED", + "GUEST_EXCEPTION", + "MACHINE_HALTED", + "MCYCLE_OVERFLOW", + "UNEXPECTED_YIELD", + }, labels) rows, err = conn.Query(ctx, ` SELECT column_name @@ -87,6 +115,22 @@ func TestPostgresSchemaExecutionOutcomeContract(t *testing.T) { require.Contains(t, columns, "advance_inc_cycles") require.Contains(t, columns, "inspect_inc_cycles") + rows, err = conn.Query(ctx, ` + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'epoch'`) + require.NoError(t, err) + epochColumns, err := pgx.CollectRows(rows, pgx.RowTo[string]) + require.NoError(t, err) + for _, column := range []string{ + "iflags_y_data_block", + "iflags_y_proof", + "htif_tohost_data_block", + "htif_tohost_proof", + } { + require.Contains(t, epochColumns, column) + } + for _, column := range []string{"advance_max_cycles", "inspect_max_cycles"} { var defaultValue string err := conn.QueryRow(ctx, ` @@ -128,6 +172,28 @@ func TestPostgresSchemaExecutionOutcomeContract(t *testing.T) { ), test.value, app.ID) requirePostgresConstraint(t, err, "execution_parameters_"+test.column+"_check") } + + seed := repotest.Seed(ctx, t, repo) + zeroHash := make([]byte, 32) + _, err = conn.Exec(ctx, ` + UPDATE epoch + SET machine_hash = $1 + WHERE application_id = $2 AND index = $3`, + zeroHash, seed.App.ID, seed.Epoch.Index) + requirePostgresConstraint(t, err, "epoch_state_proof_tuple_check") + + _, err = conn.Exec(ctx, ` + UPDATE epoch + SET machine_hash = $1::bytea, + tx_buffer_data_block = $1::bytea, + tx_buffer_proof = ARRAY[NULL::bytea] || array_fill($1::bytea, ARRAY[58]), + iflags_y_data_block = $1::bytea, + iflags_y_proof = array_fill($1::bytea, ARRAY[59]), + htif_tohost_data_block = $1::bytea, + htif_tohost_proof = array_fill($1::bytea, ARRAY[59]) + WHERE application_id = $2 AND index = $3`, + zeroHash, seed.App.ID, seed.Epoch.Index) + requirePostgresConstraint(t, err, "epoch_tx_buffer_proof_elements_check") } func requirePostgresConstraint(t *testing.T, err error, constraint string) { @@ -135,5 +201,5 @@ func requirePostgresConstraint(t *testing.T, err error, constraint string) { require.Error(t, err) var pgErr *pgconn.PgError require.ErrorAs(t, err, &pgErr) - require.Equal(t, constraint, pgErr.ConstraintName) + require.Equal(t, constraint, pgErr.ConstraintName, "PostgreSQL error: %s", pgErr.Message) } diff --git a/internal/repository/postgres/replay.go b/internal/repository/postgres/replay.go index 540b0eb55..8f518cedc 100644 --- a/internal/repository/postgres/replay.go +++ b/internal/repository/postgres/replay.go @@ -22,6 +22,8 @@ var replayCompletedStatuses = []postgres.Expression{ postgres.NewEnumValue(model.InputCompletionStatus_Rejected.String()), postgres.NewEnumValue(model.InputCompletionStatus_Exception.String()), postgres.NewEnumValue(model.InputCompletionStatus_MachineHalted.String()), + postgres.NewEnumValue(model.InputCompletionStatus_Overflow.String()), + postgres.NewEnumValue(model.InputCompletionStatus_UnexpectedYield.String()), } func replayCompletedStatus(status postgres.StringExpression) postgres.BoolExpression { @@ -361,7 +363,7 @@ func (r *PostgresRepository) ReplayPage( table.Input.Status, table.Input.ExceptionData, table.Input.MachineHash, - table.Input.OutputsHash, + table.Input.TxBufferDataBlock, ). WHERE( whereInputApp. @@ -441,7 +443,7 @@ func selectReplayInputs( &in.Status, &in.ExceptionData, &in.MachineHash, - &in.OutputsHash, + &in.TxBufferDataBlock, ); err != nil { return nil, err } diff --git a/internal/repository/postgres/replay_source_test.go b/internal/repository/postgres/replay_source_test.go index 461685cf2..9c45fcdd5 100644 --- a/internal/repository/postgres/replay_source_test.go +++ b/internal/repository/postgres/replay_source_test.go @@ -53,19 +53,13 @@ func TestPostgresReplayVerificationLevels(t *testing.T) { Status: model.InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("output")}, Reports: [][]byte{[]byte("report")}, - OutputsProof: model.OutputsProof{ - MachineHash: repotest.UniqueHash(), - OutputsHash: repotest.UniqueHash(), - }, + StateProof: *repotest.DummyStateProof(), })) require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ EpochIndex: 0, InputIndex: 1, Status: model.InputCompletionStatus_Rejected, - OutputsProof: model.OutputsProof{ - MachineHash: repotest.UniqueHash(), - OutputsHash: repotest.UniqueHash(), - }, + StateProof: *repotest.DummyStateProof(), })) canonical, err := repo.ReplaySummary( @@ -200,10 +194,10 @@ func TestPostgresReplayRejectsCompletedInputGap(t *testing.T) { WHERE epoch_application_id = $1 AND index = 1`, app.ID) require.NoError(t, err) machineHash := repotest.UniqueHash() - outputsHash := repotest.UniqueHash() + txBufferDataBlock := repotest.UniqueHash() _, err = conn.Exec(ctx, `UPDATE input - SET status = 'ACCEPTED', machine_hash = $2, outputs_hash = $3 - WHERE epoch_application_id = $1`, app.ID, machineHash.Bytes(), outputsHash.Bytes()) + SET status = 'ACCEPTED', machine_hash = $2, tx_buffer_data_block = $3 + WHERE epoch_application_id = $1`, app.ID, machineHash.Bytes(), txBufferDataBlock.Bytes()) require.NoError(t, err) _, err = conn.Exec(ctx, `UPDATE application SET processed_inputs = 2 WHERE id = $1`, app.ID) require.NoError(t, err) @@ -255,10 +249,7 @@ func TestPostgresReplayRejectsInvalidStateHashOrdering(t *testing.T) { Status: model.InputCompletionStatus_Accepted, IsDaveConsensus: true, PaddingRepetitions: 1 << 24, - OutputsProof: model.OutputsProof{ - MachineHash: repotest.UniqueHash(), - OutputsHash: repotest.UniqueHash(), - }, + StateProof: *repotest.DummyStateProof(), })) } _, err = repo.ReplaySummary(ctx, app.IApplicationAddress, repository.ReplayVerificationFull) diff --git a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql index d993f57ec..ef342341f 100644 --- a/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql +++ b/internal/repository/postgres/schema/migrations/000001_create_initial_schema.up.sql @@ -8,14 +8,24 @@ CREATE DOMAIN "uint64" AS NUMERIC(20, 0) CHECK (VALUE >= 0 AND VALUE <= 18446744 CREATE DOMAIN "hash" AS BYTEA CHECK (octet_length(VALUE) = 32); CREATE DOMAIN "data_availability" AS BYTEA CHECK (octet_length(VALUE) >= 4); -CREATE TYPE "ApplicationStatus" AS ENUM ('OK', 'FAILED', 'DIVERGED', 'CORRUPTED'); +CREATE TYPE "ApplicationStatus" AS ENUM ( + 'OK', + 'FAILED', + 'DIVERGED', + 'CORRUPTED', + 'GUEST_EXCEPTION', + 'MACHINE_HALTED', + 'MCYCLE_OVERFLOW', + 'UNEXPECTED_YIELD'); CREATE TYPE "InputCompletionStatus" AS ENUM ( 'NONE', 'ACCEPTED', 'REJECTED', 'EXCEPTION', - 'MACHINE_HALTED'); + 'MACHINE_HALTED', + 'OVERFLOW', + 'UNEXPECTED_YIELD'); CREATE TYPE "DefaultBlock" AS ENUM ('FINALIZED', 'LATEST', 'PENDING', 'SAFE'); @@ -57,7 +67,7 @@ BEGIN FOREACH elem IN ARRAY arr LOOP - IF octet_length(elem) <> 32 THEN + IF elem IS NULL OR octet_length(elem) <> 32 THEN RETURN FALSE; -- any element not 32 bytes => fail END IF; END LOOP; @@ -121,7 +131,16 @@ CREATE TABLE "application" "accounts_drive_merkle_root" hash, "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT "reason_required_for_failure_statuses" CHECK (NOT ("status" IN ('FAILED', 'DIVERGED', 'CORRUPTED') AND ("reason" IS NULL OR LENGTH("reason") = 0))), + CONSTRAINT "reason_required_for_failure_statuses" CHECK (NOT ( + "status" IN ( + 'FAILED', + 'DIVERGED', + 'CORRUPTED', + 'GUEST_EXCEPTION', + 'MACHINE_HALTED', + 'MCYCLE_OVERFLOW', + 'UNEXPECTED_YIELD') + AND ("reason" IS NULL OR LENGTH("reason") = 0))), -- The foreclose pair is populated together by the atomic foreclosure -- marker+cursor repository write (set-once, first-writer-wins via WHERE -- foreclose_block = 0). This CHECK enforces the same invariant at the @@ -156,15 +175,45 @@ FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); CREATE OR REPLACE FUNCTION validate_application_status_transition() RETURNS TRIGGER AS $$ BEGIN - -- DIVERGED and CORRUPTED are terminal: once the node detects a consensus - -- disagreement or local corruption, neither status nor reason may change. - -- Observation continues independently (it is gated on enabled, not status). - IF OLD.status IN ('DIVERGED'::"ApplicationStatus", 'CORRUPTED'::"ApplicationStatus") + -- Execution terminals are outcomes of a run from a healthy machine. They + -- cannot be entered from FAILED or another terminal status by an unrelated + -- status writer. + IF NEW.status IN ( + 'GUEST_EXCEPTION'::"ApplicationStatus", + 'MACHINE_HALTED'::"ApplicationStatus", + 'MCYCLE_OVERFLOW'::"ApplicationStatus", + 'UNEXPECTED_YIELD'::"ApplicationStatus") + AND NEW.status <> OLD.status + AND OLD.status <> 'OK'::"ApplicationStatus" + THEN + RAISE EXCEPTION 'cannot enter execution-terminal status % from application status %', NEW.status, OLD.status; + END IF; + + -- Integrity failures are final. A later corruption finding may supersede + -- a completed execution terminal because it means the stored/L1 history + -- itself is no longer trustworthy; the input row still preserves the + -- original execution outcome. + IF OLD.status IN ( + 'DIVERGED'::"ApplicationStatus", + 'CORRUPTED'::"ApplicationStatus") AND (NEW.status <> OLD.status OR NEW.reason IS DISTINCT FROM OLD.reason) THEN RAISE EXCEPTION 'cannot change status or reason of a terminal (%) application', OLD.status; END IF; + IF OLD.status IN ( + 'GUEST_EXCEPTION'::"ApplicationStatus", + 'MACHINE_HALTED'::"ApplicationStatus", + 'MCYCLE_OVERFLOW'::"ApplicationStatus", + 'UNEXPECTED_YIELD'::"ApplicationStatus") + AND NOT ( + NEW.status = 'CORRUPTED'::"ApplicationStatus" + OR (NEW.status = OLD.status AND NEW.reason IS NOT DISTINCT FROM OLD.reason) + ) + THEN + RAISE EXCEPTION 'cannot change status or reason of a terminal (%) application', OLD.status; + END IF; + -- Foreclosure is one-way and lives in foreclose_block, not in status: a -- status write must never clear the foreclosure marker. IF OLD.foreclose_block <> 0 AND NEW.foreclose_block = 0 THEN @@ -215,8 +264,12 @@ CREATE TABLE "epoch" "input_index_lower_bound" uint64 NOT NULL, "input_index_upper_bound" uint64 NOT NULL, "machine_hash" hash, - "outputs_merkle_root" hash, - "outputs_merkle_proof" BYTEA[], + "tx_buffer_data_block" hash, + "tx_buffer_proof" BYTEA[], + "iflags_y_data_block" hash, + "iflags_y_proof" BYTEA[], + "htif_tohost_data_block" hash, + "htif_tohost_proof" BYTEA[], "commitment" hash, "commitment_proof" BYTEA[], "tournament_address" ethereum_address, @@ -231,6 +284,55 @@ CREATE TABLE "epoch" CONSTRAINT "epoch_application_id_fkey" FOREIGN KEY ("application_id") REFERENCES "application"("id") ON DELETE CASCADE, CONSTRAINT "epoch_block_bounds_check" CHECK ("first_block" <= "last_block"), CONSTRAINT "epoch_input_bounds_check" CHECK ("input_index_lower_bound" <= "input_index_upper_bound"), + CONSTRAINT "epoch_tx_buffer_proof_elements_check" CHECK (check_hash_siblings("tx_buffer_proof")), + CONSTRAINT "epoch_iflags_y_proof_elements_check" CHECK (check_hash_siblings("iflags_y_proof")), + CONSTRAINT "epoch_htif_tohost_proof_elements_check" CHECK (check_hash_siblings("htif_tohost_proof")), + -- A proof is either absent or complete, even while the epoch is CLOSED. + -- This prevents a crash or ad-hoc write from leaving a mixed tuple that a + -- later reader could mistake for a usable terminal-state commitment. + CONSTRAINT "epoch_state_proof_tuple_check" CHECK ( + ( + "machine_hash" IS NULL + AND "tx_buffer_data_block" IS NULL + AND "tx_buffer_proof" IS NULL + AND "iflags_y_data_block" IS NULL + AND "iflags_y_proof" IS NULL + AND "htif_tohost_data_block" IS NULL + AND "htif_tohost_proof" IS NULL + ) + OR ( + "machine_hash" IS NOT NULL + AND "tx_buffer_data_block" IS NOT NULL + AND "tx_buffer_proof" IS NOT NULL + AND cardinality("tx_buffer_proof") = 59 + AND "iflags_y_data_block" IS NOT NULL + AND "iflags_y_proof" IS NOT NULL + AND cardinality("iflags_y_proof") = 59 + AND "htif_tohost_data_block" IS NOT NULL + AND "htif_tohost_proof" IS NOT NULL + AND cardinality("htif_tohost_proof") = 59 + ) + ), + -- Every published claim candidate carries the complete accepted-state + -- machine proof. CLAIM_FORECLOSED is excluded because foreclosure + -- may terminalize an epoch before the machine can publish a proof. + CONSTRAINT "epoch_published_validity_proof_check" CHECK ( + "status" NOT IN ('INPUTS_PROCESSED', 'CLAIM_COMPUTED', + 'CLAIM_SUBMITTED', 'CLAIM_STAGED', + 'CLAIM_ACCEPTED', 'CLAIM_REJECTED') + OR ( + "machine_hash" IS NOT NULL + AND "tx_buffer_data_block" IS NOT NULL + AND "tx_buffer_proof" IS NOT NULL + AND cardinality("tx_buffer_proof") = 59 + AND "iflags_y_data_block" IS NOT NULL + AND "iflags_y_proof" IS NOT NULL + AND cardinality("iflags_y_proof") = 59 + AND "htif_tohost_data_block" IS NOT NULL + AND "htif_tohost_proof" IS NOT NULL + AND cardinality("htif_tohost_proof") = 59 + ) + ), -- staged_at_block is set when an epoch is staged on chain and is then -- kept historically — same lifetime convention as claim_transaction_hash. -- We only enforce the forward direction: if you're in CLAIM_STAGED you @@ -276,10 +378,10 @@ FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); -- Any other transition (including backwards) is rejected. -- Same-status updates are allowed (idempotent no-ops). -- --- When transitioning to CLAIM_COMPUTED, the trigger also verifies that --- required proof fields are populated: --- All apps: machine_hash, outputs_merkle_root, outputs_merkle_proof --- PRT (DaveConsensus): additionally commitment, commitment_proof +-- The epoch_published_validity_proof_check table constraint requires the +-- complete three-leaf machine proof from INPUTS_PROCESSED onward. When +-- transitioning to CLAIM_COMPUTED, this trigger additionally requires the +-- PRT commitment and commitment proof. -- -- CLAIM_STAGED is NEVER valid for PRT apps (PRT settles via tournaments, -- not the staging flow). The trigger rejects this regardless of which @@ -323,16 +425,8 @@ BEGIN OLD.status, NEW.status; END IF; - -- Enforce required fields when entering CLAIM_COMPUTED. + -- Enforce PRT-specific claim fields when entering CLAIM_COMPUTED. IF NEW.status::text = 'CLAIM_COMPUTED' THEN - IF NEW.machine_hash IS NULL - OR NEW.outputs_merkle_root IS NULL - OR NEW.outputs_merkle_proof IS NULL THEN - RAISE EXCEPTION - 'CLAIM_COMPUTED requires machine_hash, outputs_merkle_root, ' - 'and outputs_merkle_proof to be non-null'; - END IF; - SELECT a.consensus_type::text INTO app_consensus FROM application a WHERE a.id = NEW.application_id; @@ -387,7 +481,7 @@ CREATE TABLE "input" "status" "InputCompletionStatus" NOT NULL, "exception_data" BYTEA, "machine_hash" hash, - "outputs_hash" hash, + "tx_buffer_data_block" hash, "transaction_hash" hash NOT NULL, "log_index" uint64 NOT NULL, "snapshot_uri" VARCHAR(4096), @@ -402,9 +496,9 @@ CREATE TABLE "input" CONSTRAINT "input_epoch_index_unique" UNIQUE ("epoch_application_id", "epoch_index", "index"), CONSTRAINT "input_application_id_tx_hash_log_index_unique" UNIQUE ("epoch_application_id", "transaction_hash", "log_index"), CONSTRAINT "input_completed_hashes_check" CHECK ( - ("status" = 'NONE' AND "machine_hash" IS NULL AND "outputs_hash" IS NULL) + ("status" = 'NONE' AND "machine_hash" IS NULL AND "tx_buffer_data_block" IS NULL) OR - ("status" <> 'NONE' AND "machine_hash" IS NOT NULL AND "outputs_hash" IS NOT NULL) + ("status" <> 'NONE' AND "machine_hash" IS NOT NULL AND "tx_buffer_data_block" IS NOT NULL) ), CONSTRAINT "input_epoch_id_fkey" FOREIGN KEY ("epoch_application_id", "epoch_index") REFERENCES "epoch"("application_id", "index") ON DELETE CASCADE ); @@ -432,7 +526,7 @@ BEGIN OR NEW.status IS DISTINCT FROM OLD.status OR NEW.exception_data IS DISTINCT FROM OLD.exception_data OR NEW.machine_hash IS DISTINCT FROM OLD.machine_hash - OR NEW.outputs_hash IS DISTINCT FROM OLD.outputs_hash + OR NEW.tx_buffer_data_block IS DISTINCT FROM OLD.tx_buffer_data_block ) THEN RAISE EXCEPTION 'completed input result is immutable'; @@ -444,7 +538,7 @@ $$ LANGUAGE plpgsql; CREATE TRIGGER "input_completion_immutability_check" BEFORE UPDATE OF "epoch_application_id", "epoch_index", "index", "raw_data", - "status", "exception_data", "machine_hash", "outputs_hash" ON "input" + "status", "exception_data", "machine_hash", "tx_buffer_data_block" ON "input" FOR EACH ROW EXECUTE FUNCTION enforce_input_completion_immutability(); diff --git a/internal/repository/repository.go b/internal/repository/repository.go index f98ec9a09..72351913f 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -17,6 +17,18 @@ import ( var ( ErrNotFound = errors.New("not found") ErrNoUpdate = errors.New("update did not take effect") + // ErrApplicationNotRunnable means an advance result cannot be stored because + // the application's durable status does not allow machine execution. + ErrApplicationNotRunnable = errors.New("application is not runnable") + // ErrAdvanceCursorMismatch means an advance result does not match the next + // unprocessed input selected by the application's durable cursor. + ErrAdvanceCursorMismatch = errors.New("advance result does not match application cursor") + // ErrAdvanceAfterTerminal preserves the more specific classification for a + // store rejected because the application's durable status is terminal. + ErrAdvanceAfterTerminal = errors.New("cannot store an advance result after a terminal input") + // ErrInvalidStateProof means an advance result or epoch publication did not + // include the complete three-leaf machine state proof. + ErrInvalidStateProof = errors.New("invalid machine state proof") // ErrInputLogIdentityConflict indicates an input insert conflicted with a // stored row on the L1 log identity (transaction_hash, log_index) under a @@ -170,6 +182,8 @@ type ApplicationRepository interface { UpdateExecutionParameters(ctx context.Context, ep *ExecutionParameters) error GetEventLastCheckBlock(ctx context.Context, appID int64, event MonitoredEvent) (uint64, error) + // UpdateEventLastCheckBlock advances the event cursor monotonically. Lower + // or equal block numbers are no-ops. UpdateEventLastCheckBlock(ctx context.Context, appIDs []int64, event MonitoredEvent, blockNumber uint64) error GetLastSnapshot(ctx context.Context, nameOrAddress string) (*Input, error) @@ -186,9 +200,7 @@ type EpochRepository interface { UpdateEpochClaimTransactionHash(ctx context.Context, nameOrAddress string, e *Epoch) error UpdateEpochStatus(ctx context.Context, nameOrAddress string, e *Epoch) error - UpdateEpochInputsProcessed(ctx context.Context, nameOrAddress string, epochIndex uint64) error - UpdateEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64, proof *OutputsProof) error - RepeatPreviousEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64) error + UpdateEpochInputsProcessed(ctx context.Context, nameOrAddress string, epochIndex uint64, proof *StateProof) error ListEpochs(ctx context.Context, nameOrAddress string, f EpochFilter, p Pagination, descending bool) ([]*Epoch, uint64, error) @@ -398,17 +410,25 @@ type ClaimerRepository interface { applicationID int64, index uint64, ) error - // RejectEpochAndSetApplicationDiverged atomically marks an epoch as - // CLAIM_REJECTED and the application as DIVERGED. Used when Quorum - // consensus stages or accepts a different claim before the local claim has - // staged, making the local claim unreachable. The application is always - // halted, even when no epoch row matched the reject. + // RejectEpochAndSetApplicationDiverged atomically attempts to mark an epoch + // as CLAIM_REJECTED and the application as DIVERGED. The returned result + // reports which conditional writes applied, so callers can mirror only the + // durable transitions. The application remains unchanged when it already + // has a terminal status; an epoch outside CLAIM_COMPUTED/CLAIM_SUBMITTED is + // likewise left unchanged. RejectEpochAndSetApplicationDiverged( ctx context.Context, applicationID int64, index uint64, reason string, - ) error + ) (RejectEpochAndDivergeResult, error) +} + +// RejectEpochAndDivergeResult describes the two conditional writes performed +// by RejectEpochAndSetApplicationDiverged when the returned error is nil. +type RejectEpochAndDivergeResult struct { + EpochRejected bool + ApplicationDiverged bool } type Repository interface { diff --git a/internal/repository/repotest/application_test_cases.go b/internal/repository/repotest/application_test_cases.go index 694f24805..988774286 100644 --- a/internal/repository/repotest/application_test_cases.go +++ b/internal/repository/repotest/application_test_cases.go @@ -139,6 +139,33 @@ func (s *ApplicationSuite) TestListApplications() { s.Equal(ApplicationStatus_OK, apps[0].Status) }) + s.Run("ExecutableFilterExcludesExecutionTerminalStatuses", func() { + const pageLimit = 10 + healthy := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + for _, status := range []ApplicationStatus{ + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } { + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + reason := "terminal machine outcome" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, status, &reason)) + } + + apps, total, err := s.Repo.ListApplications( + s.Ctx, + repository.ExecutableApplicationsFilter(), + repository.Pagination{Limit: pageLimit}, + false, + ) + s.Require().NoError(err) + s.Equal(uint64(1), total) + s.Require().Len(apps, 1) + s.Equal(healthy.ID, apps[0].ID) + }) + s.Run("FilterByConsensus", func() { NewApplicationBuilder().WithConsensus(Consensus_Authority).Create(s.Ctx, s.T(), s.Repo) NewApplicationBuilder().WithConsensus(Consensus_PRT).Create(s.Ctx, s.T(), s.Repo) @@ -455,6 +482,10 @@ func (s *ApplicationSuite) TestTerminalStatusIsTerminal() { terminalStatuses := []ApplicationStatus{ ApplicationStatus_Diverged, ApplicationStatus_Corrupted, + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, } for _, status := range terminalStatuses { @@ -560,6 +591,121 @@ func (s *ApplicationSuite) TestForeclosedCanBecomeTerminal() { }) } +func (s *ApplicationSuite) TestExecutionTerminalCanEscalateToCorrupted() { + for _, status := range []ApplicationStatus{ + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } { + s.Run(status.String(), func() { + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + executionReason := "guest execution terminated" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, status, &executionReason)) + + corruptionReason := "post-foreclosure L1 history mismatch" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, ApplicationStatus_Corrupted, &corruptionReason)) + + got, err := s.Repo.GetApplication(s.Ctx, app.Name) + s.Require().NoError(err) + s.Equal(ApplicationStatus_Corrupted, got.Status) + s.Require().NotNil(got.Reason) + s.Equal(corruptionReason, *got.Reason) + }) + } +} + +func (s *ApplicationSuite) TestFailedCannotEnterExecutionTerminal() { + for _, status := range []ApplicationStatus{ + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } { + s.Run(status.String(), func() { + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + failedReason := "runtime unavailable" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, ApplicationStatus_Failed, &failedReason)) + + terminalReason := "must only be produced by an advance" + err := s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, status, &terminalReason) + s.Require().Error(err) + + got, err := s.Repo.GetApplication(s.Ctx, app.Name) + s.Require().NoError(err) + s.Equal(ApplicationStatus_Failed, got.Status) + s.Require().NotNil(got.Reason) + s.Equal(failedReason, *got.Reason) + }) + } +} + +func (s *ApplicationSuite) TestExecutionTerminalRejectsOtherTerminalTransitions() { + executionStatuses := []ApplicationStatus{ + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } + for index, current := range executionStatuses { + for _, target := range []ApplicationStatus{ + ApplicationStatus_Failed, + ApplicationStatus_Diverged, + executionStatuses[(index+1)%len(executionStatuses)], + } { + s.Run(current.String()+"To"+target.String(), func() { + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + originalReason := "terminal machine outcome" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, current, &originalReason)) + + newReason := "invalid transition" + err := s.Repo.UpdateApplicationStatus(s.Ctx, app.ID, target, &newReason) + s.Require().Error(err) + + got, err := s.Repo.GetApplication(s.Ctx, app.Name) + s.Require().NoError(err) + s.Equal(current, got.Status) + s.Equal(originalReason, *got.Reason) + }) + } + } +} + +func (s *ApplicationSuite) TestIntegrityTerminalRejectsExecutionTerminalTransitions() { + for _, current := range []ApplicationStatus{ + ApplicationStatus_Diverged, + ApplicationStatus_Corrupted, + } { + for _, target := range []ApplicationStatus{ + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } { + s.Run(current.String()+"To"+target.String(), func() { + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + originalReason := "integrity failure" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, current, &originalReason)) + + newReason := "invalid execution outcome" + err := s.Repo.UpdateApplicationStatus(s.Ctx, app.ID, target, &newReason) + s.Require().Error(err) + + got, err := s.Repo.GetApplication(s.Ctx, app.Name) + s.Require().NoError(err) + s.Equal(current, got.Status) + s.Equal(originalReason, *got.Reason) + }) + } + } +} + // TestForeclosedCanBecomeFailed verifies that a foreclosed application (one // with a non-zero foreclose_block) can still transition to FAILED with a // reason; the row then reads FAILED with foreclose_block preserved. Health @@ -818,6 +964,26 @@ func (s *ApplicationSuite) TestEventLastCheckBlock() { s.Equal(uint64(42), block) }) + s.Run("DoesNotRegress", func() { + const ( + currentBlock = uint64(42) + staleBlock = uint64(41) + ) + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + + err := s.Repo.UpdateEventLastCheckBlock( + s.Ctx, []int64{app.ID}, MonitoredEvent_InputAdded, currentBlock) + s.Require().NoError(err) + err = s.Repo.UpdateEventLastCheckBlock( + s.Ctx, []int64{app.ID}, MonitoredEvent_InputAdded, staleBlock) + s.Require().NoError(err) + + block, err := s.Repo.GetEventLastCheckBlock( + s.Ctx, app.ID, MonitoredEvent_InputAdded) + s.Require().NoError(err) + s.Equal(currentBlock, block) + }) + s.Run("AllMonitoredEventTypes", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) @@ -909,9 +1075,7 @@ func (s *ApplicationSuite) TestGetProcessedInputCount() { EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - OutputsProof: OutputsProof{ - MachineHash: crypto.Keccak256Hash([]byte("machine")), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) @@ -981,9 +1145,7 @@ func (s *ApplicationSuite) TestUpdateApplication() { EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - OutputsProof: OutputsProof{ - MachineHash: crypto.Keccak256Hash([]byte("machine")), - }, + StateProof: *DummyStateProof(), })) app.EpochLength = 33 @@ -1277,9 +1439,7 @@ func (s *ApplicationSuite) TestGetLastSnapshot() { EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - OutputsProof: OutputsProof{ - MachineHash: crypto.Keccak256Hash([]byte("machine")), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) diff --git a/internal/repository/repotest/builders.go b/internal/repository/repotest/builders.go index 44563f018..f21545ed8 100644 --- a/internal/repository/repotest/builders.go +++ b/internal/repository/repotest/builders.go @@ -190,8 +190,8 @@ func (b *EpochBuilder) WithInputBounds(lower, upper uint64) *EpochBuilder { return b } -func (b *EpochBuilder) WithOutputsMerkleRoot(h common.Hash) *EpochBuilder { - b.epoch.OutputsMerkleRoot = &h +func (b *EpochBuilder) WithTxBufferDataBlock(h common.Hash) *EpochBuilder { + b.epoch.TxBufferDataBlock = &h return b } @@ -580,12 +580,25 @@ func AdvanceEpochStatus( } for _, s := range path { - // The DB trigger requires proof fields to be non-null when - // entering CLAIM_COMPUTED. Populate them with dummy values - // so tests that only care about status transitions don't need - // to set up proofs manually. + if s == EpochStatus_InputsProcessed { + proof := DummyStateProof() + err := repo.UpdateEpochInputsProcessed( + ctx, nameOrAddress, epoch.Index, proof, + ) + require.NoError(t, err) + persisted, err := repo.GetEpoch(ctx, nameOrAddress, epoch.Index) + require.NoError(t, err) + require.Equal(t, EpochStatus_InputsProcessed, persisted.Status) + epoch.Status = s + applyStateProofToEpoch(epoch, proof) + continue + } + + // PRT claims need dummy commitment fields when entering + // CLAIM_COMPUTED. The final state proof was published by the + // preceding INPUTS_PROCESSED transition. if s == EpochStatus_ClaimComputed { - setDummyProofFields(ctx, t, repo, nameOrAddress, epoch) + setDummyClaimFields(ctx, t, repo, nameOrAddress, epoch) // For PRT apps StoreClaimAndProofs already set the status // to CLAIM_COMPUTED, so skip the redundant UpdateEpochStatus. app, err := repo.GetApplication(ctx, nameOrAddress) @@ -601,12 +614,45 @@ func AdvanceEpochStatus( } } -// setDummyProofFields populates the proof fields required by the DB trigger -// for the INPUTS_PROCESSED → CLAIM_COMPUTED transition. -// For all apps: machine_hash, outputs_merkle_root, outputs_merkle_proof. -// For PRT apps: additionally commitment and commitment_proof (set via -// StoreClaimAndProofs which also transitions the status atomically). -func setDummyProofFields( +func proofSiblingsToHashes(siblings [][32]byte) []common.Hash { + hashes := make([]common.Hash, len(siblings)) + for i := range siblings { + hashes[i] = common.Hash(siblings[i]) + } + return hashes +} + +func applyStateProofToEpoch(epoch *Epoch, proof *StateProof) { + epoch.MachineHash = Pointer(proof.MachineHash) + epoch.TxBufferDataBlock = Pointer(proof.TxBufferDataBlock) + epoch.TxBufferProof = proofSiblingsToHashes(proof.TxBufferProof) + epoch.IflagsYDataBlock = Pointer(proof.IflagsYDataBlock) + epoch.IflagsYProof = proofSiblingsToHashes(proof.IflagsYProof) + epoch.HtifTohostDataBlock = Pointer(proof.HtifTohostDataBlock) + epoch.HtifTohostProof = proofSiblingsToHashes(proof.HtifTohostProof) +} + +// DummyStateProof returns a structurally complete machine state proof for +// repository tests that exercise epoch state transitions rather than proof +// cryptography. pkg/machine tests cover proof construction and verification. +func DummyStateProof() *StateProof { + proof := &StateProof{ + TxBufferDataBlock: UniqueHash(), + MachineHash: UniqueHash(), + IflagsYDataBlock: UniqueHash(), + HtifTohostDataBlock: UniqueHash(), + } + for range StateProofSiblingCount { + proof.TxBufferProof = append(proof.TxBufferProof, UniqueHash()) + proof.IflagsYProof = append(proof.IflagsYProof, UniqueHash()) + proof.HtifTohostProof = append(proof.HtifTohostProof, UniqueHash()) + } + return proof +} + +// setDummyClaimFields supplies the PRT-only fields required for the +// INPUTS_PROCESSED → CLAIM_COMPUTED transition. +func setDummyClaimFields( ctx context.Context, t *testing.T, repo repository.Repository, nameOrAddress string, @@ -614,15 +660,6 @@ func setDummyProofFields( ) { t.Helper() - proof := &OutputsProof{ - OutputsHash: UniqueHash(), - OutputsHashProof: [][32]byte{[32]byte(UniqueHash())}, - MachineHash: UniqueHash(), - } - err := repo.UpdateEpochOutputsProof( - ctx, epoch.ApplicationID, epoch.Index, proof) - require.NoError(t, err) - app, err := repo.GetApplication(ctx, nameOrAddress) require.NoError(t, err) if app.IsDaveConsensus() { diff --git a/internal/repository/repotest/bulk_test_cases.go b/internal/repository/repotest/bulk_test_cases.go index 001e996d1..81e33a41a 100644 --- a/internal/repository/repotest/bulk_test_cases.go +++ b/internal/repository/repotest/bulk_test_cases.go @@ -23,6 +23,7 @@ func NewBulkOperationsSuite(factory RepositoryFactory) *BulkOperationsSuite { return &BulkOperationsSuite{BaseSuite: BaseSuite{factory: factory}} } +//nolint:mnd // Numeric values are intentionally explicit repository fixtures. func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Run("RejectsNilResult", func() { err := s.Repo.StoreAdvanceResult(s.Ctx, 0, nil) @@ -32,7 +33,10 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Run("AcceptedInput", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("machine")) - outputsHash := crypto.Keccak256Hash([]byte("outputs")) + txBufferDataBlock := crypto.Keccak256Hash([]byte("outputs")) + proof := DummyStateProof() + proof.MachineHash = machineHash + proof.TxBufferDataBlock = txBufferDataBlock result := &AdvanceResult{ EpochIndex: 0, @@ -40,10 +44,7 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("output1"), []byte("output2")}, Reports: [][]byte{[]byte("report1")}, - OutputsProof: OutputsProof{ - OutputsHash: outputsHash, - MachineHash: machineHash, - }, + StateProof: *proof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) @@ -55,6 +56,15 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(InputCompletionStatus_Accepted, input.Status) s.Require().NotNil(input.MachineHash) s.Equal(machineHash, *input.MachineHash) + epoch, err := s.Repo.GetEpoch(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Require().NotNil(epoch.MachineHash) + s.Equal(machineHash, *epoch.MachineHash) + s.Require().NotNil(epoch.TxBufferDataBlock) + s.Equal(txBufferDataBlock, *epoch.TxBufferDataBlock) + s.True(epoch.HasCompleteStateProof()) + s.Equal(proof.IflagsYDataBlock, *epoch.IflagsYDataBlock) + s.Equal(proof.HtifTohostDataBlock, *epoch.HtifTohostDataBlock) // Verify outputs were created outputs, total, err := s.Repo.ListOutputs( @@ -76,14 +86,14 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Run("RejectedInput", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("machine-rejected")) + proof := DummyStateProof() + proof.MachineHash = machineHash result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Rejected, - OutputsProof: OutputsProof{ - MachineHash: machineHash, - }, + StateProof: *proof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) @@ -92,11 +102,17 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) s.Require().NoError(err) s.Equal(InputCompletionStatus_Rejected, input.Status) + epoch, err := s.Repo.GetEpoch(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Nil(epoch.MachineHash, "rejection must leave the pre-input epoch state unchanged") + s.Nil(epoch.TxBufferDataBlock) }) for _, status := range []InputCompletionStatus{ InputCompletionStatus_Exception, InputCompletionStatus_MachineHalted, + InputCompletionStatus_Overflow, + InputCompletionStatus_UnexpectedYield, } { s.Run("CompletedStatus/"+status.String(), func() { seed := Seed(s.Ctx, s.T(), s.Repo) @@ -104,14 +120,13 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { if status == InputCompletionStatus_Exception { exceptionData = []byte{0xff, 0x00, 0x80} } + proof := DummyStateProof() result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: status, ExceptionData: exceptionData, - OutputsProof: OutputsProof{ - MachineHash: UniqueHash(), - }, + StateProof: *proof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) @@ -120,9 +135,130 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Require().NoError(err) s.Equal(status, input.Status) s.Equal(exceptionData, input.ExceptionData) + epoch, err := s.Repo.GetEpoch(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Require().NotNil(epoch.MachineHash) + s.Equal(result.MachineHash, *epoch.MachineHash) + s.Require().NotNil(epoch.TxBufferDataBlock) + s.Equal(result.TxBufferDataBlock, *epoch.TxBufferDataBlock) + s.Equal(proofSiblingsToHashes(proof.TxBufferProof), epoch.TxBufferProof) + s.Equal(proof.IflagsYDataBlock, *epoch.IflagsYDataBlock) + s.Equal(proofSiblingsToHashes(proof.IflagsYProof), epoch.IflagsYProof) + s.Equal(proof.HtifTohostDataBlock, *epoch.HtifTohostDataBlock) + s.Equal(proofSiblingsToHashes(proof.HtifTohostProof), epoch.HtifTohostProof) + s.True(epoch.HasCompleteStateProof()) + + app, err := s.Repo.GetApplication(s.Ctx, seed.App.IApplicationAddress.String()) + s.Require().NoError(err) + expectedStatus, ok := status.TerminalApplicationStatus() + s.Require().True(ok) + s.Equal(expectedStatus, app.Status) + s.Require().NotNil(app.Reason) + s.Contains(*app.Reason, "input 0 completed with "+status.String()) }) } + s.Run("RejectsEffectsForNonacceptedInput", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Rejected, + Outputs: [][]byte{[]byte("must-not-be-stored")}, + }) + s.Require().ErrorContains(err, "must not contain outputs or reports") + }) + + s.Run("RejectsCursorAndEpochMismatches", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + base := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 1, + Status: InputCompletionStatus_Accepted, + StateProof: *DummyStateProof(), + } + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, base) + s.Require().ErrorIs(err, repository.ErrAdvanceCursorMismatch) + + base.InputIndex = 0 + base.EpochIndex = 1 + err = s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, base) + s.Require().ErrorIs(err, repository.ErrAdvanceCursorMismatch) + }) + + s.Run("RejectsResultAfterTerminalInput", func() { + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + epoch := NewEpochBuilder(app.ID). + WithIndex(0).WithStatus(EpochStatus_Closed). + WithBlocks(0, 9).WithInputBounds(0, 1).Build() + input0 := NewInputBuilder().WithIndex(0).WithBlockNumber(5).Build() + input1 := NewInputBuilder().WithIndex(1).WithBlockNumber(6).Build() + err := s.Repo.CreateEpochsAndInputs( + s.Ctx, + app.IApplicationAddress.String(), + map[*Epoch][]*Input{epoch: {input0, input1}}, + 10, + ) + s.Require().NoError(err) + err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_MachineHalted, + StateProof: *DummyStateProof(), + }) + s.Require().NoError(err) + + err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, &AdvanceResult{ + EpochIndex: 0, + InputIndex: 1, + Status: InputCompletionStatus_Accepted, + StateProof: *DummyStateProof(), + }) + s.Require().ErrorIs(err, repository.ErrAdvanceAfterTerminal) + s.Require().ErrorIs(err, repository.ErrApplicationNotRunnable) + }) + + s.Run("RejectsResultWhileApplicationFailedWithoutPartialWrites", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + reason := "runtime unavailable" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, seed.App.ID, ApplicationStatus_Failed, &reason)) + + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Accepted, + Outputs: [][]byte{[]byte("must roll back")}, + StateProof: *DummyStateProof(), + }) + s.Require().ErrorIs(err, repository.ErrApplicationNotRunnable) + s.NotErrorIs(err, repository.ErrAdvanceAfterTerminal) + + input, err := s.Repo.GetInput( + s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(InputCompletionStatus_None, input.Status) + s.Nil(input.MachineHash) + + epoch, err := s.Repo.GetEpoch( + s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Nil(epoch.MachineHash) + s.Nil(epoch.TxBufferDataBlock) + + processed, err := s.Repo.GetProcessedInputCount( + s.Ctx, seed.App.IApplicationAddress.String()) + s.Require().NoError(err) + s.Zero(processed) + + outputs, total, err := s.Repo.ListOutputs( + s.Ctx, seed.App.IApplicationAddress.String(), + repository.OutputFilter{}, repository.Pagination{Limit: 10}, false) + s.Require().NoError(err) + s.Empty(outputs) + s.Zero(total) + }) + for _, test := range []struct { name string status InputCompletionStatus @@ -138,9 +274,9 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { InputIndex: 0, Status: test.status, ExceptionData: test.exceptionData, - OutputsProof: OutputsProof{ - MachineHash: UniqueHash(), - OutputsHash: UniqueHash(), + StateProof: StateProof{ + MachineHash: UniqueHash(), + TxBufferDataBlock: UniqueHash(), }, }) s.Require().ErrorContains(err, "exception data") @@ -164,7 +300,7 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { InputIndex: 0, Status: status, Outputs: [][]byte{[]byte("must-not-be-stored")}, - OutputsProof: OutputsProof{ + StateProof: StateProof{ MachineHash: UniqueHash(), }, } @@ -180,14 +316,14 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Run("WithNoOutputsOrReports", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("machine-empty")) + proof := DummyStateProof() + proof.MachineHash = machineHash result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - OutputsProof: OutputsProof{ - MachineHash: machineHash, - }, + StateProof: *proof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) @@ -198,10 +334,9 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(InputCompletionStatus_Accepted, input.Status) }) - // Verify that a failure mid-transaction rolls back all prior changes. - // We trigger failure by providing a non-existent epoch index, causing the - // epoch outputs proof update to fail after outputs and input are written. - s.Run("RollbackOnPartialFailure", func() { + // A result for the current input but a different epoch is rejected by the + // locked-row preflight before any child rows are inserted. + s.Run("RejectsEpochMismatchBeforeWrites", func() { seed := Seed(s.Ctx, s.T(), s.Repo) result := &AdvanceResult{ @@ -210,14 +345,11 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("should-be-rolled-back")}, Reports: [][]byte{[]byte("should-be-rolled-back")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) - s.Require().Error(err) + s.Require().ErrorIs(err, repository.ErrAdvanceCursorMismatch) // Input status should remain unchanged (NONE) input, err := s.Repo.GetInput( @@ -251,7 +383,10 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Run("DaveConsensusWithStateHashes", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("dave-machine")) - outputsHash := crypto.Keccak256Hash([]byte("dave-outputs")) + txBufferDataBlock := crypto.Keccak256Hash([]byte("dave-outputs")) + proof := DummyStateProof() + proof.MachineHash = machineHash + proof.TxBufferDataBlock = txBufferDataBlock hash1 := [32]byte(crypto.Keccak256Hash([]byte("state-1"))) hash2 := [32]byte(crypto.Keccak256Hash([]byte("state-2"))) @@ -266,10 +401,7 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { PeriodicStateHashes: hashes, PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), IsDaveConsensus: true, - OutputsProof: OutputsProof{ - OutputsHash: outputsHash, - MachineHash: machineHash, - }, + StateProof: *proof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) @@ -319,6 +451,8 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { seed := Seed(s.Ctx, s.T(), s.Repo) hashes := make([][32]byte, hashesAboveExtendedProtocolParameterLimit) machineHash := UniqueHash() + proof := DummyStateProof() + proof.MachineHash = machineHash result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, @@ -326,10 +460,7 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { PeriodicStateHashes: hashes, PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), IsDaveConsensus: true, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: machineHash, - }, + StateProof: *proof, } s.Require().NoError(s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result)) @@ -356,8 +487,11 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Equal(InputHashCollectionCapacity-uint64(len(hashes)), tail.Repetitions) }) - s.Run("PRTConsensusRejectsUnnormalizedExactBoundaryHashCollection", func() { + s.Run("DaveConsensusRollsBackChildrenOnInvalidHashSpan", func() { seed := Seed(s.Ctx, s.T(), s.Repo) + proof := DummyStateProof() + proof.TxBufferDataBlock = UniqueHash() + proof.MachineHash = UniqueHash() result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, @@ -366,14 +500,14 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { Reports: [][]byte{[]byte("must-roll-back")}, PaddingRepetitions: 0, IsDaveConsensus: true, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *proof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) - s.Require().Error(err) + s.Require().ErrorContains(err, "does not cover input hash collection capacity") + // The complete proof and locked rows let this transaction insert the + // output and report before state-hash shape validation fails. Their + // absence below is the rollback witness. input, err := s.Repo.GetInput(s.Ctx, seed.App.IApplicationAddress.String(), 0) s.Require().NoError(err) @@ -409,33 +543,36 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResult() { s.Require().NoError(err) s.Empty(stateHashes) s.Zero(stateHashCount) + epoch, err := s.Repo.GetEpoch( + s.Ctx, seed.App.IApplicationAddress.String(), seed.Epoch.Index) + s.Require().NoError(err) + s.False(epoch.HasCompleteStateProof()) + app, err := s.Repo.GetApplication(s.Ctx, seed.App.IApplicationAddress.String()) + s.Require().NoError(err) + s.Equal(uint64(0), app.ProcessedInputs) + s.Equal(ApplicationStatus_OK, app.Status) }) } -func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { - // Trigger rollback by referencing a non-existent input index (the input - // doesn't exist in the DB, so updateInput will fail with sql.ErrNoRows). - // This tests that outputs inserted earlier in the same transaction are - // rolled back when a subsequent step fails. - s.Run("RollbackOnInputUpdateFailure", func() { +func (s *BulkOperationsSuite) TestStoreAdvanceResultPreflight() { + // The application cursor is locked and checked before any child rows are + // written, so an unexpected input index is a preflight rejection. + s.Run("RejectsUnexpectedInputIndex", func() { seed := Seed(s.Ctx, s.T(), s.Repo) result := &AdvanceResult{ EpochIndex: 0, InputIndex: 999, // non-existent input index Status: InputCompletionStatus_Accepted, - Outputs: [][]byte{[]byte("should-be-rolled-back")}, - Reports: [][]byte{[]byte("should-be-rolled-back")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + Outputs: [][]byte{[]byte("must-not-be-inserted")}, + Reports: [][]byte{[]byte("must-not-be-inserted")}, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) - s.Require().Error(err) + s.Require().ErrorIs(err, repository.ErrAdvanceCursorMismatch) - // Verify no outputs were persisted (rolled back) + // Verify the preflight rejected the result before inserting outputs. outputs, total, err := s.Repo.ListOutputs( s.Ctx, seed.App.IApplicationAddress.String(), repository.OutputFilter{}, repository.Pagination{Limit: 10}, false) @@ -443,7 +580,7 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { s.Empty(outputs) s.Equal(uint64(0), total) - // Verify no reports were persisted (rolled back) + // Verify the preflight rejected the result before inserting reports. reports, total, err := s.Repo.ListReports( s.Ctx, seed.App.IApplicationAddress.String(), repository.ReportFilter{}, repository.Pagination{Limit: 10}, false) @@ -464,28 +601,21 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { s.Equal(uint64(0), count) }) - // Trigger rollback by providing a valid input index but a non-existent - // app ID, so updateApp fails. This verifies that outputs, reports, and - // the input status update are all rolled back. - s.Run("RollbackOnAppUpdateFailure", func() { + // Missing applications are rejected while acquiring the aggregate row. + s.Run("RejectsUnknownApplication", func() { seed := Seed(s.Ctx, s.T(), s.Repo) result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - Outputs: [][]byte{[]byte("should-be-rolled-back")}, - Reports: [][]byte{[]byte("should-be-rolled-back")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + Outputs: [][]byte{[]byte("must-not-be-inserted")}, + Reports: [][]byte{[]byte("must-not-be-inserted")}, + StateProof: *DummyStateProof(), } - // Use a non-existent app ID -- updateApp will fail - // because no application row matches. err := s.Repo.StoreAdvanceResult(s.Ctx, 999999, result) - s.Require().Error(err) + s.Require().ErrorIs(err, repository.ErrNotFound) // Verify the original input is untouched input, err := s.Repo.GetInput( @@ -502,9 +632,9 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { s.Equal(uint64(0), total) }) - // Verify that when Dave consensus state hash insertion fails (due to - // a bad epoch index), all prior work (outputs, reports) is rolled back. - s.Run("DaveConsensusRollbackOnStateHashFailure", func() { + // A Dave result that names the wrong epoch is also rejected by locked-row + // preflight; state-hash insertion is never reached. + s.Run("DaveConsensusRejectsEpochMismatch", func() { seed := Seed(s.Ctx, s.T(), s.Repo) hashes := [][32]byte{{1}, {2}} @@ -512,20 +642,17 @@ func (s *BulkOperationsSuite) TestStoreAdvanceResultRollback() { EpochIndex: 99, // non-existent epoch InputIndex: 0, Status: InputCompletionStatus_Accepted, - Outputs: [][]byte{[]byte("should-be-rolled-back")}, + Outputs: [][]byte{[]byte("must-not-be-inserted")}, PeriodicStateHashes: hashes, PaddingRepetitions: InputHashCollectionCapacity - uint64(len(hashes)), IsDaveConsensus: true, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) - s.Require().Error(err) + s.Require().ErrorIs(err, repository.ErrAdvanceCursorMismatch) - // Verify no outputs were persisted (rolled back) + // Verify the preflight rejected the result before inserting outputs. outputs, total, err := s.Repo.ListOutputs( s.Ctx, seed.App.IApplicationAddress.String(), repository.OutputFilter{}, repository.Pagination{Limit: 10}, false) @@ -545,28 +672,28 @@ func (s *BulkOperationsSuite) TestStoreClaimAndProofs() { s.Run("StoresClaimAndOutputProofs", func() { seed := Seed(s.Ctx, s.T(), s.Repo) - // Advance epoch to INPUTS_PROCESSED so StoreClaimAndProofs can set CLAIM_COMPUTED - AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, - seed.App.IApplicationAddress.String(), seed.Epoch, EpochStatus_InputsProcessed) - - // First store an advance result to create outputs + // First store an advance result to create outputs. machineHash := crypto.Keccak256Hash([]byte("machine")) outputData := []byte("output-for-claim") - outputsHash := crypto.Keccak256Hash([]byte("outputs-merkle")) + txBufferDataBlock := crypto.Keccak256Hash([]byte("outputs-merkle")) + stateProof := DummyStateProof() + stateProof.MachineHash = machineHash + stateProof.TxBufferDataBlock = txBufferDataBlock result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{outputData}, - OutputsProof: OutputsProof{ - OutputsHash: outputsHash, - MachineHash: machineHash, - }, + StateProof: *stateProof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) + // Publish the final state proof only after all inputs are stored. + AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, + seed.App.IApplicationAddress.String(), seed.Epoch, EpochStatus_InputsProcessed) + // Now store claim and proofs using Commitment/CommitmentProof fields commitmentHash := crypto.Keccak256Hash([]byte("commitment")) seed.Epoch.Commitment = &commitmentHash @@ -722,6 +849,7 @@ func (s *BulkOperationsSuite) TestStoreTournamentEvents() { }) } +//nolint:mnd // Numeric values are intentionally explicit concurrency fixtures. func (s *BulkOperationsSuite) TestConcurrentStoreAdvanceResult() { // Verify that concurrent StoreAdvanceResult calls for different // applications succeed independently without corrupting data. @@ -737,18 +865,18 @@ func (s *BulkOperationsSuite) TestConcurrentStoreAdvanceResult() { wg.Add(1) go func() { defer wg.Done() + stateProof := DummyStateProof() + stateProof.TxBufferDataBlock = crypto.Keccak256Hash( + []byte(fmt.Sprintf("outputs-%d", i))) + stateProof.MachineHash = crypto.Keccak256Hash( + []byte(fmt.Sprintf("machine-%d", i))) result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte(fmt.Sprintf("output-%d", i))}, Reports: [][]byte{[]byte(fmt.Sprintf("report-%d", i))}, - OutputsProof: OutputsProof{ - OutputsHash: crypto.Keccak256Hash( - []byte(fmt.Sprintf("outputs-%d", i))), - MachineHash: crypto.Keccak256Hash( - []byte(fmt.Sprintf("machine-%d", i))), - }, + StateProof: *stateProof, } errs[i] = s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) }() @@ -809,17 +937,17 @@ func (s *BulkOperationsSuite) TestConcurrentStoreAdvanceResult() { wg.Add(1) go func() { defer wg.Done() + stateProof := DummyStateProof() + stateProof.TxBufferDataBlock = crypto.Keccak256Hash( + []byte(fmt.Sprintf("outputs-%d", i))) + stateProof.MachineHash = crypto.Keccak256Hash( + []byte(fmt.Sprintf("machine-%d", i))) result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte(fmt.Sprintf("output-%d", i))}, - OutputsProof: OutputsProof{ - OutputsHash: crypto.Keccak256Hash( - []byte(fmt.Sprintf("outputs-%d", i))), - MachineHash: crypto.Keccak256Hash( - []byte(fmt.Sprintf("machine-%d", i))), - }, + StateProof: *stateProof, } errs[i] = s.Repo.StoreAdvanceResult( s.Ctx, seed.App.ID, result) @@ -833,8 +961,7 @@ func (s *BulkOperationsSuite) TestConcurrentStoreAdvanceResult() { successCount++ } } - s.GreaterOrEqual(successCount, 1, - "at least one concurrent store should succeed") + s.Equal(1, successCount, "exactly one concurrent store may advance the cursor") // Verify data integrity: the input must be in Accepted state input, err := s.Repo.GetInput( @@ -846,7 +973,71 @@ func (s *BulkOperationsSuite) TestConcurrentStoreAdvanceResult() { count, err := s.Repo.GetProcessedInputCount( s.Ctx, seed.App.IApplicationAddress.String()) s.Require().NoError(err) - s.GreaterOrEqual(count, uint64(1)) + s.Equal(uint64(1), count) + }) + + // Input ingestion and advance persistence both touch the current epoch, + // input rows, and application cursor. They must share a lock order so the + // event reader can index a later input while the advancer completes the + // preceding one. + s.Run("ConcurrentInputIngestion", func() { + for _, status := range []InputCompletionStatus{ + InputCompletionStatus_Accepted, + InputCompletionStatus_MachineHalted, + } { + s.Run(status.String(), func() { + const attempts = 10 + for range attempts { + seed := Seed(s.Ctx, s.T(), s.Repo) + nextEpoch := NewEpochBuilder(seed.App.ID). + WithIndex(0). + WithStatus(EpochStatus_Closed). + WithBlocks(0, 9). + WithInputBounds(0, 1). + Build() + nextInput := NewInputBuilder(). + WithIndex(1). + WithBlockNumber(6). + Build() + result := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: status, + StateProof: *DummyStateProof(), + } + + start := make(chan struct{}) + var wg sync.WaitGroup + errs := make([]error, 2) + wg.Add(2) + go func() { + defer wg.Done() + <-start + errs[0] = s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) + }() + go func() { + defer wg.Done() + <-start + errs[1] = s.Repo.CreateEpochsAndInputs( + s.Ctx, + seed.App.IApplicationAddress.String(), + map[*Epoch][]*Input{nextEpoch: {nextInput}}, + 11, + ) + }() + close(start) + wg.Wait() + + s.Require().NoError(errs[0], "store advance result") + s.Require().NoError(errs[1], "index later input") + + stored, err := s.Repo.GetInput( + s.Ctx, seed.App.IApplicationAddress.String(), nextInput.Index) + s.Require().NoError(err) + s.Equal(InputCompletionStatus_None, stored.Status) + } + }) + } }) } @@ -862,10 +1053,7 @@ func (s *BulkOperationsSuite) TestStoreClaimAndProofsRollback() { InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("output")}, - OutputsProof: OutputsProof{ - OutputsHash: crypto.Keccak256Hash([]byte("outputs")), - MachineHash: crypto.Keccak256Hash([]byte("machine")), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) @@ -895,26 +1083,22 @@ func (s *BulkOperationsSuite) TestStoreClaimAndProofsRollback() { s.Run("RollbackOnOutputProofUpdateFailure", func() { seed := Seed(s.Ctx, s.T(), s.Repo) - // Advance epoch to INPUTS_PROCESSED so updateEpochClaim can - // set CLAIM_COMPUTED (the trigger rejects other transitions). - AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, - seed.App.IApplicationAddress.String(), seed.Epoch, - EpochStatus_InputsProcessed) - // Store advance result to create one output (index 0) result := &AdvanceResult{ EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("real-output")}, - OutputsProof: OutputsProof{ - OutputsHash: crypto.Keccak256Hash([]byte("outputs")), - MachineHash: crypto.Keccak256Hash([]byte("machine")), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) + // Publish the final state proof only after all inputs are stored. + AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, + seed.App.IApplicationAddress.String(), seed.Epoch, + EpochStatus_InputsProcessed) + // Prepare a valid epoch claim (this part would succeed) commitmentHash := crypto.Keccak256Hash([]byte("commitment")) seed.Epoch.Commitment = &commitmentHash @@ -1094,10 +1278,7 @@ func (s *BulkOperationsSuite) TestContextCancellation() { Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("should-not-persist")}, Reports: [][]byte{[]byte("should-not-persist")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(cancelledCtx, seed.App.ID, result) @@ -1152,10 +1333,7 @@ func (s *BulkOperationsSuite) TestContextCancellation() { InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("output")}, - OutputsProof: OutputsProof{ - OutputsHash: crypto.Keccak256Hash([]byte("outputs")), - MachineHash: crypto.Keccak256Hash([]byte("machine")), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) diff --git a/internal/repository/repotest/claimer_test_cases.go b/internal/repository/repotest/claimer_test_cases.go index e57d728d4..4c7674b8e 100644 --- a/internal/repository/repotest/claimer_test_cases.go +++ b/internal/repository/repotest/claimer_test_cases.go @@ -664,8 +664,10 @@ func (s *ClaimerSuite) TestRejectEpochAndSetApplicationDiverged() { app := s.createAppWithClaimComputedEpoch() reason := "quorum_divergence_at_acceptance: rejected computed epoch" - err := s.Repo.RejectEpochAndSetApplicationDiverged(s.Ctx, app.ID, 0, reason) + result, err := s.Repo.RejectEpochAndSetApplicationDiverged(s.Ctx, app.ID, 0, reason) s.Require().NoError(err) + s.True(result.EpochRejected) + s.True(result.ApplicationDiverged) assertRejected(app, reason) }) @@ -676,12 +678,44 @@ func (s *ClaimerSuite) TestRejectEpochAndSetApplicationDiverged() { s.Require().NoError(err) reason := "quorum_divergence_at_staging: rejected submitted epoch" - err = s.Repo.RejectEpochAndSetApplicationDiverged(s.Ctx, app.ID, 0, reason) + result, err := s.Repo.RejectEpochAndSetApplicationDiverged(s.Ctx, app.ID, 0, reason) s.Require().NoError(err) + s.True(result.EpochRejected) + s.True(result.ApplicationDiverged) assertRejected(app, reason) }) + for _, terminalStatus := range []ApplicationStatus{ + ApplicationStatus_MachineHalted, + ApplicationStatus_Corrupted, + } { + s.Run("Preserves"+terminalStatus.String()+"WhileRejectingEpoch", func() { + app := s.createAppWithClaimComputedEpoch() + originalReason := "earlier terminal cause" + s.Require().NoError(s.Repo.UpdateApplicationStatus( + s.Ctx, app.ID, terminalStatus, &originalReason)) + + result, err := s.Repo.RejectEpochAndSetApplicationDiverged( + s.Ctx, app.ID, 0, "later claim disagreement") + s.Require().NoError(err) + s.True(result.EpochRejected) + s.False(result.ApplicationDiverged) + + gotEpoch, err := s.Repo.GetEpoch( + s.Ctx, app.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(EpochStatus_ClaimRejected, gotEpoch.Status) + + gotApp, err := s.Repo.GetApplication( + s.Ctx, app.IApplicationAddress.String()) + s.Require().NoError(err) + s.Equal(terminalStatus, gotApp.Status) + s.Require().NotNil(gotApp.Reason) + s.Equal(originalReason, *gotApp.Reason) + }) + } + // A CLAIM_STAGED epoch is outside the COMPUTED/SUBMITTED set, so the // best-effort epoch reject matches no row and the epoch stays CLAIM_STAGED. // The application halt is unconditional: it still becomes DIVERGED so a @@ -694,8 +728,10 @@ func (s *ClaimerSuite) TestRejectEpochAndSetApplicationDiverged() { s.Require().NoError(err) reason := "quorum_divergence_at_acceptance: staged epoch is not a normal rejection source" - err = s.Repo.RejectEpochAndSetApplicationDiverged(s.Ctx, app.ID, 0, reason) + result, err := s.Repo.RejectEpochAndSetApplicationDiverged(s.Ctx, app.ID, 0, reason) s.Require().NoError(err) + s.False(result.EpochRejected) + s.True(result.ApplicationDiverged) gotEpoch, err := s.Repo.GetEpoch(s.Ctx, app.IApplicationAddress.String(), 0) s.Require().NoError(err) @@ -722,9 +758,11 @@ func (s *ClaimerSuite) TestRejectEpochAndSetApplicationDiverged() { s.Require().NoError(err) reason := "divergence detected against a non-rejectable epoch" - err = s.Repo.RejectEpochAndSetApplicationDiverged( + result, err := s.Repo.RejectEpochAndSetApplicationDiverged( s.Ctx, app.ID, 0, reason) s.Require().NoError(err) + s.False(result.EpochRejected) + s.True(result.ApplicationDiverged) gotEpoch, err := s.Repo.GetEpoch(s.Ctx, app.IApplicationAddress.String(), 0) s.Require().NoError(err) @@ -740,7 +778,7 @@ func (s *ClaimerSuite) TestRejectEpochAndSetApplicationDiverged() { // A missing application row surfaces ErrNotFound, distinguishing a genuine // "no such app" from the best-effort epoch reject matching no row. s.Run("ReturnsNotFoundWhenApplicationMissing", func() { - err := s.Repo.RejectEpochAndSetApplicationDiverged( + _, err := s.Repo.RejectEpochAndSetApplicationDiverged( s.Ctx, int64(99_999_999), 0, "missing application") s.Require().ErrorIs(err, repository.ErrNotFound) }) diff --git a/internal/repository/repotest/epoch_test_cases.go b/internal/repository/repotest/epoch_test_cases.go index 7d22efdbb..05d74679f 100644 --- a/internal/repository/repotest/epoch_test_cases.go +++ b/internal/repository/repotest/epoch_test_cases.go @@ -9,7 +9,6 @@ import ( . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/repository" - "github.com/ethereum/go-ethereum/common" ) type EpochSuite struct { @@ -131,7 +130,7 @@ func (s *EpochSuite) TestGetEpoch() { s.Equal(EpochStatus_Closed, got.Status) s.Equal(uint64(0), got.VirtualIndex) s.Nil(got.MachineHash) - s.Nil(got.OutputsMerkleRoot) + s.Nil(got.TxBufferDataBlock) s.Nil(got.ClaimTransactionHash) s.Nil(got.Commitment) s.False(got.CreatedAt.IsZero(), "CreatedAt should be set") @@ -195,12 +194,13 @@ func (s *EpochSuite) TestGetEpochByVirtualIndex() { }) } +//nolint:mnd // Numeric values are intentionally explicit repository fixtures. func (s *EpochSuite) TestGetLastAcceptedEpochIndex() { s.Run("WithAcceptedEpoch", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) epoch0 := NewEpochBuilder(app.ID). - WithIndex(0).WithStatus(EpochStatus_ClaimAccepted).WithBlocks(0, 9).Build() + WithIndex(0).WithStatus(EpochStatus_Closed).WithBlocks(0, 9).Build() input0 := NewInputBuilder().WithIndex(0).WithBlockNumber(5).Build() epoch1 := NewEpochBuilder(app.ID). @@ -211,6 +211,10 @@ func (s *EpochSuite) TestGetLastAcceptedEpochIndex() { s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch0: {input0}, epoch1: {input1}}, 20) s.Require().NoError(err) + AdvanceEpochStatus( + s.Ctx, s.T(), s.Repo, app.IApplicationAddress.String(), epoch0, + EpochStatus_ClaimAccepted, + ) idx, err := s.Repo.GetLastAcceptedEpochIndex(s.Ctx, app.IApplicationAddress.String()) s.Require().NoError(err) @@ -275,6 +279,7 @@ func (s *EpochSuite) TestGetLastNonOpenEpoch() { }) } +//nolint:mnd // Numeric values are intentionally explicit repository fixtures. func (s *EpochSuite) TestListEpochs() { s.Run("EmptyResult", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) @@ -313,6 +318,40 @@ func (s *EpochSuite) TestListEpochs() { s.Equal(EpochStatus_Closed, epochs[0].Status) }) + s.Run("ReturnsStateProofDataBlocks", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + proof := DummyStateProof() + err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Accepted, + StateProof: *proof, + }) + s.Require().NoError(err) + + epochs, total, err := s.Repo.ListEpochs( + s.Ctx, + seed.App.IApplicationAddress.String(), + repository.EpochFilter{}, + repository.Pagination{Limit: 10}, + false, + ) + s.Require().NoError(err) + s.Equal(uint64(1), total) + s.Require().Len(epochs, 1) + s.Require().NotNil(epochs[0].MachineHash) + s.Require().NotNil(epochs[0].TxBufferDataBlock) + s.Require().NotNil(epochs[0].IflagsYDataBlock) + s.Require().NotNil(epochs[0].HtifTohostDataBlock) + s.Equal(proof.MachineHash, *epochs[0].MachineHash) + s.Equal(proof.TxBufferDataBlock, *epochs[0].TxBufferDataBlock) + s.Equal(proof.IflagsYDataBlock, *epochs[0].IflagsYDataBlock) + s.Equal(proof.HtifTohostDataBlock, *epochs[0].HtifTohostDataBlock) + s.Nil(epochs[0].TxBufferProof) + s.Nil(epochs[0].IflagsYProof) + s.Nil(epochs[0].HtifTohostProof) + }) + s.Run("FilterByBeforeBlock", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) @@ -434,7 +473,7 @@ func (s *EpochSuite) TestListEpochs() { WithIndex(1).WithStatus(EpochStatus_Closed). WithBlocks(10, 19).WithInputBounds(1, 1).Build() epoch2 := NewEpochBuilder(app.ID). - WithIndex(2).WithStatus(EpochStatus_InputsProcessed). + WithIndex(2).WithStatus(EpochStatus_ClaimForeclosed). WithBlocks(20, 29).WithInputBounds(2, 2).Build() input0 := NewInputBuilder().WithIndex(0).WithBlockNumber(5).Build() @@ -446,11 +485,11 @@ func (s *EpochSuite) TestListEpochs() { map[*Epoch][]*Input{epoch0: {input0}, epoch1: {input1}, epoch2: {input2}}, 30) s.Require().NoError(err) - // Filter for both Closed and InputsProcessed + // Filter for two distinct terminal/non-open statuses. epochs, total, err := s.Repo.ListEpochs( s.Ctx, app.IApplicationAddress.String(), repository.EpochFilter{ - Status: []EpochStatus{EpochStatus_Closed, EpochStatus_InputsProcessed}, + Status: []EpochStatus{EpochStatus_Closed, EpochStatus_ClaimForeclosed}, }, repository.Pagination{Limit: 10}, false) s.Require().NoError(err) @@ -458,7 +497,7 @@ func (s *EpochSuite) TestListEpochs() { s.Equal(uint64(2), total) for _, e := range epochs { s.True( - e.Status == EpochStatus_Closed || e.Status == EpochStatus_InputsProcessed, + e.Status == EpochStatus_Closed || e.Status == EpochStatus_ClaimForeclosed, "unexpected status: %s", e.Status) } }) @@ -468,14 +507,14 @@ func (s *EpochSuite) TestUpdateEpochStatus() { s.Run("UpdatesStatus", func() { seed := Seed(s.Ctx, s.T(), s.Repo) epoch := seed.Epoch - epoch.Status = EpochStatus_InputsProcessed + epoch.Status = EpochStatus_ClaimForeclosed err := s.Repo.UpdateEpochStatus(s.Ctx, seed.App.IApplicationAddress.String(), epoch) s.Require().NoError(err) got, err := s.Repo.GetEpoch(s.Ctx, seed.App.IApplicationAddress.String(), 0) s.Require().NoError(err) - s.Equal(EpochStatus_InputsProcessed, got.Status) + s.Equal(EpochStatus_ClaimForeclosed, got.Status) }) s.Run("NotFoundForNonExistentEpoch", func() { @@ -489,23 +528,75 @@ func (s *EpochSuite) TestUpdateEpochStatus() { }) } +//nolint:mnd // Numeric values are intentionally explicit repository fixtures. func (s *EpochSuite) TestUpdateEpochInputsProcessed() { s.Run("MarksEpochProcessed", func() { seed := Seed(s.Ctx, s.T(), s.Repo) + proof := DummyStateProof() err := s.Repo.UpdateEpochInputsProcessed( - s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Ctx, seed.App.IApplicationAddress.String(), 0, proof) s.Require().NoError(err) got, err := s.Repo.GetEpoch(s.Ctx, seed.App.IApplicationAddress.String(), 0) s.Require().NoError(err) s.Equal(EpochStatus_InputsProcessed, got.Status) + s.True(got.HasCompleteStateProof()) + s.Equal(proof.MachineHash, *got.MachineHash) + s.Equal(proof.TxBufferDataBlock, *got.TxBufferDataBlock) + s.Equal(proofSiblingsToHashes(proof.TxBufferProof), got.TxBufferProof) + s.Equal(proof.IflagsYDataBlock, *got.IflagsYDataBlock) + s.Equal(proofSiblingsToHashes(proof.IflagsYProof), got.IflagsYProof) + s.Equal(proof.HtifTohostDataBlock, *got.HtifTohostDataBlock) + s.Equal(proofSiblingsToHashes(proof.HtifTohostProof), got.HtifTohostProof) + }) + + s.Run("RejectsIncompleteProofBeforeUpdate", func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + + err := s.Repo.UpdateEpochInputsProcessed( + s.Ctx, seed.App.IApplicationAddress.String(), 0, &StateProof{}) + s.Require().ErrorIs(err, repository.ErrInvalidStateProof) + + got, err := s.Repo.GetEpoch(s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(EpochStatus_Closed, got.Status) + s.False(got.HasCompleteStateProof()) }) - // The update should be a no-op when the previous epoch is still Open + for _, status := range []ApplicationStatus{ + ApplicationStatus_Failed, + ApplicationStatus_Diverged, + ApplicationStatus_Corrupted, + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } { + s.Run("RejectsApplicationStatus/"+status.String(), func() { + seed := Seed(s.Ctx, s.T(), s.Repo) + reason := "application cannot publish a claimable epoch" + err := s.Repo.UpdateApplicationStatus( + s.Ctx, seed.App.ID, status, &reason, + ) + s.Require().NoError(err) + + err = s.Repo.UpdateEpochInputsProcessed( + s.Ctx, seed.App.IApplicationAddress.String(), 0, DummyStateProof()) + s.Require().ErrorIs(err, repository.ErrNoUpdate) + + got, err := s.Repo.GetEpoch( + s.Ctx, seed.App.IApplicationAddress.String(), 0) + s.Require().NoError(err) + s.Equal(EpochStatus_Closed, got.Status) + s.False(got.HasCompleteStateProof()) + }) + } + + // The update should fail when the previous epoch is still Open // (i.e., not yet past Closed). The SQL condition requires that the // previous epoch status is NOT IN (Open, Closed). - s.Run("NoOpWhenPreviousEpochStillOpen", func() { + s.Run("RejectsWhenPreviousEpochStillOpen", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) epoch0 := NewEpochBuilder(app.ID). @@ -524,23 +615,30 @@ func (s *EpochSuite) TestUpdateEpochInputsProcessed() { map[*Epoch][]*Input{epoch0: {input0}, epoch1: {input1}}, 20) s.Require().NoError(err) - // Process input1 so the inputs-present condition is satisfied - result := &AdvanceResult{ + // Advance the durable cursor in order, while deliberately leaving + // epoch0's status Open to exercise the previous-epoch gate. + result0 := &AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: InputCompletionStatus_Accepted, + StateProof: *DummyStateProof(), + } + err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, result0) + s.Require().NoError(err) + + result1 := &AdvanceResult{ EpochIndex: 1, InputIndex: 1, Status: InputCompletionStatus_Accepted, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } - err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, result) + err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, result1) s.Require().NoError(err) // Try to mark epoch1 as InputsProcessed; previous epoch0 is Open err = s.Repo.UpdateEpochInputsProcessed( - s.Ctx, app.IApplicationAddress.String(), 1) - s.Require().NoError(err) // returns nil (no-op), not an error + s.Ctx, app.IApplicationAddress.String(), 1, DummyStateProof()) + s.Require().ErrorIs(err, repository.ErrNoUpdate) got, err := s.Repo.GetEpoch(s.Ctx, app.IApplicationAddress.String(), 1) s.Require().NoError(err) @@ -548,9 +646,9 @@ func (s *EpochSuite) TestUpdateEpochInputsProcessed() { s.Equal(EpochStatus_Closed, got.Status) }) - // The update should be a no-op when the epoch still has pending + // The update should fail when the epoch still has pending // (unprocessed) inputs. The SQL requires pending_count == 0. - s.Run("NoOpWhenPendingInputsRemain", func() { + s.Run("RejectsWhenPendingInputsRemain", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) // Create a single-epoch setup with 2 inputs @@ -571,18 +669,15 @@ func (s *EpochSuite) TestUpdateEpochInputsProcessed() { EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, result) s.Require().NoError(err) // Try to mark epoch as InputsProcessed; input1 is still pending err = s.Repo.UpdateEpochInputsProcessed( - s.Ctx, app.IApplicationAddress.String(), 0) - s.Require().NoError(err) // returns nil (no-op) + s.Ctx, app.IApplicationAddress.String(), 0, DummyStateProof()) + s.Require().ErrorIs(err, repository.ErrNoUpdate) got, err := s.Repo.GetEpoch(s.Ctx, app.IApplicationAddress.String(), 0) s.Require().NoError(err) @@ -590,9 +685,9 @@ func (s *EpochSuite) TestUpdateEpochInputsProcessed() { s.Equal(EpochStatus_Closed, got.Status) }) - // The update should be a no-op when not all expected inputs are present. + // The update should fail when not all expected inputs are present. // total_count != (upper_bound - lower_bound). - s.Run("NoOpWhenInputCountDoesNotMatchBounds", func() { + s.Run("RejectsWhenInputCountDoesNotMatchBounds", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) // Epoch expects 3 inputs (bounds 0..3) but we only provide 1. @@ -612,18 +707,15 @@ func (s *EpochSuite) TestUpdateEpochInputsProcessed() { EpochIndex: 0, InputIndex: 0, Status: InputCompletionStatus_Accepted, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, result) s.Require().NoError(err) // Try to mark epoch as InputsProcessed err = s.Repo.UpdateEpochInputsProcessed( - s.Ctx, app.IApplicationAddress.String(), 0) - s.Require().NoError(err) // returns nil (no-op) + s.Ctx, app.IApplicationAddress.String(), 0, DummyStateProof()) + s.Require().ErrorIs(err, repository.ErrNoUpdate) got, err := s.Repo.GetEpoch(s.Ctx, app.IApplicationAddress.String(), 0) s.Require().NoError(err) @@ -631,13 +723,12 @@ func (s *EpochSuite) TestUpdateEpochInputsProcessed() { s.Equal(EpochStatus_Closed, got.Status) }) - // Non-existent epoch should be a silent no-op (returns nil). - s.Run("NoOpForNonExistentEpoch", func() { + s.Run("RejectsNonExistentEpoch", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) err := s.Repo.UpdateEpochInputsProcessed( - s.Ctx, app.IApplicationAddress.String(), 99) - s.Require().NoError(err) + s.Ctx, app.IApplicationAddress.String(), 99, DummyStateProof()) + s.Require().ErrorIs(err, repository.ErrNoUpdate) }) } @@ -669,90 +760,11 @@ func (s *EpochSuite) TestUpdateEpochClaimTransactionHash() { }) } -func (s *EpochSuite) TestUpdateEpochOutputsProof() { - s.Run("SetsOutputsProof", func() { - seed := Seed(s.Ctx, s.T(), s.Repo) - proof := &OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - OutputsHashProof: [][32]byte{ - [32]byte(common.HexToHash("0xaabb")), - }, - } - - err := s.Repo.UpdateEpochOutputsProof(s.Ctx, seed.App.ID, 0, proof) - s.Require().NoError(err) - - got, err := s.Repo.GetEpoch(s.Ctx, seed.App.IApplicationAddress.String(), 0) - s.Require().NoError(err) - s.Require().NotNil(got.OutputsMerkleRoot) - }) -} - -func (s *EpochSuite) TestRepeatPreviousEpochOutputsProof() { - s.Run("CopiesProofFromPreviousEpoch", func() { - app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) - - epoch0 := NewEpochBuilder(app.ID). - WithIndex(0).WithStatus(EpochStatus_Closed). - WithBlocks(0, 9).WithInputBounds(0, 0).Build() - epoch1 := NewEpochBuilder(app.ID). - WithIndex(1).WithStatus(EpochStatus_Closed). - WithBlocks(10, 19).WithInputBounds(1, 1).Build() - - input0 := NewInputBuilder().WithIndex(0).WithBlockNumber(5).Build() - input1 := NewInputBuilder().WithIndex(1).WithEpochIndex(1).WithBlockNumber(15).Build() - - err := s.Repo.CreateEpochsAndInputs( - s.Ctx, app.IApplicationAddress.String(), - map[*Epoch][]*Input{epoch0: {input0}, epoch1: {input1}}, 20) - s.Require().NoError(err) - - // Set proof on epoch 0 - outputsHash := UniqueHash() - machineHash := UniqueHash() - proof := &OutputsProof{ - OutputsHash: outputsHash, - MachineHash: machineHash, - OutputsHashProof: [][32]byte{ - [32]byte(UniqueHash()), - }, - } - err = s.Repo.UpdateEpochOutputsProof(s.Ctx, app.ID, 0, proof) - s.Require().NoError(err) - - // Copy proof from epoch 0 to epoch 1 - err = s.Repo.RepeatPreviousEpochOutputsProof(s.Ctx, app.ID, 1) - s.Require().NoError(err) - - // Verify epoch 1 has epoch 0's proof - got, err := s.Repo.GetEpoch(s.Ctx, app.IApplicationAddress.String(), 1) - s.Require().NoError(err) - s.Require().NotNil(got.OutputsMerkleRoot) - s.Equal(outputsHash, *got.OutputsMerkleRoot) - s.Require().NotNil(got.MachineHash) - s.Equal(machineHash, *got.MachineHash) - }) - - s.Run("ErrorsForEpochZero", func() { - seed := Seed(s.Ctx, s.T(), s.Repo) - - err := s.Repo.RepeatPreviousEpochOutputsProof(s.Ctx, seed.App.ID, 0) - s.Require().Error(err) - s.Contains(err.Error(), "epoch 0") - }) - - s.Run("ErrorsForNonExistentEpoch", func() { - seed := Seed(s.Ctx, s.T(), s.Repo) - - err := s.Repo.RepeatPreviousEpochOutputsProof(s.Ctx, seed.App.ID, 99) - s.Require().Error(err) - }) -} - // TestUpsertPreservesNonOpenEpoch verifies the CASE/WHEN crash-recovery guard // in CreateEpochsAndInputs: re-upserting an epoch that has advanced past OPEN // must preserve the existing row's fields (status, block range, input bounds). +// +//nolint:mnd // Numeric values are intentionally explicit repository fixtures. func (s *EpochSuite) TestUpsertPreservesNonOpenEpoch() { s.Run("PreservesClosedEpochFields", func() { app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) @@ -794,13 +806,17 @@ func (s *EpochSuite) TestUpsertPreservesNonOpenEpoch() { epoch := NewEpochBuilder(app.ID). WithIndex(0).WithStatus(EpochStatus_Closed). - WithBlocks(0, 50).WithInputBounds(0, 3).Build() + WithBlocks(0, 50).WithInputBounds(0, 1).Build() input := NewInputBuilder().WithIndex(0).WithBlockNumber(5).Build() err := s.Repo.CreateEpochsAndInputs( s.Ctx, app.IApplicationAddress.String(), map[*Epoch][]*Input{epoch: {input}}, 51) s.Require().NoError(err) + StoreAdvanceResult( + s.Ctx, s.T(), s.Repo, app.ID, 0, 0, + InputCompletionStatus_Accepted, nil, nil, + ) // Advance past CLOSED so it is no longer OPEN. AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, @@ -824,7 +840,7 @@ func (s *EpochSuite) TestUpsertPreservesNonOpenEpoch() { "status should be preserved, not overwritten") s.Equal(uint64(50), got.LastBlock, "LastBlock should be preserved from original epoch") - s.Equal(uint64(3), got.InputIndexUpperBound, + s.Equal(uint64(1), got.InputIndexUpperBound, "InputIndexUpperBound should be preserved from original epoch") }) @@ -1079,21 +1095,21 @@ func (s *EpochSuite) TestEpochStatusTransitionTrigger() { s.Require().NoError(err) }) - // Verify the trigger rejects CLAIM_COMPUTED when proof fields are missing. - s.Run("RejectsClaimComputedWithoutProofFields", func() { + // Verify the schema rejects publication when proof fields are missing. + s.Run("RejectsInputsProcessedWithoutProofFields", func() { seed := Seed(s.Ctx, s.T(), s.Repo) - AdvanceEpochStatus(s.Ctx, s.T(), s.Repo, - seed.App.IApplicationAddress.String(), seed.Epoch, - EpochStatus_InputsProcessed) - - // Try INPUTS_PROCESSED -> CLAIM_COMPUTED without setting - // machine_hash, outputs_merkle_root, outputs_merkle_proof. - seed.Epoch.Status = EpochStatus_ClaimComputed + // Bypass the atomic publication API and try CLOSED -> + // INPUTS_PROCESSED without the three proof leaves. + seed.Epoch.Status = EpochStatus_InputsProcessed err := s.Repo.UpdateEpochStatus( s.Ctx, seed.App.IApplicationAddress.String(), seed.Epoch) s.Require().Error(err) - s.Contains(err.Error(), "CLAIM_COMPUTED requires") + + got, err := s.Repo.GetEpoch( + s.Ctx, seed.App.IApplicationAddress.String(), seed.Epoch.Index) + s.Require().NoError(err) + s.Equal(EpochStatus_Closed, got.Status) }) // Verify CLAIM_COMPUTED succeeds when all required fields are present. @@ -1104,18 +1120,8 @@ func (s *EpochSuite) TestEpochStatusTransitionTrigger() { seed.App.IApplicationAddress.String(), seed.Epoch, EpochStatus_InputsProcessed) - // Set the required proof fields. - proof := &OutputsProof{ - OutputsHash: UniqueHash(), - OutputsHashProof: [][32]byte{[32]byte(UniqueHash())}, - MachineHash: UniqueHash(), - } - err := s.Repo.UpdateEpochOutputsProof( - s.Ctx, seed.App.ID, seed.Epoch.Index, proof) - s.Require().NoError(err) - seed.Epoch.Status = EpochStatus_ClaimComputed - err = s.Repo.UpdateEpochStatus( + err := s.Repo.UpdateEpochStatus( s.Ctx, seed.App.IApplicationAddress.String(), seed.Epoch) s.Require().NoError(err) @@ -1141,21 +1147,13 @@ func (s *EpochSuite) TestEpochStatusTransitionTrigger() { map[*Epoch][]*Input{epoch: {input}}, 10) s.Require().NoError(err) - // Advance to INPUTS_PROCESSED. - epoch.Status = EpochStatus_InputsProcessed - err = s.Repo.UpdateEpochStatus( - s.Ctx, app.IApplicationAddress.String(), epoch) - s.Require().NoError(err) - - // Set base proof fields but NOT commitment. - proof := &OutputsProof{ - OutputsHash: UniqueHash(), - OutputsHashProof: [][32]byte{[32]byte(UniqueHash())}, - MachineHash: UniqueHash(), - } - err = s.Repo.UpdateEpochOutputsProof( - s.Ctx, app.ID, epoch.Index, proof) + // Publish the machine proof but not the PRT commitment. + err = s.Repo.UpdateEpochInputsProcessed( + s.Ctx, app.IApplicationAddress.String(), epoch.Index, + DummyStateProof(), + ) s.Require().NoError(err) + epoch.Status = EpochStatus_InputsProcessed // INPUTS_PROCESSED -> CLAIM_COMPUTED without commitment — must fail. epoch.Status = EpochStatus_ClaimComputed diff --git a/internal/repository/repotest/input_test_cases.go b/internal/repository/repotest/input_test_cases.go index 82254ced2..3d42326a7 100644 --- a/internal/repository/repotest/input_test_cases.go +++ b/internal/repository/repotest/input_test_cases.go @@ -30,7 +30,7 @@ func (s *InputSuite) TestGetInput() { s.Equal(seed.Input.TransactionHash, got.TransactionHash) s.Equal(seed.Input.LogIndex, got.LogIndex) s.Nil(got.MachineHash) - s.Nil(got.OutputsHash) + s.Nil(got.TxBufferDataBlock) s.Nil(got.SnapshotURI) s.False(got.CreatedAt.IsZero(), "CreatedAt should be set") s.False(got.UpdatedAt.IsZero(), "UpdatedAt should be set") diff --git a/internal/repository/repotest/output_test_cases.go b/internal/repository/repotest/output_test_cases.go index 4cf7ca506..996a887bf 100644 --- a/internal/repository/repotest/output_test_cases.go +++ b/internal/repository/repotest/output_test_cases.go @@ -88,10 +88,7 @@ func (s *OutputSuite) TestListOutputs() { InputIndex: 0, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("epoch-output")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) @@ -145,10 +142,7 @@ func (s *OutputSuite) TestListOutputs() { InputIndex: i, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("output-data")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err = s.Repo.StoreAdvanceResult(s.Ctx, app.ID, result) s.Require().NoError(err) @@ -200,10 +194,7 @@ func (s *OutputSuite) TestListOutputs() { InputIndex: e.input, Status: InputCompletionStatus_Accepted, Outputs: [][]byte{[]byte("output-data")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } s.Require().NoError(s.Repo.StoreAdvanceResult(s.Ctx, app.ID, result)) } diff --git a/internal/repository/repotest/report_test_cases.go b/internal/repository/repotest/report_test_cases.go index 2791bef91..f90672dc7 100644 --- a/internal/repository/repotest/report_test_cases.go +++ b/internal/repository/repotest/report_test_cases.go @@ -88,10 +88,7 @@ func (s *ReportSuite) TestListReports() { InputIndex: 0, Status: InputCompletionStatus_Accepted, Reports: [][]byte{[]byte("epoch-report")}, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) s.Require().NoError(err) diff --git a/internal/repository/repotest/repotest.go b/internal/repository/repotest/repotest.go index 4cbb529e5..45562afa5 100644 --- a/internal/repository/repotest/repotest.go +++ b/internal/repository/repotest/repotest.go @@ -49,10 +49,7 @@ func StoreAdvanceResult( Status: status, Outputs: outputs, Reports: reports, - OutputsProof: OutputsProof{ - OutputsHash: UniqueHash(), - MachineHash: UniqueHash(), - }, + StateProof: *DummyStateProof(), } if status == InputCompletionStatus_Exception { result.ExceptionData = []byte{} diff --git a/internal/repository/repotest/state_hash_test_cases.go b/internal/repository/repotest/state_hash_test_cases.go index 59992c708..bf42f36b5 100644 --- a/internal/repository/repotest/state_hash_test_cases.go +++ b/internal/repository/repotest/state_hash_test_cases.go @@ -45,7 +45,10 @@ func (s *StateHashSuite) TestListStateHashes() { s.Run("ReturnsStateHashesFromDaveConsensus", func() { seed := Seed(s.Ctx, s.T(), s.Repo) machineHash := crypto.Keccak256Hash([]byte("dave-list-machine")) - outputsHash := crypto.Keccak256Hash([]byte("dave-list-outputs")) + txBufferDataBlock := crypto.Keccak256Hash([]byte("dave-list-outputs")) + proof := DummyStateProof() + proof.MachineHash = machineHash + proof.TxBufferDataBlock = txBufferDataBlock hash1 := [32]byte(crypto.Keccak256Hash([]byte("list-state-1"))) hash2 := [32]byte(crypto.Keccak256Hash([]byte("list-state-2"))) @@ -58,10 +61,7 @@ func (s *StateHashSuite) TestListStateHashes() { PeriodicStateHashes: collectedHashes, PaddingRepetitions: InputHashCollectionCapacity - uint64(len(collectedHashes)), IsDaveConsensus: true, - OutputsProof: OutputsProof{ - OutputsHash: outputsHash, - MachineHash: machineHash, - }, + StateProof: *proof, } err := s.Repo.StoreAdvanceResult(s.Ctx, seed.App.ID, result) diff --git a/test/integration/divergent_claim_test.go b/test/integration/divergent_claim_test.go index 06a474d8b..997673051 100644 --- a/test/integration/divergent_claim_test.go +++ b/test/integration/divergent_claim_test.go @@ -203,11 +203,11 @@ func (s *DivergentClaimSuite) TestDivergentClaimReplay() { // semantic correctness, so we can splice it into the divergent payload. epoch1, err := readEpoch(s.ctx, appAName, inputEpochs[1]) r.NoError(err, "read epoch 1") - r.NotEmpty(epoch1.OutputsMerkleProof, + r.NotEmpty(epoch1.TxBufferProof, "epoch 1 must have an outputs merkle proof to reuse for the attack") epochLen := epoch1.LastBlock - epoch1.FirstBlock + 1 s.T().Logf(" epoch length = %d blocks; epoch 1 proof = %d siblings", - epochLen, len(epoch1.OutputsMerkleProof)) + epochLen, len(epoch1.TxBufferProof)) // ─── Phase 1.5: stop the node so the attacker cannot lose the race ── s.T().Log("--- Phase 1.5: stop node, then send input 2 and submit divergent claim ---") @@ -261,7 +261,7 @@ func (s *DivergentClaimSuite) TestDivergentClaimReplay() { r.NoError(err, "bind iauthority") divergentOutputs := randomBytes32(s.T()) - proof := merkleProofToBytes32(epoch1.OutputsMerkleProof) + proof := merkleProofToBytes32(epoch1.TxBufferProof) s.T().Logf(" attacker submitting divergent claim: lpbn=%d outputs=0x%x proof_siblings=%d", targetEpochLastBlock, divergentOutputs, len(proof)) submitTx, err := authorityBinding.SubmitClaim(attackerOpts, appAddr, diff --git a/test/integration/echo_authority_staging_test.go b/test/integration/echo_authority_staging_test.go index 1f6abfe85..34f230014 100644 --- a/test/integration/echo_authority_staging_test.go +++ b/test/integration/echo_authority_staging_test.go @@ -169,5 +169,5 @@ func (s *EchoAuthorityStagingSuite) TestEchoAuthorityForecloseStagedClaim() { foreclosedCancel() r.NoError(err, "foreclosed staged claim should become CLAIM_FORECLOSED without waiting for staging-period expiry") r.NotNil(epoch.StagedAtBlock, "staged_at_block should be preserved after CLAIM_FORECLOSED") - r.NotNil(epoch.OutputsMerkleRoot, "local claim data should be preserved when terminalizing as CLAIM_FORECLOSED") + r.NotNil(epoch.TxBufferDataBlock, "local claim data should be preserved when terminalizing as CLAIM_FORECLOSED") } diff --git a/test/integration/echo_quorum_test.go b/test/integration/echo_quorum_test.go index 418f11c96..e471294d6 100644 --- a/test/integration/echo_quorum_test.go +++ b/test/integration/echo_quorum_test.go @@ -139,8 +139,8 @@ func (s *EchoQuorumSuite) TestEchoQuorumLifecycle() { submittedCancel() r.NoError(err, "wait for node to submit quorum claim") - s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.OutputsMerkleRoot) - s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.OutputsMerkleRoot) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.TxBufferDataBlock) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.TxBufferDataBlock) s.waitForQuorumAccepted(app.appName, epoch.Index) verifyClaimAndExecute(s.ctx, s.T(), r, verifyAndExecuteConfig{ @@ -162,8 +162,8 @@ func (s *EchoQuorumSuite) TestNodeVoteFirstThenOtherValidatorsStageAndAccept() { submittedCancel() s.Require().NoError(err, "wait for node to submit quorum claim") - s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.OutputsMerkleRoot) - s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.OutputsMerkleRoot) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.TxBufferDataBlock) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.TxBufferDataBlock) s.waitForQuorumAccepted(app.appName, epoch.Index) } @@ -193,7 +193,7 @@ func (s *EchoQuorumSuite) TestExternalValidatorThenNodeVoteStagesAndAccepts() { s.Require().Equal(model.EpochStatus_ClaimComputed, epoch.Status, "node should compute the claim before the slowed claimer polling interval submits it") - s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.OutputsMerkleRoot) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.TxBufferDataBlock) stopSharedNode(s.T()) startSharedNode(s.T()) @@ -227,8 +227,8 @@ func (s *EchoQuorumSuite) TestExternalMajorityStagesBeforeNodeVoteThenNodeAccept s.Require().Equal(model.EpochStatus_ClaimComputed, epoch.Status, "node should compute the claim before the slowed claimer polling interval submits it") - s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.OutputsMerkleRoot) - s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.OutputsMerkleRoot) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.TxBufferDataBlock) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.TxBufferDataBlock) stopSharedNode(s.T()) startSharedNode(s.T()) @@ -246,17 +246,17 @@ func (s *EchoQuorumSuite) TestDivergentMinorityVoteDoesNotBlockAcceptance() { submittedCancel() s.Require().NoError(err, "wait for node to submit quorum claim") - divergentOutputs := randomOutputsMerkleRoot(s.T(), *epoch.OutputsMerkleRoot) - s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.OutputsMerkleRoot) + divergentOutputs := randomOutputsMerkleRoot(s.T(), *epoch.TxBufferDataBlock) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.TxBufferDataBlock) s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, divergentOutputs) s.waitForQuorumAccepted(app.appName, epoch.Index) } -func (s *EchoQuorumSuite) TestDivergentMajorityMarksApplicationInoperable() { +func (s *EchoQuorumSuite) TestDivergentMajorityMarksApplicationDiverged() { s.SetExpectedLogs(s.T(), ExpectedLog{ - Pattern: regexp.MustCompile(`marking application as diverged.*quorum_divergence_at_staging`), + Pattern: regexp.MustCompile(`claim divergence detected.*quorum_divergence_at_staging`), Level: LevelError, Reason: "expected DIVERGED transition after a divergent Quorum majority stages a different claim", }, @@ -275,7 +275,7 @@ func (s *EchoQuorumSuite) TestDivergentMajorityMarksApplicationInoperable() { submittedCancel() s.Require().NoError(err, "wait for node to submit quorum claim") - divergentOutputs := randomOutputsMerkleRoot(s.T(), *epoch.OutputsMerkleRoot) + divergentOutputs := randomOutputsMerkleRoot(s.T(), *epoch.TxBufferDataBlock) s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, divergentOutputs) s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, divergentOutputs) @@ -343,7 +343,7 @@ func (s *EchoQuorumSuite) TestForecloseQuorumClaimBeforeAcceptanceMarksClaimFore epoch, err = waitForEpochStatus(foreclosedCtx, s.T(), app.appName, epoch.Index, model.EpochStatus_ClaimForeclosed) foreclosedCancel() r.NoError(err, "foreclosed quorum claim should become CLAIM_FORECLOSED instead of stalling in CLAIM_SUBMITTED") - r.NotNil(epoch.OutputsMerkleRoot, "local claim data should be preserved when terminalizing as CLAIM_FORECLOSED") + r.NotNil(epoch.TxBufferDataBlock, "local claim data should be preserved when terminalizing as CLAIM_FORECLOSED") // Ordinary foreclosure keeps the app's health status OK, keeps it // enabled for L1 observation, and surfaces the marker in `app status`. @@ -377,8 +377,8 @@ func (s *EchoQuorumSuite) TestForecloseQuorumOutputExecutionAfterForeclosureIsRe submittedCancel() r.NoError(err, "wait for node to submit quorum claim") - s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.OutputsMerkleRoot) - s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.OutputsMerkleRoot) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexA, *epoch.TxBufferDataBlock) + s.submitQuorumClaim(app, epoch, quorumValidatorIndexB, *epoch.TxBufferDataBlock) s.waitForQuorumAccepted(app.appName, epoch.Index) r.NoError(guardianForeclose(s.ctx, app.appName, guardianIndex), "guardian foreclose") @@ -540,7 +540,7 @@ func (s *EchoQuorumSuite) waitForEpochWithClaim(appName string, epochIndex uint6 } return false, fmt.Errorf("poll epoch %d claim: %w", epochIndex, err) } - if epoch.OutputsMerkleRoot != nil && epoch.MachineHash != nil && isQuorumClaimReadyStatus(epoch.Status) { + if epoch.TxBufferDataBlock != nil && epoch.MachineHash != nil && isQuorumClaimReadyStatus(epoch.Status) { result = epoch return true, nil } @@ -601,7 +601,7 @@ func (s *EchoQuorumSuite) submitQuorumClaim( outputsMerkleRoot [32]byte, ) common.Hash { r := s.Require() - r.NotNil(epoch.OutputsMerkleRoot, "epoch %d missing outputs merkle root", epoch.Index) + r.NotNil(epoch.TxBufferDataBlock, "epoch %d missing outputs merkle root", epoch.Index) key, err := ethutil.MnemonicToPrivateKey(ethutil.FoundryMnemonic, accountIndex) r.NoError(err, "derive validator key %d", accountIndex) @@ -615,7 +615,7 @@ func (s *EchoQuorumSuite) submitQuorumClaim( app.appAddress, new(big.Int).SetUint64(epoch.LastBlock), outputsMerkleRoot, - merkleProofToBytes32(epoch.OutputsMerkleProof), + merkleProofToBytes32(epoch.TxBufferProof), ) r.NoError(err, "validator %d submit quorum claim", accountIndex) diff --git a/test/integration/same_block_inputs_test.go b/test/integration/same_block_inputs_test.go index 66abda816..e96348038 100644 --- a/test/integration/same_block_inputs_test.go +++ b/test/integration/same_block_inputs_test.go @@ -483,11 +483,11 @@ func (s *SameBlockInputsSuite) readEpochSettlementData( if err != nil { return false, err } - if epoch == nil || epoch.OutputsMerkleRoot == nil { + if epoch == nil || epoch.TxBufferDataBlock == nil { return false, nil } - root = *epoch.OutputsMerkleRoot - proof = epoch.OutputsMerkleProof + root = *epoch.TxBufferDataBlock + proof = epoch.TxBufferProof return true, nil }) r.NoError(err, "wait for epoch %d settlement data (outputs merkle root)", epochIndex) diff --git a/test/integration/withdrawal_lifecycle_test.go b/test/integration/withdrawal_lifecycle_test.go index 60fffcdd7..93dddbfba 100644 --- a/test/integration/withdrawal_lifecycle_test.go +++ b/test/integration/withdrawal_lifecycle_test.go @@ -351,8 +351,8 @@ func (s *WithdrawalLifecycleSuite) finalizeQuorumEpoch( case model.EpochStatus_ClaimStaged: return s.waitForQuorumAccepted(deployment.appName, epochIndex) case model.EpochStatus_ClaimComputed, model.EpochStatus_ClaimSubmitted: - s.submitQuorumClaim(deployment, epoch, quorumValidatorIndexA, *epoch.OutputsMerkleRoot) - s.submitQuorumClaim(deployment, epoch, quorumValidatorIndexB, *epoch.OutputsMerkleRoot) + s.submitQuorumClaim(deployment, epoch, quorumValidatorIndexA, *epoch.TxBufferDataBlock) + s.submitQuorumClaim(deployment, epoch, quorumValidatorIndexB, *epoch.TxBufferDataBlock) return s.waitForQuorumAccepted(deployment.appName, epochIndex) default: s.Require().FailNowf("unexpected quorum epoch status", @@ -377,7 +377,7 @@ func (s *WithdrawalLifecycleSuite) waitForQuorumEpochWithClaim(appName string, e } return false, fmt.Errorf("poll epoch %d claim: %w", epochIndex, err) } - if epoch.OutputsMerkleRoot != nil && epoch.MachineHash != nil && isQuorumClaimReadyStatus(epoch.Status) { + if epoch.TxBufferDataBlock != nil && epoch.MachineHash != nil && isQuorumClaimReadyStatus(epoch.Status) { result = epoch return true, nil } @@ -398,7 +398,7 @@ func (s *WithdrawalLifecycleSuite) submitQuorumClaim( outputsMerkleRoot [32]byte, ) { r := s.Require() - r.NotNil(epoch.OutputsMerkleRoot, "epoch %d missing outputs merkle root", epoch.Index) + r.NotNil(epoch.TxBufferDataBlock, "epoch %d missing outputs merkle root", epoch.Index) r.NotNil(deployment.quorum, "quorum binding is required") key, err := ethutil.MnemonicToPrivateKey(ethutil.FoundryMnemonic, accountIndex) @@ -412,7 +412,7 @@ func (s *WithdrawalLifecycleSuite) submitQuorumClaim( deployment.appAddress, new(big.Int).SetUint64(epoch.LastBlock), outputsMerkleRoot, - merkleProofToBytes32(epoch.OutputsMerkleProof), + merkleProofToBytes32(epoch.TxBufferProof), ) r.NoError(err, "validator %d submit quorum claim", accountIndex) receipt, err := bind.WaitMined(s.ctx, s.client, tx) From 39f653057a2363bfe71c792be7e8a606963a79fb Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:31:20 -0300 Subject: [PATCH 03/11] feat(manager): preserve terminal post-run states --- internal/manager/instance.go | 209 +++++++++++++-------- internal/manager/instance_test.go | 291 ++++++++++++++++++++---------- internal/manager/manager.go | 80 ++++---- internal/manager/manager_test.go | 84 +++++---- internal/manager/types.go | 18 +- 5 files changed, 441 insertions(+), 241 deletions(-) diff --git a/internal/manager/instance.go b/internal/manager/instance.go index 53ef50392..530fce80c 100644 --- a/internal/manager/instance.go +++ b/internal/manager/instance.go @@ -15,7 +15,7 @@ import ( "time" "github.com/cartesi/rollups-node/internal/manager/pmutex" - . "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/pkg/machine" "github.com/ethereum/go-ethereum/common" "golang.org/x/sync/semaphore" @@ -47,7 +47,7 @@ var ( // LLock for inspect (read-only fork). HLock starves LLock by design. // - inspectSemaphore: Bounds concurrent inspect operations. type MachineInstanceImpl struct { - application *Application + application *model.Application runtime machine.Machine // How many inputs were processed by the machine. @@ -82,7 +82,7 @@ var ( // NewMachineInstance creates a new machine instance for an application func NewMachineInstance( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, checkTemplateHash bool, ) (MachineInstance, error) { @@ -93,7 +93,7 @@ func NewMachineInstance( // NewMachineInstanceFromSnapshot creates a new machine instance from a snapshot func NewMachineInstanceFromSnapshot( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, snapshotPath string, expectedHash common.Hash, @@ -106,7 +106,7 @@ func NewMachineInstanceFromSnapshot( func newMachineInstanceFromSnapshot( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, snapshotPath string, expectedHash common.Hash, @@ -127,7 +127,7 @@ func newMachineInstanceFromSnapshot( // NewMachineInstanceWithFactory creates a new machine instance with a custom factory func NewMachineInstanceWithFactory( ctx context.Context, - app *Application, + app *model.Application, processedInputs uint64, logger *slog.Logger, factory MachineRuntimeFactory, @@ -184,7 +184,7 @@ func NewMachineInstanceWithFactory( return instance, nil } -func (m *MachineInstanceImpl) Application() *Application { +func (m *MachineInstanceImpl) Application() *model.Application { return m.application } @@ -201,7 +201,6 @@ func (m *MachineInstanceImpl) forkForAdvance(ctx context.Context, index uint64) if m.runtime == nil { return nil, ErrMachineClosed } - // Verify input index current := m.processedInputs.Load() if current != index { @@ -216,10 +215,19 @@ func (m *MachineInstanceImpl) forkForAdvance(ctx context.Context, index uint64) // Advance treats a machine fork as the execution transaction for one input. // It executes on the fork, selects the canonical root from the typed completion // status, and advances processedInputs exactly once for every completed input. -// Accepted adopts the fork; currently nonaccepted completions keep the -// predecessor runtime and close the fork. Incomplete execution returns an error -// and adopts neither the fork nor a canonical result. -func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIndex uint64, index uint64, computeHashes bool) (*AdvanceResult, error) { +// Accepted completions adopt the post-run fork, rejected inputs keep the +// predecessor runtime, and terminal completions dispose both runtimes after +// collecting the post-run proof. Incomplete execution returns an error and +// adopts neither the fork nor a canonical result. Disposing a terminal runtime +// here prevents inspect from observing it before the durable application status +// is committed by the advancer. +func (m *MachineInstanceImpl) Advance( + ctx context.Context, + input []byte, + epochIndex uint64, + index uint64, + computeHashes bool, +) (*model.AdvanceResult, error) { // Only one advance can be active at a time m.advanceMutex.Lock() defer m.advanceMutex.Unlock() @@ -233,18 +241,17 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn return nil, err } - // Get the machine state before processing - prevMachineHash, err := fork.Hash(ctx) + // Every input starts from the accepted state that can receive it. Keep its + // proof as the canonical result if the input rejects and the fork is + // discarded. + prevMachineProof, err := fork.StateProof(ctx) if err != nil { return nil, errors.Join(err, fork.Close()) } - - prevOutputsHash, err := fork.OutputsHash(ctx) - if err != nil { + if err := machine.ValidateAcceptedState(prevMachineProof); err != nil { return nil, errors.Join(err, fork.Close()) } - - prevOutputsHashProof, err := fork.OutputsHashProof(ctx) + prevProof, err := stateProofFromMachine(prevMachineProof) if err != nil { return nil, errors.Join(err, fork.Close()) } @@ -254,7 +261,7 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn defer cancel() // Process the input - advanceResp, err := fork.Advance(advanceCtx, input, prevMachineHash, computeHashes) + advanceResp, err := fork.Advance(advanceCtx, input, prevProof.MachineHash, computeHashes) if err != nil { return nil, errors.Join(err, fork.Close()) } @@ -270,12 +277,10 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn } // Create the result - result := &AdvanceResult{ + result := &model.AdvanceResult{ EpochIndex: epochIndex, InputIndex: index, Status: status, - Outputs: advanceResp.Outputs, - Reports: advanceResp.Reports, ExceptionData: advanceResp.ExceptionData, PeriodicStateHashes: advanceResp.PeriodicStateHashes, PaddingRepetitions: advanceResp.PaddingRepetitions, @@ -285,27 +290,43 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn // Resolve the canonical result and fork disposition once. Validation below // must succeed before the selected disposition mutates the live instance. adoptFork := false + terminalCompletion := false switch result.Status { - case InputCompletionStatus_Accepted: - // Get the machine hash after processing - result.MachineHash, err = fork.Hash(ctx) - if err != nil { - return nil, errors.Join(err, fork.Close()) + case model.InputCompletionStatus_Accepted: + postMachineProof, proofErr := fork.StateProof(ctx) + if proofErr != nil { + return nil, errors.Join(proofErr, fork.Close()) } - result.OutputsHash = advanceResp.OutputsHash - result.OutputsHashProof, err = fork.OutputsHashProof(ctx) - if err != nil { - return nil, errors.Join(err, fork.Close()) + if proofErr := machine.ValidateAcceptedState(postMachineProof); proofErr != nil { + return nil, errors.Join(proofErr, fork.Close()) } + postProof, proofErr := stateProofFromMachine(postMachineProof) + if proofErr != nil { + return nil, errors.Join(proofErr, fork.Close()) + } + result.StateProof = *postProof + result.Outputs = advanceResp.Outputs + result.Reports = advanceResp.Reports adoptFork = true - case InputCompletionStatus_Rejected, - InputCompletionStatus_Exception, - InputCompletionStatus_MachineHalted: - // Use the previous state for currently nonaccepted inputs. - result.MachineHash = prevMachineHash - result.OutputsHash = prevOutputsHash - result.OutputsHashProof = prevOutputsHashProof - case InputCompletionStatus_None: + case model.InputCompletionStatus_Rejected: + // Rejected execution has no canonical state transition or effects. + result.StateProof = *prevProof + case model.InputCompletionStatus_Exception, + model.InputCompletionStatus_MachineHalted, + model.InputCompletionStatus_Overflow, + model.InputCompletionStatus_UnexpectedYield: + // Terminal execution preserves and proves the actual post-run state. + postMachineProof, proofErr := fork.StateProof(ctx) + if proofErr != nil { + return nil, errors.Join(proofErr, fork.Close()) + } + postProof, proofErr := stateProofFromMachine(postMachineProof) + if proofErr != nil { + return nil, errors.Join(proofErr, fork.Close()) + } + result.StateProof = *postProof + terminalCompletion = true + case model.InputCompletionStatus_None: return nil, errors.Join( fmt.Errorf("cannot resolve advance result for status %q", result.Status), fork.Close(), @@ -327,7 +348,27 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn } } - if adoptFork { + switch { + case terminalCompletion: + // A completed terminal input has no runnable successor. Make the + // runtime unavailable before returning so concurrent inspect requests + // cannot fork the terminal state while its database transaction is + // waiting to commit. The complete post-run proof above remains in the + // returned result; if persistence fails, the advancer shuts down and a + // restart reconstructs execution from persisted state. + m.mutex.HLock() + oldRuntime := m.runtime + m.runtime = nil + m.processedInputs.Add(1) + m.mutex.Unlock() + + if err := oldRuntime.Close(); err != nil { + m.logger.Warn("Failed to close predecessor machine runtime after terminal completion", "error", err) + } + if err := fork.Close(); err != nil { + m.logger.Warn("Failed to close terminal machine runtime", "error", err) + } + case adoptFork: // Replace the current machine with the fork m.mutex.HLock() oldRuntime := m.runtime @@ -338,7 +379,7 @@ func (m *MachineInstanceImpl) Advance(ctx context.Context, input []byte, epochIn if err := oldRuntime.Close(); err != nil { m.logger.Warn("Failed to close old machine runtime", "error", err) } - } else { + default: // Close the fork since we're not using it if err := fork.Close(); err != nil { @@ -517,7 +558,7 @@ func (m *MachineInstanceImpl) Hash(ctx context.Context) ([32]byte, error) { return hash, nil } -func (m *MachineInstanceImpl) OutputsProof(ctx context.Context) (*OutputsProof, error) { +func (m *MachineInstanceImpl) StateProof(ctx context.Context) (*model.StateProof, error) { // Acquire the advance mutex to ensure no advance operations are in progress m.advanceMutex.Lock() defer m.advanceMutex.Unlock() @@ -530,7 +571,7 @@ func (m *MachineInstanceImpl) OutputsProof(ctx context.Context) (*OutputsProof, return nil, ErrMachineClosed } - m.logger.Debug("Retrieving machine hash, outputs merkle root and outputs merkle proof") + m.logger.Debug("Retrieving accepted machine state proof") proofCtx, cancel := context.WithTimeout(ctx, m.application.ExecutionParameters.LoadDeadline) defer cancel() @@ -538,29 +579,44 @@ func (m *MachineInstanceImpl) OutputsProof(ctx context.Context) (*OutputsProof, // The runtime is a local child process — errors here indicate the process // crashed, ran out of resources, or is otherwise unrecoverable. // Close the runtime to avoid leaving a broken process alive. - machineHash, err := m.runtime.Hash(proofCtx) + machineProof, err := m.runtime.StateProof(proofCtx) if err != nil { - return nil, m.destroyRuntime(fmt.Errorf("failed to get machine hash: %w", err)) + return nil, m.destroyRuntime(fmt.Errorf("failed to get machine state proof: %w", err)) } - - outputsHash, err := m.runtime.OutputsHash(proofCtx) - if err != nil { - return nil, m.destroyRuntime(fmt.Errorf("failed to get outputs hash: %w", err)) + if err := machine.ValidateAcceptedState(machineProof); err != nil { + return nil, m.destroyRuntime(fmt.Errorf("machine is not at an accepted state: %w", err)) } - - outputsHashProof, err := m.runtime.OutputsHashProof(proofCtx) + proof, err := stateProofFromMachine(machineProof) if err != nil { - return nil, m.destroyRuntime(fmt.Errorf("failed to get outputs hash proof: %w", err)) + return nil, m.destroyRuntime(err) } - proof := &OutputsProof{ - MachineHash: machineHash, - OutputsHash: outputsHash, - OutputsHashProof: outputsHashProof, - } + m.logger.Debug("Accepted machine state proof retrieved successfully", + "hash", "0x"+hex.EncodeToString(proof.MachineHash[:])) + return proof, nil +} - m.logger.Debug("Machine hash, outputs merkle root and outputs merkle proof retrieved successfully", - "hash", "0x"+hex.EncodeToString(machineHash[:])) +func stateProofFromMachine(machineProof *machine.StateProof) (*model.StateProof, error) { + if machineProof == nil { + return nil, fmt.Errorf( + "machine returned no state proof: %w", machine.ErrInvalidMachineProof, + ) + } + proof := &model.StateProof{ + MachineHash: machineProof.MachineHash, + TxBufferDataBlock: machineProof.TxBufferProof.DataBlock, + TxBufferProof: machineProof.TxBufferProof.Siblings, + IflagsYDataBlock: machineProof.IflagsYProof.DataBlock, + IflagsYProof: machineProof.IflagsYProof.Siblings, + HtifTohostDataBlock: machineProof.HtifTohostProof.DataBlock, + HtifTohostProof: machineProof.HtifTohostProof.Siblings, + } + if !proof.IsComplete() { + return nil, fmt.Errorf( + "machine returned an incomplete state proof: %w", + machine.ErrInvalidMachineProof, + ) + } return proof, nil } @@ -606,19 +662,24 @@ func (m *MachineInstanceImpl) Close() error { // fail fast with ErrMachineClosed instead of talking to a broken process. // Must be called while holding the appropriate locks. func (m *MachineInstanceImpl) destroyRuntime(cause error) error { - if m.runtime == nil { + // Cancellation is checked before backend calls and does not imply that the + // child process is unhealthy. Preserve it for graceful shutdown/retry. + if errors.Is(cause, machine.ErrCanceled) { return cause } + if m.runtime == nil { + return errors.Join(ErrMachineClosed, cause) + } closeErr := m.runtime.Close() m.runtime = nil - return errors.Join(cause, closeErr) + return errors.Join(ErrMachineClosed, cause, closeErr) } // MachineRuntimeFactory defines an interface for creating machine runtimes type MachineRuntimeFactory interface { CreateMachineRuntime( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, ) (machine.Machine, error) } @@ -632,7 +693,7 @@ type machineLoader func( // createMachineRuntimeCommon contains the shared logic for creating machine runtimes func createMachineRuntimeCommon( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, verifyExpectedHash bool, machinePath string, @@ -713,7 +774,7 @@ type DefaultMachineRuntimeFactory struct { // CreateMachineRuntime creates a new machine runtime for an application func (f *DefaultMachineRuntimeFactory) CreateMachineRuntime( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, ) (machine.Machine, error) { return createMachineRuntimeCommon( @@ -738,7 +799,7 @@ type SnapshotMachineRuntimeFactory struct { // CreateMachineRuntime creates a new machine runtime from a snapshot func (f *SnapshotMachineRuntimeFactory) CreateMachineRuntime( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, ) (machine.Machine, error) { return createMachineRuntimeCommon( @@ -755,20 +816,24 @@ func (f *SnapshotMachineRuntimeFactory) CreateMachineRuntime( // toInputStatus converts only completed, deterministic machine statuses to // canonical input statuses. Infrastructure interruptions never reach here. -func toInputStatus(status machine.CompletionStatus) (InputCompletionStatus, error) { +func toInputStatus(status machine.CompletionStatus) (model.InputCompletionStatus, error) { switch status { case machine.CompletionStatusAccepted: - return InputCompletionStatus_Accepted, nil + return model.InputCompletionStatus_Accepted, nil case machine.CompletionStatusRejected: - return InputCompletionStatus_Rejected, nil + return model.InputCompletionStatus_Rejected, nil case machine.CompletionStatusException: - return InputCompletionStatus_Exception, nil + return model.InputCompletionStatus_Exception, nil case machine.CompletionStatusHalted: - return InputCompletionStatus_MachineHalted, nil + return model.InputCompletionStatus_MachineHalted, nil + case machine.CompletionStatusOverflow: + return model.InputCompletionStatus_Overflow, nil + case machine.CompletionStatusUnexpectedYield: + return model.InputCompletionStatus_UnexpectedYield, nil case machine.CompletionStatusUnknown: // Intentionally empty. } - return InputCompletionStatus_None, fmt.Errorf( + return model.InputCompletionStatus_None, fmt.Errorf( "unknown completed machine status %d: %w", status, ErrIncompleteAdvance, diff --git a/internal/manager/instance_test.go b/internal/manager/instance_test.go index d40cbf025..abfb22982 100644 --- a/internal/manager/instance_test.go +++ b/internal/manager/instance_test.go @@ -28,16 +28,10 @@ func TestMachineInstance(t *testing.T) { type MachineInstanceSuite struct{ suite.Suite } -func (s *MachineInstanceSuite) TestMcycleOverflowRemainsIncomplete() { - require := s.Require() - - // Cycle exhaustion surfaces as an error from machine.Advance, never as a - // completed CompletionStatus, so no input completion status can exist for it. - // The zero-value status is the closest representable input and must be - // rejected rather than mapped to a completed status. - status, err := toInputStatus(machine.CompletionStatusUnknown) - require.ErrorIs(err, ErrIncompleteAdvance) - require.Equal(model.InputCompletionStatus_None, status) +func (s *MachineInstanceSuite) TestOverflowCompletionStatusMapsToInputOverflow() { + status, err := toInputStatus(machine.CompletionStatusOverflow) + s.Require().NoError(err) + s.Require().Equal(model.InputCompletionStatus_Overflow, status) } // MockMachineRuntimeFactory implements MachineRuntimeFactory for testing @@ -393,8 +387,9 @@ func (s *MachineInstanceSuite) TestAdvance() { require.Equal(model.InputCompletionStatus_Accepted, res.Status) require.Equal(expectedOutputs, res.Outputs) require.Equal(expectedReports1, res.Reports) - require.Equal(newHash(1), res.OutputsHash) + require.Equal(newHash(1), res.TxBufferDataBlock) require.Equal(newHash(2), res.MachineHash) + require.True(res.IsComplete()) require.Equal(uint64(6), machine.processedInputs.Load()) }) @@ -410,10 +405,11 @@ func (s *MachineInstanceSuite) TestAdvance() { require.Same(inner, instance.runtime) require.Equal(model.InputCompletionStatus_Rejected, res.Status) - require.Equal(expectedOutputs, res.Outputs) - require.Equal(expectedReports1, res.Reports) - require.Equal(newHash(1), res.OutputsHash) + require.Empty(res.Outputs) + require.Empty(res.Reports) + require.Equal(newHash(1), res.TxBufferDataBlock) require.Equal(newHash(2), res.MachineHash) + require.True(res.IsComplete()) require.Equal(uint64(6), instance.processedInputs.Load()) }) @@ -429,19 +425,35 @@ func (s *MachineInstanceSuite) TestAdvance() { fork.CompletionStatusReturn = machineStatus fork.ExceptionDataReturn = exceptionData fork.CloseError = nil + preProof := fork.StateProofReturn + postProof := acceptedStateProof(newHash(2), newHash(1)) + postProof.IflagsYProof.DataBlock = machine.Hash{} + proofCalls := 0 + fork.StateProofFunc = func(context.Context) (*machine.StateProof, error) { + proofCalls++ + if proofCalls == 1 { + return preProof, nil + } + return postProof, nil + } res, err := instance.Advance(context.Background(), []byte{}, 0, 5, false) require.Nil(err) require.NotNil(res) - require.Same(inner, instance.runtime) + require.Nil(instance.runtime) require.Equal(inputStatus, res.Status) require.Equal(exceptionData, res.ExceptionData) - require.Equal(expectedOutputs, res.Outputs) - require.Equal(expectedReports1, res.Reports) - require.Equal(newHash(1), res.OutputsHash) + require.Empty(res.Outputs) + require.Empty(res.Reports) + require.Equal(newHash(1), res.TxBufferDataBlock) require.Equal(newHash(2), res.MachineHash) + require.True(res.IsComplete()) require.Equal(uint64(6), instance.processedInputs.Load()) + require.Equal(int64(1), inner.CloseCalls.Load()) + require.Equal(int64(1), fork.CloseCalls.Load()) + _, inspectErr := instance.Inspect(context.Background(), nil) + require.ErrorIs(inspectErr, ErrMachineClosed) }) } @@ -454,6 +466,16 @@ func (s *MachineInstanceSuite) TestAdvance() { machine.CompletionStatusHalted, model.InputCompletionStatus_MachineHalted, nil) + + testCompletedStatus("Overflow", + machine.CompletionStatusOverflow, + model.InputCompletionStatus_Overflow, + nil) + + testCompletedStatus("UnexpectedYield", + machine.CompletionStatusUnexpectedYield, + model.InputCompletionStatus_UnexpectedYield, + nil) }) s.Run("Error", func() { @@ -567,39 +589,63 @@ func (s *MachineInstanceSuite) TestAdvance() { require.Equal(uint64(5), machine.processedInputs.Load()) }) - s.Run("Hash", func() { + s.Run("StateProof", func() { require := s.Require() inner, fork, machine := s.setupAdvance() - errHash := errors.New("Hash error") - fork.HashError = errHash + errProof := errors.New("state proof error") + fork.StateProofError = errProof fork.CloseError, inner.CloseError = inner.CloseError, fork.CloseError res, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) require.Error(err) require.Nil(res) - require.ErrorIs(err, errHash) + require.ErrorIs(err, errProof) require.NotErrorIs(err, errUnreachable) require.Equal(uint64(5), machine.processedInputs.Load()) }) - s.Run("HashAndClose", func() { + s.Run("StateProofAndClose", func() { require := s.Require() inner, fork, machine := s.setupAdvance() - errHash := errors.New("Hash error") + errProof := errors.New("state proof error") errClose := errors.New("Close error") - fork.HashError = errHash + fork.StateProofError = errProof fork.CloseError = errClose inner.CloseError = nil res, err := machine.Advance(context.Background(), []byte{}, 0, 5, false) require.Error(err) require.Nil(res) - require.ErrorIs(err, errHash) + require.ErrorIs(err, errProof) require.ErrorIs(err, errClose) require.NotErrorIs(err, errUnreachable) require.Equal(uint64(5), machine.processedInputs.Load()) }) + s.Run("PostAdvanceStateProof", func() { + require := s.Require() + inner, fork, instance := s.setupAdvance() + errProof := errors.New("post-advance state proof error") + preProof := fork.StateProofReturn + proofCalls := 0 + fork.StateProofFunc = func(context.Context) (*machine.StateProof, error) { + proofCalls++ + if proofCalls == 1 { + return preProof, nil + } + return nil, errProof + } + fork.CloseError = nil + + res, err := instance.Advance(context.Background(), nil, 0, 5, false) + require.Nil(res) + require.ErrorIs(err, errProof) + require.Same(inner, instance.runtime) + require.Equal(uint64(5), instance.processedInputs.Load()) + require.Equal(int64(1), fork.CloseCalls.Load()) + require.Zero(inner.CloseCalls.Load()) + }) + s.Run("Close", func() { s.Run("Inner", func() { require := s.Require() @@ -668,8 +714,8 @@ func (s *MachineInstanceSuite) TestAdvance() { fork2.CompletionStatusReturn = machine.CompletionStatusAccepted fork2.AdvanceOutputsReturn = expectedOutputs fork2.AdvanceReportsReturn = expectedReports1 - fork2.OutputsHashReturn = newHash(1) fork2.HashReturn = newHash(2) + fork2.StateProofReturn = acceptedStateProof(newHash(2), newHash(1)) fork2.CloseError = errUnreachable // old runtime close for second advance fork2.ForkReturn = nil fork.ForkReturn = fork2 @@ -698,6 +744,8 @@ func (s *MachineInstanceSuite) TestInspect() { {"Reject", machine.CompletionStatusRejected}, {"Exception", machine.CompletionStatusException}, {"Halted", machine.CompletionStatusHalted}, + {"Overflow", machine.CompletionStatusOverflow}, + {"UnexpectedYield", machine.CompletionStatusUnexpectedYield}, } { s.Run(test.name, func() { require := s.Require() @@ -853,11 +901,24 @@ func (s *MachineInstanceSuite) TestCreateSnapshot() { err := machine.CreateSnapshot(context.Background(), 5, "/tmp/snapshot") require.Error(err) require.ErrorIs(err, errStore) + require.ErrorIs(err, ErrMachineClosed) // Runtime should be destroyed after a store error. require.Nil(machine.runtime) }) + s.Run("CanceledPreservesRuntime", func() { + require := s.Require() + inner, _, machineInst := s.setupAdvance() + inner.StoreError = machine.ErrCanceled + + err := machineInst.CreateSnapshot(context.Background(), 5, "/tmp/snapshot") + require.ErrorIs(err, machine.ErrCanceled) + require.NotErrorIs(err, ErrMachineClosed) + require.Same(inner, machineInst.runtime) + require.Zero(inner.CloseCalls.Load()) + }) + s.Run("ErrorAndCloseError", func() { require := s.Require() inner, _, machine := s.setupAdvance() @@ -870,6 +931,7 @@ func (s *MachineInstanceSuite) TestCreateSnapshot() { require.Error(err) require.ErrorIs(err, errStore) require.ErrorIs(err, errClose) + require.ErrorIs(err, ErrMachineClosed) require.Nil(machine.runtime) }) @@ -895,7 +957,7 @@ func (s *MachineInstanceSuite) TestCreateSnapshot() { func (s *MachineInstanceSuite) TestHash() { s.Run("Ok", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() + inner, machineInst := s.setupStateProof() hash, err := machineInst.Hash(context.Background()) require.NoError(err) @@ -907,7 +969,7 @@ func (s *MachineInstanceSuite) TestHash() { s.Run("MachineClosed", func() { require := s.Require() - _, machineInst := s.setupOutputsProof() + _, machineInst := s.setupStateProof() machineInst.runtime = nil hash, err := machineInst.Hash(context.Background()) @@ -918,7 +980,7 @@ func (s *MachineInstanceSuite) TestHash() { s.Run("Error", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() + inner, machineInst := s.setupStateProof() errHash := errors.New("Hash error") inner.HashError = errHash inner.CloseError = nil @@ -926,6 +988,7 @@ func (s *MachineInstanceSuite) TestHash() { hash, err := machineInst.Hash(context.Background()) require.Error(err) require.ErrorIs(err, errHash) + require.ErrorIs(err, ErrMachineClosed) require.Equal([32]byte{}, hash) // Runtime should be destroyed after a hash error. @@ -934,7 +997,7 @@ func (s *MachineInstanceSuite) TestHash() { s.Run("ErrorAndCloseError", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() + inner, machineInst := s.setupStateProof() errHash := errors.New("Hash error") errClose := errors.New("Close error") inner.HashError = errHash @@ -944,23 +1007,39 @@ func (s *MachineInstanceSuite) TestHash() { require.Error(err) require.ErrorIs(err, errHash) require.ErrorIs(err, errClose) + require.ErrorIs(err, ErrMachineClosed) require.Equal([32]byte{}, hash) require.Nil(machineInst.runtime) }) + + s.Run("CanceledPreservesRuntime", func() { + require := s.Require() + inner, machineInst := s.setupStateProof() + inner.HashError = machine.ErrCanceled + + hash, err := machineInst.Hash(context.Background()) + require.Equal([32]byte{}, hash) + require.ErrorIs(err, machine.ErrCanceled) + require.NotErrorIs(err, ErrMachineClosed) + require.Same(inner, machineInst.runtime) + require.Zero(inner.CloseCalls.Load()) + }) } -func (s *MachineInstanceSuite) TestOutputsProof() { +func (s *MachineInstanceSuite) TestStateProof() { s.Run("Ok", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() + inner, machineInst := s.setupStateProof() - proof, err := machineInst.OutputsProof(context.Background()) + proof, err := machineInst.StateProof(context.Background()) require.NoError(err) require.NotNil(proof) require.Equal(newHash(1), proof.MachineHash) - require.Equal(newHash(2), proof.OutputsHash) - require.Equal(expectedOutputsHashProof, proof.OutputsHashProof) + require.Equal(newHash(2), proof.TxBufferDataBlock) + require.True(proof.IsComplete()) + require.Equal(common.Hash(inner.StateProofReturn.IflagsYProof.DataBlock), proof.IflagsYDataBlock) + require.Equal(common.Hash(inner.StateProofReturn.HtifTohostProof.DataBlock), proof.HtifTohostDataBlock) // Runtime should still be alive after a successful call. require.Same(inner, machineInst.runtime) @@ -968,78 +1047,89 @@ func (s *MachineInstanceSuite) TestOutputsProof() { s.Run("MachineClosed", func() { require := s.Require() - _, machineInst := s.setupOutputsProof() + _, machineInst := s.setupStateProof() machineInst.runtime = nil - proof, err := machineInst.OutputsProof(context.Background()) + proof, err := machineInst.StateProof(context.Background()) require.Nil(proof) require.Error(err) require.Equal(ErrMachineClosed, err) }) - s.Run("HashError", func() { + s.Run("StateProofError", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() - errHash := errors.New("Hash error") - inner.HashError = errHash + inner, machineInst := s.setupStateProof() + errProof := errors.New("state proof error") + inner.StateProofError = errProof inner.CloseError = nil - proof, err := machineInst.OutputsProof(context.Background()) + proof, err := machineInst.StateProof(context.Background()) require.Nil(proof) require.Error(err) - require.ErrorIs(err, errHash) + require.ErrorIs(err, errProof) + require.ErrorIs(err, ErrMachineClosed) - // Runtime should be destroyed after a hash error. + // Runtime should be destroyed after a proof error. require.Nil(machineInst.runtime) }) - s.Run("HashErrorAndCloseError", func() { + s.Run("StateProofErrorAndCloseError", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() - errHash := errors.New("Hash error") + inner, machineInst := s.setupStateProof() + errProof := errors.New("state proof error") errClose := errors.New("Close error") - inner.HashError = errHash + inner.StateProofError = errProof inner.CloseError = errClose - proof, err := machineInst.OutputsProof(context.Background()) + proof, err := machineInst.StateProof(context.Background()) require.Nil(proof) require.Error(err) - require.ErrorIs(err, errHash) + require.ErrorIs(err, errProof) require.ErrorIs(err, errClose) + require.ErrorIs(err, ErrMachineClosed) // Runtime should be destroyed even when Close also fails. require.Nil(machineInst.runtime) }) - s.Run("OutputsHashError", func() { + s.Run("CanceledPreservesRuntime", func() { + require := s.Require() + inner, machineInst := s.setupStateProof() + inner.StateProofError = machine.ErrCanceled + + proof, err := machineInst.StateProof(context.Background()) + require.Nil(proof) + require.ErrorIs(err, machine.ErrCanceled) + require.NotErrorIs(err, ErrMachineClosed) + require.Same(inner, machineInst.runtime) + require.Zero(inner.CloseCalls.Load()) + }) + + s.Run("NilStateProof", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() - errOutputsHash := errors.New("OutputsHash error") - inner.OutputsHashError = errOutputsHash + inner, machineInst := s.setupStateProof() + inner.StateProofReturn = nil inner.CloseError = nil - proof, err := machineInst.OutputsProof(context.Background()) + proof, err := machineInst.StateProof(context.Background()) require.Nil(proof) - require.Error(err) - require.ErrorIs(err, errOutputsHash) + require.ErrorIs(err, machine.ErrInvalidMachineProof) + require.ErrorIs(err, ErrMachineClosed) - // Runtime should be destroyed after an outputs hash error. require.Nil(machineInst.runtime) }) - s.Run("OutputsHashProofError", func() { + s.Run("IncompleteStateProof", func() { require := s.Require() - inner, machineInst := s.setupOutputsProof() - errProof := errors.New("OutputsHashProof error") - inner.OutputsHashProofError = errProof + inner, machineInst := s.setupStateProof() + inner.StateProofReturn.TxBufferProof.Siblings = nil inner.CloseError = nil - proof, err := machineInst.OutputsProof(context.Background()) + proof, err := machineInst.StateProof(context.Background()) require.Nil(proof) - require.Error(err) - require.ErrorIs(err, errProof) + require.ErrorIs(err, machine.ErrInvalidMachineProof) + require.ErrorIs(err, ErrMachineClosed) - // Runtime should be destroyed after an outputs hash proof error. require.Nil(machineInst.runtime) }) } @@ -1150,11 +1240,6 @@ var ( newBytes(33, 300), newBytes(34, 300), } - expectedOutputsHashProof = []machine.Hash{ - newHash(3), - newHash(4), - newHash(5), - } ) func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollupsMachine, *MachineInstanceImpl) { @@ -1194,11 +1279,11 @@ func (s *MachineInstanceSuite) setupAdvance() (*MockRollupsMachine, *MockRollups newBytes(21, 200), newBytes(22, 200), } - fork.OutputsHashReturn = newHash(1) fork.AdvanceError = nil fork.HashReturn = newHash(2) fork.HashError = nil + fork.StateProofReturn = acceptedStateProof(newHash(2), newHash(1)) fork.InspectResponseReturn = &machine.InspectResponse{ Status: machine.CompletionStatusAccepted, @@ -1262,7 +1347,7 @@ func (s *MachineInstanceSuite) setupInspect() (*MockRollupsMachine, *MockRollups return inner, fork, machineInst } -func (s *MachineInstanceSuite) setupOutputsProof() (*MockRollupsMachine, *MachineInstanceImpl) { +func (s *MachineInstanceSuite) setupStateProof() (*MockRollupsMachine, *MachineInstanceImpl) { app := &model.Application{ ExecutionParameters: model.ExecutionParameters{ AdvanceMaxDeadline: decisecond, @@ -1286,20 +1371,41 @@ func (s *MachineInstanceSuite) setupOutputsProof() (*MockRollupsMachine, *Machin machineInst.processedInputs.Store(5) inner.HashReturn = newHash(1) - inner.HashError = nil - inner.OutputsHashReturn = newHash(2) - inner.OutputsHashError = nil - inner.OutputsHashProofReturn = []machine.Hash{ - newHash(3), - newHash(4), - newHash(5), - } - inner.OutputsHashProofError = nil + inner.StateProofReturn = acceptedStateProof(newHash(1), newHash(2)) inner.CloseError = errUnreachable return inner, machineInst } +func testValidityLeaf(dataBlock, sibling machine.Hash) machine.LeafProof { + siblings := make([]machine.Hash, model.StateProofSiblingCount) + for i := range siblings { + siblings[i] = sibling + } + return machine.LeafProof{DataBlock: dataBlock, Siblings: siblings} +} + +func acceptedStateProof(machineHash, outputsHash machine.Hash) *machine.StateProof { + iflagsYData := newHash(6) + for index := 8; index < 16; index++ { + iflagsYData[index] = 0 + } + iflagsYData[8] = 1 + htifTohostData := newHash(7) + for index := 16; index < 24; index++ { + htifTohostData[index] = 0 + } + htifTohostData[20] = 1 + htifTohostData[22] = 1 + htifTohostData[23] = 2 + return &machine.StateProof{ + MachineHash: machineHash, + IflagsYProof: testValidityLeaf(iflagsYData, newHash(8)), + HtifTohostProof: testValidityLeaf(htifTohostData, newHash(9)), + TxBufferProof: testValidityLeaf(outputsHash, newHash(10)), + } +} + // ------------------------------------------------------------------------------------------------ const ( @@ -1340,10 +1446,9 @@ type MockRollupsMachine struct { AdvanceReportsReturn []machine.Report AdvanceLeafsReturn []machine.Hash AdvanceRemainingReturn uint64 - OutputsHashReturn machine.Hash - OutputsHashError error - OutputsHashProofReturn []machine.Hash - OutputsHashProofError error + StateProofReturn *machine.StateProof + StateProofError error + StateProofFunc func(context.Context) (*machine.StateProof, error) AdvanceError error LastAdvanceComputeHashes bool @@ -1368,12 +1473,11 @@ func (m *MockRollupsMachine) Hash(_ context.Context) (machine.Hash, error) { return m.HashReturn, m.HashError } -func (m *MockRollupsMachine) OutputsHash(_ context.Context) (machine.Hash, error) { - return m.OutputsHashReturn, m.OutputsHashError -} - -func (m *MockRollupsMachine) OutputsHashProof(_ context.Context) ([]machine.Hash, error) { - return m.OutputsHashProofReturn, m.OutputsHashProofError +func (m *MockRollupsMachine) StateProof(ctx context.Context) (*machine.StateProof, error) { + if m.StateProofFunc != nil { + return m.StateProofFunc(ctx) + } + return m.StateProofReturn, m.StateProofError } func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash, computeHashes bool) (*machine.AdvanceResponse, error) { @@ -1388,7 +1492,6 @@ func (m *MockRollupsMachine) Advance(_ context.Context, _ []byte, _ machine.Hash Reports: m.AdvanceReportsReturn, PeriodicStateHashes: m.AdvanceLeafsReturn, PaddingRepetitions: m.AdvanceRemainingReturn, - OutputsHash: m.OutputsHashReturn, }, nil } diff --git a/internal/manager/manager.go b/internal/manager/manager.go index fc8fb921e..54aec6588 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -14,7 +14,7 @@ import ( "sync" "github.com/cartesi/rollups-node/internal/appstatus" - . "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/internal/replay" "github.com/cartesi/rollups-node/internal/repository" "github.com/cartesi/rollups-node/pkg/machine" @@ -31,22 +31,37 @@ type MachineRepository interface { repository.ReplayRepository // ListApplications retrieves applications based on filter criteria - ListApplications(ctx context.Context, f repository.ApplicationFilter, p repository.Pagination, descending bool) ([]*Application, uint64, error) + ListApplications( + ctx context.Context, + f repository.ApplicationFilter, + p repository.Pagination, + descending bool, + ) ([]*model.Application, uint64, error) HasUndrainedEpochsBeforeBlock(ctx context.Context, appID int64, blockBound uint64) (bool, error) // GetLastSnapshot retrieves the most recent input with a snapshot for the given application - GetLastSnapshot(ctx context.Context, nameOrAddress string) (*Input, error) - GetApplication(ctx context.Context, nameOrAddress string) (*Application, error) + GetLastSnapshot(ctx context.Context, nameOrAddress string) (*model.Input, error) + GetApplication(ctx context.Context, nameOrAddress string) (*model.Application, error) // UpdateApplicationStatus persists an application's health status. - UpdateApplicationStatus(ctx context.Context, appID int64, status ApplicationStatus, reason *string) error + UpdateApplicationStatus( + ctx context.Context, + appID int64, + status model.ApplicationStatus, + reason *string, + ) error } // MachineInstanceFactory creates MachineInstance values from applications. // Implementations decide whether to load from a template or snapshot. type MachineInstanceFactory interface { - NewFromTemplate(ctx context.Context, app *Application, logger *slog.Logger, checkTemplateHash bool) (MachineInstance, error) - NewFromSnapshot(ctx context.Context, app *Application, logger *slog.Logger, + NewFromTemplate( + ctx context.Context, + app *model.Application, + logger *slog.Logger, + checkTemplateHash bool, + ) (MachineInstance, error) + NewFromSnapshot(ctx context.Context, app *model.Application, logger *slog.Logger, snapshotPath string, expectedHash common.Hash, inputIndex uint64) (MachineInstance, error) } @@ -54,13 +69,13 @@ type MachineInstanceFactory interface { type DefaultMachineInstanceFactory struct{} func (f *DefaultMachineInstanceFactory) NewFromTemplate( - ctx context.Context, app *Application, logger *slog.Logger, checkTemplateHash bool, + ctx context.Context, app *model.Application, logger *slog.Logger, checkTemplateHash bool, ) (MachineInstance, error) { return NewMachineInstance(ctx, app, logger, checkTemplateHash) } func (f *DefaultMachineInstanceFactory) NewFromSnapshot( - ctx context.Context, app *Application, logger *slog.Logger, + ctx context.Context, app *model.Application, logger *slog.Logger, snapshotPath string, expectedHash common.Hash, inputIndex uint64, ) (MachineInstance, error) { return NewMachineInstanceFromSnapshot(ctx, app, logger, snapshotPath, expectedHash, inputIndex) @@ -84,7 +99,7 @@ type MachineManager struct { // cannot be confirmed. It is deliberately private: this is a short-lived retry // queue, not process-local application health state. type pendingApplicationFailure struct { - application *Application + application *model.Application reason string } @@ -96,14 +111,7 @@ func WithInstanceFactory(f MachineInstanceFactory) Option { return func(m *MachineManager) { m.instanceFactory = f } } -// withReplayRun overrides replay execution for manager policy tests. -func withReplayRun( - run func(context.Context, repository.ReplayRepository, replay.Executor, replay.Options) (replay.Result, error), -) Option { - return func(m *MachineManager) { m.replayRun = run } -} - -func snapshotProcessedInputs(app *Application, snapshot *Input) (uint64, error) { +func snapshotProcessedInputs(app *model.Application, snapshot *model.Input) (uint64, error) { if snapshot.Index == math.MaxUint64 { return 0, fmt.Errorf("%w: snapshot input index cannot be incremented", ErrInvalidSnapshotPoint) } @@ -132,7 +140,7 @@ func closeMachineCandidate( func (m *MachineManager) tryLoadSnapshotInstance( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, ) MachineInstance { snapshot, err := m.repository.GetLastSnapshot(ctx, app.IApplicationAddress.String()) @@ -223,7 +231,7 @@ func (m *MachineManager) tryLoadSnapshotInstance( func (m *MachineManager) tryLoadTemplateInstance( ctx context.Context, - app *Application, + app *model.Application, logger *slog.Logger, ) MachineInstance { candidate, err := m.instanceFactory.NewFromTemplate(ctx, app, m.logger, m.checkTemplateHash) @@ -393,11 +401,14 @@ func (m *MachineManager) UpdateMachines(ctx context.Context) error { return persistenceErr } -func excludeFencedApplications(apps []*Application, fenced map[int64]struct{}) []*Application { +func excludeFencedApplications( + apps []*model.Application, + fenced map[int64]struct{}, +) []*model.Application { if len(fenced) == 0 { return apps } - active := make([]*Application, 0, len(apps)) + active := make([]*model.Application, 0, len(apps)) for _, app := range apps { if _, excluded := fenced[app.ID]; !excluded { active = append(active, app) @@ -410,7 +421,7 @@ func excludeFencedApplications(apps []*Application, fenced map[int64]struct{}) [ // normalized FAILED reason for a later durability retry. Callers use this only // after their initial status write failed; this method does not write the // repository itself. -func (m *MachineManager) FenceApplicationFailure(app *Application, reason string) { +func (m *MachineManager) FenceApplicationFailure(app *model.Application, reason string) { reason = appstatus.NormalizeReason(reason) pending := &pendingApplicationFailure{ application: app, @@ -496,20 +507,17 @@ func (m *MachineManager) persistApplicationFailure(ctx context.Context, appID in return nil } -func applicationFailureAlreadyResolved(app *Application, reason string) bool { +func applicationFailureAlreadyResolved(app *model.Application, reason string) bool { if app == nil { return false } - switch app.Status { - case ApplicationStatus_Diverged, ApplicationStatus_Corrupted: + if app.Status.IsTerminal() { return true - case ApplicationStatus_Failed: + } + if app.Status == model.ApplicationStatus_Failed { return app.Reason != nil && *app.Reason == reason - case ApplicationStatus_OK: - return false - default: - return false } + return false } func (m *MachineManager) deletePendingApplicationFailure( @@ -567,7 +575,7 @@ func (m *MachineManager) addMachine(appID int64, machine MachineInstance) bool { } // RemoveMachines removes machines for applications not in the provided list -func (m *MachineManager) removeMachines(apps []*Application) { +func (m *MachineManager) removeMachines(apps []*model.Application) { m.mutex.Lock() defer m.mutex.Unlock() @@ -595,11 +603,11 @@ func (m *MachineManager) removeMachines(apps []*Application) { // Applications returns the list of applications with active machines, // sorted by ID for deterministic iteration order. -func (m *MachineManager) Applications() []*Application { +func (m *MachineManager) Applications() []*model.Application { m.mutex.RLock() defer m.mutex.RUnlock() - apps := make([]*Application, 0, len(m.machines)) + apps := make([]*model.Application, 0, len(m.machines)) for _, machine := range m.machines { apps = append(apps, machine.Application()) } @@ -646,7 +654,7 @@ func (m *MachineManager) Close() error { return errors.Join(errs...) } -func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*Application, error) { +func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*model.Application, error) { apps, _, err := repo.ListApplications(ctx, repository.ExecutableApplicationsFilter(), repository.Pagination{}, false) if err != nil { return nil, err @@ -684,7 +692,7 @@ func getMachineApplications(ctx context.Context, repo MachineRepository) ([]*App func foreclosedMachineDrainFilter() repository.ApplicationFilter { return repository.ApplicationFilter{ Enabled: new(true), - Status: new(ApplicationStatus_OK), + Status: new(model.ApplicationStatus_OK), ForeclosureRecorded: new(true), } } diff --git a/internal/manager/manager_test.go b/internal/manager/manager_test.go index 9fd6665ff..c65cd2616 100644 --- a/internal/manager/manager_test.go +++ b/internal/manager/manager_test.go @@ -34,11 +34,18 @@ type MachineManagerSuite struct { suite.Suite } +const ( + replayMachineHashField = "machine_hash" + replayTxBufferDataBlockField = "tx_buffer_data_block" + replayExecutionLimitReason = "execution limit" + testApplicationName = "App1" +) + func newForkableMock() *MockRollupsMachine { runtime := &MockRollupsMachine{ CompletionStatusReturn: machine.CompletionStatusAccepted, HashReturn: newHash(1), - OutputsHashReturn: newHash(2), + StateProofReturn: acceptedStateProof(newHash(1), newHash(2)), } runtime.ForkFunc = func(context.Context) (machine.Machine, error) { return newForkableMock(), nil @@ -70,6 +77,12 @@ func newTestMachineManager( ) } +func withReplayRun( + run func(context.Context, repository.ReplayRepository, replay.Executor, replay.Options) (replay.Result, error), +) Option { + return func(m *MachineManager) { m.replayRun = run } +} + type nilSingleUnwrapperError struct{} func (nilSingleUnwrapperError) Error() string { return "nil single unwrapper" } @@ -251,7 +264,10 @@ func (s *MachineManagerSuite) TestUpdateMachines() { instance := &DummyMachineInstanceMock{ application: app, - replayErr: &replay.ContradictionError{Application: app.Name, Field: "machine_hash"}, + replayErr: &replay.ContradictionError{ + Application: app.Name, + Field: replayMachineHashField, + }, } factory := &MockMachineInstanceFactory{Instance: instance} manager := newTestMachineManager( @@ -312,7 +328,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { instance := &DummyMachineInstanceMock{ application: app, replayErr: &replay.ContradictionError{ - Application: app.Name, InputIndex: 9, Field: "outputs_hash", + Application: app.Name, InputIndex: 9, Field: replayTxBufferDataBlockField, Expected: "0x01", Actual: "0x02", }, } @@ -370,7 +386,10 @@ func (s *MachineManagerSuite) TestUpdateMachines() { instance := &DummyMachineInstanceMock{ application: app, - replayErr: &replay.ContradictionError{Application: app.Name, Field: "machine_hash"}, + replayErr: &replay.ContradictionError{ + Application: app.Name, + Field: replayMachineHashField, + }, } factory := &MockMachineInstanceFactory{Instance: instance} manager := newTestMachineManager( @@ -398,7 +417,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, ) manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ - Application: app.Name, Field: "machine_hash", + Application: app.Name, Field: replayMachineHashField, })) unrelatedReason := "machine process crashed" @@ -429,7 +448,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, ) manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ - Application: app.Name, Field: "machine_hash", + Application: app.Name, Field: replayMachineHashField, })) pending := manager.pendingApplicationFailures[app.ID] @@ -460,7 +479,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { ) detail := &replay.ContradictionError{ Application: app.Name, - Field: "machine_hash", + Field: replayMachineHashField, Expected: strings.Repeat("e", 5000), Actual: "different", } @@ -500,7 +519,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, ) manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ - Application: app.Name, Field: "machine_hash", + Application: app.Name, Field: replayMachineHashField, })) repo.On("UpdateApplicationStatus", mock.Anything, app.ID, model.ApplicationStatus_Failed, mock.Anything). @@ -546,7 +565,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, ) manager.FenceApplicationFailure(oldApp, replayContradictionReason(&replay.ContradictionError{ - Application: oldApp.Name, Field: "machine_hash", + Application: oldApp.Name, Field: replayMachineHashField, })) replacement := *oldApp replacement.ID = 83 @@ -588,7 +607,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo, slog.New(slog.NewTextHandler(io.Discard, nil)), false, 10, ) manager.FenceApplicationFailure(app, replayContradictionReason(&replay.ContradictionError{ - Application: app.Name, Field: "machine_hash", + Application: app.Name, Field: replayMachineHashField, })) writeErr := errors.New("status write unavailable") readErr := errors.New("status read unavailable") @@ -806,7 +825,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { instance := &DummyMachineInstanceMock{application: app} if app.ID == badApp.ID { instance.replayErr = &replay.ContradictionError{ - Application: app.Name, InputIndex: 4, Field: "outputs_hash", + Application: app.Name, InputIndex: 4, Field: replayTxBufferDataBlockField, Expected: "0x01", Actual: "0x02", } } @@ -861,7 +880,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { instance := &DummyMachineInstanceMock{application: app} if app.ID == badApp.ID { instance.replayErr = &replay.ContradictionError{ - Application: app.Name, InputIndex: 4, Field: "outputs_hash", + Application: app.Name, InputIndex: 4, Field: replayTxBufferDataBlockField, } } instances[app.ID] = instance @@ -889,10 +908,10 @@ func (s *MachineManagerSuite) TestUpdateMachines() { err error reasonContains string }{ - {"PayloadLengthLimit", machine.ErrPayloadLengthLimitExceeded, "execution limit"}, - {"OutputsLimit", machine.ErrOutputsLimitExceeded, "execution limit"}, - {"ReportsLimit", machine.ErrReportsLimitExceeded, "execution limit"}, - {"McycleLimit", machine.ErrReachedLimitMcycle, "execution limit"}, + {"PayloadLengthLimit", machine.ErrPayloadLengthLimitExceeded, replayExecutionLimitReason}, + {"OutputsLimit", machine.ErrOutputsLimitExceeded, replayExecutionLimitReason}, + {"ReportsLimit", machine.ErrReportsLimitExceeded, replayExecutionLimitReason}, + {"McycleLimit", machine.ErrReachedLimitMcycle, replayExecutionLimitReason}, {"Deadline", machine.ErrDeadlineExceeded, "execution deadline"}, {"IncompleteAdvance", ErrIncompleteAdvance, "incomplete advance result"}, {"MachineInternal", machine.ErrMachineInternal, "failed internally"}, @@ -958,7 +977,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { repo := &MockMachineRepository{} app1 := &model.Application{ ID: 1, - Name: "App1", + Name: testApplicationName, IApplicationAddress: common.HexToAddress("0x1"), Status: model.ApplicationStatus_OK, ExecutionParameters: model.ExecutionParameters{ @@ -1094,7 +1113,7 @@ func (s *MachineManagerSuite) TestUpdateMachines() { manager := newTestMachineManager(repo, testLogger, false, 500) // Add mock machines - app1 := &model.Application{ID: 1, Name: "App1"} + app1 := &model.Application{ID: 1, Name: testApplicationName} app2 := &model.Application{ID: 2, Name: "App2"} app3 := &model.Application{ID: 3, Name: "App3"} @@ -1155,21 +1174,20 @@ func (s *MachineManagerSuite) TestSnapshotStartingStateVerification() { repo.replayCount = app.ProcessedInputs repo.replayRecords = []*model.ReplayRecord{{ Input: model.ReplayInput{ - ApplicationID: app.ID, - EpochIndex: 0, - InputIndex: 0, - RawData: []byte("replayed input"), - Status: model.InputCompletionStatus_Accepted, - MachineHash: &machineHash, - OutputsHash: &outputsHash, + ApplicationID: app.ID, + EpochIndex: 0, + InputIndex: 0, + RawData: []byte("replayed input"), + Status: model.InputCompletionStatus_Accepted, + MachineHash: &machineHash, + TxBufferDataBlock: &outputsHash, }, }} return machineHash } newReplayTemplate := func(app *model.Application) MachineInstance { base := &MockRollupsMachine{ - HashReturn: newHash(0), - OutputsHashReturn: newHash(0), + HashReturn: newHash(0), } base.ForkReturn = newForkableMock() instance, err := NewMachineInstanceWithFactory( @@ -1554,7 +1572,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { app := &model.Application{ ID: 1, - Name: "App1", + Name: testApplicationName, IApplicationAddress: common.HexToAddress("0x1"), Status: model.ApplicationStatus_OK, ExecutionParameters: model.ExecutionParameters{ @@ -1591,7 +1609,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { app := &model.Application{ ID: 1, - Name: "App1", + Name: testApplicationName, IApplicationAddress: common.HexToAddress("0x1"), Status: model.ApplicationStatus_OK, ExecutionParameters: model.ExecutionParameters{ @@ -1622,7 +1640,7 @@ func (s *MachineManagerSuite) TestUpdateMachinesErrors() { app := &model.Application{ ID: 1, - Name: "App1", + Name: testApplicationName, IApplicationAddress: common.HexToAddress("0x1"), Status: model.ApplicationStatus_OK, ProcessedInputs: 3, @@ -1694,7 +1712,7 @@ func (s *MachineManagerSuite) TestApplications() { manager := newTestMachineManager(repo, nil, false, 500) // Add machines - app1 := &model.Application{ID: 1, Name: "App1"} + app1 := &model.Application{ID: 1, Name: testApplicationName} app2 := &model.Application{ID: 2, Name: "App2"} machine1 := &DummyMachineInstanceMock{application: app1} @@ -1715,7 +1733,7 @@ func (s *MachineManagerSuite) TestApplications() { require.Contains(appMap, int64(1)) require.Contains(appMap, int64(2)) - require.Equal("App1", appMap[1].Name) + require.Equal(testApplicationName, appMap[1].Name) require.Equal("App2", appMap[2].Name) } @@ -1901,7 +1919,7 @@ func (m *DummyMachineInstanceMock) ProcessedInputs() uint64 { return m.processedInputs } -func (m *DummyMachineInstanceMock) OutputsProof(ctx context.Context) (*model.OutputsProof, error) { +func (m *DummyMachineInstanceMock) StateProof(_ context.Context) (*model.StateProof, error) { return nil, nil } diff --git a/internal/manager/types.go b/internal/manager/types.go index 91570fff8..c46685eed 100644 --- a/internal/manager/types.go +++ b/internal/manager/types.go @@ -6,7 +6,7 @@ package manager import ( "context" - . "github.com/cartesi/rollups-node/internal/model" + "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/pkg/machine" ) @@ -23,13 +23,19 @@ type InspectResult struct { // MachineInstance defines the interface for a machine instance type MachineInstance interface { - Application() *Application - Advance(ctx context.Context, input []byte, epochIndex uint64, inputIndex uint64, computeHashes bool) (*AdvanceResult, error) + Application() *model.Application + Advance( + ctx context.Context, + input []byte, + epochIndex uint64, + inputIndex uint64, + computeHashes bool, + ) (*model.AdvanceResult, error) Inspect(ctx context.Context, query []byte) (*InspectResult, error) CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error ProcessedInputs() uint64 Hash(ctx context.Context) ([32]byte, error) - OutputsProof(ctx context.Context) (*OutputsProof, error) + StateProof(ctx context.Context) (*model.StateProof, error) Close() error } @@ -39,7 +45,7 @@ type MachineProvider interface { GetMachine(appID int64) (MachineInstance, bool) // Applications returns the list of applications with active machines - Applications() []*Application + Applications() []*model.Application // UpdateMachines refreshes the list of machines UpdateMachines(ctx context.Context) error @@ -47,7 +53,7 @@ type MachineProvider interface { // FenceApplicationFailure fences an application whose initial FAILED // status write could not be confirmed. It queues a later durability retry // without duplicating the initial repository write. - FenceApplicationFailure(app *Application, reason string) + FenceApplicationFailure(app *model.Application, reason string) // HasMachine checks if a machine exists for the given application ID HasMachine(appID int64) bool From c14baeec1f022c369b1d256c3e52d462a73c31b8 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:31:39 -0300 Subject: [PATCH 04/11] fix(replay): reject inputs after terminal completion --- internal/replay/compare.go | 8 +- internal/replay/compare_test.go | 28 +++---- internal/replay/run.go | 10 +++ internal/replay/run_test.go | 75 ++++++++++++++++--- .../repository/postgres/replay_source_test.go | 54 +++++++++++++ test/integration/foreclose_replay_test.go | 4 +- 6 files changed, 148 insertions(+), 31 deletions(-) diff --git a/internal/replay/compare.go b/internal/replay/compare.go index 6a97e17a7..01e7081bf 100644 --- a/internal/replay/compare.go +++ b/internal/replay/compare.go @@ -86,11 +86,11 @@ func compareRecord( if actual.MachineHash != *record.Input.MachineHash { return contradiction("machine_hash", record.Input.MachineHash.Hex(), actual.MachineHash.Hex()) } - if record.Input.OutputsHash == nil { - return contradiction("outputs_hash", "persisted hash", "missing") + if record.Input.TxBufferDataBlock == nil { + return contradiction("tx_buffer_data_block", "persisted hash", "missing") } - if actual.OutputsHash != *record.Input.OutputsHash { - return contradiction("outputs_hash", record.Input.OutputsHash.Hex(), actual.OutputsHash.Hex()) + if actual.TxBufferDataBlock != *record.Input.TxBufferDataBlock { + return contradiction("tx_buffer_data_block", record.Input.TxBufferDataBlock.Hex(), actual.TxBufferDataBlock.Hex()) } if verification == repository.ReplayVerificationCanonical { return nil diff --git a/internal/replay/compare_test.go b/internal/replay/compare_test.go index 830ec186c..5136374c8 100644 --- a/internal/replay/compare_test.go +++ b/internal/replay/compare_test.go @@ -20,26 +20,26 @@ func replayFixture(status model.InputCompletionStatus, consensus model.Consensus *model.AdvanceResult, ) { machineHash := common.HexToHash("0x11") - outputsHash := common.HexToHash("0x22") + txBufferDataBlock := common.HexToHash("0x22") app := &model.Application{ID: 7, Name: "replay-app", ConsensusType: consensus} record := &model.ReplayRecord{ Input: model.ReplayInput{ - ApplicationID: app.ID, - EpochIndex: 3, - InputIndex: 9, - RawData: []byte("input"), - Status: status, - MachineHash: &machineHash, - OutputsHash: &outputsHash, + ApplicationID: app.ID, + EpochIndex: 3, + InputIndex: 9, + RawData: []byte("input"), + Status: status, + MachineHash: &machineHash, + TxBufferDataBlock: &txBufferDataBlock, }, } actual := &model.AdvanceResult{ EpochIndex: 3, InputIndex: 9, Status: status, - OutputsProof: model.OutputsProof{ - MachineHash: machineHash, - OutputsHash: outputsHash, + StateProof: model.StateProof{ + MachineHash: machineHash, + TxBufferDataBlock: txBufferDataBlock, }, } if status == model.InputCompletionStatus_Exception { @@ -202,7 +202,7 @@ func TestCompareReplayRecordAcceptedMutationTable(t *testing.T) { a.Status = model.InputCompletionStatus_Rejected }}, {"machine-root", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.MachineHash[0]++ }}, - {"outputs-root", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.OutputsHash[0]++ }}, + {"tx-buffer-data-block", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.TxBufferDataBlock[0]++ }}, {"outputs-count", func(_ *model.Application, _ *model.ReplayRecord, a *model.AdvanceResult) { a.Outputs = a.Outputs[:1] }}, @@ -254,9 +254,9 @@ func TestCompareReplayRecordPersistedRecordValidation(t *testing.T) { ErrContradiction, ) }) - t.Run("missing outputs root", func(t *testing.T) { + t.Run("missing TX buffer data block", func(t *testing.T) { app, record, actual := replayFixture(model.InputCompletionStatus_Rejected, model.Consensus_Authority) - record.Input.OutputsHash = nil + record.Input.TxBufferDataBlock = nil require.ErrorIs(t, compareRecord(app.Name, app.ID, app.IsDaveConsensus(), repository.ReplayVerificationFull, record, actual), ErrContradiction, diff --git a/internal/replay/run.go b/internal/replay/run.go index c89b31f86..a14031783 100644 --- a/internal/replay/run.go +++ b/internal/replay/run.go @@ -116,6 +116,16 @@ func Run( ); err != nil { return Result{}, err } + if input.Status.IsTerminal() && input.InputIndex != summary.ProcessedInputs-1 { + return Result{}, newContradiction( + applicationLabel, + knownEpochIndex(input.EpochIndex), + input.InputIndex, + "terminal_status.position", + summary.ProcessedInputs-1, + input.InputIndex, + ) + } replayed++ if expected := input.InputIndex + 1; executor.ProcessedInputs() != expected { return Result{}, contradiction( diff --git a/internal/replay/run_test.go b/internal/replay/run_test.go index 69c59a5f2..c7272f922 100644 --- a/internal/replay/run_test.go +++ b/internal/replay/run_test.go @@ -67,6 +67,7 @@ type fakeExecutor struct { computeHashes []bool wrongResultPos bool fullPRTResult bool + statuses map[uint64]model.InputCompletionStatus } func (executor *fakeExecutor) ProcessedInputs() uint64 { return executor.processed } @@ -96,11 +97,14 @@ func (executor *fakeExecutor) Advance( EpochIndex: epochIndex, InputIndex: resultIndex, Status: model.InputCompletionStatus_Accepted, - OutputsProof: model.OutputsProof{ - MachineHash: common.BigToHash(newBig(inputIndex + 1)), - OutputsHash: common.BigToHash(newBig(inputIndex + 100)), + StateProof: model.StateProof{ + MachineHash: common.BigToHash(newBig(inputIndex + 1)), + TxBufferDataBlock: common.BigToHash(newBig(inputIndex + 100)), }, } + if status, ok := executor.statuses[inputIndex]; ok { + result.Status = status + } if executor.fullPRTResult { result.PaddingRepetitions = 1 << 24 } @@ -113,15 +117,15 @@ func replayRecords(count uint64) []*model.ReplayRecord { records := make([]*model.ReplayRecord, count) for index := range count { machineHash := common.BigToHash(newBig(index + 1)) - outputsHash := common.BigToHash(newBig(index + 100)) + txBufferDataBlock := common.BigToHash(newBig(index + 100)) records[index] = &model.ReplayRecord{Input: model.ReplayInput{ - ApplicationID: 7, - EpochIndex: index / 2, - InputIndex: index, - RawData: []byte{byte(index)}, - Status: model.InputCompletionStatus_Accepted, - MachineHash: &machineHash, - OutputsHash: &outputsHash, + ApplicationID: 7, + EpochIndex: index / 2, + InputIndex: index, + RawData: []byte{byte(index)}, + Status: model.InputCompletionStatus_Accepted, + MachineHash: &machineHash, + TxBufferDataBlock: &txBufferDataBlock, }} } return records @@ -210,6 +214,55 @@ func TestRunCaughtUpStillValidatesSummary(t *testing.T) { require.Empty(t, source.pageRequests) } +func TestRunRejectsCompletedInputAfterTerminalStatus(t *testing.T) { + records := replayRecords(2) + records[0].Input.Status = model.InputCompletionStatus_MachineHalted + source := &fakeSource{ + summary: model.ReplaySummary{ + ApplicationID: 7, ProcessedInputs: 2, Consensus: model.Consensus_Authority, + }, + records: records, + } + executor := &fakeExecutor{statuses: map[uint64]model.InputCompletionStatus{ + 0: model.InputCompletionStatus_MachineHalted, + }} + + _, err := Run( + context.Background(), source, executor, + replayOptions(model.Consensus_Authority, 0, 2), + ) + require.ErrorIs(t, err, ErrContradiction) + var contradiction *ContradictionError + require.ErrorAs(t, err, &contradiction) + require.Equal(t, "terminal_status.position", contradiction.Field) + require.Equal(t, "1", contradiction.Expected) + require.Equal(t, "0", contradiction.Actual) + require.Len(t, executor.advanceCalls, 1) +} + +func TestRunAcceptsTerminalStatusAtEndOfReplay(t *testing.T) { + records := replayRecords(2) + records[1].Input.Status = model.InputCompletionStatus_UnexpectedYield + source := &fakeSource{ + summary: model.ReplaySummary{ + ApplicationID: 7, ProcessedInputs: 2, Consensus: model.Consensus_Authority, + }, + records: records, + } + executor := &fakeExecutor{statuses: map[uint64]model.InputCompletionStatus{ + 1: model.InputCompletionStatus_UnexpectedYield, + }} + + result, err := Run( + context.Background(), source, executor, + replayOptions(model.Consensus_Authority, 0, 2), + ) + + require.NoError(t, err) + require.Equal(t, uint64(2), result.ReplayedInputs) + require.Equal(t, uint64(2), executor.ProcessedInputs()) +} + func TestRunRejectsMalformedPagesBeforeExecution(t *testing.T) { tests := []struct { name string diff --git a/internal/repository/postgres/replay_source_test.go b/internal/repository/postgres/replay_source_test.go index 9c45fcdd5..53b4e1c17 100644 --- a/internal/repository/postgres/replay_source_test.go +++ b/internal/repository/postgres/replay_source_test.go @@ -156,6 +156,60 @@ func TestPostgresReplayVerificationLevels(t *testing.T) { require.Equal(t, uint64(2), violation.CompletedInputCount) } +func TestPostgresReplayIncludesNewTerminalStatuses(t *testing.T) { + endpoint, err := db.GetTestDatabaseEndpoint() + if err != nil { + t.Skipf("Skipping: %v", err) + } + require.NoError(t, db.SetupTestPostgres(endpoint)) + + ctx := context.Background() + repo, err := factory.NewRepositoryFromConnectionString(ctx, endpoint) + require.NoError(t, err) + t.Cleanup(repo.Close) + + for _, status := range []model.InputCompletionStatus{ + model.InputCompletionStatus_Overflow, + model.InputCompletionStatus_UnexpectedYield, + } { + t.Run(status.String(), func(t *testing.T) { + app := repotest.NewApplicationBuilder().Create(ctx, t, repo) + epoch := repotest.NewEpochBuilder(app.ID). + WithStatus(model.EpochStatus_Closed). + WithInputBounds(0, 0). + Build() + input := repotest.NewInputBuilder().WithIndex(0).Build() + require.NoError(t, repo.CreateEpochsAndInputs( + ctx, + app.IApplicationAddress.String(), + map[*model.Epoch][]*model.Input{epoch: {input}}, + 10, + )) + require.NoError(t, repo.StoreAdvanceResult(ctx, app.ID, &model.AdvanceResult{ + EpochIndex: 0, + InputIndex: 0, + Status: status, + StateProof: *repotest.DummyStateProof(), + })) + + summary, err := repo.ReplaySummary( + ctx, app.IApplicationAddress, repository.ReplayVerificationCanonical) + require.NoError(t, err) + require.Equal(t, uint64(1), summary.ProcessedInputs) + page, err := repo.ReplayPage(ctx, repository.ReplayPageRequest{ + ApplicationID: summary.ApplicationID, + FromInput: 0, + ToInputExclusive: 1, + Limit: 1, + Verification: repository.ReplayVerificationCanonical, + }) + require.NoError(t, err) + require.Len(t, page, 1) + require.Equal(t, status, page[0].Input.Status) + }) + } +} + func TestPostgresReplayRejectsCompletedInputGap(t *testing.T) { endpoint, err := db.GetTestDatabaseEndpoint() if err != nil { diff --git a/test/integration/foreclose_replay_test.go b/test/integration/foreclose_replay_test.go index d43165a9b..860f9aa26 100644 --- a/test/integration/foreclose_replay_test.go +++ b/test/integration/foreclose_replay_test.go @@ -156,7 +156,7 @@ func (s *ForecloseReplaySuite) TestForecloseReregisterReplay() { epoch, err := waitForEpochStatus(claimCtx, s.T(), appAName, ep, model.EpochStatus_ClaimAccepted) claimCancel() r.NoError(err, "epoch %d should reach CLAIM_ACCEPTED", ep) - r.NotNil(epoch.OutputsMerkleRoot, "epoch %d outputs merkle root", ep) + r.NotNil(epoch.TxBufferDataBlock, "epoch %d outputs merkle root", ep) s.T().Logf(" epoch %d accepted", ep) } @@ -495,7 +495,7 @@ func compareReplayedEpoch(t testing.TB, r *require.Assertions, a, b *model.Epoch r.Equal(a.LastBlock, b.LastBlock, "epoch %d: last block", ep) r.Equal(a.InputIndexLowerBound, b.InputIndexLowerBound, "epoch %d: input lower bound", ep) r.Equal(a.InputIndexUpperBound, b.InputIndexUpperBound, "epoch %d: input upper bound", ep) - r.Equal(a.OutputsMerkleRoot, b.OutputsMerkleRoot, "epoch %d: outputs merkle root", ep) + r.Equal(a.TxBufferDataBlock, b.TxBufferDataBlock, "epoch %d: outputs merkle root", ep) r.Equal(a.MachineHash, b.MachineHash, "epoch %d: machine hash", ep) } From 2c773067a8d677bf453de521adccc83fc86ab96b Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:31:54 -0300 Subject: [PATCH 05/11] feat(advancer): stop after durable terminal outcomes --- internal/advancer/advancer.go | 197 +++++--- internal/advancer/advancer_test.go | 434 ++++++++++++------ internal/advancer/determinism_test.go | 108 +++-- test/integration/lifecycle_test.go | 51 +- test/integration/polling_helpers_test.go | 29 ++ test/integration/reject_exception_prt_test.go | 13 +- test/integration/reject_exception_test.go | 14 +- 7 files changed, 582 insertions(+), 264 deletions(-) diff --git a/internal/advancer/advancer.go b/internal/advancer/advancer.go index 30f8aa5c6..a2767a964 100644 --- a/internal/advancer/advancer.go +++ b/internal/advancer/advancer.go @@ -31,9 +31,7 @@ type AdvancerRepository interface { ListInputs(ctx context.Context, nameOrAddress string, f repository.InputFilter, p repository.Pagination, descending bool) ([]*Input, uint64, error) GetLastInput(ctx context.Context, appAddress string, epochIndex uint64) (*Input, error) StoreAdvanceResult(ctx context.Context, appID int64, ar *AdvanceResult) error - UpdateEpochInputsProcessed(ctx context.Context, nameOrAddress string, epochIndex uint64) error - UpdateEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64, proof *OutputsProof) error - RepeatPreviousEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64) error + UpdateEpochInputsProcessed(ctx context.Context, nameOrAddress string, epochIndex uint64, proof *StateProof) error UpdateApplicationStatus(ctx context.Context, appID int64, status ApplicationStatus, reason *string) error GetEpoch(ctx context.Context, nameOrAddress string, index uint64) (*Epoch, error) UpdateInputSnapshotURI(ctx context.Context, appId int64, inputIndex uint64, snapshotURI string) error @@ -118,6 +116,9 @@ func (s *Service) Step(ctx context.Context) (bool, error) { // the tick even when it has many small epochs. func (s *Service) stepApp(ctx context.Context, app *Application) (bool, error) { appAddress := app.IApplicationAddress.String() + if !s.machineManager.HasMachine(app.ID) { + return false, fmt.Errorf("%w: %d", ErrNoApp, app.ID) + } epochs, _, err := getUnprocessedEpochs(ctx, s.repository, appAddress) if err != nil { @@ -126,12 +127,19 @@ func (s *Service) stepApp(ctx context.Context, app *Application) (bool, error) { budgetRemaining := s.inputBatchSize for _, epoch := range epochs { - moreInputs, processed, err := s.processEpochInputs(ctx, app, epoch.Index, budgetRemaining) + moreInputs, processed, terminal, err := s.processEpochInputs( + ctx, app, epoch.Index, budgetRemaining, + ) if err != nil { return false, err } budgetRemaining -= processed + if terminal { + // The terminal result itself is durable, but its epoch remains CLOSED: + // there is no accepted state from which to publish a validity proof. + return false, nil + } // More inputs remain in this epoch — reschedule immediately. if moreInputs { @@ -165,13 +173,19 @@ func (s *Service) finalizeEpoch(ctx context.Context, app *Application, epoch *Ep return nil } - if err := s.handleEpochAfterInputsProcessed(ctx, app, epoch); err != nil { + proof, err := s.prepareEpochPublication(ctx, app, epoch) + if err != nil { return err } appAddress := app.IApplicationAddress.String() - if err := s.repository.UpdateEpochInputsProcessed(ctx, appAddress, epoch.Index); err != nil { - return err + if err := s.repository.UpdateEpochInputsProcessed(ctx, appAddress, epoch.Index, proof); err != nil { + return fmt.Errorf( + "publishing state proof for application %s epoch %d: %w", + app.Name, + epoch.Index, + err, + ) } s.Logger.Info("Epoch updated to Inputs Processed", @@ -184,24 +198,24 @@ func (s *Service) finalizeEpoch(ctx context.Context, app *Application, epoch *Ep // and the number of inputs actually processed. func (s *Service) processEpochInputs( ctx context.Context, app *Application, epochIndex uint64, budget uint64, -) (bool, uint64, error) { +) (bool, uint64, bool, error) { appAddress := app.IApplicationAddress.String() inputs, total, err := getUnprocessedInputs(ctx, s.repository, appAddress, epochIndex, budget) if err != nil { - return false, 0, err + return false, 0, false, err } if len(inputs) == 0 { - return false, 0, nil + return false, 0, false, nil } s.Logger.Debug("Processing inputs", "application", app.Name, "epoch_index", epochIndex, "count", len(inputs), "total", total) - if err := s.processInputs(ctx, app, inputs); err != nil { - return false, 0, err + processed, terminal, err := s.processInputs(ctx, app, inputs) + if err != nil { + return false, processed, false, err } - processed := uint64(len(inputs)) // More work remains if total exceeds what we just processed. - return total > processed, processed, nil + return total > processed, processed, terminal, nil } func (s *Service) isAllEpochInputsProcessed(app *Application, epoch *Epoch) (bool, error) { @@ -219,24 +233,31 @@ func (s *Service) isAllEpochInputsProcessed(app *Application, epoch *Epoch) (boo return false, nil } -// processInputs handles the processing of inputs for an application -func (s *Service) processInputs(ctx context.Context, app *Application, inputs []*Input) error { +// processInputs stores the completed prefix of inputs and stops immediately +// after a terminal completion. Rejected inputs are nonterminal and therefore +// do not truncate the batch. +func (s *Service) processInputs( + ctx context.Context, + app *Application, + inputs []*Input, +) (uint64, bool, error) { // Skip if there are no inputs to process if len(inputs) == 0 { - return nil + return 0, false, nil } // Get the machine instance for this application machine, exists := s.machineManager.GetMachine(app.ID) if !exists { - return fmt.Errorf("%w: %d", ErrNoApp, app.ID) + return 0, false, fmt.Errorf("%w: %d", ErrNoApp, app.ID) } + var processed uint64 // Process each input sequentially for _, input := range inputs { // Check for context cancellation before processing each input if err := ctx.Err(); err != nil { - return err + return processed, false, err } s.Logger.Info("Processing input", @@ -257,7 +278,7 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] "application", app.Name, "index", input.Index, "error", err) - return err + return processed, false, err } // Anything else, including a deadline, is an execution failure. @@ -279,7 +300,7 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] } } - return err + return processed, false, err } // log advance result hashes s.Logger.Info("Processing input finished", @@ -296,6 +317,24 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] // Store the result in the database err = s.repository.StoreAdvanceResult(ctx, input.EpochApplicationID, result) if err != nil { + if errors.Is(err, repository.ErrApplicationNotRunnable) { + // Another service durably fenced this application after the + // machine began its advance. Discard only this stale runtime; the + // database already contains the authoritative app-local outcome. + s.Logger.Warn("Advance result lost an application status race; closing stale machine", + "application", app.Name, + "epoch", input.EpochIndex, + "index", input.Index, + "error", err) + closeErr := machine.Close() + if closeErr != nil { + s.Logger.Error("Could not close stale machine after application status race", + "application", app.Name, + "error", closeErr) + } + return processed, false, errors.Join(err, closeErr) + } + // Advance has already changed the live machine, but the transaction // did not confirm that its result was saved. The database may still // show this input as pending. Reusing this machine could then execute @@ -323,18 +362,40 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] "application", app.Name, "error", closeErr) } - return errors.Join(err, closeErr) + return processed, false, errors.Join(err, closeErr) + } + processed++ + if result.Status.IsTerminal() { + applicationStatus, _ := result.Status.TerminalApplicationStatus() + s.Logger.Error("Application execution terminated", + "application", app.Name, + "address", app.IApplicationAddress, + "epoch", result.EpochIndex, + "input", result.InputIndex, + "completion_status", result.Status, + "application_status", applicationStatus, + ) + return processed, true, nil } // Create a snapshot if needed if result.Status == InputCompletionStatus_Accepted { err = s.handleSnapshot(ctx, app, machine, input) if err != nil { - if errors.Is(err, context.Canceled) { + switch { + case errors.Is(ctx.Err(), context.Canceled): s.Logger.Debug("Snapshot creation cancelled due to shutdown", "application", app.Name, "index", input.Index) - } else { + return processed, false, err + case errors.Is(err, manager.ErrMachineClosed): + s.Logger.Error("Snapshot failure destroyed the machine runtime", + "application", app.Name, + "index", input.Index, + "error", err) + s.markApplicationFailed(ctx, app, err.Error()) + return processed, false, err + default: s.Logger.Error("Failed to create snapshot", "application", app.Name, "index", input.Index, @@ -345,7 +406,7 @@ func (s *Service) processInputs(ctx context.Context, app *Application, inputs [] } } - return nil + return processed, false, nil } // markApplicationFailed persists FAILED or installs a local fence when the @@ -391,60 +452,52 @@ func (s *Service) isEpochLastInput(ctx context.Context, app *Application, input return false, nil } -// handleEpochAfterInputsProcessed handles the snapshot creation after when an epoch is closed after an input was processed -func (s *Service) handleEpochAfterInputsProcessed(ctx context.Context, app *Application, epoch *Epoch) error { - // if epoch has inputs, all data is updated after advance, just check for snapshot - if epoch.InputIndexLowerBound != epoch.InputIndexUpperBound { - // Get the machine instance for this application - machine, exists := s.machineManager.GetMachine(app.ID) - if !exists { - return fmt.Errorf("%w: %d", ErrNoApp, app.ID) - } - - // Check if this is the last processed input - lastProcessedInput, err := s.repository.GetLastProcessedInput(ctx, app.IApplicationAddress.String()) - if err != nil { - return fmt.Errorf("failed to get last input: %w", err) - } +// prepareEpochPublication acquires a fresh proof from the resident accepted +// machine state for every epoch, including empty and rejection-only epochs, +// and performs any epoch-boundary snapshot before the proof is published. +func (s *Service) prepareEpochPublication( + ctx context.Context, + app *Application, + epoch *Epoch, +) (*StateProof, error) { + machine, exists := s.machineManager.GetMachine(app.ID) + if !exists { + return nil, fmt.Errorf("%w: %d", ErrNoApp, app.ID) + } - // Check if the application has a epoch snapshot policy - if lastProcessedInput != nil && app.ExecutionParameters.SnapshotPolicy == SnapshotPolicy_EveryEpoch { - // Handle the snapshot - return s.handleSnapshot(ctx, app, machine, lastProcessedInput) + proof, err := machine.StateProof(ctx) + if err != nil { + // If the runtime was destroyed (e.g., child process crashed), mark the + // app as failed to avoid an infinite retry loop. + if errors.Is(err, manager.ErrMachineClosed) { + s.markApplicationFailed(ctx, app, err.Error()) } + return nil, fmt.Errorf("failed to get final state proof from machine: %w", err) + } + if !proof.IsComplete() { + return nil, fmt.Errorf( + "machine returned an incomplete final state proof: %w", + repository.ErrInvalidStateProof, + ) + } - return nil + if epoch.InputIndexLowerBound == epoch.InputIndexUpperBound || + app.ExecutionParameters.SnapshotPolicy != SnapshotPolicy_EveryEpoch { + return proof, nil } - // if epoch has no inputs, we need to copy previous epoch Outputs Proof - // first epoch we need to get it from the template - if epoch.Index == 0 { - // Get the machine instance for this application - machine, exists := s.machineManager.GetMachine(app.ID) - if !exists { - return fmt.Errorf("%w: %d", ErrNoApp, app.ID) - } - outputsProof, err := machine.OutputsProof(ctx) - if err != nil { - // If the runtime was destroyed (e.g., child process crashed), - // mark the app as failed to avoid an infinite retry loop. - if errors.Is(err, manager.ErrMachineClosed) { - s.markApplicationFailed(ctx, app, err.Error()) - } - return fmt.Errorf("failed to get outputs proof from machine: %w", err) - } - err = s.repository.UpdateEpochOutputsProof(ctx, app.ID, epoch.Index, outputsProof) - if err != nil { - return fmt.Errorf("failed to store outputs proof for epoch 0: %w", err) - } - } else { - err := s.repository.RepeatPreviousEpochOutputsProof(ctx, app.ID, epoch.Index) - if err != nil { - return fmt.Errorf("failed to repeat previous epoch outputs proof: %w", err) + lastProcessedInput, err := s.repository.GetLastProcessedInput( + ctx, app.IApplicationAddress.String(), + ) + if err != nil { + return nil, fmt.Errorf("failed to get last input: %w", err) + } + if lastProcessedInput != nil { + if err := s.handleSnapshot(ctx, app, machine, lastProcessedInput); err != nil { + return nil, err } } - - return nil + return proof, nil } // handleSnapshot creates a snapshot based on the application's snapshot policy diff --git a/internal/advancer/advancer_test.go b/internal/advancer/advancer_test.go index efa70f142..7a5816f1b 100644 --- a/internal/advancer/advancer_test.go +++ b/internal/advancer/advancer_test.go @@ -209,6 +209,8 @@ func (s *AdvancerSuite) TestStep() { _, err := env.service.Step(context.Background()) require.Error(err) require.Contains(err.Error(), "update epochs error") + require.Contains(err.Error(), env.app.Application.Name) + require.Contains(err.Error(), "epoch 0") }) s.Run("Error/UpdateMachines", func() { @@ -424,7 +426,7 @@ func (s *AdvancerSuite) TestProcess() { newInput(env.app.Application.ID, 0, 0, []byte("advance error")), } - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Equal(1, env.repo.ApplicationStatusUpdates) require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) @@ -440,7 +442,7 @@ func (s *AdvancerSuite) TestProcess() { } env.repo.UpdateApplicationStatusError = errors.New("update state error") - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "advance error") }) @@ -466,7 +468,7 @@ func (s *AdvancerSuite) TestProcess() { newInput(env.app.Application.ID, 0, 0, []byte("input")), } - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.ErrorIs(err, interruption.err) require.Empty(env.repo.StoredResults) require.Equal(1, env.repo.ApplicationStatusUpdates) @@ -489,17 +491,73 @@ func (s *AdvancerSuite) TestProcess() { newInput(env.app.Application.ID, 2, 6, marshal(randomAdvanceResult(6))), } - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Nil(err) require.Len(env.repo.StoredResults, 7) }) + for _, status := range []InputCompletionStatus{ + InputCompletionStatus_Exception, + InputCompletionStatus_MachineHalted, + InputCompletionStatus_Overflow, + InputCompletionStatus_UnexpectedYield, + } { + s.Run("StopsAfterTerminal/"+status.String(), func() { + require := s.Require() + env := s.setupOneApp() + accepted := randomAdvanceResult(0) + terminal := randomAdvanceResult(1) + terminal.Status = status + terminal.Outputs = nil + terminal.Reports = nil + if status == InputCompletionStatus_Exception { + terminal.ExceptionData = []byte("guest exception") + } + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, marshal(accepted)), + newInput(env.app.Application.ID, 0, 1, marshal(terminal)), + newInput(env.app.Application.ID, 0, 2, []byte("must not execute")), + } + + processed, stopped, err := env.service.processInputs( + context.Background(), env.app.Application, inputs, + ) + require.NoError(err) + require.Equal(uint64(2), processed) + require.True(stopped) + require.Len(env.repo.StoredResults, 2) + require.Equal(status, env.repo.StoredResults[1].Status) + require.Equal([]byte("must not execute"), inputs[2].RawData) + }) + } + + s.Run("RejectedDoesNotStopBatch", func() { + require := s.Require() + env := s.setupOneApp() + rejected := randomAdvanceResult(0) + rejected.Status = InputCompletionStatus_Rejected + rejected.Outputs = nil + rejected.Reports = nil + accepted := randomAdvanceResult(1) + + processed, stopped, err := env.service.processInputs( + context.Background(), env.app.Application, []*Input{ + newInput(env.app.Application.ID, 0, 0, marshal(rejected)), + newInput(env.app.Application.ID, 0, 1, marshal(accepted)), + }, + ) + require.NoError(err) + require.Equal(uint64(2), processed) + require.False(stopped) + require.Len(env.repo.StoredResults, 2) + }) + s.Run("Noop", func() { s.Run("NoInputs", func() { require := s.Require() env := s.setupOneApp() - err := env.service.processInputs(context.Background(), env.app.Application, []*Input{}) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, []*Input{}) require.Nil(err) }) }) @@ -511,7 +569,7 @@ func (s *AdvancerSuite) TestProcess() { invalidApp := Application{ID: 999} inputs := randomInputs(1, 0, 3) - err := env.service.processInputs(context.Background(), &invalidApp, inputs) + _, _, err := env.service.processInputs(context.Background(), &invalidApp, inputs) expected := fmt.Sprintf("%v: %v", ErrNoApp, invalidApp.ID) require.EqualError(err, expected) }) @@ -525,7 +583,7 @@ func (s *AdvancerSuite) TestProcess() { newInput(env.app.Application.ID, 0, 2, []byte("unreachable")), } - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "advance error") require.Len(env.repo.StoredResults, 1) @@ -547,7 +605,7 @@ func (s *AdvancerSuite) TestProcess() { } env.repo.StoreAdvanceError = errors.New("store-advance error") - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "store-advance error") require.Empty(env.repo.StoredResults) @@ -571,7 +629,7 @@ func (s *AdvancerSuite) TestProcess() { } env.repo.StoreAdvanceCommitError = errors.New("commit response lost") - err := env.service.processInputs( + _, _, err := env.service.processInputs( context.Background(), env.app.Application, []*Input{pending}, ) require.ErrorContains(err, "commit response lost") @@ -589,6 +647,55 @@ func (s *AdvancerSuite) TestProcess() { require.Empty(unprocessed, "a restart must not execute an input whose result already committed") }) + + s.Run("StoreAdvanceApplicationNotRunnableIsAppLocal", func() { + require := s.Require() + env := s.setupOneApp() + pending := newInput( + env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0)), + ) + env.repo.StoreAdvanceError = fmt.Errorf( + "status changed: %w", repository.ErrApplicationNotRunnable) + + _, _, err := env.service.processInputs( + context.Background(), env.app.Application, []*Input{pending}, + ) + require.ErrorIs(err, repository.ErrApplicationNotRunnable) + require.NoError(env.service.Context.Err(), + "an application-local conflict must not stop unrelated services") + require.Equal(1, env.mm.Map[env.app.Application.ID].closeCalls) + require.Zero(env.repo.ApplicationStatusUpdates, + "the repository's durable status is already authoritative") + }) + }) + + s.Run("SnapshotFailureCannotLeaveSilentZombie", func() { + require := s.Require() + env := s.setupOneApp() + env.app.Application.ExecutionParameters.SnapshotPolicy = SnapshotPolicy_EveryInput + env.service.snapshotsDir = s.T().TempDir() + instance := env.mm.Map[env.app.Application.ID] + instance.createSnapshotError = errors.New("snapshot runtime crashed") + instance.destroyAfterSnapshotError = true + inputs := []*Input{ + newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + newInput(env.app.Application.ID, 0, 1, marshal(randomAdvanceResult(1))), + } + + processed, stopped, err := env.service.processInputs( + context.Background(), + env.app.Application, + inputs, + ) + require.ErrorIs(err, manager.ErrMachineClosed) + require.ErrorContains(err, "snapshot runtime crashed") + require.Equal(uint64(1), processed) + require.False(stopped) + require.Len(env.repo.StoredResults, 1) + require.Equal(1, instance.advanceCalls, + "the destroyed runtime must not be handed the next input") + require.Equal(1, env.repo.ApplicationStatusUpdates) + require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) }) } @@ -639,7 +746,8 @@ func (s *AdvancerSuite) TestContextCancellation() { errCh := make(chan error) go func() { - errCh <- env.service.processInputs(ctx, env.app.Application, inputs) + _, _, err := env.service.processInputs(ctx, env.app.Application, inputs) + errCh <- err }() // Cancel the context after a short delay @@ -676,7 +784,7 @@ func (s *AdvancerSuite) TestContextCancellation() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer cancel() - err := env.service.processInputs(ctx, env.app.Application, inputs) + _, _, err := env.service.processInputs(ctx, env.app.Application, inputs) require.ErrorIs(err, context.DeadlineExceeded) require.Empty(env.repo.StoredResults) // The expired context prevents the immediate database write, so the @@ -719,7 +827,7 @@ func (s *AdvancerSuite) TestContextCancellation() { } ctx := context.Background() - err := env.service.processInputs(ctx, env.app.Application, inputs) + _, _, err := env.service.processInputs(ctx, env.app.Application, inputs) require.ErrorIs(err, context.Canceled) require.NoError(ctx.Err(), "the caller context must remain active") @@ -748,7 +856,7 @@ func (s *AdvancerSuite) TestLargeNumberOfInputs() { inputs[i] = newInput(env.app.Application.ID, 0, uint64(i), marshal(randomAdvanceResult(uint64(i)))) } - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Nil(err) require.Len(env.repo.StoredResults, inputCount) }) @@ -766,7 +874,7 @@ func (s *AdvancerSuite) TestErrorRecovery() { newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), } - err := env.service.processInputs(context.Background(), env.app.Application, inputs) + _, _, err := env.service.processInputs(context.Background(), env.app.Application, inputs) require.Error(err) require.Contains(err.Error(), "temporary failure") require.Empty(env.repo.StoredResults) @@ -792,7 +900,7 @@ func (s *AdvancerSuite) TestContextCancelledBeforeProcessing() { newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), } - err := env.service.processInputs(ctx, env.app.Application, inputs) + _, _, err := env.service.processInputs(ctx, env.app.Application, inputs) require.ErrorIs(err, context.Canceled) }) } @@ -941,141 +1049,133 @@ func (s *AdvancerSuite) TestIsEpochLastInput() { } // --------------------------------------------------------------------------- -// handleEpochAfterInputsProcessed tests +// prepareEpochPublication tests // --------------------------------------------------------------------------- -func (s *AdvancerSuite) TestHandleEpochAfterInputsProcessed() { - s.Run("EmptyEpochIndex0GetsOutputsProofFromMachine", func() { - require := s.Require() - env := s.setupOneApp() - - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Nil(err) - require.True(env.repo.OutputsProofUpdated) - }) - - s.Run("EmptyEpochIndex0ErrorOnOutputsProof", func() { - require := s.Require() - env := s.setupOneApp() - env.app.OutputsProofError = errors.New("proof error") +func (s *AdvancerSuite) TestPrepareEpochPublication() { + s.Run("EveryEmptyEpochGetsFreshMachineProof", func() { + for _, index := range []uint64{0, 2} { + env := s.setupOneApp() + epoch := &Epoch{Index: index, Status: EpochStatus_Closed} - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Error(err) - require.Contains(err.Error(), "proof error") + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, epoch, + ) + s.Require().NoError(err) + s.Require().Same(env.app.StateProofReturn, proof) + s.True(proof.IsComplete()) + } }) - s.Run("EmptyEpochIndex0ErrMachineClosedMarksAppFailed", func() { - require := s.Require() + s.Run("ProofError", func() { env := s.setupOneApp() - env.app.OutputsProofError = manager.ErrMachineClosed + env.app.StateProofError = errors.New("proof error") - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Error(err) - require.ErrorIs(err, manager.ErrMachineClosed) - require.Equal(1, env.repo.ApplicationStatusUpdates) - require.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, &Epoch{}, + ) + s.Require().Nil(proof) + s.Require().ErrorContains(err, "proof error") }) - s.Run("EmptyEpochIndex0ErrMachineClosedWriteFailureQueuesDurableFence", func() { - require := s.Require() + s.Run("IncompleteProof", func() { env := s.setupOneApp() - env.app.OutputsProofError = manager.ErrMachineClosed - env.repo.UpdateApplicationStatusError = errors.New("FAILED write unavailable") - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} - - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) + env.app.StateProofReturn = &StateProof{} - require.ErrorIs(err, manager.ErrMachineClosed) - require.Equal( - appstatus.NormalizeReason(manager.ErrMachineClosed.Error()), - env.mm.RecordedApplicationFailures[env.app.Application.ID], + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, &Epoch{}, ) - require.False(env.service.Ready()) + s.Require().Nil(proof) + s.Require().ErrorIs(err, repository.ErrInvalidStateProof) }) - s.Run("EmptyEpochIndexGt0RepeatsPreviousProof", func() { - require := s.Require() + s.Run("MachineClosedMarksAppFailed", func() { env := s.setupOneApp() + env.app.StateProofError = manager.ErrMachineClosed - epoch := &Epoch{Index: 2, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Nil(err) - require.True(env.repo.RepeatOutputsProofCalled) + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, &Epoch{}, + ) + s.Require().Nil(proof) + s.Require().ErrorIs(err, manager.ErrMachineClosed) + s.Equal(1, env.repo.ApplicationStatusUpdates) + s.Equal(ApplicationStatus_Failed, env.repo.LastApplicationStatus) }) - s.Run("EmptyEpochIndexGt0RepeatError", func() { - require := s.Require() + s.Run("MachineClosedWriteFailureQueuesDurableFence", func() { env := s.setupOneApp() - env.repo.RepeatOutputsProofError = errors.New("repeat error") + env.app.StateProofError = manager.ErrMachineClosed + env.repo.UpdateApplicationStatusError = errors.New("FAILED write unavailable") - epoch := &Epoch{Index: 2, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 0} - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Error(err) - require.Contains(err.Error(), "repeat error") + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, &Epoch{}, + ) + s.Require().Nil(proof) + s.Require().ErrorIs(err, manager.ErrMachineClosed) + s.Equal( + appstatus.NormalizeReason(manager.ErrMachineClosed.Error()), + env.mm.RecordedApplicationFailures[env.app.Application.ID], + ) + s.False(env.service.Ready()) }) s.Run("NonEmptyEpochWithEveryEpochSnapshotPolicy", func() { - require := s.Require() env := s.setupOneApp() env.app.Application.ExecutionParameters.SnapshotPolicy = SnapshotPolicy_EveryEpoch env.service.snapshotsDir = s.T().TempDir() - - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 3} + epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexUpperBound: 3} lastInput := repotest.NewInputBuilder().WithIndex(2).WithEpochIndex(0). WithStatus(InputCompletionStatus_Accepted).Build() lastInput.EpochApplicationID = env.app.Application.ID env.repo.GetLastProcessedInputReturn = lastInput env.repo.GetLastInputReturn = lastInput - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Nil(err) - require.True(env.repo.SnapshotURIUpdated) + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, epoch, + ) + s.Require().NoError(err) + s.True(proof.IsComplete()) + s.True(env.repo.SnapshotURIUpdated) }) s.Run("NonEmptyEpochNoSnapshotPolicy", func() { - require := s.Require() env := s.setupOneApp() env.app.Application.ExecutionParameters.SnapshotPolicy = SnapshotPolicy_None + epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexUpperBound: 3} - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 3} - lastInput := repotest.NewInputBuilder().WithIndex(2).WithEpochIndex(0). - WithStatus(InputCompletionStatus_Accepted).Build() - lastInput.EpochApplicationID = env.app.Application.ID - env.repo.GetLastProcessedInputReturn = lastInput - - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Nil(err) - require.False(env.repo.SnapshotURIUpdated) + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, epoch, + ) + s.Require().NoError(err) + s.True(proof.IsComplete()) + s.False(env.repo.SnapshotURIUpdated) }) s.Run("NoMachineReturnsError", func() { - require := s.Require() mm := newMockMachineManager() svc, err := newMockAdvancerService(mm, &MockRepository{}) - require.Nil(err) - + s.Require().NoError(err) app := repotest.NewApplicationBuilder().Build() app.ID = 999 - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 3} - err = svc.handleEpochAfterInputsProcessed(context.Background(), app, epoch) - require.Error(err) - require.ErrorIs(err, ErrNoApp) + proof, err := svc.prepareEpochPublication( + context.Background(), app, &Epoch{InputIndexUpperBound: 3}, + ) + s.Require().Nil(proof) + s.Require().ErrorIs(err, ErrNoApp) }) s.Run("GetLastProcessedInputError", func() { - require := s.Require() env := s.setupOneApp() env.app.Application.ExecutionParameters.SnapshotPolicy = SnapshotPolicy_EveryEpoch env.repo.GetLastProcessedInputError = errors.New("db connection lost") - epoch := &Epoch{Index: 0, Status: EpochStatus_Closed, InputIndexLowerBound: 0, InputIndexUpperBound: 3} - err := env.service.handleEpochAfterInputsProcessed(context.Background(), env.app.Application, epoch) - require.Error(err) - require.Contains(err.Error(), "db connection lost") + proof, err := env.service.prepareEpochPublication( + context.Background(), env.app.Application, + &Epoch{InputIndexUpperBound: 3}, + ) + s.Require().Nil(proof) + s.Require().ErrorContains(err, "db connection lost") }) } @@ -1458,6 +1558,47 @@ func (s *AdvancerSuite) TestSingleBatchEnforcement() { "should process exactly one batch (batchSize=5)") } +func (s *AdvancerSuite) TestTerminalInputStopsEpochAndFutureTicks() { + require := s.Require() + env := s.setupOneApp() + terminal := randomAdvanceResult(1) + terminal.Status = InputCompletionStatus_MachineHalted + terminal.Outputs = nil + terminal.Reports = nil + address := env.app.Application.IApplicationAddress + env.repo.GetEpochsReturn = map[common.Address][]*Epoch{ + address: {{ + Index: 0, + Status: EpochStatus_Closed, + InputIndexLowerBound: 0, + InputIndexUpperBound: 3, + }}, + } + env.repo.GetInputsReturn = map[common.Address][]*Input{ + address: { + newInput(env.app.Application.ID, 0, 0, marshal(randomAdvanceResult(0))), + newInput(env.app.Application.ID, 0, 1, marshal(terminal)), + newInput(env.app.Application.ID, 0, 2, []byte("must not execute")), + }, + } + + hadWork, err := env.service.Step(context.Background()) + require.NoError(err) + require.False(hadWork) + require.Len(env.repo.StoredResults, 2) + require.Zero(env.repo.EpochInputsProcessedCount, + "a terminal epoch must remain closed because it has no accepted terminal state") + + terminalAppStatus, ok := terminal.Status.TerminalApplicationStatus() + require.True(ok) + env.app.Application.Status = terminalAppStatus + hadWork, err = env.service.Step(context.Background()) + require.NoError(err) + require.False(hadWork) + require.Len(env.repo.StoredResults, 2, + "a terminal machine must not receive work on later ticks") +} + // More-work signal accuracy. func (s *AdvancerSuite) TestMoreWorkSignal() { s.Run("TrueWhenInputsRemain", func() { @@ -1997,11 +2138,12 @@ type MockFullRepository struct { } type MockMachineImpl struct { - Application *Application - AdvanceBlock bool - AdvanceError error - OutputsProofError error - processedInputs uint64 + Application *Application + AdvanceBlock bool + AdvanceError error + StateProofReturn *StateProof + StateProofError error + processedInputs uint64 } func (mock *MockMachineImpl) Advance( @@ -2046,6 +2188,7 @@ func newMockMachine(id int64) *MockMachineImpl { ID: id, IApplicationAddress: randomAddress(), }, + StateProofReturn: randomCompleteStateProof(), } } @@ -2082,7 +2225,16 @@ func (mock *MockMachineManager) GetMachine(appID int64) (manager.MachineInstance } func (mock *MockMachineManager) UpdateMachines(ctx context.Context) error { - return mock.UpdateMachinesError + if mock.UpdateMachinesError != nil { + return mock.UpdateMachinesError + } + for id, instance := range mock.Map { + status := instance.application.Status + if status != "" && status != ApplicationStatus_OK { + delete(mock.Map, id) + } + } + return nil } func (mock *MockMachineManager) FenceApplicationFailure(app *Application, reason string) { @@ -2117,14 +2269,17 @@ func (mock *MockMachineManager) Close() error { // MockMachineInstance is a test implementation of manager.MachineInstance type MockMachineInstance struct { - application *Application - machineImpl *MockMachineImpl - createSnapshotError error - closeCalls int + application *Application + machineImpl *MockMachineImpl + createSnapshotError error + destroyAfterSnapshotError bool + closeCalls int + advanceCalls int } // Advance implements the MachineInstance interface for testing func (m *MockMachineInstance) Advance(ctx context.Context, input []byte, epochIndex uint64, index uint64, leafs bool) (*AdvanceResult, error) { + m.advanceCalls++ return m.machineImpl.Advance(ctx, input, epochIndex, index, leafs) } @@ -2143,18 +2298,19 @@ func (m *MockMachineInstance) ProcessedInputs() uint64 { return m.machineImpl.processedInputs } -func (m *MockMachineInstance) OutputsProof(ctx context.Context) (*OutputsProof, error) { - if m.machineImpl.OutputsProofError != nil { - return nil, m.machineImpl.OutputsProofError +func (m *MockMachineInstance) StateProof(_ context.Context) (*StateProof, error) { + if m.machineImpl.StateProofError != nil { + return nil, m.machineImpl.StateProofError } - return &OutputsProof{ - OutputsHash: randomHash(), - MachineHash: randomHash(), - }, nil + return m.machineImpl.StateProofReturn, nil } // CreateSnapshot implements the MachineInstance interface for testing func (m *MockMachineInstance) CreateSnapshot(ctx context.Context, processInputs uint64, path string) error { + if m.createSnapshotError != nil && m.destroyAfterSnapshotError { + m.machineImpl.AdvanceError = manager.ErrMachineClosed + return errors.Join(manager.ErrMachineClosed, m.createSnapshotError) + } return m.createSnapshotError } @@ -2184,10 +2340,8 @@ type MockRepository struct { StoreAdvanceFailCount int UpdateApplicationStatusError error UpdateEpochsError error - UpdateOutputsProofError error GetLastSnapshotReturn *Input GetLastSnapshotError error - RepeatOutputsProofError error GetEpochReturn *Epoch GetEpochError error GetLastInputReturn *Input @@ -2201,8 +2355,7 @@ type MockRepository struct { ApplicationStatusUpdates int LastApplicationStatus ApplicationStatus LastApplicationStatusReason *string - OutputsProofUpdated bool - RepeatOutputsProofCalled bool + PublishedStateProof *StateProof SnapshotURIUpdated bool EpochInputsProcessedCount int @@ -2330,21 +2483,19 @@ func (mock *MockRepository) StoreAdvanceResult( return mock.StoreAdvanceCommitError } -func (mock *MockRepository) UpdateEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64, proof *OutputsProof) error { - if ctx.Err() != nil { - return ctx.Err() - } - mock.OutputsProofUpdated = true - return mock.UpdateOutputsProofError -} - -func (mock *MockRepository) UpdateEpochInputsProcessed(ctx context.Context, nameOrAddress string, epochIndex uint64) error { +func (mock *MockRepository) UpdateEpochInputsProcessed( + ctx context.Context, + _ string, + _ uint64, + proof *StateProof, +) error { // Check for context cancellation if ctx.Err() != nil { return ctx.Err() } mock.EpochInputsProcessedCount++ + mock.PublishedStateProof = proof return mock.UpdateEpochsError } @@ -2444,14 +2595,6 @@ func (mock *MockRepository) GetLastSnapshot(ctx context.Context, nameOrAddress s return mock.GetLastSnapshotReturn, mock.GetLastSnapshotError } -func (mock *MockRepository) RepeatPreviousEpochOutputsProof(ctx context.Context, appID int64, epochIndex uint64) error { - if ctx.Err() != nil { - return ctx.Err() - } - mock.RepeatOutputsProofCalled = true - return mock.RepeatOutputsProofError -} - // ------------------------------------------------------------------------------------------------ func randomAddress() common.Address { @@ -2472,6 +2615,21 @@ func randomHash() common.Hash { return common.BytesToHash(hash) } +func randomCompleteStateProof() *StateProof { + proof := &StateProof{ + TxBufferDataBlock: randomHash(), + MachineHash: randomHash(), + IflagsYDataBlock: randomHash(), + HtifTohostDataBlock: randomHash(), + } + for range StateProofSiblingCount { + proof.TxBufferProof = append(proof.TxBufferProof, randomHash()) + proof.IflagsYProof = append(proof.IflagsYProof, randomHash()) + proof.HtifTohostProof = append(proof.HtifTohostProof, randomHash()) + } + return proof +} + func randomBytes() []byte { size := mrand.Intn(100) + 1 bytes := make([]byte, size) @@ -2514,9 +2672,9 @@ func randomAdvanceResult(inputIndex uint64) *AdvanceResult { Status: InputCompletionStatus_Accepted, Outputs: randomSliceOfBytes(), Reports: randomSliceOfBytes(), - OutputsProof: OutputsProof{ - OutputsHash: randomHash(), - MachineHash: randomHash(), + StateProof: StateProof{ + TxBufferDataBlock: randomHash(), + MachineHash: randomHash(), }, } return res diff --git a/internal/advancer/determinism_test.go b/internal/advancer/determinism_test.go index eb8bf99ed..c355808c2 100644 --- a/internal/advancer/determinism_test.go +++ b/internal/advancer/determinism_test.go @@ -52,6 +52,16 @@ func TestProcessInputs_RetryFromNonzeroPredecessorMatchesUninterruptedResult(t * targetPayload: []byte("halt:application-finished"), wantStatus: model.InputCompletionStatus_MachineHalted, }, + { + name: "mcycle overflow", + targetPayload: []byte("overflow:cycle-ceiling"), + wantStatus: model.InputCompletionStatus_Overflow, + }, + { + name: "unexpected yield", + targetPayload: []byte("unexpected-yield:unknown-reason"), + wantStatus: model.InputCompletionStatus_UnexpectedYield, + }, } for _, tt := range tests { @@ -110,7 +120,12 @@ func determinismBaseline( require.Len(t, repo.StoredResults, 2) target := cloneDeterminismResult(repo.StoredResults[1]) require.Equal(t, uint64(2), harness.instance.ProcessedInputs()) - require.Equal(t, machine.Hash(target.MachineHash), harness.runtimeHash(t)) + if target.Status.IsTerminal() { + _, err := harness.instance.Hash(context.Background()) + require.ErrorIs(t, err, manager.ErrMachineClosed) + } else { + require.Equal(t, machine.Hash(target.MachineHash), harness.runtimeHash(t)) + } return prefix, predecessor, target } @@ -337,24 +352,38 @@ func requireDeterminismTarget( require.Equal(t, uint64(1), target.InputIndex) require.Equal(t, wantStatus, target.Status) require.True(t, target.IsDaveConsensus) - require.NotEmpty(t, target.OutputsHashProof) require.Len(t, target.PeriodicStateHashes, 2, "a PRT result must retain its periodic state hashes") require.Equal(t, machine.InputEntryCapacity-uint64(len(target.PeriodicStateHashes)), target.PaddingRepetitions) if wantStatus == model.InputCompletionStatus_Accepted { + require.True(t, target.IsComplete()) require.Equal(t, [][]byte{append([]byte("output:"), targetPayload...)}, target.Outputs) require.Equal(t, [][]byte{append([]byte("report:"), targetPayload...)}, target.Reports) require.NotEqual(t, prefix.MachineHash, target.MachineHash) - require.NotEqual(t, prefix.OutputsHash, target.OutputsHash) + require.NotEqual(t, prefix.TxBufferDataBlock, target.TxBufferDataBlock) return } require.Empty(t, target.Outputs, "effects are canonical only for accepted inputs") require.Empty(t, target.Reports, "effects are canonical only for accepted inputs") - require.Equal(t, prefix.MachineHash, target.MachineHash, - "a nonaccepted candidate must not replace the predecessor") - require.Equal(t, prefix.OutputsHash, target.OutputsHash) - require.Equal(t, prefix.OutputsHashProof, target.OutputsHashProof) + if wantStatus.IsTerminal() { + require.True(t, target.IsComplete(), + "a terminal result must preserve its actual post-run proof") + require.NotEqual(t, prefix.MachineHash, target.MachineHash, + "a terminal completion must preserve its post-run machine root") + require.NotEqual(t, prefix.TxBufferDataBlock, target.TxBufferDataBlock, + "a terminal completion must preserve its post-run TX buffer") + } else { + require.True(t, target.IsComplete()) + require.Equal(t, prefix.MachineHash, target.MachineHash, + "a rejected candidate must not replace the predecessor") + } + if !wantStatus.IsTerminal() { + require.Equal(t, prefix.TxBufferDataBlock, target.TxBufferDataBlock) + } + if !wantStatus.IsTerminal() { + require.Equal(t, prefix.StateProof, target.StateProof) + } } func requireDiscardedMutatedFork( @@ -378,11 +407,18 @@ func requireSuccessfulRetryState( ) { t.Helper() require.Equal(t, uint64(2), harness.instance.ProcessedInputs()) - require.Equal(t, machine.Hash(wantTarget.MachineHash), harness.runtimeHash(t)) lastCandidate := harness.lastFork(t) + if wantTarget.Status.IsTerminal() { + _, err := harness.instance.Hash(context.Background()) + require.ErrorIs(t, err, manager.ErrMachineClosed) + require.True(t, predecessor.isClosed()) + require.True(t, lastCandidate.isClosed(), "the terminal candidate must be disposed") + return + } + require.Equal(t, machine.Hash(wantTarget.MachineHash), harness.runtimeHash(t)) if wantTarget.Status == model.InputCompletionStatus_Accepted { require.True(t, predecessor.isClosed()) - require.False(t, lastCandidate.isClosed(), "the accepted candidate must be adopted") + require.False(t, lastCandidate.isClosed(), "the state-producing candidate must be adopted") return } require.False(t, predecessor.isClosed(), "rejection must keep the predecessor live") @@ -427,7 +463,9 @@ func waitDeterminismError(t *testing.T, errCh <-chan error) error { func cloneDeterminismResult(result *model.AdvanceResult) *model.AdvanceResult { clone := *result - clone.OutputsHashProof = append([][32]byte(nil), result.OutputsHashProof...) + clone.TxBufferProof = append([][32]byte(nil), result.TxBufferProof...) + clone.IflagsYProof = append([][32]byte(nil), result.IflagsYProof...) + clone.HtifTohostProof = append([][32]byte(nil), result.HtifTohostProof...) clone.Outputs = cloneDeterminismBytes(result.Outputs) clone.Reports = cloneDeterminismBytes(result.Reports) clone.ExceptionData = append([]byte(nil), result.ExceptionData...) @@ -528,12 +566,13 @@ func newDeterminismHarness( } func (h *determinismHarness) process(ctx context.Context, index uint64, payload []byte) error { - return h.service.processInputs(ctx, h.app, []*model.Input{{ + _, _, err := h.service.processInputs(ctx, h.app, []*model.Input{{ EpochApplicationID: h.app.ID, EpochIndex: 7, Index: index, RawData: append([]byte(nil), payload...), }}) + return err } func (h *determinismHarness) waitForMutation(t *testing.T) *determinismRuntime { @@ -584,7 +623,6 @@ type determinismMachineState struct { step uint64 machineHash machine.Hash outputsHash machine.Hash - outputsProof []machine.Hash checkpointHash machine.Hash } @@ -592,14 +630,12 @@ func newDeterminismMachineState() determinismMachineState { machineHash := determinismHash("base-machine") outputsHash := determinismHash("base-outputs") return determinismMachineState{ - machineHash: machineHash, - outputsHash: outputsHash, - outputsProof: determinismProof(outputsHash), + machineHash: machineHash, + outputsHash: outputsHash, } } func (s determinismMachineState) clone() determinismMachineState { - s.outputsProof = append([]machine.Hash(nil), s.outputsProof...) return s } @@ -710,22 +746,24 @@ func (m *determinismRuntime) Hash(ctx context.Context) (machine.Hash, error) { return m.state.machineHash, nil } -func (m *determinismRuntime) OutputsHash(ctx context.Context) (machine.Hash, error) { - m.mu.Lock() - defer m.mu.Unlock() - if err := m.checkOpenLocked(ctx); err != nil { - return machine.Hash{}, err - } - return m.state.outputsHash, nil -} - -func (m *determinismRuntime) OutputsHashProof(ctx context.Context) ([]machine.Hash, error) { +func (m *determinismRuntime) StateProof(ctx context.Context) (*machine.StateProof, error) { m.mu.Lock() defer m.mu.Unlock() if err := m.checkOpenLocked(ctx); err != nil { return nil, err } - return append([]machine.Hash(nil), m.state.outputsProof...), nil + var iflagsYData machine.Hash + iflagsYData[8] = 1 + var htifTohostData machine.Hash + htifTohostData[20] = 1 + htifTohostData[22] = 1 + htifTohostData[23] = 2 + return &machine.StateProof{ + MachineHash: m.state.machineHash, + IflagsYProof: determinismValidityLeaf("iflags-y", iflagsYData), + HtifTohostProof: determinismValidityLeaf("htif-tohost", htifTohostData), + TxBufferProof: determinismValidityLeaf("tx-buffer", m.state.outputsHash), + }, nil } func (m *determinismRuntime) Advance( @@ -767,6 +805,10 @@ func (m *determinismRuntime) Advance( status = machine.CompletionStatusException case bytes.HasPrefix(input, []byte("halt:")): status = machine.CompletionStatusHalted + case bytes.HasPrefix(input, []byte("overflow:")): + status = machine.CompletionStatusOverflow + case bytes.HasPrefix(input, []byte("unexpected-yield:")): + status = machine.CompletionStatusUnexpectedYield } output := append([]byte("output:"), input...) report := append([]byte("report:"), input...) @@ -777,13 +819,11 @@ func (m *determinismRuntime) Advance( "machine", previous.machineHash[:], checkpointHash[:], input, ) m.state.outputsHash = determinismHash("outputs", previous.outputsHash[:], output) - m.state.outputsProof = determinismProof(m.state.outputsHash) hashes := []machine.Hash{firstHash, finalHash} response := &machine.AdvanceResponse{ Status: status, PeriodicStateHashes: hashes, PaddingRepetitions: machine.InputEntryCapacity - uint64(len(hashes)), - OutputsHash: m.state.outputsHash, } if status == machine.CompletionStatusAccepted { response.Outputs = []machine.Output{output} @@ -873,11 +913,13 @@ func determinismHash(label string, values ...[]byte) machine.Hash { return result } -func determinismProof(outputsHash machine.Hash) []machine.Hash { - return []machine.Hash{ - determinismHash("proof-0", outputsHash[:]), - determinismHash("proof-1", outputsHash[:]), +func determinismValidityLeaf(label string, dataBlock machine.Hash) machine.LeafProof { + const canonicalMachineProofDepth = 59 + siblings := make([]machine.Hash, canonicalMachineProofDepth) + for i := range siblings { + siblings[i] = determinismHash(label, dataBlock[:], []byte{byte(i)}) } + return machine.LeafProof{DataBlock: dataBlock, Siblings: siblings} } type determinismMachineProvider struct { diff --git a/test/integration/lifecycle_test.go b/test/integration/lifecycle_test.go index 2b1d51236..696a0c6d5 100644 --- a/test/integration/lifecycle_test.go +++ b/test/integration/lifecycle_test.go @@ -186,8 +186,9 @@ type rejectExceptionLifecycleConfig struct { } // runRejectExceptionLifecycleTest runs the reject/exception pipeline for a dapp -// that rejects or throws on input index 1. Works for both Authority and PRT -// consensus, controlled by the config. +// that rejects or throws on input index 1. A rejection reverts that input and +// permits input 2 to execute; an exception terminates execution before input 2. +// Works for both Authority and PRT consensus. func runRejectExceptionLifecycleTest( ctx context.Context, t testing.TB, @@ -215,7 +216,12 @@ func runRejectExceptionLifecycleTest( // --- L1 -> Machine: send 3 inputs where input #1 will be rejected/exception --- - t.Logf("Sending 3 inputs — the dapp will %s input #1 while accepting #0 and #2...", cfg.FailStatus) + terminalStatus, terminal := cfg.FailStatus.TerminalApplicationStatus() + if terminal { + t.Logf("Sending 3 inputs — the dapp will %s input #1 and stop before executing #2...", cfg.FailStatus) + } else { + t.Logf("Sending 3 inputs — the dapp will %s input #1 while accepting #0 and #2...", cfg.FailStatus) + } const numInputs = 3 for i := range numInputs { payload := fmt.Sprintf("%s-payload-%d", cfg.TestName, i) @@ -226,24 +232,42 @@ func runRejectExceptionLifecycleTest( } func() { - defer timed(t, "wait for input processing (3 inputs)")() - t.Log("Waiting for the advancer to process all 3 inputs through the Cartesi Machine...") + defer timed(t, "wait for input processing")() + t.Log("Waiting for the advancer to reach the configured input outcome...") processCtx, processCancel := context.WithTimeout(ctx, inputProcessingTimeout) defer processCancel() expectedStatuses := map[uint64]model.InputCompletionStatus{ 0: model.InputCompletionStatus_Accepted, 1: cfg.FailStatus, - 2: model.InputCompletionStatus_Accepted, + } + if !terminal { + expectedStatuses[2] = model.InputCompletionStatus_Accepted } - for i := range uint64(numInputs) { + processedInputs := uint64(numInputs) + if terminal { + processedInputs = 2 + } + for i := range processedInputs { input, err := waitForInputProcessed(processCtx, t, cfg.AppName, i) require.NoError(err, "wait for input %d processing", i) require.Equal(expectedStatuses[i], input.Status, "input %d: expected status %s, got %s", i, expectedStatuses[i], input.Status) t.Logf(" input %d: %s", i, input.Status) } + + if terminal { + require.NoError( + waitForApplicationStatus(processCtx, t, cfg.AppName, terminalStatus.String()), + "wait for terminal application status", + ) + pending, err := waitForInputIndexed(processCtx, t, cfg.AppName, 2) + require.NoError(err, "wait for input 2 indexing") + require.Equal(model.InputCompletionStatus_None, pending.Status, + "input 2 must remain indexed but unprocessed after terminal execution") + t.Logf(" input 2: indexed but not executed; application status: %s", terminalStatus) + } }() // --- Verify off-chain results: only accepted inputs produce outputs --- @@ -253,6 +277,9 @@ func runRejectExceptionLifecycleTest( outputsResp, err := readOutputs(ctx, cfg.AppName) require.NoError(err, "read outputs") numAccepted := uint64(2) + if terminal { + numAccepted = 1 + } require.Equal(numAccepted*rejectOutputsPerAcceptedInput, outputsResp.Pagination.TotalCount, "expected %d outputs (%d per accepted input x %d accepted inputs)", numAccepted*rejectOutputsPerAcceptedInput, rejectOutputsPerAcceptedInput, numAccepted) @@ -272,6 +299,12 @@ func runRejectExceptionLifecycleTest( numAccepted*rejectReportsPerAcceptedInput, rejectReportsPerAcceptedInput, numAccepted) t.Logf(" %d reports found — correct", numAccepted*rejectReportsPerAcceptedInput) + if terminal { + t.Logf("=== %s test complete: %s terminalized execution before the later input ===", + cfg.TestName, cfg.FailStatus) + return + } + // --- Optional pre-claim hook (e.g. PRT tournament settlement) --- if cfg.PreClaimHook != nil { @@ -383,8 +416,8 @@ func verifyClaimAndExecute( epoch, err := waitForEpochStatus( claimCtx, t, cfg.AppName, cfg.EpochIndex, model.EpochStatus_ClaimAccepted) require.NoError(err, "wait for claim accepted") - require.NotNil(epoch.OutputsMerkleRoot, "epoch claim should be set") - t.Logf(" epoch %d claim accepted (hash=%s)", cfg.EpochIndex, *epoch.OutputsMerkleRoot) + require.NotNil(epoch.TxBufferDataBlock, "epoch claim should be set") + t.Logf(" epoch %d claim accepted (hash=%s)", cfg.EpochIndex, *epoch.TxBufferDataBlock) }() // --- Verify Merkle proofs --- diff --git a/test/integration/polling_helpers_test.go b/test/integration/polling_helpers_test.go index 655badc5b..f4810c553 100644 --- a/test/integration/polling_helpers_test.go +++ b/test/integration/polling_helpers_test.go @@ -79,6 +79,35 @@ func waitForInputProcessed( return result, err } +// waitForInputIndexed polls until the EVM reader has stored the input, without +// requiring the advancer to process it. +func waitForInputIndexed( + ctx context.Context, + t testing.TB, + appName string, + inputIndex uint64, +) (*model.Input, error) { + var lastErr error + var result *model.Input + err := pollUntil(ctx, 2*time.Second, func() (bool, error) { + input, err := readInput(ctx, appName, inputIndex) + if err != nil { + if isCLIExitError(err) { + lastErr = err + t.Logf("poll indexed input %d: %v (retrying)", inputIndex, err) + return false, nil + } + return false, fmt.Errorf("poll indexed input %d: %w", inputIndex, err) + } + result = input + return true, nil + }) + if err != nil && lastErr != nil { + return nil, fmt.Errorf("%w (last poll error: %v)", err, lastErr) + } + return result, err +} + // waitForEpochStatus polls until the epoch at the given index reaches the // desired status using the CLI. // diff --git a/test/integration/reject_exception_prt_test.go b/test/integration/reject_exception_prt_test.go index 93fc31e7d..74868135b 100644 --- a/test/integration/reject_exception_prt_test.go +++ b/test/integration/reject_exception_prt_test.go @@ -94,13 +94,11 @@ func (s *RejectExceptionPrtSuite) TestRejectInputPrt() { } // TestExceptionInputPrt deploys an exception-loop-dapp with PRT consensus, -// sends 3 inputs, and verifies that input 1 is EXCEPTION while inputs 0 and 2 -// are ACCEPTED. Then settles tournaments and executes outputs on L1. +// sends 3 inputs, and verifies that input 1 terminates execution with EXCEPTION +// before input 2 can execute. func (s *RejectExceptionPrtSuite) TestExceptionInputPrt() { - s.SetExpectedLogs(s.T(), prtBlockOutOfRangeAllowlist) + s.SetExpectedLogs(s.T(), terminalExecutionExpectedLog) - ethClient := s.ethClient - prtEpoch := uint64(1) appName := uniqueAppName("exception-prt-loop") s.appNames = append(s.appNames, appName) runRejectExceptionLifecycleTest(s.ctx, s.T(), s.Require(), rejectExceptionLifecycleConfig{ @@ -109,10 +107,5 @@ func (s *RejectExceptionPrtSuite) TestExceptionInputPrt() { TestName: "exception", FailStatus: model.InputCompletionStatus_Exception, ExtraDeployArgs: []string{"--prt"}, - EpochIndex: &prtEpoch, - PreClaimHook: func(ctx context.Context, t testing.TB, require *require.Assertions, appName string) { - settleTournament(ctx, t, require, ethClient, appName, 0) - settleTournament(ctx, t, require, ethClient, appName, 1) - }, }) } diff --git a/test/integration/reject_exception_test.go b/test/integration/reject_exception_test.go index b563033bd..f0e39221c 100644 --- a/test/integration/reject_exception_test.go +++ b/test/integration/reject_exception_test.go @@ -7,6 +7,7 @@ package integration import ( "context" + "regexp" "testing" "time" @@ -14,6 +15,13 @@ import ( "github.com/stretchr/testify/suite" ) +var terminalExecutionExpectedLog = ExpectedLog{ + Pattern: regexp.MustCompile(`Application execution terminated`), + Level: LevelError, + Reason: "the guest exception durably terminates application execution", + Required: true, +} + type RejectExceptionSuite struct { suite.Suite LogChecker @@ -64,9 +72,11 @@ func (s *RejectExceptionSuite) TestRejectInput() { } // TestExceptionInput deploys an exception-loop-dapp (ioctl-echo-loop --exception=1), -// sends 3 inputs, and verifies that input 1 is EXCEPTION while inputs 0 and 2 -// are ACCEPTED with correct outputs and reports. +// sends 3 inputs, and verifies that input 1 terminates execution with EXCEPTION +// before input 2 can execute. func (s *RejectExceptionSuite) TestExceptionInput() { + s.SetExpectedLogs(s.T(), terminalExecutionExpectedLog) + appName := uniqueAppName("exception-loop") s.appNames = append(s.appNames, appName) runRejectExceptionLifecycleTest(s.ctx, s.T(), s.Require(), rejectExceptionLifecycleConfig{ From 122dd299a68654fe65e7281ceb0ba1148dcdc246 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:32:09 -0300 Subject: [PATCH 06/11] test(validator): use complete state proof fixtures --- internal/validator/validator.go | 26 +++---- internal/validator/validator_test.go | 61 +++++++-------- test/validator/validator_test.go | 111 +++++++++++++-------------- 3 files changed, 97 insertions(+), 101 deletions(-) diff --git a/internal/validator/validator.go b/internal/validator/validator.go index dade2609d..fbc8de60c 100644 --- a/internal/validator/validator.go +++ b/internal/validator/validator.go @@ -176,7 +176,7 @@ func (s *Service) validateApplication(ctx context.Context, app *Application) err return s.setApplicationCorrupted(ctx, app, "epoch %v (%v) has no machine hash", epoch.Index, epoch.VirtualIndex) } - if epoch.OutputsMerkleRoot == nil { + if epoch.TxBufferDataBlock == nil { return s.setApplicationCorrupted(ctx, app, "epoch %v (%v) has no outputs merkle root", epoch.Index, epoch.VirtualIndex) } @@ -204,10 +204,10 @@ func (s *Service) validateApplication(ctx context.Context, app *Application) err // tree after each input. Therefore, the root hash calculated after the // last input in the epoch must match the one calculated by the Validator // So we need to validate the application state. - if *epoch.OutputsMerkleRoot != *merkleRoot { + if *epoch.TxBufferDataBlock != *merkleRoot { return s.setApplicationCorrupted(ctx, app, "epoch %v outputs merkle root does not match computed one. Expected: %v, Got %v", - epoch.Index, *epoch.OutputsMerkleRoot, *merkleRoot) + epoch.Index, *epoch.TxBufferDataBlock, *merkleRoot) } input, err := s.repository.GetLastInput(ctx, appAddress, epoch.Index) @@ -219,17 +219,17 @@ func (s *Service) validateApplication(ctx context.Context, app *Application) err } if input != nil { - if input.OutputsHash == nil { + if input.TxBufferDataBlock == nil { return s.setApplicationCorrupted(ctx, app, "inconsistent state: epoch %v last input (%v) outputs merkle root is not defined", epoch.Index, input.Index) } // ...and compare it to the hash calculated by the Validator - if *epoch.OutputsMerkleRoot != *input.OutputsHash { + if *epoch.TxBufferDataBlock != *input.TxBufferDataBlock { return s.setApplicationCorrupted(ctx, app, "computed outputs merkle root does not match epoch %v last input %v merkle root. Expected: %v, Got %v", - epoch.Index, input.Index, *input.OutputsHash, *epoch.OutputsMerkleRoot) + epoch.Index, input.Index, *input.TxBufferDataBlock, *epoch.TxBufferDataBlock) } if input.MachineHash == nil { @@ -273,14 +273,14 @@ func (s *Service) validateApplication(ctx context.Context, app *Application) err "epoch %v machine hash does not match previous epoch %v machine hash. Expected: %v, Got %v", epoch.Index, previousEpoch.Index, *previousEpoch.MachineHash, *epoch.MachineHash) } - if previousEpoch.OutputsMerkleRoot == nil { + if previousEpoch.TxBufferDataBlock == nil { return s.setApplicationCorrupted(ctx, app, "previous epoch %v (%v) outputs merkle root is not defined", previousEpoch.Index, previousEpoch.VirtualIndex) } - if *epoch.OutputsMerkleRoot != *previousEpoch.OutputsMerkleRoot { + if *epoch.TxBufferDataBlock != *previousEpoch.TxBufferDataBlock { return s.setApplicationCorrupted(ctx, app, "epoch %v outputs merkle root does not match previous epoch %v one. Expected: %v, Got %v", - epoch.Index, previousEpoch.Index, *previousEpoch.OutputsMerkleRoot, *epoch.OutputsMerkleRoot) + epoch.Index, previousEpoch.Index, *previousEpoch.TxBufferDataBlock, *epoch.TxBufferDataBlock) } } else { // first epoch if *epoch.MachineHash != app.TemplateHash { @@ -288,10 +288,10 @@ func (s *Service) validateApplication(ctx context.Context, app *Application) err "epoch %v machine hash does not match for application template hash. Expected: %v, Got %v", epoch.Index, app.TemplateHash, *epoch.MachineHash) } - if *epoch.OutputsMerkleRoot != s.pristineRootHash { + if *epoch.TxBufferDataBlock != s.pristineRootHash { return s.setApplicationCorrupted(ctx, app, "epoch %v outputs merkle root does not match pristine root hash. Expected: %v, Got %v", - epoch.Index, s.pristineRootHash, *epoch.OutputsMerkleRoot) + epoch.Index, s.pristineRootHash, *epoch.TxBufferDataBlock) } } } @@ -480,12 +480,12 @@ func (s *Service) computeMerkleTreeAndProofs( return &s.pristineRootHash, nil, nil } // if there are no outputs and there is a previous epoch, return its claim - if previousEpoch.OutputsMerkleRoot == nil { + if previousEpoch.TxBufferDataBlock == nil { return nil, nil, s.setApplicationCorrupted(ctx, app, "invalid application state for epoch %v (%v) of application %v. Previous epoch has no claim.", epoch.Index, epoch.VirtualIndex, appAddress) } - return previousEpoch.OutputsMerkleRoot, nil, nil + return previousEpoch.TxBufferDataBlock, nil, nil } // Build the pre-context for the cumulative outputs tree from the output that diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index 5f9888259..b3a1e1a18 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -53,9 +53,10 @@ func (s *ValidatorSuite) SetupSubTest() { serviceArgs := &service.CreateInfo{Name: "validator", Impl: validator} err := service.Create(context.Background(), serviceArgs, &validator.Service) s.Require().Nil(err) - dummyOutputsMerkleRoot := common.HexToHash("0x0a162946e56158bac0673e6dd3bdfdc1e4a0e7744a120fdb640050c8d7abe1c6") + dummyTxBufferDataBlock := common.HexToHash("0x0a162946e56158bac0673e6dd3bdfdc1e4a0e7744a120fdb640050c8d7abe1c6") dummyEpochs = []Epoch{ - {Index: 0, VirtualIndex: 0, FirstBlock: 0, LastBlock: 9, OutputsMerkleRoot: &dummyOutputsMerkleRoot, MachineHash: &validator.pristineRootHash}, + {Index: 0, VirtualIndex: 0, FirstBlock: 0, LastBlock: 9, + TxBufferDataBlock: &dummyTxBufferDataBlock, MachineHash: &validator.pristineRootHash}, {Index: 1, VirtualIndex: 1, FirstBlock: 10, LastBlock: 19}, {Index: 2, VirtualIndex: 2, FirstBlock: 20, LastBlock: 29}, {Index: 3, VirtualIndex: 3, FirstBlock: 30, LastBlock: 39}, @@ -188,7 +189,7 @@ func (s *ValidatorSuite) TestCreateClaimAndProofSuccess() { claimHash, _, err := validator.computeMerkleTreeAndProofs(ctx, &app, &dummyEpochs[1]) s.NoError(err) - s.Equal(dummyEpochs[0].OutputsMerkleRoot, claimHash) + s.Equal(dummyEpochs[0].TxBufferDataBlock, claimHash) repo.AssertExpectations(s.T()) }) @@ -260,7 +261,7 @@ func (s *ValidatorSuite) TestCreateClaimAndProofFailures() { ).Return([]*Output{}, uint64(0), nil).Once() invalidEpoch := dummyEpochs[0] - invalidEpoch.OutputsMerkleRoot = nil + invalidEpoch.TxBufferDataBlock = nil repo.On("GetEpochByVirtualIndex", mock.Anything, mock.Anything, mock.Anything, ).Return(&invalidEpoch, nil).Once() @@ -354,7 +355,7 @@ func (s *ValidatorSuite) TestValidateApplicationSuccess() { input := Input{ EpochApplicationID: app.ID, MachineHash: &validator.pristineRootHash, - OutputsHash: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } repo.On("ListEpochs", @@ -446,7 +447,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { s.Run("GetLastInputFailure", func() { input := Input{ EpochApplicationID: app.ID, - OutputsHash: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } repo.On("ListEpochs", @@ -469,7 +470,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { s.Run("InvalidInputFailure", func() { input := Input{ EpochApplicationID: app.ID, - OutputsHash: nil, // <- this is invalid + TxBufferDataBlock: nil, // <- this is invalid } repo.On("ListEpochs", @@ -494,7 +495,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { s.Run("NilInputMachineHash", func() { input := Input{ EpochApplicationID: app.ID, - OutputsHash: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, MachineHash: nil, // <- trigger nil guard } @@ -521,7 +522,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { invalidClaim := common.Hash{} input := Input{ EpochApplicationID: app.ID, - OutputsHash: &invalidClaim, + TxBufferDataBlock: &invalidClaim, } repo.On("ListEpochs", @@ -546,7 +547,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { s.Run("StoreClaimAndProofsFailure", func() { input := Input{ EpochApplicationID: app.ID, - OutputsHash: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, MachineHash: &validator.pristineRootHash, } @@ -579,7 +580,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { VirtualIndex: 0, FirstBlock: 0, LastBlock: 9, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, MachineHash: nil, // <- nil triggers the new guard } repo.On("ListEpochs", @@ -593,14 +594,14 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { repo.AssertExpectations(s.T()) }) - s.Run("NilEpochOutputsMerkleRoot", func() { + s.Run("NilEpochTxBufferDataBlock", func() { epoch := Epoch{ Index: 0, VirtualIndex: 0, FirstBlock: 0, LastBlock: 9, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: nil, // <- nil triggers the new guard + TxBufferDataBlock: nil, // <- nil triggers the new guard } repo.On("ListEpochs", mock.Anything, app.IApplicationAddress.String(), mock.Anything, mock.Anything, false, @@ -624,7 +625,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { FirstBlock: 0, LastBlock: 9, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: dummyEpochs[0].OutputsMerkleRoot, + TxBufferDataBlock: dummyEpochs[0].TxBufferDataBlock, } repo.On("ListEpochs", mock.Anything, app.IApplicationAddress.String(), mock.Anything, mock.Anything, false, @@ -658,7 +659,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { FirstBlock: 10, LastBlock: 19, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: dummyEpochs[0].OutputsMerkleRoot, + TxBufferDataBlock: dummyEpochs[0].TxBufferDataBlock, } repo.On("ListEpochs", mock.Anything, app.IApplicationAddress.String(), mock.Anything, mock.Anything, false, @@ -702,7 +703,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { FirstBlock: 10, LastBlock: 19, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: dummyEpochs[0].OutputsMerkleRoot, + TxBufferDataBlock: dummyEpochs[0].TxBufferDataBlock, } repo.On("ListEpochs", mock.Anything, app.IApplicationAddress.String(), mock.Anything, mock.Anything, false, @@ -726,7 +727,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { Index: 0, VirtualIndex: 0, MachineHash: nil, // <- nil triggers the guard - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } repo.On("GetEpochByVirtualIndex", mock.Anything, app.IApplicationAddress.String(), uint64(0), @@ -739,7 +740,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { repo.AssertExpectations(s.T()) }) - s.Run("EmptyEpochPreviousEpochOutputsMerkleRootNil", func() { + s.Run("EmptyEpochPreviousEpochTxBufferDataBlockNil", func() { app := Application{ Name: "dummy-application-name", ConsensusType: Consensus_PRT, @@ -750,7 +751,7 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { FirstBlock: 10, LastBlock: 19, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: dummyEpochs[0].OutputsMerkleRoot, + TxBufferDataBlock: dummyEpochs[0].TxBufferDataBlock, } repo.On("ListEpochs", mock.Anything, app.IApplicationAddress.String(), mock.Anything, mock.Anything, false, @@ -769,12 +770,12 @@ func (s *ValidatorSuite) TestValidateApplicationFailure() { mock.Anything, app.IApplicationAddress.String(), epoch.Index, ).Return((*Input)(nil), nil).Once() - // 2nd GetEpochByVirtualIndex: MachineHash matches but OutputsMerkleRoot is nil + // 2nd GetEpochByVirtualIndex: MachineHash matches but TxBufferDataBlock is nil prev := Epoch{ Index: 0, VirtualIndex: 0, MachineHash: &validator.pristineRootHash, // matches epoch.MachineHash - OutputsMerkleRoot: nil, // <- nil triggers the guard + TxBufferDataBlock: nil, // <- nil triggers the guard } repo.On("GetEpochByVirtualIndex", mock.Anything, app.IApplicationAddress.String(), uint64(0), @@ -817,7 +818,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 0, InputIndexUpperBound: 5, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } // 5 inputs, each with one state hash covering the full @@ -856,7 +857,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 7, InputIndexUpperBound: 7, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } // No ListStateHashes call expected (inputCount==0 branch is skipped) @@ -879,7 +880,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 5, InputIndexUpperBound: 3, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } expectCorrupted(app, "lower bound") @@ -906,7 +907,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 0, InputIndexUpperBound: MaxAdvanceStatesPerEpoch + 1, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } expectCorrupted(app, "input count is too large") @@ -930,7 +931,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 0, InputIndexUpperBound: 2, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } repo.On("ListStateHashes", @@ -956,7 +957,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 0, InputIndexUpperBound: 1, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } repo.On("ListStateHashes", @@ -982,7 +983,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 0, InputIndexUpperBound: 1, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } repo.On("ListStateHashes", @@ -1008,7 +1009,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 0, InputIndexUpperBound: 2, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } stateHashes := []*StateHash{ {MachineHash: validator.pristineRootHash, Repetitions: 1}, @@ -1038,7 +1039,7 @@ func (s *ValidatorSuite) TestBuildCommitment() { InputIndexLowerBound: 0, InputIndexUpperBound: 1, MachineHash: &validator.pristineRootHash, - OutputsMerkleRoot: &validator.pristineRootHash, + TxBufferDataBlock: &validator.pristineRootHash, } repetitions := (uint64(1) << Log2EpochComputationHashLeafCount) + InputHashCollectionCapacity diff --git a/test/validator/validator_test.go b/test/validator/validator_test.go index 6fa78fc50..1001fc08f 100644 --- a/test/validator/validator_test.go +++ b/test/validator/validator_test.go @@ -122,7 +122,7 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsPristineClaim() { ApplicationID: 1, Index: 0, VirtualIndex: 0, - Status: model.EpochStatus_InputsProcessed, + Status: model.EpochStatus_Closed, FirstBlock: 0, LastBlock: 9, } @@ -145,12 +145,9 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsPristineClaim() { advanceResult := model.AdvanceResult{ InputIndex: input.Index, Status: model.InputCompletionStatus_Accepted, - OutputsProof: model.OutputsProof{ - OutputsHash: pristineRootHash, - MachineHash: machinehash1, - }, + StateProof: completeStateProof(machinehash1, pristineRootHash), } - err = s.repository.StoreAdvanceResult(s.ctx, 1, &advanceResult) + err = s.storeAdvanceAndPublishEpoch(app, &advanceResult) s.Require().Nil(err) errs := s.validator.Tick() @@ -159,12 +156,12 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsPristineClaim() { updatedEpoch, err := s.repository.GetEpoch(s.ctx, app.IApplicationAddress.String(), epoch.Index) s.Require().Nil(err) s.Require().NotNil(updatedEpoch) - s.Require().NotNil(updatedEpoch.OutputsMerkleRoot) + s.Require().NotNil(updatedEpoch.TxBufferDataBlock) // epoch status was updated s.Equal(model.EpochStatus_ClaimComputed, updatedEpoch.Status) // claim is pristine claim - s.Equal(pristineRootHash, *updatedEpoch.OutputsMerkleRoot) + s.Equal(pristineRootHash, *updatedEpoch.TxBufferDataBlock) }) } @@ -216,7 +213,7 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsPreviousClaim() { ApplicationID: 1, Index: 1, VirtualIndex: 1, - Status: model.EpochStatus_InputsProcessed, + Status: model.EpochStatus_Closed, FirstBlock: 10, LastBlock: 19, } @@ -236,27 +233,18 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsPreviousClaim() { err = s.repository.CreateEpochsAndInputs(s.ctx, app.IApplicationAddress.String(), epochInputMap, 20) s.Require().Nil(err) - // Advance first epoch to INPUTS_PROCESSED so StoreClaimAndProofs can - // transition it to CLAIM_COMPUTED. - firstEpoch.Status = model.EpochStatus_InputsProcessed - err = s.repository.UpdateEpochStatus(s.ctx, app.IApplicationAddress.String(), &firstEpoch) - s.Require().Nil(err) - // Store the input advance result machinehash1 := crypto.Keccak256Hash([]byte("machine-hash1")) advanceResult := model.AdvanceResult{ EpochIndex: firstEpochInput.EpochIndex, InputIndex: firstEpochInput.Index, Status: model.InputCompletionStatus_Accepted, - OutputsProof: model.OutputsProof{ - OutputsHash: firstEpochClaim, - MachineHash: machinehash1, - }, + StateProof: completeStateProof(machinehash1, firstEpochClaim), } - err = s.repository.StoreAdvanceResult(s.ctx, 1, &advanceResult) + err = s.storeAdvanceAndPublishEpoch(app, &advanceResult) s.Require().Nil(err) - firstEpoch.OutputsMerkleRoot = &firstEpochClaim + firstEpoch.TxBufferDataBlock = &firstEpochClaim err = s.repository.StoreClaimAndProofs(s.ctx, &firstEpoch, []*model.Output{}) s.Require().Nil(err) @@ -267,13 +255,10 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsPreviousClaim() { InputIndex: secondEpochInput.Index, Status: model.InputCompletionStatus_Accepted, // since there are no new outputs in the second epoch, - // the machine OutputsHash will remain the same - OutputsProof: model.OutputsProof{ - OutputsHash: firstEpochClaim, - MachineHash: machinehash2, - }, + // the machine TxBufferDataBlock will remain the same + StateProof: completeStateProof(machinehash2, firstEpochClaim), } - err = s.repository.StoreAdvanceResult(s.ctx, 1, &advanceResult) + err = s.storeAdvanceAndPublishEpoch(app, &advanceResult) s.Require().Nil(err) errs := s.validator.Tick() @@ -282,12 +267,12 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsPreviousClaim() { updatedEpoch, err := s.repository.GetEpoch(s.ctx, app.IApplicationAddress.String(), secondEpoch.Index) s.Require().Nil(err) s.Require().NotNil(updatedEpoch) - s.Require().NotNil(updatedEpoch.OutputsMerkleRoot) + s.Require().NotNil(updatedEpoch.TxBufferDataBlock) // epoch status was updated s.Equal(model.EpochStatus_ClaimComputed, updatedEpoch.Status) // claim is the same from previous epoch - s.Equal(firstEpochClaim, *updatedEpoch.OutputsMerkleRoot) + s.Equal(firstEpochClaim, *updatedEpoch.TxBufferDataBlock) }) } @@ -312,7 +297,7 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() ApplicationID: 1, Index: 0, VirtualIndex: 0, - Status: model.EpochStatus_InputsProcessed, + Status: model.EpochStatus_Closed, FirstBlock: 10, LastBlock: 19, } @@ -351,12 +336,9 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() InputIndex: input.Index, Status: model.InputCompletionStatus_Accepted, Outputs: [][]byte{outputRawData}, - OutputsProof: model.OutputsProof{ - OutputsHash: expectedClaim, - MachineHash: machinehash1, - }, + StateProof: completeStateProof(machinehash1, expectedClaim), } - err = s.repository.StoreAdvanceResult(s.ctx, 1, &advanceResult) + err = s.storeAdvanceAndPublishEpoch(app, &advanceResult) s.Require().Nil(err) errs := s.validator.Tick() @@ -365,12 +347,12 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() updatedEpoch, err := s.repository.GetEpoch(s.ctx, app.IApplicationAddress.String(), epoch.Index) s.Require().Nil(err) s.Require().NotNil(updatedEpoch) - s.Require().NotNil(updatedEpoch.OutputsMerkleRoot) + s.Require().NotNil(updatedEpoch.TxBufferDataBlock) // epoch status was updated s.Equal(model.EpochStatus_ClaimComputed, updatedEpoch.Status) // claim is the expected new claim - s.Equal(expectedClaim, *updatedEpoch.OutputsMerkleRoot) + s.Equal(expectedClaim, *updatedEpoch.TxBufferDataBlock) updatedOutput, err := s.repository.GetOutput(s.ctx, app.IApplicationAddress.String(), output.Index) s.Require().Nil(err) @@ -421,12 +403,6 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() err = s.repository.CreateEpochsAndInputs(s.ctx, app.IApplicationAddress.String(), epochInputMap, 10) s.Require().Nil(err) - // Advance first epoch to INPUTS_PROCESSED so StoreClaimAndProofs can - // transition it to CLAIM_COMPUTED. - firstEpoch.Status = model.EpochStatus_InputsProcessed - err = s.repository.UpdateEpochStatus(s.ctx, app.IApplicationAddress.String(), &firstEpoch) - s.Require().Nil(err) - firstOutputData := []byte("output1") firstOutputHash := crypto.Keccak256Hash(firstOutputData) firstOutput := model.Output{ @@ -450,16 +426,13 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() InputIndex: firstInput.Index, Status: model.InputCompletionStatus_Accepted, Outputs: [][]byte{firstOutputData}, - OutputsProof: model.OutputsProof{ - MachineHash: machinehash1, - OutputsHash: firstEpochClaim, - }, + StateProof: completeStateProof(machinehash1, firstEpochClaim), } - err = s.repository.StoreAdvanceResult(s.ctx, 1, &advanceResult) + err = s.storeAdvanceAndPublishEpoch(app, &advanceResult) s.Require().Nil(err) // update epoch with its claim and insert it in the db - firstEpoch.OutputsMerkleRoot = &firstEpochClaim + firstEpoch.TxBufferDataBlock = &firstEpochClaim firstOutput.OutputHashesSiblings = firstEpochProofs err = s.repository.StoreClaimAndProofs(s.ctx, &firstEpoch, []*model.Output{&firstOutput}) s.Require().Nil(err) @@ -469,7 +442,7 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() ApplicationID: 1, Index: 1, VirtualIndex: 1, - Status: model.EpochStatus_InputsProcessed, + Status: model.EpochStatus_Closed, FirstBlock: 10, LastBlock: 19, } @@ -505,12 +478,9 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() InputIndex: secondInput.Index, Status: model.InputCompletionStatus_Accepted, Outputs: [][]byte{secondOutputData}, - OutputsProof: model.OutputsProof{ - OutputsHash: expectedEpochClaim, - MachineHash: machinehash2, - }, + StateProof: completeStateProof(machinehash2, expectedEpochClaim), } - err = s.repository.StoreAdvanceResult(s.ctx, 1, &advanceResult) + err = s.storeAdvanceAndPublishEpoch(app, &advanceResult) s.Require().Nil(err) errs := s.validator.Tick() @@ -523,13 +493,13 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() ) s.Require().Nil(err) s.Require().NotNil(updatedSecondEpoch) - s.Require().NotNil(updatedSecondEpoch.OutputsMerkleRoot) + s.Require().NotNil(updatedSecondEpoch.TxBufferDataBlock) // assert epoch status was changed s.Equal(model.EpochStatus_ClaimComputed, updatedSecondEpoch.Status) // assert second epoch claim is a new claim - s.NotEqual(firstEpochClaim, *updatedSecondEpoch.OutputsMerkleRoot) - s.Equal(expectedEpochClaim, *updatedSecondEpoch.OutputsMerkleRoot) + s.NotEqual(firstEpochClaim, *updatedSecondEpoch.TxBufferDataBlock) + s.Equal(expectedEpochClaim, *updatedSecondEpoch.TxBufferDataBlock) updatedSecondOutput, err := s.repository.GetOutput( s.ctx, @@ -546,3 +516,28 @@ func (s *ValidatorRepositoryIntegrationSuite) TestItReturnsANewClaimAndProofs() s.Len(updatedSecondOutput.OutputHashesSiblings, MAX_OUTPUT_TREE_HEIGHT) }) } + +func completeStateProof(machineHash, txBufferDataBlock common.Hash) model.StateProof { + return model.StateProof{ + MachineHash: machineHash, + TxBufferDataBlock: txBufferDataBlock, + TxBufferProof: make([][32]byte, model.StateProofSiblingCount), + IflagsYProof: make([][32]byte, model.StateProofSiblingCount), + HtifTohostProof: make([][32]byte, model.StateProofSiblingCount), + } +} + +func (s *ValidatorRepositoryIntegrationSuite) storeAdvanceAndPublishEpoch( + app *model.Application, + result *model.AdvanceResult, +) error { + if err := s.repository.StoreAdvanceResult(s.ctx, 1, result); err != nil { + return err + } + return s.repository.UpdateEpochInputsProcessed( + s.ctx, + app.IApplicationAddress.String(), + result.EpochIndex, + &result.StateProof, + ) +} From 5e93c6a3b64d6899b80fdebb93f373b89b5f99a4 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:44:25 -0300 Subject: [PATCH 07/11] fix(evmreader): observe terminal apps until foreclosure --- internal/appstatus/appstatus.go | 38 ++++- internal/appstatus/appstatus_test.go | 96 ++++++++++++ internal/evmreader/block_scan_plan.go | 20 +-- internal/evmreader/block_scan_plan_test.go | 28 +++- internal/evmreader/mocks_test.go | 2 +- internal/evmreader/output.go | 40 +++-- internal/evmreader/output_test.go | 140 +++++++++++++++++- .../evmreader/post_foreclosure_withdrawal.go | 35 ++++- .../post_foreclosure_withdrawal_test.go | 85 ++++++++++- internal/evmreader/sealedepochs.go | 25 +++- internal/evmreader/sealedepochs_test.go | 77 +++++++++- internal/model/application_lifecycle_test.go | 24 ++- internal/model/execution_parameters_test.go | 7 + internal/model/models.go | 45 +++--- internal/prt/handle_foreclosed_test.go | 9 +- internal/repository/postgres/epoch.go | 8 +- internal/repository/repository.go | 5 +- .../repository/repotest/epoch_test_cases.go | 25 ++++ internal/validator/validator_test.go | 5 + 19 files changed, 619 insertions(+), 95 deletions(-) diff --git a/internal/appstatus/appstatus.go b/internal/appstatus/appstatus.go index 29ba5f3cc..470b373d6 100644 --- a/internal/appstatus/appstatus.go +++ b/internal/appstatus/appstatus.go @@ -33,7 +33,11 @@ type Repository interface { // - replay.Run will correctly verify inputs from the snapshot point. // // The reason parameter must be a pre-formatted string describing the failure. -// Returns the database error if the status update fails; returns nil on success. +// Execution-terminal and integrity-terminal statuses are preserved: execution +// terminals are entered atomically by repository.StoreAdvanceResult, while +// integrity terminals carry stronger evidence than a recoverable runtime +// failure. Returns the database error if the status update fails; returns nil +// on success or when an existing terminal status is preserved. func SetFailed( ctx context.Context, logger *slog.Logger, @@ -41,6 +45,14 @@ func SetFailed( app *Application, reason string, ) error { + if app.Status.IsTerminal() { + logger.Debug("preserving existing terminal application status", + "application", app.Name, + "address", app.IApplicationAddress.String(), + "current_status", app.Status, + "requested_status", ApplicationStatus_Failed) + return nil + } return setApplicationStatus(ctx, logger, repo, app, ApplicationStatus_Failed, reason) } @@ -135,8 +147,24 @@ func setTerminalStatus( reason string, ) error { reason = NormalizeReason(reason) - dbErr := setApplicationStatus(ctx, logger, repo, app, status, reason) reasonErr := errors.New(reason) + + // Integrity terminals are immutable. Execution terminals preserve their + // deterministic machine outcome unless later observation proves local state + // corrupted, which is the one permitted escalation. + executionTerminalEscalation := app.Status.IsExecutionTerminal() && + status == ApplicationStatus_Corrupted + preserveExistingTerminal := app.Status.IsTerminal() && !executionTerminalEscalation + if preserveExistingTerminal { + logger.Debug("preserving existing terminal application status", + "application", app.Name, + "address", app.IApplicationAddress.String(), + "current_status", app.Status, + "requested_status", status) + return reasonErr + } + + dbErr := setApplicationStatus(ctx, logger, repo, app, status, reason) if dbErr != nil { return errors.Join(reasonErr, dbErr) } @@ -161,6 +189,12 @@ func setApplicationStatus( status ApplicationStatus, reason string, ) error { + if status.IsExecutionTerminal() { + return fmt.Errorf( + "execution-terminal status %s must be written atomically by repository.StoreAdvanceResult", + status, + ) + } reason = NormalizeReason(reason) switch status { diff --git a/internal/appstatus/appstatus_test.go b/internal/appstatus/appstatus_test.go index cc27ee211..9ac78c9ec 100644 --- a/internal/appstatus/appstatus_test.go +++ b/internal/appstatus/appstatus_test.go @@ -93,6 +93,54 @@ func (s *AppStatusSuite) TestSetFailedf() { require.Equal(ApplicationStatus_Failed, app.Status) } +func (s *AppStatusSuite) TestSetFailedPreservesTerminalStatus() { + for _, status := range []ApplicationStatus{ + ApplicationStatus_Diverged, + ApplicationStatus_Corrupted, + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } { + s.Run(status.String(), func() { + app := newTestApp() + app.Status = status + originalReason := "terminal reason" + app.Reason = &originalReason + repo := &mockRepo{} + + err := SetFailed( + context.Background(), slog.Default(), repo, app, "later runtime failure") + + s.Require().NoError(err) + s.Zero(repo.callCount) + s.Equal(status, app.Status) + s.Equal(originalReason, *app.Reason) + }) + } +} + +func (s *AppStatusSuite) TestRejectsExecutionTerminalTarget() { + for _, status := range []ApplicationStatus{ + ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield, + } { + s.Run(status.String(), func() { + app := newTestApp() + repo := &mockRepo{} + + err := setApplicationStatus( + context.Background(), slog.Default(), repo, app, status, "terminal outcome") + + s.Require().ErrorContains(err, "must be written atomically by repository.StoreAdvanceResult") + s.Zero(repo.callCount) + s.Equal(ApplicationStatus_OK, app.Status) + }) + } +} + func (s *AppStatusSuite) TestSetDiverged() { require := s.Require() repo := &mockRepo{} @@ -139,6 +187,54 @@ func (s *AppStatusSuite) TestSetCorrupted() { require.Equal("machine snapshot missing", *app.Reason) } +func (s *AppStatusSuite) TestExecutionTerminalCanEscalateOnlyToCorrupted() { + require := s.Require() + logger := slog.Default() + + escalated := newTestApp() + escalated.Status = ApplicationStatus_MachineHalted + originalReason := "machine halted" + escalated.Reason = &originalReason + repo := &mockRepo{} + err := SetCorrupted(context.Background(), logger, repo, escalated, "L1 output mismatch") + require.Error(err) + require.Equal(1, repo.callCount) + require.Equal(ApplicationStatus_Corrupted, escalated.Status) + require.Equal("L1 output mismatch", *escalated.Reason) + + preserved := newTestApp() + preserved.Status = ApplicationStatus_MachineHalted + preserved.Reason = &originalReason + repo = &mockRepo{} + err = SetDiverged(context.Background(), logger, repo, preserved, "later claim disagreement") + require.ErrorContains(err, "later claim disagreement") + require.Zero(repo.callCount) + require.Equal(ApplicationStatus_MachineHalted, preserved.Status) + require.Equal(originalReason, *preserved.Reason) +} + +func (s *AppStatusSuite) TestIntegrityTerminalStatusWritesAreIdempotent() { + for _, current := range []ApplicationStatus{ + ApplicationStatus_Diverged, + ApplicationStatus_Corrupted, + } { + s.Run(current.String(), func() { + app := newTestApp() + app.Status = current + originalReason := "first integrity finding" + app.Reason = &originalReason + repo := &mockRepo{} + + err := SetCorrupted( + context.Background(), slog.Default(), repo, app, "repeated finding") + s.Require().ErrorContains(err, "repeated finding") + s.Zero(repo.callCount) + s.Equal(current, app.Status) + s.Equal(originalReason, *app.Reason) + }) + } +} + func (s *AppStatusSuite) TestSetFailedDBError() { require := s.Require() dbErr := errors.New("db connection failed") diff --git a/internal/evmreader/block_scan_plan.go b/internal/evmreader/block_scan_plan.go index cb326429e..d88f8162f 100644 --- a/internal/evmreader/block_scan_plan.go +++ b/internal/evmreader/block_scan_plan.go @@ -16,7 +16,7 @@ func buildBlockScanPlan(apps []appContracts) blockScanPlan { var plan blockScanPlan for _, app := range apps { application := app.application - if application == nil { + if application == nil || !application.NeedsL1Observation() { continue } @@ -25,7 +25,8 @@ func buildBlockScanPlan(apps []appContracts) blockScanPlan { plan.postForeclosureTargets = append(plan.postForeclosureTargets, app) if application.IsDaveConsensus() { - if application.LastEpochCheckBlock < application.ForecloseBlock { + if application.LastEpochCheckBlock < application.ForecloseBlock || + application.LastInputCheckBlock < application.ForecloseBlock { plan.daveEpochTargets = append(plan.daveEpochTargets, app) } continue @@ -38,13 +39,14 @@ func buildBlockScanPlan(apps []appContracts) blockScanPlan { continue } - if application.CanExecute() { - plan.outputTargets = append(plan.outputTargets, app) - if application.IsDaveConsensus() { - plan.daveEpochTargets = append(plan.daveEpochTargets, app) - } else { - plan.iConsensusInputTargets = append(plan.iConsensusInputTargets, app) - } + // Execution health controls machine work, not L1 observation. Keep + // indexing inputs/epochs and output executions while an external + // watchdog observes the durable failure and eventually forecloses. + plan.outputTargets = append(plan.outputTargets, app) + if application.IsDaveConsensus() { + plan.daveEpochTargets = append(plan.daveEpochTargets, app) + } else { + plan.iConsensusInputTargets = append(plan.iConsensusInputTargets, app) } } return plan diff --git a/internal/evmreader/block_scan_plan_test.go b/internal/evmreader/block_scan_plan_test.go index 95b7d9b88..a46c30156 100644 --- a/internal/evmreader/block_scan_plan_test.go +++ b/internal/evmreader/block_scan_plan_test.go @@ -43,10 +43,18 @@ func TestBuildBlockScanPlan_RoutesScannerTargets(t *testing.T) { wantOutput: []int64{3}, }, { - name: "diverged app without foreclosure is not routed", - apps: []appContracts{planApp(4, planAppConfig{ - status: ApplicationStatus_Diverged, - })}, + name: "non-executing apps remain observable before foreclosure", + apps: []appContracts{ + planApp(40, planAppConfig{status: ApplicationStatus_Failed}), + planApp(41, planAppConfig{status: ApplicationStatus_Diverged}), + planApp(42, planAppConfig{status: ApplicationStatus_Corrupted}), + planApp(43, planAppConfig{status: ApplicationStatus_GuestException}), + planApp(44, planAppConfig{status: ApplicationStatus_MachineHalted}), + planApp(45, planAppConfig{status: ApplicationStatus_McycleOverflow}), + planApp(46, planAppConfig{status: ApplicationStatus_UnexpectedYield}), + }, + wantIConsensusInput: []int64{40, 41, 42, 43, 44, 45, 46}, + wantOutput: []int64{40, 41, 42, 43, 44, 45, 46}, }, { name: "foreclosed IConsensus app with input cursor behind gets final input catch-up", @@ -79,6 +87,18 @@ func TestBuildBlockScanPlan_RoutesScannerTargets(t *testing.T) { wantOutput: []int64{7}, wantPostForeclosure: []int64{7}, }, + { + name: "foreclosed DaveConsensus app with open-input cursor behind gets input catch-up", + apps: []appContracts{planApp(71, planAppConfig{ + consensus: Consensus_PRT, + forecloseBlock: 100, + lastEpochCheckBlock: 100, + lastInputCheckBlock: 99, + })}, + wantDaveEpoch: []int64{71}, + wantOutput: []int64{71}, + wantPostForeclosure: []int64{71}, + }, { name: "foreclosed diverged app still catches up pre-foreclosure work", apps: []appContracts{planApp(8, planAppConfig{ diff --git a/internal/evmreader/mocks_test.go b/internal/evmreader/mocks_test.go index 28f19db2b..476bfe19f 100644 --- a/internal/evmreader/mocks_test.go +++ b/internal/evmreader/mocks_test.go @@ -274,7 +274,7 @@ func (m *MockRepository) SetupDefaultBehavior() *MockRepository { FirstBlock: 11, LastBlock: 20, Status: EpochStatus_Open, - OutputsMerkleRoot: nil, + TxBufferDataBlock: nil, ClaimTransactionHash: nil, }, nil).Twice() diff --git a/internal/evmreader/output.go b/internal/evmreader/output.go index 596e729b4..b45778da3 100644 --- a/internal/evmreader/output.go +++ b/internal/evmreader/output.go @@ -236,17 +236,37 @@ func (r *Service) readAndUpdateOutputs( } if !bytes.Equal(output.RawData, event.Output) { - // setApplicationDiverged always returns non-nil (the reason text itself). - // The DB error case is already logged inside setApplicationStatus. - // On DB success the app is marked inoperable and won't reappear next tick. - // On DB failure the app reappears as Enabled next tick, retrying this path. - _ = r.setApplicationDiverged(ctx, app.application, - "Output mismatch. Application is in an invalid state. Output Index %d, raw data %s != event data %s", + reasonFmt := + "Output mismatch. Application is in an invalid state. Output Index %d, raw data %s != event data %s" + args := []any{ output.Index, - "0x"+hex.EncodeToString(output.RawData), - "0x"+hex.EncodeToString(event.Output), - ) - return + "0x" + hex.EncodeToString(output.RawData), + "0x" + hex.EncodeToString(event.Output), + } + + switch { + case app.application.Status == ApplicationStatus_OK || + app.application.Status == ApplicationStatus_Failed: + _ = r.setApplicationDiverged(ctx, app.application, reasonFmt, args...) + if app.application.Status != ApplicationStatus_Diverged { + return // persistence failed; retry this event next tick + } + case app.application.Status.IsExecutionTerminal(): + _ = r.setApplicationCorrupted(ctx, app.application, reasonFmt, args...) + if app.application.Status != ApplicationStatus_Corrupted { + return // persistence failed; retry this event next tick + } + case app.application.Status == ApplicationStatus_Diverged || + app.application.Status == ApplicationStatus_Corrupted: + // The integrity failure is already durable. Do not retry an + // impossible status rewrite on every polling tick. + default: + r.Logger.Error("Output mismatch found for application with unknown status", + "application", app.application.Name, + "status", app.application.Status) + return + } + continue } r.Logger.Info("Output executed", diff --git a/internal/evmreader/output_test.go b/internal/evmreader/output_test.go index 15c175a2d..a9f81cd40 100644 --- a/internal/evmreader/output_test.go +++ b/internal/evmreader/output_test.go @@ -381,16 +381,148 @@ func (s *EvmReaderSuite) TestOutputExecutionMismatchMarksApplicationDiverged() { ApplicationStatus_Diverged, mock.Anything, ).Return(nil).Once() + s.repository.On("UpdateOutputsExecution", + mock.Anything, + foreclosedApp.IApplicationAddress.Hex(), + mock.MatchedBy(func(outputs []*Output) bool { return len(outputs) == 0 }), + uint64(0x13), + ).Return(nil).Once() s.evmReader.checkForOutputExecution(s.ctx, []appContracts{ {application: foreclosedApp, applicationContract: applicationContract}, }, 0x13) + s.Equal(ApplicationStatus_Diverged, foreclosedApp.Status) + s.repository.AssertNumberOfCalls(s.T(), "UpdateOutputsExecution", 1) + s.repository.AssertExpectations(s.T()) + applicationContract.AssertExpectations(s.T()) +} + +func (s *EvmReaderSuite) TestOutputExecutionMismatchEscalatesExecutionTerminalToCorrupted() { + s.repository = newMockRepository() + s.evmReader.repository = s.repository + applicationContract := newMockApplicationContract() + + app := copyApplications(applications)[0] + app.ID = 1 + app.Status = ApplicationStatus_MachineHalted + app.ForecloseBlock = 0x12 + app.LastOutputCheckBlock = 0x12 + + mismatchedOutput := &Output{ + Index: outputExecution0.OutputIndex, + RawData: common.Hex2Bytes("FFBBCCDDEE"), + } + applicationContract.On("GetNumberOfExecutedOutputs", blockFrom(0x13)). + Return(new(big.Int).SetUint64(1), nil) + applicationContract.On("RetrieveOutputExecutionEvents", + mock.MatchedBy(func(opts *bind.FilterOpts) bool { return opts.Start == 0x13 }), + ).Return([]*iapplication.IApplicationOutputExecuted{outputExecution0}, nil).Once() + s.repository.On("GetNumberOfExecutedOutputs", + mock.Anything, app.IApplicationAddress.String()).Return(uint64(0), nil).Once() + s.repository.On("GetOutput", + mock.Anything, app.IApplicationAddress.Hex(), outputExecution0.OutputIndex). + Return(mismatchedOutput, nil).Once() + s.repository.On("UpdateApplicationStatus", + mock.Anything, app.ID, ApplicationStatus_Corrupted, mock.Anything). + Return(nil).Once() + s.repository.On("UpdateOutputsExecution", + mock.Anything, + app.IApplicationAddress.Hex(), + mock.MatchedBy(func(outputs []*Output) bool { return len(outputs) == 0 }), + uint64(0x13), + ).Return(nil).Once() + + s.evmReader.checkForOutputExecution(s.ctx, []appContracts{ + {application: app, applicationContract: applicationContract}, + }, 0x13) + + s.Equal(ApplicationStatus_Corrupted, app.Status) + s.repository.AssertExpectations(s.T()) + applicationContract.AssertExpectations(s.T()) +} + +func (s *EvmReaderSuite) TestOutputExecutionMismatchRetriesWhenStatusWriteFails() { + s.repository = newMockRepository() + s.evmReader.repository = s.repository + applicationContract := newMockApplicationContract() + + app := copyApplications(applications)[0] + app.ID = 1 + app.Status = ApplicationStatus_OK + app.ForecloseBlock = 0x12 + app.LastOutputCheckBlock = 0x12 + mismatchedOutput := &Output{ + Index: outputExecution0.OutputIndex, + RawData: common.Hex2Bytes("FFBBCCDDEE"), + } + dbErr := errors.New("database unavailable") + + applicationContract.On("GetNumberOfExecutedOutputs", blockFrom(0x13)). + Return(new(big.Int).SetUint64(1), nil) + applicationContract.On("RetrieveOutputExecutionEvents", + mock.MatchedBy(func(opts *bind.FilterOpts) bool { return opts.Start == 0x13 }), + ).Return([]*iapplication.IApplicationOutputExecuted{outputExecution0}, nil).Once() + s.repository.On("GetNumberOfExecutedOutputs", + mock.Anything, app.IApplicationAddress.String()).Return(uint64(0), nil).Once() + s.repository.On("GetOutput", + mock.Anything, app.IApplicationAddress.Hex(), outputExecution0.OutputIndex). + Return(mismatchedOutput, nil).Once() + s.repository.On("UpdateApplicationStatus", + mock.Anything, app.ID, ApplicationStatus_Diverged, mock.Anything). + Return(dbErr).Once() + + s.evmReader.checkForOutputExecution(s.ctx, []appContracts{ + {application: app, applicationContract: applicationContract}, + }, 0x13) + + s.Equal(ApplicationStatus_OK, app.Status) s.repository.AssertNumberOfCalls(s.T(), "UpdateOutputsExecution", 0) s.repository.AssertExpectations(s.T()) applicationContract.AssertExpectations(s.T()) } +func (s *EvmReaderSuite) TestOutputExecutionMismatchDoesNotRewriteIntegrityTerminal() { + s.repository = newMockRepository() + s.evmReader.repository = s.repository + applicationContract := newMockApplicationContract() + + app := copyApplications(applications)[0] + app.ID = 1 + app.Status = ApplicationStatus_Corrupted + app.ForecloseBlock = 0x12 + app.LastOutputCheckBlock = 0x12 + + mismatchedOutput := &Output{ + Index: outputExecution0.OutputIndex, + RawData: common.Hex2Bytes("FFBBCCDDEE"), + } + applicationContract.On("GetNumberOfExecutedOutputs", blockFrom(0x13)). + Return(new(big.Int).SetUint64(1), nil) + applicationContract.On("RetrieveOutputExecutionEvents", + mock.MatchedBy(func(opts *bind.FilterOpts) bool { return opts.Start == 0x13 }), + ).Return([]*iapplication.IApplicationOutputExecuted{outputExecution0}, nil).Once() + s.repository.On("GetNumberOfExecutedOutputs", + mock.Anything, app.IApplicationAddress.String()).Return(uint64(0), nil).Once() + s.repository.On("GetOutput", + mock.Anything, app.IApplicationAddress.Hex(), outputExecution0.OutputIndex). + Return(mismatchedOutput, nil).Once() + s.repository.On("UpdateOutputsExecution", + mock.Anything, + app.IApplicationAddress.Hex(), + mock.MatchedBy(func(outputs []*Output) bool { return len(outputs) == 0 }), + uint64(0x13), + ).Return(nil).Once() + + s.evmReader.checkForOutputExecution(s.ctx, []appContracts{ + {application: app, applicationContract: applicationContract}, + }, 0x13) + + s.repository.AssertNumberOfCalls(s.T(), "UpdateApplicationStatus", 0) + s.repository.AssertExpectations(s.T()) + applicationContract.AssertExpectations(s.T()) +} + func (s *EvmReaderSuite) TestCheckOutputFailsWhenRetrieveOutputsFails() { s.setupOutputExecution() @@ -711,6 +843,12 @@ func (s *EvmReaderSuite) setupOutputMismatchTest() { ApplicationStatus_Diverged, mock.Anything, ).Return(nil).Once() + s.repository.On("UpdateOutputsExecution", + mock.Anything, + applications[0].IApplicationAddress.Hex(), + mock.MatchedBy(func(outputs []*Output) bool { return len(outputs) == 0 }), + uint64(0x11), + ).Return(nil).Once() s.applicationContract1.On("GetDeploymentBlockNumber", mock.Anything, @@ -763,7 +901,7 @@ func (s *EvmReaderSuite) TestCheckOutputFailsWhenOutputMismatches() { s.Require().True(waitNotification(called), "evmreader did not read new header") - s.repository.AssertNumberOfCalls(s.T(), "UpdateOutputsExecution", 0) + s.repository.AssertNumberOfCalls(s.T(), "UpdateOutputsExecution", 1) s.repository.AssertExpectations(s.T()) s.inputBox.AssertExpectations(s.T()) diff --git a/internal/evmreader/post_foreclosure_withdrawal.go b/internal/evmreader/post_foreclosure_withdrawal.go index f3919c536..a2972867d 100644 --- a/internal/evmreader/post_foreclosure_withdrawal.go +++ b/internal/evmreader/post_foreclosure_withdrawal.go @@ -8,12 +8,21 @@ import ( "errors" "fmt" "math/big" + "strings" . "github.com/cartesi/rollups-node/internal/model" "github.com/cartesi/rollups-node/pkg/ethutil" "github.com/ethereum/go-ethereum/accounts/abi/bind" ) +const withdrawalLedgerDivergenceReasonPrefix = "withdrawal_ledger_divergence:" + +func hasWithdrawalLedgerDivergence(app *Application) bool { + return app.Status == ApplicationStatus_Corrupted && + app.Reason != nil && + strings.HasPrefix(*app.Reason, withdrawalLedgerDivergenceReasonPrefix) +} + // checkForPostForeclosureWithdrawals runs once per evmreader tick for each // foreclosed app whose accounts drive has been proved. It performs a // FindTransitions search on the on-chain `getNumberOfWithdrawals()` counter @@ -33,11 +42,11 @@ func (r *Service) checkForPostForeclosureWithdrawals( app appContracts, mostRecentBlockNumber uint64, ) { - // A withdrawal-count divergence (detected below) is terminal state - // corruption: once it has marked the app CORRUPTED, stop scanning rather - // than re-deriving the same divergence every tick. The local withdrawal - // ledger can no longer be trusted, so there is nothing left to index. - if app.application.Status == ApplicationStatus_Corrupted { + // A withdrawal-ledger divergence (detected below) means this local ledger + // cannot be trusted. Stop re-deriving that specific failure every tick, but + // keep indexing when CORRUPTED came from another subsystem: an output or + // sealed-epoch disagreement says nothing about the withdrawal ledger. + if hasWithdrawalLedgerDivergence(app.application) { return } @@ -99,10 +108,20 @@ func (r *Service) checkForPostForeclosureWithdrawals( // deliberately do not advance the cursor or re-seed from the chain; // either would silently rewrite committed withdrawal history, which // is never acceptable for a ledger of fund movements. - // setApplicationCorrupted always returns non-nil (the reason text); - // the DB-error case is logged inside setApplicationStatus. + r.Logger.Error("Withdrawal ledger divergence detected", + "application", app.application.Name, + "address", app.application.IApplicationAddress, + "start_block", startBlock, + "end_block", mostRecentBlockNumber, + "error", err) + // setApplicationCorrupted always returns non-nil (the reason text), + // and logs any DB failure. An existing integrity-terminal status keeps + // its first cause, so this branch will deliberately detect and report + // the withdrawal divergence again on a later tick rather than relying + // on process-local suppression. _ = r.setApplicationCorrupted(ctx, app.application, - "withdrawal count divergence while scanning from block %d: %v", + withdrawalLedgerDivergenceReasonPrefix+ + " local withdrawal count exceeds chain while scanning from block %d: %v", startBlock, err) return } diff --git a/internal/evmreader/post_foreclosure_withdrawal_test.go b/internal/evmreader/post_foreclosure_withdrawal_test.go index ac72d8d46..f4ed37c04 100644 --- a/internal/evmreader/post_foreclosure_withdrawal_test.go +++ b/internal/evmreader/post_foreclosure_withdrawal_test.go @@ -4,8 +4,10 @@ package evmreader import ( + "bytes" "context" "errors" + "log/slog" "math/big" "testing" @@ -454,15 +456,17 @@ func TestCheckForWithdrawals_CountDivergenceMarksCorrupted(t *testing.T) { assert.Equal(t, ApplicationStatus_Corrupted, app.Status, "count divergence must mark the app CORRUPTED") + assert.True(t, hasWithdrawalLedgerDivergence(app), + "the durable reason must identify withdrawal-ledger corruption") assert.Equal(t, uint64(119), app.LastWithdrawalCheckBlock, "cursor must not advance on a detected divergence") } -// TestCheckForWithdrawals_SkipsAlreadyCorruptedApp verifies that once an app -// is CORRUPTED the withdrawal scan stops entirely: no RPC, no DB read, no -// persistence. This is what turns the previous infinite retry into a clean -// halt. The mock has no expectations — any call trips the test. -func TestCheckForWithdrawals_SkipsAlreadyCorruptedApp(t *testing.T) { +// TestCheckForWithdrawals_SkipsPersistedWithdrawalLedgerDivergence verifies +// that once this specific ledger failure is durable, the scan stops entirely: +// no RPC, no DB read, and no persistence. The mocks have no expectations, so +// any call trips the test. +func TestCheckForWithdrawals_SkipsPersistedWithdrawalLedgerDivergence(t *testing.T) { s, c, repo := newPostForeclosureFixture(t) defer c.AssertExpectations(t) defer repo.AssertExpectations(t) @@ -470,13 +474,82 @@ func TestCheckForWithdrawals_SkipsAlreadyCorruptedApp(t *testing.T) { app := postForeclosureWithdrawalApp(1, 100, 110) app.LastWithdrawalCheckBlock = 119 app.Status = ApplicationStatus_Corrupted + reason := withdrawalLedgerDivergenceReasonPrefix + " test fixture" + app.Reason = &reason const head = uint64(130) s.checkForPostForeclosureWithdrawals(context.Background(), appContracts{application: app, applicationContract: c}, head) assert.Equal(t, uint64(119), app.LastWithdrawalCheckBlock, - "a corrupted app's cursor must stay frozen") + "the diverged withdrawal ledger's cursor must stay frozen") +} + +// TestCheckForWithdrawals_ExistingIntegrityTerminalReportsLedgerDivergence +// verifies the honest fallback when the first terminal cause is immutable: +// preserve it, leave the cursor frozen, and report every detected fund-ledger +// divergence rather than hiding it behind a process-local latch. +func TestCheckForWithdrawals_ExistingIntegrityTerminalReportsLedgerDivergence(t *testing.T) { + for _, status := range []ApplicationStatus{ + ApplicationStatus_Diverged, + ApplicationStatus_Corrupted, + } { + t.Run(status.String(), func(t *testing.T) { + s, c, repo := newPostForeclosureFixture(t) + defer c.AssertExpectations(t) + defer repo.AssertExpectations(t) + + var logs bytes.Buffer + s.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + + app := postForeclosureWithdrawalApp(1, 100, 110) + app.LastWithdrawalCheckBlock = 119 + app.Status = status + originalReason := "earlier integrity failure" + app.Reason = &originalReason + + repo.On("GetNumberOfWithdrawals", mock.Anything, app.ID). + Return(uint64(5), nil).Once() + c.On("GetNumberOfWithdrawals", mock.Anything). + Return(big.NewInt(2), nil).Once() + + s.checkForPostForeclosureWithdrawals(context.Background(), + appContracts{application: app, applicationContract: c}, 130) + + assert.Contains(t, logs.String(), "Withdrawal ledger divergence detected") + assert.Equal(t, status, app.Status) + assert.Equal(t, originalReason, *app.Reason) + assert.Equal(t, uint64(119), app.LastWithdrawalCheckBlock) + repo.AssertNumberOfCalls(t, "UpdateApplicationStatus", 0) + }) + } +} + +// TestCheckForWithdrawals_OtherCorruptionCauseStillIndexes verifies that the +// shared CORRUPTED status does not suppress a healthy withdrawal ledger when +// another observer, such as output indexing, discovered the inconsistency. +func TestCheckForWithdrawals_OtherCorruptionCauseStillIndexes(t *testing.T) { + s, c, repo := newPostForeclosureFixture(t) + defer c.AssertExpectations(t) + defer repo.AssertExpectations(t) + + app := postForeclosureWithdrawalApp(1, 100, 110) + app.Status = ApplicationStatus_Corrupted + reason := "Output mismatch. Application is in an invalid state." + app.Reason = &reason + const head = uint64(130) + + c.On("GetNumberOfWithdrawals", mock.Anything).Return(big.NewInt(0), nil) + repo.On("StoreWithdrawalEvents", + mock.Anything, app.ID, mock.MatchedBy(func(ws []*Withdrawal) bool { + return len(ws) == 0 + }), head).Return(nil).Once() + + s.checkForPostForeclosureWithdrawals(context.Background(), + appContracts{application: app, applicationContract: c}, head) + + assert.Equal(t, head, app.LastWithdrawalCheckBlock, + "an unrelated corruption cause must not stop withdrawal observation") } // --------------------------------------------------------------------------- diff --git a/internal/evmreader/sealedepochs.go b/internal/evmreader/sealedepochs.go index 1a8b57618..62030327e 100644 --- a/internal/evmreader/sealedepochs.go +++ b/internal/evmreader/sealedepochs.go @@ -95,12 +95,24 @@ func (r *Service) scanDaveConsensusEpochsAndInputs( // Process each application individually since each has its own DaveConsensus contract for _, app := range applications { + if app.inputSource == nil { + // Alpha.6 permits data-availability encodings that this node does not + // support. A missing adapter is therefore a capability mismatch, not + // evidence that the application's persisted state is corrupted. + r.Logger.Error("Cannot scan DaveConsensus epochs: configured input source is unsupported", + "application", app.application.Name, + "address", app.application.IApplicationAddress, + "input_box", app.application.IInputBoxAddress, + "data_availability", app.application.DataAvailability, + ) + continue + } r.Logger.Debug("Processing DaveConsensus application", "application", app.application.Name, "consensus_address", app.application.IConsensusAddress) - sealedEpochEndBlock := foreclosureBoundedEndBlock(app.application, mostRecentBlockNumber) - err := r.processApplicationSealedEpochs(ctx, app, sealedEpochEndBlock) + observationEndBlock := foreclosureBoundedEndBlock(app.application, mostRecentBlockNumber) + err := r.processApplicationSealedEpochs(ctx, app, observationEndBlock) if err != nil { if errors.Is(err, context.Canceled) { return // shutting down @@ -112,11 +124,10 @@ func (r *Service) scanDaveConsensusEpochsAndInputs( continue } - if !app.application.CanExecute() { - continue - } - - err = r.processApplicationOpenEpoch(ctx, app, mostRecentBlockNumber) + // Open-epoch input ingestion is L1 observation, not machine execution. + // Keep it active for non-executing applications and bound it at the + // foreclosure block just like the sealed-epoch scan above. + err = r.processApplicationOpenEpoch(ctx, app, observationEndBlock) if err != nil { if errors.Is(err, context.Canceled) { return // shutting down diff --git a/internal/evmreader/sealedepochs_test.go b/internal/evmreader/sealedepochs_test.go index 4f45f799d..739f7661a 100644 --- a/internal/evmreader/sealedepochs_test.go +++ b/internal/evmreader/sealedepochs_test.go @@ -4,7 +4,9 @@ package evmreader import ( + "bytes" "context" + "log/slog" "math/big" "testing" @@ -196,7 +198,7 @@ func (s *SealedEpochsSuite) TestCatchUpForeclosedSealedEpochsAdvancesCursor() { ConsensusType: Consensus_PRT, ForecloseBlock: forecloseBlock, LastEpochCheckBlock: lastEpochCheckBlock, - LastInputCheckBlock: lastEpochCheckBlock, + LastInputCheckBlock: forecloseBlock, LastOutputCheckBlock: lastEpochCheckBlock, DataAvailability: DataAvailability_InputBox[:], }, @@ -237,7 +239,7 @@ func (s *SealedEpochsSuite) TestCatchUpForeclosedSealedEpochsAdvancesCursor() { s.dave.AssertExpectations(s.T()) } -func (s *SealedEpochsSuite) TestForeclosedDaveConsensusAppDoesNotProcessOpenEpoch() { +func (s *SealedEpochsSuite) TestTerminalDaveConsensusAppProcessesOpenEpochToForeclosure() { const forecloseBlock uint64 = 70 s.evmReader.inputReaderEnabled = true @@ -248,18 +250,83 @@ func (s *SealedEpochsSuite) TestForeclosedDaveConsensusAppDoesNotProcessOpenEpoc IApplicationAddress: app1Addr, IConsensusAddress: consensusAddr, ConsensusType: Consensus_PRT, - Status: ApplicationStatus_OK, + Status: ApplicationStatus_MachineHalted, ForecloseBlock: forecloseBlock, LastEpochCheckBlock: forecloseBlock, + LastInputCheckBlock: forecloseBlock - 1, DataAvailability: DataAvailability_InputBox[:], }, daveConsensus: s.dave, inputSource: s.inputBox, } + s.repository.On("GetLastNonOpenEpoch", + mock.Anything, app.application.IApplicationAddress.String()). + Return(&Epoch{ + Index: 2, + LastBlock: 50, + InputIndexUpperBound: 0, + }, nil).Once() + s.repository.On("GetEpoch", + mock.Anything, app.application.IApplicationAddress.Hex(), uint64(3)). + Return(nil, nil).Once() + s.repository.On("GetEventLastCheckBlock", + mock.Anything, app.application.ID, MonitoredEvent_InputAdded). + Return(forecloseBlock-1, nil).Once() + s.repository.On("GetNumberOfInputs", + mock.Anything, app.application.IApplicationAddress.String()). + Return(uint64(0), nil).Once() + s.inputBox.On("GetNumberOfInputs", mock.Anything, app.application.IApplicationAddress). + Return(big.NewInt(0), nil) + s.repository.On("CreateEpochsAndInputs", + mock.Anything, + app.application.IApplicationAddress.String(), + mock.MatchedBy(func(epochInputs map[*Epoch][]*Input) bool { + if len(epochInputs) != 1 { + return false + } + for epoch, inputs := range epochInputs { + return epoch.Status == EpochStatus_Open && + epoch.Index == 3 && + epoch.LastBlock == forecloseBlock && + len(inputs) == 0 + } + return false + }), + forecloseBlock, + ).Return(nil).Once() + s.evmReader.scanDaveConsensusEpochsAndInputs(s.ctx, []appContracts{app}, forecloseBlock+10) - s.repository.AssertNumberOfCalls(s.T(), "GetLastNonOpenEpoch", 0) - s.repository.AssertNumberOfCalls(s.T(), "CreateEpochsAndInputs", 0) + s.repository.AssertExpectations(s.T()) + s.inputBox.AssertExpectations(s.T()) s.dave.AssertNumberOfCalls(s.T(), "GetCurrentSealedEpoch", 0) } + +func (s *SealedEpochsSuite) TestDaveConsensusWithUnsupportedInputSourceDoesNotPanic() { + s.evmReader.inputReaderEnabled = true + var logs bytes.Buffer + s.evmReader.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + app := appContracts{ + application: &Application{ + ID: 1, + Name: "unsupported-input-source-prt-app", + IApplicationAddress: app1Addr, + ConsensusType: Consensus_PRT, + Status: ApplicationStatus_OK, + // Alpha.6's abandoned InputBoxAndEspresso experiment. The node + // intentionally supports only the official InputBox source. + DataAvailability: []byte{0x85, 0x79, 0xfd, 0x0c}, + }, + daveConsensus: s.dave, + } + + s.NotPanics(func() { + s.evmReader.scanDaveConsensusEpochsAndInputs( + s.ctx, []appContracts{app}, 100) + }) + s.Contains(logs.String(), "configured input source is unsupported") + s.Equal(ApplicationStatus_OK, app.application.Status) + s.dave.AssertNotCalled(s.T(), "GetCurrentSealedEpoch") + s.repository.AssertNumberOfCalls(s.T(), "UpdateApplicationStatus", 0) +} diff --git a/internal/model/application_lifecycle_test.go b/internal/model/application_lifecycle_test.go index d4f194692..27363c2ea 100644 --- a/internal/model/application_lifecycle_test.go +++ b/internal/model/application_lifecycle_test.go @@ -9,35 +9,25 @@ import ( "github.com/stretchr/testify/require" ) -func TestApplicationLifecycleHelpers(t *testing.T) { +func TestApplicationObservationHelper(t *testing.T) { app := &Application{Enabled: true, Status: ApplicationStatus_OK} - require.True(t, app.CanExecute()) require.True(t, app.NeedsL1Observation()) - require.True(t, app.NeedsForeclosureObservation()) - require.False(t, app.NeedsPostForeclosureObservation()) app.ForecloseBlock = 42 - require.False(t, app.CanExecute()) require.True(t, app.NeedsL1Observation()) - require.False(t, app.NeedsForeclosureObservation()) - require.True(t, app.NeedsPostForeclosureObservation()) app.ForecloseBlock = 0 app.Status = ApplicationStatus_Diverged - require.False(t, app.CanExecute()) require.True(t, app.NeedsL1Observation()) - require.True(t, app.NeedsForeclosureObservation()) app.Enabled = false - require.False(t, app.CanExecute()) require.False(t, app.NeedsL1Observation()) - require.False(t, app.NeedsForeclosureObservation()) } // TestForeclosureScanCaughtUp pins the single drain-readiness definition shared // by the claimer, PRT, and manager: it consults last_input_check_block for -// IConsensus apps and last_epoch_check_block for DaveConsensus apps, and the +// IConsensus apps and both epoch/input cursors for DaveConsensus apps, and the // foreclose_block boundary is inclusive (cursor == foreclose_block is caught up). func TestForeclosureScanCaughtUp(t *testing.T) { t.Run("IConsensus uses last_input_check_block", func(t *testing.T) { @@ -58,18 +48,22 @@ func TestForeclosureScanCaughtUp(t *testing.T) { require.False(t, app.ForeclosureScanCaughtUp(), "epoch cursor is ignored for IConsensus") }) - t.Run("DaveConsensus uses last_epoch_check_block", func(t *testing.T) { + t.Run("DaveConsensus requires both scan cursors", func(t *testing.T) { app := &Application{ConsensusType: Consensus_PRT, ForecloseBlock: 100} app.LastEpochCheckBlock = 99 + app.LastInputCheckBlock = 100 require.False(t, app.ForeclosureScanCaughtUp(), "below bound: not caught up") app.LastEpochCheckBlock = 100 + app.LastInputCheckBlock = 100 require.True(t, app.ForeclosureScanCaughtUp(), "at bound: caught up (inclusive)") - // The input cursor must not influence a DaveConsensus app. + app.LastInputCheckBlock = 99 + require.False(t, app.ForeclosureScanCaughtUp(), "open-epoch input scan is still behind") + app.LastEpochCheckBlock = 99 app.LastInputCheckBlock = 1000 - require.False(t, app.ForeclosureScanCaughtUp(), "input cursor is ignored for DaveConsensus") + require.False(t, app.ForeclosureScanCaughtUp(), "sealed-epoch scan is still behind") }) } diff --git a/internal/model/execution_parameters_test.go b/internal/model/execution_parameters_test.go index c0622bf9e..102c4a546 100644 --- a/internal/model/execution_parameters_test.go +++ b/internal/model/execution_parameters_test.go @@ -197,6 +197,13 @@ func TestApplicationStatusContract(t *testing.T) { value != ApplicationStatus_OK && value != ApplicationStatus_Failed, value.IsTerminal(), ) + require.Equal(t, + value == ApplicationStatus_GuestException || + value == ApplicationStatus_MachineHalted || + value == ApplicationStatus_McycleOverflow || + value == ApplicationStatus_UnexpectedYield, + value.IsExecutionTerminal(), + ) }) } } diff --git a/internal/model/models.go b/internal/model/models.go index ebe9ed03a..97d1beafc 100644 --- a/internal/model/models.go +++ b/internal/model/models.go @@ -61,22 +61,10 @@ func (a *Application) IsForeclosed() bool { return a.ForecloseBlock != 0 } -func (a *Application) CanExecute() bool { - return a.Enabled && a.Status == ApplicationStatus_OK && !a.IsForeclosed() -} - func (a *Application) NeedsL1Observation() bool { return a.Enabled } -func (a *Application) NeedsForeclosureObservation() bool { - return a.NeedsL1Observation() && !a.IsForeclosed() -} - -func (a *Application) NeedsPostForeclosureObservation() bool { - return a.NeedsL1Observation() && a.IsForeclosed() -} - // ForeclosureScanCaughtUp reports whether the historical L1 scan has reached // foreclose_block, so the pre-foreclosure drain queries — which read the // inputs/epochs already ingested into the DB — can be trusted. @@ -84,15 +72,18 @@ func (a *Application) NeedsPostForeclosureObservation() bool { // A freshly bootstrapped node can record foreclose_block before it has ingested // the historical inputs/epochs. Until the scan catches up the drain tables are // incomplete, and a "nothing left to drain" answer would be premature. Each -// consensus type advances a different cursor: DaveConsensus ingestion is driven -// by EpochSealed scans (last_epoch_check_block), while IConsensus ingestion is -// driven by InputAdded scans (last_input_check_block). This is the single -// definition of drain-readiness shared by the claimer, PRT, and manager. +// IConsensus ingestion is driven by InputAdded scans (last_input_check_block). +// DaveConsensus has two independent scans: sealed epochs advance +// last_epoch_check_block, while inputs in the current open epoch advance +// last_input_check_block. Both must reach the foreclosure boundary before its +// historical state is complete. This is the single definition of +// drain-readiness shared by the claimer, PRT, and manager. // // Only meaningful for a foreclosed app (foreclose_block != 0). func (a *Application) ForeclosureScanCaughtUp() bool { if a.IsDaveConsensus() { - return a.LastEpochCheckBlock >= a.ForecloseBlock + return a.LastEpochCheckBlock >= a.ForecloseBlock && + a.LastInputCheckBlock >= a.ForecloseBlock } return a.LastInputCheckBlock >= a.ForecloseBlock } @@ -397,6 +388,26 @@ func (e ApplicationStatus) IsTerminal() bool { } } +// IsExecutionTerminal reports whether machine execution ended deterministically +// and must not be retried. Unlike DIVERGED and CORRUPTED, these states may still +// escalate to CORRUPTED when later L1 observation disproves local history. +func (e ApplicationStatus) IsExecutionTerminal() bool { + switch e { + case ApplicationStatus_GuestException, + ApplicationStatus_MachineHalted, + ApplicationStatus_McycleOverflow, + ApplicationStatus_UnexpectedYield: + return true + case ApplicationStatus_OK, + ApplicationStatus_Failed, + ApplicationStatus_Diverged, + ApplicationStatus_Corrupted: + return false + default: + return false + } +} + func (e *ApplicationStatus) Scan(value any) error { var enumValue string switch val := value.(type) { diff --git a/internal/prt/handle_foreclosed_test.go b/internal/prt/handle_foreclosed_test.go index 0c6a9b5be..2be4a46ae 100644 --- a/internal/prt/handle_foreclosed_test.go +++ b/internal/prt/handle_foreclosed_test.go @@ -27,10 +27,11 @@ func prtForeclosedApp(id int64, block uint64) *model.Application { Status: model.ApplicationStatus_OK, ForecloseBlock: block, ForecloseTransaction: &txHash, - // LastEpochCheckBlock defaults to the foreclose block so callers - // who don't care about the bootstrap guard skip past it. Tests - // that exercise the guard override this field explicitly. + // Both DaveConsensus observation cursors default to the foreclose + // block so callers that don't care about the bootstrap guard skip it. + // Tests that exercise the guard override one cursor explicitly. LastEpochCheckBlock: block, + LastInputCheckBlock: block, } } @@ -137,7 +138,7 @@ func TestHandleForeclosedApp_SurfacesDrainCheckError(t *testing.T) { // drain gate would then see an empty input table and incorrectly return // false, making the app look drained before any pre-foreclosure epoch is // observed locally. The guard must defer the drain check until -// LastEpochCheckBlock >= ForecloseBlock. +// both DaveConsensus observation cursors reach ForecloseBlock. // // The mock has no HasUndrainedEpochsBeforeBlock or UpdateApplicationStatus // expectation registered; testify/mock panics on an unexpected call, so diff --git a/internal/repository/postgres/epoch.go b/internal/repository/postgres/epoch.go index 43159e328..6cb5da893 100644 --- a/internal/repository/postgres/epoch.go +++ b/internal/repository/postgres/epoch.go @@ -227,7 +227,9 @@ func (r *PostgresRepository) CreateEpochsAndInputs( } } - // Update last processed block + // Update the input cursor monotonically. Dave sealed-epoch catch-up may + // revisit an older block after open-epoch ingestion has already scanned + // farther ahead; that historical write must not rewind drain readiness. appUpdateStmt := table.Application. UPDATE( table.Application.LastInputCheckBlock, @@ -235,7 +237,9 @@ func (r *PostgresRepository) CreateEpochsAndInputs( SET( uint64Expr(blockNumber), ). - WHERE(whereClause) + WHERE(whereClause.AND( + table.Application.LastInputCheckBlock.LT(uint64Expr(blockNumber)), + )) sqlStr, args := appUpdateStmt.Sql() _, err = tx.Exec(ctx, sqlStr, args...) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 72351913f..d79c1618d 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,10 +59,7 @@ type ApplicationFilter struct { } // ExecutableApplicationsFilter selects apps that may run normal machine work. -// -// This is the repository-side equivalent of Application.CanExecute. Keep this -// helper shared because manager and validator both need the exact same -// database predicate before they create machines or compute claims. +// The manager uses this exact database predicate before creating machines. func ExecutableApplicationsFilter() ApplicationFilter { return ApplicationFilter{ Enabled: new(true), diff --git a/internal/repository/repotest/epoch_test_cases.go b/internal/repository/repotest/epoch_test_cases.go index 05d74679f..3bb013dc7 100644 --- a/internal/repository/repotest/epoch_test_cases.go +++ b/internal/repository/repotest/epoch_test_cases.go @@ -114,6 +114,31 @@ func (s *EpochSuite) TestCreateEpochsAndInputs() { s.Require().NoError(err) s.Equal(EpochStatus_Closed, got.Status) }) + + s.Run("HistoricalCatchUpDoesNotRewindInputCursor", func() { + const ( + currentBlock uint64 = 100 + historicalBlock uint64 = 90 + ) + app := NewApplicationBuilder().Create(s.Ctx, s.T(), s.Repo) + epoch := NewEpochBuilder(app.ID). + WithIndex(0).WithStatus(EpochStatus_Closed).WithBlocks(0, currentBlock-1).Build() + + err := s.Repo.CreateEpochsAndInputs( + s.Ctx, app.IApplicationAddress.String(), + map[*Epoch][]*Input{epoch: {}}, currentBlock) + s.Require().NoError(err) + + err = s.Repo.CreateEpochsAndInputs( + s.Ctx, app.IApplicationAddress.String(), + map[*Epoch][]*Input{epoch: {}}, historicalBlock) + s.Require().NoError(err) + + block, err := s.Repo.GetEventLastCheckBlock( + s.Ctx, app.ID, MonitoredEvent_InputAdded) + s.Require().NoError(err) + s.Equal(currentBlock, block) + }) } func (s *EpochSuite) TestGetEpoch() { diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index b3a1e1a18..9db68b22e 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -34,6 +34,11 @@ var ( ) func expectCorrupted(app *Application, reasonSubstring string) { + // Subtests intentionally reuse application fixtures. Reset health so each + // case independently exercises its own corruption transition rather than + // inheriting the previous subtest's now-idempotent terminal status. + app.Status = ApplicationStatus_OK + app.Reason = nil repo.On("UpdateApplicationStatus", mock.Anything, app.ID, ApplicationStatus_Corrupted, mock.MatchedBy(func(reason *string) bool { From 47565da4568886677716101eea86aea69d1ffe78 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:44:52 -0300 Subject: [PATCH 08/11] feat(inspect): sanitize terminal machine outcomes --- api/openapi/inspect.yaml | 7 ++- internal/inspect/hardening_test.go | 4 +- internal/inspect/inspect.go | 42 ++++++++++++++++- internal/inspect/inspect_test.go | 73 ++++++++++++++++++++++++++++-- 4 files changed, 118 insertions(+), 8 deletions(-) diff --git a/api/openapi/inspect.yaml b/api/openapi/inspect.yaml index 2a88cf557..07c83e201 100644 --- a/api/openapi/inspect.yaml +++ b/api/openapi/inspect.yaml @@ -60,8 +60,9 @@ paths: This can happen when the application is registered but its Cartesi Machine instance has not been initialized yet, when the application has been foreclosed and its machine is no longer - available for live inspect requests, or when the application's - inspect capacity is exhausted. + available for live inspect requests, when application execution + has terminated, or when the application's inspect capacity is + exhausted. content: text/plain: schema: @@ -71,6 +72,8 @@ paths: value: Machine not ready foreclosed: value: Application was foreclosed; machine unavailable + terminal: + value: Application is terminal; inspect unavailable atCapacity: value: Application inspect at capacity diff --git a/internal/inspect/hardening_test.go b/internal/inspect/hardening_test.go index fa6991609..87fe42c09 100644 --- a/internal/inspect/hardening_test.go +++ b/internal/inspect/hardening_test.go @@ -103,8 +103,8 @@ func (m *erroringMachine) Advance(ctx context.Context, input []byte, a, b uint64 } func (m *erroringMachine) Application() *Application { return m.inner.Application() } func (m *erroringMachine) ProcessedInputs() uint64 { return m.inner.ProcessedInputs() } -func (m *erroringMachine) OutputsProof(ctx context.Context) (*OutputsProof, error) { - return m.inner.OutputsProof(ctx) +func (m *erroringMachine) StateProof(ctx context.Context) (*StateProof, error) { + return m.inner.StateProof(ctx) } func (m *erroringMachine) CreateSnapshot(ctx context.Context, processedInputs uint64, path string) error { return m.inner.CreateSnapshot(ctx, processedInputs, path) diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go index 54c3e4c2e..d93a9b2fd 100644 --- a/internal/inspect/inspect.go +++ b/internal/inspect/inspect.go @@ -42,6 +42,7 @@ var ( ErrNoApp = errors.New("no application") ErrMachineNotReady = errors.New("machine not ready for application") ErrForeclosedAppNoMachine = errors.New("application was foreclosed; machine unavailable") + ErrTerminalAppNoInspect = errors.New("application is terminal; inspect unavailable") ) type IInspectMachines interface { @@ -204,6 +205,12 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { app, machine, resolveErr := inspect.resolveApp(r.Context(), dapp) if resolveErr != nil { + if errors.Is(resolveErr, ErrTerminalAppNoInspect) { + inspect.Logger.Info("Terminal application inspect unavailable", + "application", dapp, "err", resolveErr) + http.Error(w, "Application is terminal; inspect unavailable", http.StatusServiceUnavailable) + return + } if errors.Is(resolveErr, ErrMachineNotReady) { inspect.Logger.Warn("Machine not ready", "application", dapp, "err", resolveErr) http.Error(w, "Machine not ready", http.StatusServiceUnavailable) @@ -233,11 +240,17 @@ func (inspect *Inspector) ServeHTTP(w http.ResponseWriter, r *http.Request) { result, err := machine.Inspect(ctx, payload) if err != nil { - if errors.Is(err, manager.ErrInspectAtCapacity) { + switch { + case errors.Is(err, manager.ErrInspectAtCapacity): inspect.Logger.Info("Application inspect at capacity", "application", dapp) http.Error(w, "Application inspect at capacity", http.StatusServiceUnavailable) return + case errors.Is(err, manager.ErrMachineClosed): + inspect.Logger.Info("Application machine unavailable", + "application", dapp) + http.Error(w, "Machine not ready", http.StatusServiceUnavailable) + return } service.WriteInternalError(ctx, w, inspect.Logger, fmt.Errorf("inspect processing failed: %w", err)) @@ -312,6 +325,25 @@ func (inspect *Inspector) buildInspectResponse( "application", dapp, "request_id", requestID, ) + case pkgmachine.CompletionStatusOverflow: + // The current public inspect schema predates the machine overflow + // condition. Preserve its sanitized failure contract until that API is + // versioned to expose overflow directly. + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + inspect.Logger.Debug("Machine reached mcycle overflow while inspecting", + "application", dapp, + "request_id", requestID, + ) + case pkgmachine.CompletionStatusUnexpectedYield: + // As with overflow, keep the existing public inspect contract stable and + // expose the new outcome only through trusted logs for now. + response.Status = inspectStatusFailed + response.Error = inspectFailureMessage + inspect.Logger.Debug("Machine returned an unexpected manual yield while inspecting", + "application", dapp, + "request_id", requestID, + ) case pkgmachine.CompletionStatusUnknown: response.Status = inspectStatusFailed response.Error = inspectFailureMessage @@ -359,6 +391,14 @@ func (inspect *Inspector) resolveApp( } return nil, nil, fmt.Errorf("%w %s", ErrNoApp, nameOrAddress) } + if app.Status.IsTerminal() { + return nil, nil, fmt.Errorf( + "%w: application %s has status %s", + ErrTerminalAppNoInspect, + nameOrAddress, + app.Status, + ) + } machine, exists := inspect.GetMachine(app.ID) if !exists { if app.IsForeclosed() { diff --git a/internal/inspect/inspect_test.go b/internal/inspect/inspect_test.go index 9abb0f0a5..98164a91d 100644 --- a/internal/inspect/inspect_test.go +++ b/internal/inspect/inspect_test.go @@ -153,6 +153,49 @@ func (s *InspectSuite) TestPostForeclosedMachineUnavailable() { s.Contains(string(body), "Application was foreclosed; machine unavailable") } +func (s *InspectSuite) TestPostTerminalApplicationUnavailable() { + for _, status := range ApplicationStatusAllValues { + if !status.IsTerminal() { + continue + } + s.Run(status.String(), func() { + inspect, app, _ := s.setup() + app.Status = status + request := httptest.NewRequest( + http.MethodPost, + "/inspect/"+app.Name, + bytes.NewBufferString("query"), + ) + request.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + + inspect.ServeHTTP(recorder, request) + + s.Equal(http.StatusServiceUnavailable, recorder.Code) + s.Contains(recorder.Body.String(), "Application is terminal; inspect unavailable") + }) + } +} + +func (s *InspectSuite) TestPostMachineClosedDuringInspectIsUnavailable() { + inspect, app, _ := s.setup() + machine := inspect.IInspectMachines.(*MachinesMock).Map[app.ID] + machine.inspectError = manager.ErrMachineClosed + inspect.IInspectMachines.(*MachinesMock).Map[app.ID] = machine + request := httptest.NewRequest( + http.MethodPost, + "/inspect/"+app.Name, + bytes.NewBufferString("query"), + ) + request.SetPathValue("dapp", app.Name) + recorder := httptest.NewRecorder() + + inspect.ServeHTTP(recorder, request) + + s.Equal(http.StatusServiceUnavailable, recorder.Code) + s.Contains(recorder.Body.String(), "Machine not ready") +} + func (s *InspectSuite) TestPostMaxPayloadSize() { inspect, app, _ := s.setup() @@ -244,6 +287,26 @@ func (s *InspectSuite) TestPostResponseMatchesGeneratedClientContract() { }, wantStatus: inspectclient.MachineHalted, }, + { + name: "overflow", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusOverflow, + Reports: [][]byte{{0xab, 0xcd}}, + ProcessedInputs: 52, + }, + wantStatus: inspectclient.Failed, + wantError: inspectFailureMessage, + }, + { + name: "unexpected yield", + result: manager.InspectResult{ + Status: pkgmachine.CompletionStatusUnexpectedYield, + Reports: [][]byte{{0xab, 0xce}}, + ProcessedInputs: 53, + }, + wantStatus: inspectclient.Failed, + wantError: inspectFailureMessage, + }, { name: "failed", result: manager.InspectResult{ @@ -253,7 +316,7 @@ func (s *InspectSuite) TestPostResponseMatchesGeneratedClientContract() { Error: errors.New("backend disconnected"), }, wantStatus: inspectclient.Failed, - wantError: "The node could not complete the inspection", + wantError: inspectFailureMessage, }, } @@ -317,7 +380,7 @@ func (s *InspectSuite) TestCycleLimitIsSanitizedFailedResultWithoutApplicationFa s.Require().NoError(json.NewDecoder(recorder.Body).Decode(&got)) s.Equal(inspectclient.Failed, got.Status) s.Require().NotNil(got.Error) - s.Equal("The node could not complete the inspection", *got.Error) + s.Equal(inspectFailureMessage, *got.Error) s.NotContains(*got.Error, "absolute_mcycle") s.NotContains(*got.Error, "configured_cap") s.Contains(logs.String(), "absolute_mcycle=123456") @@ -415,12 +478,16 @@ func (mock *MachinesMock) GetMachine(appId int64) (manager.MachineInstance, bool type MockMachine struct { application *Application inspectResult *manager.InspectResult + inspectError error } func (mock *MockMachine) Inspect( _ context.Context, query []byte, ) (*manager.InspectResult, error) { + if mock.inspectError != nil { + return nil, mock.inspectError + } if mock.inspectResult != nil { result := *mock.inspectResult return &result, nil @@ -457,7 +524,7 @@ func (mock *MockMachine) ProcessedInputs() uint64 { return 0 } -func (m *MockMachine) OutputsProof(ctx context.Context) (*OutputsProof, error) { +func (mock *MockMachine) StateProof(_ context.Context) (*StateProof, error) { return nil, nil } From ffa6a09efb0efbc296747923fb1ab531a63e9e22 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:47:29 -0300 Subject: [PATCH 09/11] feat(jsonrpc): expose terminal states and proof leaves --- .../execution_outcome_contract_test.go | 44 ++++++++++++- internal/jsonrpc/jsonrpc-discover.json | 62 +++++++++++++++++-- internal/jsonrpc/jsonrpc_test.go | 10 +-- internal/jsonrpc/util_test.go | 47 +++++++++++++- 4 files changed, 150 insertions(+), 13 deletions(-) diff --git a/internal/jsonrpc/execution_outcome_contract_test.go b/internal/jsonrpc/execution_outcome_contract_test.go index 811da54be..5ca2b2a97 100644 --- a/internal/jsonrpc/execution_outcome_contract_test.go +++ b/internal/jsonrpc/execution_outcome_contract_test.go @@ -25,13 +25,55 @@ func TestDiscoverySchemaExecutionOutcomeContract(t *testing.T) { Enum []string `json:"enum"` } require.NoError(t, json.Unmarshal(spec.Components.Schemas["InputCompletionStatus"], &completionStatus)) - require.Equal(t, []string{"NONE", "ACCEPTED", "REJECTED", "EXCEPTION", "MACHINE_HALTED"}, completionStatus.Enum) + require.Equal(t, []string{ + "NONE", + "ACCEPTED", + "REJECTED", + "EXCEPTION", + "MACHINE_HALTED", + "OVERFLOW", + "UNEXPECTED_YIELD", + }, completionStatus.Enum) + + var applicationStatus struct { + Enum []string `json:"enum"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["ApplicationStatus"], &applicationStatus)) + require.Equal(t, []string{ + "OK", + "FAILED", + "DIVERGED", + "CORRUPTED", + "GUEST_EXCEPTION", + "MACHINE_HALTED", + "MCYCLE_OVERFLOW", + "UNEXPECTED_YIELD", + }, applicationStatus.Enum) var input struct { Properties map[string]json.RawMessage `json:"properties"` } require.NoError(t, json.Unmarshal(spec.Components.Schemas["Input"], &input)) require.Contains(t, input.Properties, "exception_data") + require.Contains(t, input.Properties, "tx_buffer_data_block") + require.NotContains(t, input.Properties, "outputs_hash") + + var epoch struct { + Properties map[string]json.RawMessage `json:"properties"` + } + require.NoError(t, json.Unmarshal(spec.Components.Schemas["Epoch"], &epoch)) + for _, field := range []string{ + "tx_buffer_data_block", + "tx_buffer_proof", + "iflags_y_data_block", + "iflags_y_proof", + "htif_tohost_data_block", + "htif_tohost_proof", + } { + require.Contains(t, epoch.Properties, field) + } + require.NotContains(t, epoch.Properties, "outputs_merkle_root") + require.NotContains(t, epoch.Properties, "outputs_merkle_proof") var executionParameters struct { Properties map[string]json.RawMessage `json:"properties"` diff --git a/internal/jsonrpc/jsonrpc-discover.json b/internal/jsonrpc/jsonrpc-discover.json index 5d72106e2..f304b7d17 100644 --- a/internal/jsonrpc/jsonrpc-discover.json +++ b/internal/jsonrpc/jsonrpc-discover.json @@ -1827,7 +1827,7 @@ "string", "null" ], - "description": "Human-readable failure description. Non-null when status is FAILED, DIVERGED, or CORRUPTED; null otherwise. Foreclosure is reported separately via foreclose_block, not via status." + "description": "Human-readable terminal or failure description. Null only while status is OK. Foreclosure is reported separately via foreclose_block, not via status." }, "iinputbox_block": { "$ref": "#/components/schemas/UnsignedInteger" @@ -1955,7 +1955,8 @@ } ] }, - "outputs_merkle_root": { + "tx_buffer_data_block": { + "description": "The 32-byte CMIO TX-buffer memory block in the proved machine state.", "oneOf": [ { "$ref": "#/components/schemas/Hash" @@ -1965,7 +1966,50 @@ } ] }, - "outputs_merkle_proof": { + "tx_buffer_proof": { + "description": "Merkle siblings proving the TX-buffer data block against machine_hash.", + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/Hash" + } + }, + "iflags_y_data_block": { + "description": "The 32-byte memory block containing the machine iflags.Y register.", + "oneOf": [ + { + "$ref": "#/components/schemas/Hash" + }, + { + "type": "null" + } + ] + }, + "iflags_y_proof": { + "description": "Merkle siblings proving the iflags.Y data block against machine_hash.", + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/components/schemas/Hash" + } + }, + "htif_tohost_data_block": { + "description": "The 32-byte memory block containing the machine HTIF tohost register.", + "oneOf": [ + { + "$ref": "#/components/schemas/Hash" + }, + { + "type": "null" + } + ] + }, + "htif_tohost_proof": { + "description": "Merkle siblings proving the HTIF tohost data block against machine_hash.", "type": [ "array", "null" @@ -2068,7 +2112,9 @@ "ACCEPTED", "REJECTED", "EXCEPTION", - "MACHINE_HALTED" + "MACHINE_HALTED", + "OVERFLOW", + "UNEXPECTED_YIELD" ] }, "Input": { @@ -2120,7 +2166,7 @@ } ] }, - "outputs_hash": { + "tx_buffer_data_block": { "oneOf": [ { "$ref": "#/components/schemas/Hash" @@ -2596,7 +2642,11 @@ "OK", "FAILED", "DIVERGED", - "CORRUPTED" + "CORRUPTED", + "GUEST_EXCEPTION", + "MACHINE_HALTED", + "MCYCLE_OVERFLOW", + "UNEXPECTED_YIELD" ] }, "Consensus": { diff --git a/internal/jsonrpc/jsonrpc_test.go b/internal/jsonrpc/jsonrpc_test.go index 0f65c1836..626883048 100644 --- a/internal/jsonrpc/jsonrpc_test.go +++ b/internal/jsonrpc/jsonrpc_test.go @@ -947,7 +947,7 @@ func TestMethod(t *testing.T) { appID := s.newTestApplication(ctx, t, app) epoch := repotest.NewEpochBuilder(appID). WithIndex(0). - WithStatus(model.EpochStatus_ClaimAccepted). + WithStatus(model.EpochStatus_Closed). Build() inputs := []*model.Input{ repotest.NewInputBuilder().WithIndex(0).WithRawData(emptyInput()).Build(), @@ -1515,9 +1515,9 @@ func TestMethod(t *testing.T) { nr := uint64(1) appID := s.newTestApplication(ctx, t, nr) for i, status := range []model.EpochStatus{ - model.EpochStatus_Open, - model.EpochStatus_Closed, model.EpochStatus_ClaimAccepted, + model.EpochStatus_Closed, + model.EpochStatus_Open, } { s.createTestEpoch(ctx, t, numberToName(nr), repotest.NewEpochBuilder(appID). @@ -1540,8 +1540,8 @@ func TestMethod(t *testing.T) { assert.Nil(t, json.Unmarshal(body, &resp)) assert.Nil(t, resp.Error) assert.Len(t, resp.Result.Data, 2) - assert.Equal(t, model.EpochStatus_Open, resp.Result.Data[0].Status) - assert.Equal(t, model.EpochStatus_Closed, resp.Result.Data[1].Status) + assert.Equal(t, model.EpochStatus_Closed, resp.Result.Data[0].Status) + assert.Equal(t, model.EpochStatus_Open, resp.Result.Data[1].Status) }) // success: many epochs is in the database -> limit diff --git a/internal/jsonrpc/util_test.go b/internal/jsonrpc/util_test.go index aadfbe8f5..d2f73cd07 100644 --- a/internal/jsonrpc/util_test.go +++ b/internal/jsonrpc/util_test.go @@ -172,9 +172,16 @@ func emptyVoucher() []byte { // createTestEpoch creates an epoch using production CreateEpochsAndInputs. func (s *Service) createTestEpoch(ctx context.Context, t *testing.T, appName string, epoch *model.Epoch) { t.Helper() + targetStatus := epoch.Status + if requiresPublishedProof(targetStatus) { + epoch.Status = model.EpochStatus_Closed + } err := s.repository.CreateEpochsAndInputs(ctx, appName, map[*model.Epoch][]*model.Input{epoch: {}}, 10) require.NoError(t, err) + if epoch.Status != targetStatus { + repotest.AdvanceEpochStatus(ctx, t, s.repository, appName, epoch, targetStatus) + } } // createTestEpochWithInput creates an epoch with one input using production CreateEpochsAndInputs. @@ -183,9 +190,47 @@ func (s *Service) createTestEpochWithInput( epoch *model.Epoch, input *model.Input, ) { t.Helper() + // Inputs and outputs must be stored before the final state proof is + // published. These JSON-RPC fixtures do not depend on the epoch's claim + // status, so keep it CLOSED while advanceInput populates its contents. + if requiresPublishedProof(epoch.Status) { + epoch.Status = model.EpochStatus_Closed + } + inputs := make([]*model.Input, 0, input.Index+1) + for index := uint64(0); index < input.Index; index++ { + inputs = append(inputs, repotest.NewInputBuilder(). + WithIndex(index). + WithEpochIndex(epoch.Index). + Build()) + } + inputs = append(inputs, input) err := s.repository.CreateEpochsAndInputs(ctx, appName, - map[*model.Epoch][]*model.Input{epoch: {input}}, 10) + map[*model.Epoch][]*model.Input{epoch: inputs}, 10) require.NoError(t, err) + for index := uint64(0); index < input.Index; index++ { + repotest.StoreAdvanceResult( + ctx, t, s.repository, epoch.ApplicationID, epoch.Index, index, + model.InputCompletionStatus_Accepted, nil, nil, + ) + } +} + +func requiresPublishedProof(status model.EpochStatus) bool { + switch status { + case model.EpochStatus_InputsProcessed, + model.EpochStatus_ClaimComputed, + model.EpochStatus_ClaimSubmitted, + model.EpochStatus_ClaimStaged, + model.EpochStatus_ClaimAccepted, + model.EpochStatus_ClaimRejected: + return true + case model.EpochStatus_Open, + model.EpochStatus_Closed, + model.EpochStatus_ClaimForeclosed: + return false + default: + return false + } } // advanceInput stores an advance result (outputs/reports) for an input using production StoreAdvanceResult. From 1d28a0d3fc00148ba2a4d14c583ab95c6533ec71 Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:08:36 -0300 Subject: [PATCH 10/11] test(integration): cover durable terminal machine states --- Makefile | 27 +- test/compose/compose.integration.yaml | 12 + test/integration/reject_exception_test.go | 2 +- .../terminal_machine_states_test.go | 247 +++++++++++++++ test/tooling/terminalmachine/main.go | 282 ++++++++++++++++++ 5 files changed, 567 insertions(+), 3 deletions(-) create mode 100644 test/integration/terminal_machine_states_test.go create mode 100644 test/tooling/terminalmachine/main.go diff --git a/Makefile b/Makefile index 452a3ba79..4e5ba22df 100644 --- a/Makefile +++ b/Makefile @@ -350,6 +350,12 @@ reject-loop-dapp: applications/reject-loop-dapp ## Reject loop dapp exception-loop-dapp: applications/exception-loop-dapp ## Exception loop dapp +halt-loop-dapp: applications/halt-loop-dapp ## Halt loop dapp + +mcycle-overflow-dapp: applications/mcycle-overflow-dapp ## MCYCLE overflow dapp + +unexpected-yield-dapp: applications/unexpected-yield-dapp ## Unexpected-yield dapp + erc20-withdrawal-dapp: applications/erc20-withdrawal-dapp ## ERC-20 withdrawal test dapp applications/reject-loop-dapp: ## Create reject-loop-dapp test application @@ -362,6 +368,19 @@ applications/exception-loop-dapp: ## Create exception-loop-dapp test application @mkdir -p applications @cartesi-machine --ram-length=128Mi --store=applications/exception-loop-dapp --final-hash -- ioctl-echo-loop --vouchers=1 --notices=1 --reports=1 --exception=1 --verbose=1 +applications/halt-loop-dapp: ## Create halt-loop-dapp test application + @echo "Creating halt-loop-dapp test application" + @mkdir -p applications + @cartesi-machine --ram-length=128Mi --store=applications/halt-loop-dapp --final-hash -- "rollup accept && rollup accept" + +applications/mcycle-overflow-dapp: applications/echo-dapp ## Create MCYCLE overflow test application + @echo "Creating mcycle-overflow-dapp test application" + @go run $(GO_BUILD_PARAMS) ./test/tooling/terminalmachine mcycle-overflow --source=$< --output=$@ + +applications/unexpected-yield-dapp: ## Create unexpected-yield test application + @echo "Creating unexpected-yield-dapp test application" + @go run $(GO_BUILD_PARAMS) ./test/tooling/terminalmachine unexpected-yield --output=$@ + applications/erc20-withdrawal-dapp: test/dapps/erc20-withdrawal/install.sh ## Create ERC-20 withdrawal test application @echo "Creating ERC-20 withdrawal test application" @mkdir -p applications @@ -587,7 +606,7 @@ check-license: ## Verify license headers on Go source files # dependency for the check to build on the CI setup runner. INTEGRATION_SHARDS := basic quorum prt replay restart withdrawal awskms -INTEGRATION_SHARD_basic := ^Test(EchoAuthority|RejectException|MultiApp|EchoAuthorityStaging)$$ +INTEGRATION_SHARD_basic := ^Test(EchoAuthority|RejectException|TerminalMachineStates|MultiApp|EchoAuthorityStaging)$$ INTEGRATION_SHARD_quorum := ^Test(EchoQuorum|SameBlockInputs)$$ INTEGRATION_SHARD_prt := ^Test(EchoPrt|RejectExceptionPrt|ForeclosePrt)$$ INTEGRATION_SHARD_replay := ^Test(Foreclose|ForecloseReplay|DivergentClaim)$$ @@ -723,7 +742,7 @@ test-with-compose: ## Run all tests using docker compose with auto-shutdown @$(MAKE) unit-test-with-compose @$(MAKE) integration-test-with-compose -integration-test-local: build cartesi-rollups-machine-tool echo-dapp reject-loop-dapp exception-loop-dapp erc20-withdrawal-dapp ## Run integration tests on the host (NODE_TOPOLOGY=, SHARD=; requires: make start && eval $$(make env); CLEAN_STALE_LOCAL_NODE=true to stop test-port listeners) +integration-test-local: build cartesi-rollups-machine-tool echo-dapp reject-loop-dapp exception-loop-dapp halt-loop-dapp mcycle-overflow-dapp unexpected-yield-dapp erc20-withdrawal-dapp ## Run integration tests on the host (NODE_TOPOLOGY=, SHARD=; requires: make start && eval $$(make env); CLEAN_STALE_LOCAL_NODE=true to stop test-port listeners) @set -e; first=1; for t in $(TOPOLOGIES_SELECTED); do \ if [ "$$first" = 1 ]; then first=0; else echo "=== resetting dev DB + devnet between topologies ==="; $(MAKE) restart; fi; \ $(MAKE) _local-topology-$$t; \ @@ -759,6 +778,9 @@ _local-topology-%: export CARTESI_TEST_DAPP_PATH=$(CURDIR)/applications/echo-dapp; \ export CARTESI_TEST_REJECT_DAPP_PATH=$(CURDIR)/applications/reject-loop-dapp; \ export CARTESI_TEST_EXCEPTION_DAPP_PATH=$(CURDIR)/applications/exception-loop-dapp; \ + export CARTESI_TEST_HALT_DAPP_PATH=$(CURDIR)/applications/halt-loop-dapp; \ + export CARTESI_TEST_MCYCLE_OVERFLOW_DAPP_PATH=$(CURDIR)/applications/mcycle-overflow-dapp; \ + export CARTESI_TEST_UNEXPECTED_YIELD_DAPP_PATH=$(CURDIR)/applications/unexpected-yield-dapp; \ export CARTESI_TEST_ERC20_WITHDRAWAL_DAPP_PATH=$(CURDIR)/applications/erc20-withdrawal-dapp; \ NODE_TOPOLOGY='$*' TEST_PATTERN="$$pattern" $(MAKE) integration-test @@ -828,5 +850,6 @@ build-debian-package: install devnet image tester-image debian-packager run-with-compose shutdown-compose \ start start-devnet start-postgres stop stop-devnet stop-postgres restart restart-devnet restart-postgres \ install copy-debian-package build-debian-package \ + mcycle-overflow-dapp unexpected-yield-dapp \ deploy-erc20-withdrawal-dapp fund-wallet withdraw-wallet \ env help version diff --git a/test/compose/compose.integration.yaml b/test/compose/compose.integration.yaml index 20e846e03..5054be4c6 100644 --- a/test/compose/compose.integration.yaml +++ b/test/compose/compose.integration.yaml @@ -67,12 +67,21 @@ services: make reject-loop-dapp echo "Building exception-loop-dapp machine snapshot..." make exception-loop-dapp + echo "Building halt-loop-dapp machine snapshot..." + make halt-loop-dapp + echo "Building mcycle-overflow-dapp machine snapshot..." + make mcycle-overflow-dapp + echo "Building unexpected-yield-dapp machine snapshot..." + make unexpected-yield-dapp echo "Building erc20-withdrawal-dapp machine snapshot..." make erc20-withdrawal-dapp echo "Copying to shared volume..." cp -r applications/echo-dapp /dapps/echo-dapp cp -r applications/reject-loop-dapp /dapps/reject-loop-dapp cp -r applications/exception-loop-dapp /dapps/exception-loop-dapp + cp -r applications/halt-loop-dapp /dapps/halt-loop-dapp + cp -r applications/mcycle-overflow-dapp /dapps/mcycle-overflow-dapp + cp -r applications/unexpected-yield-dapp /dapps/unexpected-yield-dapp cp -r applications/erc20-withdrawal-dapp /dapps/erc20-withdrawal-dapp echo "DApp images built successfully." ' @@ -133,6 +142,9 @@ services: CARTESI_TEST_DAPP_PATH: /var/lib/cartesi-rollups-node/dapps/echo-dapp CARTESI_TEST_REJECT_DAPP_PATH: /var/lib/cartesi-rollups-node/dapps/reject-loop-dapp CARTESI_TEST_EXCEPTION_DAPP_PATH: /var/lib/cartesi-rollups-node/dapps/exception-loop-dapp + CARTESI_TEST_HALT_DAPP_PATH: /var/lib/cartesi-rollups-node/dapps/halt-loop-dapp + CARTESI_TEST_MCYCLE_OVERFLOW_DAPP_PATH: /var/lib/cartesi-rollups-node/dapps/mcycle-overflow-dapp + CARTESI_TEST_UNEXPECTED_YIELD_DAPP_PATH: /var/lib/cartesi-rollups-node/dapps/unexpected-yield-dapp CARTESI_TEST_ERC20_WITHDRAWAL_DAPP_PATH: /var/lib/cartesi-rollups-node/dapps/erc20-withdrawal-dapp CARTESI_TEST_NODE_LOG_FILE: /var/lib/cartesi-rollups-node/logs/node.log CARTESI_INSPECT_URL: http://localhost:10012/ diff --git a/test/integration/reject_exception_test.go b/test/integration/reject_exception_test.go index f0e39221c..4d094fba5 100644 --- a/test/integration/reject_exception_test.go +++ b/test/integration/reject_exception_test.go @@ -18,7 +18,7 @@ import ( var terminalExecutionExpectedLog = ExpectedLog{ Pattern: regexp.MustCompile(`Application execution terminated`), Level: LevelError, - Reason: "the guest exception durably terminates application execution", + Reason: "a durable terminal machine outcome stops application execution", Required: true, } diff --git a/test/integration/terminal_machine_states_test.go b/test/integration/terminal_machine_states_test.go new file mode 100644 index 000000000..46fc4c21a --- /dev/null +++ b/test/integration/terminal_machine_states_test.go @@ -0,0 +1,247 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +//go:build endtoendtests + +package integration + +import ( + "context" + "fmt" + "regexp" + "testing" + "time" + + "github.com/cartesi/rollups-node/internal/jsonrpc/api" + "github.com/cartesi/rollups-node/internal/model" + jsonrpcclient "github.com/cartesi/rollups-node/pkg/jsonrpc/client" + "github.com/stretchr/testify/suite" +) + +type TerminalMachineStatesSuite struct { + suite.Suite + LogChecker + ctx context.Context + cancel context.CancelFunc + appName string +} + +type terminalMachineStateCase struct { + namePrefix string + dappPathEnv string + defaultDappPath string + payloadPrefix string + description string + terminalInput uint64 + inputStatus model.InputCompletionStatus + applicationStatus model.ApplicationStatus +} + +const ( + terminalObservationWindow = 6 * time.Second + terminalObservationInterval = 500 * time.Millisecond + terminalObservationRPCTimeout = 2 * time.Second +) + +var terminalMachineRestartExpectedLog = ExpectedLog{ + Pattern: regexp.MustCompile(`service=(?:claimer|evm-reader).*context canceled`), + Level: LevelError, + Reason: "benign service cancellation while deliberately restarting the node", +} + +func TestTerminalMachineStates(t *testing.T) { + if !isNodeSelfManaged() { + t.Skip("skipping: durable terminal-state test requires a test-managed node restart") + } + suite.Run(t, new(TerminalMachineStatesSuite)) +} + +func (s *TerminalMachineStatesSuite) SetupSuite() { + s.ctx, s.cancel = context.WithTimeout(context.Background(), 12*time.Minute) +} + +func (s *TerminalMachineStatesSuite) TearDownSuite() { + // Restore the shared node if the test failed between stop and restart. + if sharedNode == nil { + s.T().Log("Restarting shared node for subsequent tests...") + startSharedNode(s.T()) + } + s.cancel() +} + +func (s *TerminalMachineStatesSuite) SetupTest() { + s.StartLogCapture() + s.appName = "" +} + +func (s *TerminalMachineStatesSuite) TearDownTest() { + if s.appName != "" { + s.T().Logf("Disabling application %s", s.appName) + if err := disableApplication(s.ctx, s.appName); err != nil { + s.T().Errorf("failed to disable application %s: %v", s.appName, err) + } + } + s.CheckLogs(s.T()) +} + +func (s *TerminalMachineStatesSuite) TestMachineHaltSurvivesRestart() { + s.runTerminalMachineState(terminalMachineStateCase{ + namePrefix: "halt-loop", + dappPathEnv: "CARTESI_TEST_HALT_DAPP_PATH", + defaultDappPath: "applications/halt-loop-dapp", + payloadPrefix: "halt", + description: "a guest that accepts input 0 and exits while handling input 1", + terminalInput: 1, + inputStatus: model.InputCompletionStatus_MachineHalted, + applicationStatus: model.ApplicationStatus_MachineHalted, + }) +} + +func (s *TerminalMachineStatesSuite) TestMcycleOverflowSurvivesRestart() { + s.runTerminalMachineState(terminalMachineStateCase{ + namePrefix: "mcycle-overflow", + dappPathEnv: "CARTESI_TEST_MCYCLE_OVERFLOW_DAPP_PATH", + defaultDappPath: "applications/mcycle-overflow-dapp", + payloadPrefix: "overflow", + description: "an accepted-yield machine with mcycle near UINT64_MAX", + terminalInput: 0, + inputStatus: model.InputCompletionStatus_Overflow, + applicationStatus: model.ApplicationStatus_McycleOverflow, + }) +} + +func (s *TerminalMachineStatesSuite) TestUnexpectedYieldSurvivesRestart() { + s.runTerminalMachineState(terminalMachineStateCase{ + namePrefix: "unexpected-yield", + dappPathEnv: "CARTESI_TEST_UNEXPECTED_YIELD_DAPP_PATH", + defaultDappPath: "applications/unexpected-yield-dapp", + payloadPrefix: "unexpected-yield", + description: "a guest that issues unsupported manual yield reason 9", + terminalInput: 0, + inputStatus: model.InputCompletionStatus_UnexpectedYield, + applicationStatus: model.ApplicationStatus_UnexpectedYield, + }) +} + +func (s *TerminalMachineStatesSuite) runTerminalMachineState(tc terminalMachineStateCase) { + s.T().Helper() + s.SetExpectedLogs(s.T(), terminalExecutionExpectedLog, terminalMachineRestartExpectedLog) + require := s.Require() + s.appName = uniqueAppName(tc.namePrefix) + dappPath := envOrDefault(tc.dappPathEnv, tc.defaultDappPath) + + s.T().Logf("Deploying %s...", tc.description) + _, err := deployApplication(s.ctx, s.appName, dappPath, "--salt", uniqueSalt()) + require.NoError(err, "deploy %s", tc.defaultDappPath) + + for index := range tc.terminalInput + 1 { + got, _, sendErr := sendInput(s.ctx, s.appName, + fmt.Sprintf("%s-payload-%d", tc.payloadPrefix, index)) + require.NoError(sendErr, "send input %d", index) + require.Equal(index, got, "input index mismatch") + } + + processCtx, processCancel := context.WithTimeout(s.ctx, inputProcessingTimeout) + defer processCancel() + for index := range tc.terminalInput { + accepted, waitErr := waitForInputProcessed(processCtx, s.T(), s.appName, index) + require.NoError(waitErr, "wait for accepted input %d", index) + require.Equal(model.InputCompletionStatus_Accepted, accepted.Status) + } + + terminal, err := waitForInputProcessed(processCtx, s.T(), s.appName, tc.terminalInput) + require.NoError(err, "wait for terminal input") + require.Equal(tc.inputStatus, terminal.Status) + + rpc := jsonrpcclient.NewClient( + envOrDefault("CARTESI_JSONRPC_API_URL", "http://localhost:10011/rpc"), + ) + app := s.getApplication(rpc) + require.Equal(tc.applicationStatus, app.Status) + require.NotNil(app.Reason) + require.Equal(fmt.Sprintf("input %d completed with %s", tc.terminalInput, tc.inputStatus), *app.Reason) + + var epochResponse api.SingleResponse[*model.Epoch] + err = rpc.Call(s.ctx, "cartesi_getEpoch", api.GetEpochParams{ + Application: s.appName, + EpochIndex: fmt.Sprintf("0x%x", terminal.EpochIndex), + }, &epochResponse) + require.NoError(err, "read terminal epoch through JSON-RPC") + require.NotNil(epochResponse.Data) + require.True(epochResponse.Data.HasCompleteStateProof(), + "terminal epoch must expose all three machine-state proof leaves") + + outputs, err := readOutputs(s.ctx, s.appName) + require.NoError(err, "read outputs") + require.Zero(outputs.Pagination.TotalCount, "terminal fixture must not emit outputs") + reports, err := readReports(s.ctx, s.appName) + require.NoError(err, "read reports") + require.Zero(reports.Pagination.TotalCount, "terminal fixture must not emit reports") + + s.T().Logf("Restarting the node after durable %s...", tc.inputStatus) + stopSharedNode(s.T()) + startSharedNode(s.T()) + + app = s.getApplication(rpc) + require.Equal(tc.applicationStatus, app.Status) + + inputIndex, _, err := sendInput(s.ctx, s.appName, "post-terminal-payload") + require.NoError(err, "send input after restart") + require.Equal(tc.terminalInput+1, inputIndex) + + indexCtx, indexCancel := context.WithTimeout(s.ctx, inputProcessingTimeout) + pending, err := waitForInputIndexed(indexCtx, s.T(), s.appName, inputIndex) + indexCancel() + require.NoError(err, "wait for post-halt input indexing") + require.Equal(model.InputCompletionStatus_None, pending.Status) + + s.requireInputRemainsPending(rpc, inputIndex) + + app = s.getApplication(rpc) + require.Equal(tc.applicationStatus, app.Status) + s.T().Logf("The %s app remained observable, but execution did not restart", tc.inputStatus) +} + +func (s *TerminalMachineStatesSuite) requireInputRemainsPending( + rpc *jsonrpcclient.Client, + inputIndex uint64, +) { + s.T().Helper() + require := s.Require() + timer := time.NewTimer(terminalObservationWindow) + defer timer.Stop() + ticker := time.NewTicker(terminalObservationInterval) + defer ticker.Stop() + + for { + select { + case <-timer.C: + return + case <-ticker.C: + callCtx, cancel := context.WithTimeout(s.ctx, terminalObservationRPCTimeout) + var response api.SingleResponse[*model.Input] + err := rpc.Call(callCtx, "cartesi_getInput", api.GetInputParams{ + Application: s.appName, + InputIndex: fmt.Sprintf("0x%x", inputIndex), + }, &response) + cancel() + require.NoError(err, "observe post-terminal input") + require.NotNil(response.Data, "post-terminal input disappeared") + require.Equal(model.InputCompletionStatus_None, response.Data.Status, + "a restarted node must not execute inputs after a durable terminal outcome") + } + } +} + +func (s *TerminalMachineStatesSuite) getApplication( + rpc *jsonrpcclient.Client, +) *model.Application { + s.T().Helper() + var response api.SingleResponse[*model.Application] + err := rpc.Call(s.ctx, "cartesi_getApplication", api.GetApplicationParams{ + Application: s.appName, + }, &response) + s.Require().NoError(err, "read application through JSON-RPC") + s.Require().NotNil(response.Data) + return response.Data +} diff --git a/test/tooling/terminalmachine/main.go b/test/tooling/terminalmachine/main.go new file mode 100644 index 000000000..8da8a3231 --- /dev/null +++ b/test/tooling/terminalmachine/main.go @@ -0,0 +1,282 @@ +// (c) Cartesi and individual authors (see AUTHORS) +// SPDX-License-Identifier: Apache-2.0 (see LICENSE) + +// Command terminalmachine creates machine snapshots that deterministically +// reach terminal execution outcomes during integration tests. +package main + +import ( + "encoding/binary" + "encoding/json" + "errors" + "flag" + "fmt" + "math" + "os" + "path/filepath" + "time" + + "github.com/cartesi/rollups-node/pkg/emulator" +) + +const ( + minimumArgumentCount = 2 + ramStart = uint64(0x80000000) + unexpectedYieldReason = uint64(9) + unexpectedYieldData = uint64(23) + htifYieldDevice = uint64(2) + mcycleOverflowHeadroom = uint64(255) + unexpectedYieldRAMSize = uint64(4096) + snapshotDirectoryMode = os.FileMode(0o755) + storedRootHashOffset = int64(0x60) + + machineTimeout = 5 * time.Minute +) + +var unexpectedYieldProgram = []uint32{ + 0x400082b7, // lui t0,0x40008: load the HTIF base address + 0x0002b423, // sd zero,8(t0): clear fromhost + 0x0132b023, // sd x19,0(t0): write the prepared request to tohost +} + +func main() { + if len(os.Args) < minimumArgumentCount { + fatalf("usage: terminalmachine [options]") + } + + var err error + switch os.Args[1] { + case "mcycle-overflow": + err = runMcycleOverflow(os.Args[2:]) + case "unexpected-yield": + err = runUnexpectedYield(os.Args[2:]) + default: + err = fmt.Errorf("unknown fixture %q", os.Args[1]) + } + if err != nil { + fatalf("%v", err) + } +} + +func runMcycleOverflow(args []string) error { + flags := flag.NewFlagSet("mcycle-overflow", flag.ContinueOnError) + source := flags.String("source", "", "accepted-yield machine snapshot to clone") + output := flags.String("output", "", "snapshot output directory") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 0 || *source == "" || *output == "" { + return errors.New("mcycle-overflow requires --source and --output") + } + if err := requireAbsent(*output); err != nil { + return err + } + + machine, _, _, err := emulator.SpawnServer("127.0.0.1:0", machineTimeout) + if err != nil { + return fmt.Errorf("spawn emulator server: %w", err) + } + defer machine.Delete() + defer func() { _ = machine.ShutdownServer() }() + + if err := machine.Load(*source, ""); err != nil { + return fmt.Errorf("load source snapshot: %w", err) + } + if err := requireAcceptedYield(&machine.Machine); err != nil { + return fmt.Errorf("source snapshot: %w", err) + } + + // The emulator saturates the per-input cycle limit at UINT64_MAX. Starting + // this close to the boundary makes normal advance-state delivery exercise + // CM_BREAK_REASON_MCYCLE_OVERFLOW without changing the guest program. + if err := machine.WriteReg( + emulator.REG_MCYCLE, math.MaxUint64-mcycleOverflowHeadroom, + ); err != nil { + return fmt.Errorf("set mcycle: %w", err) + } + if err := store(&machine.Machine, *output); err != nil { + return err + } + if err := sendAdvanceAndRun(&machine.Machine, emulator.BreakReasonMcycleOverflow); err != nil { + _ = os.RemoveAll(*output) + return fmt.Errorf("validate stored fixture: %w", err) + } + return nil +} + +func runUnexpectedYield(args []string) error { + flags := flag.NewFlagSet("unexpected-yield", flag.ContinueOnError) + output := flags.String("output", "", "snapshot output directory") + if err := flags.Parse(args); err != nil { + return err + } + if flags.NArg() != 0 || *output == "" { + return errors.New("unexpected-yield requires --output") + } + if err := requireAbsent(*output); err != nil { + return err + } + + config, err := json.Marshal(map[string]any{ + "processor": map[string]any{ + "registers": map[string]any{"pc": ramStart}, + }, + "ram": map[string]any{"length": unexpectedYieldRAMSize}, + "cmio": map[string]any{ + "rx_buffer": map[string]any{}, + "tx_buffer": map[string]any{}, + }, + }) + if err != nil { + return fmt.Errorf("encode machine config: %w", err) + } + + machine, err := emulator.CreateMachine(string(config), "", "") + if err != nil { + return fmt.Errorf("create machine: %w", err) + } + defer machine.Delete() + defer func() { _ = machine.Destroy() }() + + program := make([]byte, 4*len(unexpectedYieldProgram)) + for i, instruction := range unexpectedYieldProgram { + binary.LittleEndian.PutUint32(program[4*i:], instruction) + } + if err := machine.WriteMemory(ramStart, program); err != nil { + return fmt.Errorf("write guest program: %w", err) + } + + request := htifYieldDevice<<56 | + uint64(emulator.YieldManual)<<48 | + unexpectedYieldReason<<32 | + unexpectedYieldData + registers := []struct { + id emulator.RegID + value uint64 + }{ + {emulator.REG_X19, request}, + {emulator.REG_IFLAGS_Y, 1}, + {emulator.REG_HTIF_TOHOST_DEV, htifYieldDevice}, + {emulator.REG_HTIF_TOHOST_CMD, uint64(emulator.YieldManual)}, + {emulator.REG_HTIF_TOHOST_REASON, uint64(emulator.ManualYieldReasonAccepted)}, + {emulator.REG_HTIF_TOHOST_DATA, 0}, + } + for _, register := range registers { + if err := machine.WriteReg(register.id, register.value); err != nil { + return fmt.Errorf("initialize register %d: %w", register.id, err) + } + } + if err := requireAcceptedYield(machine); err != nil { + return fmt.Errorf("generated snapshot: %w", err) + } + if err := store(machine, *output); err != nil { + return err + } + if err := sendAdvanceAndRun(machine, emulator.BreakReasonYieldedManually); err != nil { + _ = os.RemoveAll(*output) + return fmt.Errorf("validate stored fixture: %w", err) + } + command, reason, _, err := machine.ReceiveCmioRequest() + if err != nil { + _ = os.RemoveAll(*output) + return fmt.Errorf("read terminal CMIO request: %w", err) + } + if command != uint8(emulator.YieldManual) || reason != uint16(unexpectedYieldReason) { + _ = os.RemoveAll(*output) + return fmt.Errorf("expected manual yield reason %d, got command %d reason %d", + unexpectedYieldReason, command, reason) + } + return nil +} + +func sendAdvanceAndRun(machine *emulator.Machine, expected emulator.BreakReason) error { + revertRootHash, err := machine.GetRootHash() + if err != nil { + return fmt.Errorf("read accepted-state root hash: %w", err) + } + if err := machine.SendCmioResponse( + uint16(emulator.YieldReasonAdvanceState), nil, &revertRootHash, + ); err != nil { + return fmt.Errorf("send advance-state response: %w", err) + } + result, err := machine.Run(math.MaxUint64) + if err != nil { + return fmt.Errorf("run machine: %w", err) + } + if result != expected { + return fmt.Errorf("expected %s, got %s", expected, result) + } + return nil +} + +func requireAcceptedYield(machine *emulator.Machine) error { + command, reason, _, err := machine.ReceiveCmioRequest() + if err != nil { + return fmt.Errorf("read initial CMIO request: %w", err) + } + if command != uint8(emulator.YieldManual) || + reason != uint16(emulator.ManualYieldReasonAccepted) { + return fmt.Errorf("expected manual accepted yield, got command %d reason %d", command, reason) + } + return nil +} + +func requireAbsent(path string) error { + _, err := os.Stat(path) + switch { + case err == nil: + return fmt.Errorf("output %q already exists", path) + case !errors.Is(err, os.ErrNotExist): + return fmt.Errorf("inspect output %q: %w", path, err) + } + return nil +} + +func store(machine *emulator.Machine, output string) error { + if err := os.MkdirAll(filepath.Dir(output), snapshotDirectoryMode); err != nil { + return fmt.Errorf("create output parent: %w", err) + } + + // Force the emulator to materialize dirty Merkle nodes before storing. + // The deploy CLI reads the cached root at hash_tree.sht offset 0x60, while + // the advancer asks the emulator for the live root after loading. Comparing + // both representations here prevents a fixture from registering one hash + // and loading as another. + expectedRootHash, err := machine.GetRootHash() + if err != nil { + return fmt.Errorf("calculate snapshot root hash: %w", err) + } + if err := machine.Store(output); err != nil { + _ = os.RemoveAll(output) + return fmt.Errorf("store snapshot: %w", err) + } + storedRootHash, err := readStoredRootHash(output) + if err != nil { + _ = os.RemoveAll(output) + return err + } + if storedRootHash != expectedRootHash { + _ = os.RemoveAll(output) + return fmt.Errorf("stored root hash %x does not match emulator root %x", + storedRootHash, expectedRootHash) + } + return nil +} + +func readStoredRootHash(output string) (emulator.Hash, error) { + var rootHash emulator.Hash + file, err := os.Open(filepath.Join(output, "hash_tree.sht")) + if err != nil { + return rootHash, fmt.Errorf("open stored hash tree: %w", err) + } + defer file.Close() + if _, err := file.ReadAt(rootHash[:], storedRootHashOffset); err != nil { + return rootHash, fmt.Errorf("read stored root hash: %w", err) + } + return rootHash, nil +} + +func fatalf(format string, args ...any) { + fmt.Fprintf(os.Stderr, "terminalmachine: "+format+"\n", args...) + os.Exit(1) +} From fad13fb01c78e529c284ce6de615116aaa4fc91f Mon Sep 17 00:00:00 2001 From: Victor Fusco <1221933+vfusco@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:15:21 -0300 Subject: [PATCH 11/11] fix(prt): handle replayed tournament creation Ignore an exact tournament-key replay when shutdown leaves the first insert with an uncertain result. Other constraint conflicts remain errors. Suppress nested PRT error logs only for context cancellation while the service is stopping. --- internal/prt/prt.go | 30 ++++++++------ internal/prt/service.go | 12 ++++++ internal/prt/validation_test.go | 39 +++++++++++++++++++ internal/repository/postgres/tournament.go | 14 ++++++- internal/repository/repository.go | 3 ++ .../repotest/tournament_test_cases.go | 34 ++++++++++++++++ 6 files changed, 119 insertions(+), 13 deletions(-) diff --git a/internal/prt/prt.go b/internal/prt/prt.go index bf2dd5c66..a4b72abea 100644 --- a/internal/prt/prt.go +++ b/internal/prt/prt.go @@ -320,8 +320,9 @@ func (s *Service) createTournament( err = s.repository.CreateTournament(ctx, app.IApplicationAddress.Hex(), t) if err != nil { - s.Logger.Error("failed to create tournament in database", "level", level, "application", app.Name, - "epoch", epoch.Index, "tournament_address", tournamentAddress.String(), "error", err) + s.logErrorUnlessShutdown("failed to create tournament in database", err, + "level", level, "application", app.Name, + "epoch", epoch.Index, "tournament_address", tournamentAddress.String()) return nil, err } return t, nil @@ -413,8 +414,9 @@ func (s *Service) checkEpochs(ctx context.Context, app *Application, mostRecentB if epoch.ClaimTransactionHash == nil { // epoch not claimed on-chain yet err = s.fetchTournamentData(ctx, app, epoch, RootLevel, nil, nil, *epoch.TournamentAddress, mostRecentBlock) if err != nil { - s.Logger.Error("failed to fetch root tournament data", "application", app.Name, - "epoch", epoch.Index, "tournament", epoch.TournamentAddress.String(), "error", err) + s.logErrorUnlessShutdown("failed to fetch root tournament data", err, + "application", app.Name, "epoch", epoch.Index, + "tournament", epoch.TournamentAddress.String()) return err } // if this epoch is not claimed on-chain yet, all other epochs with higher index should not be claimed either, so we can @@ -460,8 +462,9 @@ func (s *Service) checkEpochs(ctx context.Context, app *Application, mostRecentB err = s.fetchTournamentData(ctx, app, epoch, RootLevel, nil, nil, *epoch.TournamentAddress, mostRecentBlock) if err != nil { - s.Logger.Error("failed to fetch tournament data", "application", app.Name, - "epoch", epoch.Index, "tournament", epoch.TournamentAddress.String(), "error", err) + s.logErrorUnlessShutdown("failed to fetch tournament data", err, + "application", app.Name, "epoch", epoch.Index, + "tournament", epoch.TournamentAddress.String()) return err } @@ -512,8 +515,9 @@ func (s *Service) fetchTournamentData( t, err = s.createTournament(ctx, app, epoch, level, parentMatchIDHash, parentTournamentAddress, tournamentAddress) if err != nil { - s.Logger.Error("failed to create new tournament", "level", level, "application", app.Name, - "epoch", epoch.Index, "tournament_address", tournamentAddress.String(), "error", err) + s.logErrorUnlessShutdown("failed to create new tournament", err, + "level", level, "application", app.Name, + "epoch", epoch.Index, "tournament_address", tournamentAddress.String()) return err } } else if t.FinishedAtBlock == 0 { @@ -599,8 +603,9 @@ func (s *Service) fetchTournamentData( err = s.fetchTournamentData(ctx, app, epoch, nextLevel, i.ParentMatchIDHash, &tournamentAddress, i.Address, mostRecentBlock) if err != nil { - s.Logger.Error("failed to fetch tournament data", "level", nextLevel, "application", app.Name, - "tournament", i.Address.String(), "error", err) + s.logErrorUnlessShutdown("failed to fetch tournament data", err, + "level", nextLevel, "application", app.Name, + "tournament", i.Address.String()) return err } } @@ -613,8 +618,9 @@ func (s *Service) fetchTournamentData( err = s.fetchTournamentData(ctx, app, epoch, nextLevel, &hashID, &tournamentAddress, childAddress, mostRecentBlock) if err != nil { - s.Logger.Error("failed to fetch tournament data", "level", nextLevel, "application", app.Name, - "tournament", childAddress.String(), "error", err) + s.logErrorUnlessShutdown("failed to fetch tournament data", err, + "level", nextLevel, "application", app.Name, + "tournament", childAddress.String()) return err } } diff --git a/internal/prt/service.go b/internal/prt/service.go index 3279dee9c..05899fd31 100644 --- a/internal/prt/service.go +++ b/internal/prt/service.go @@ -131,6 +131,18 @@ func (s *Service) Alive() bool { return true } func (s *Service) Ready() bool { return true } func (s *Service) Reload() []error { return nil } +// logErrorUnlessShutdown keeps an in-flight shutdown cancellation from being +// reported as an operational failure. DeadlineExceeded and cancellations while +// the service is running remain errors. +func (s *Service) logErrorUnlessShutdown(message string, err error, args ...any) { + if s.IsStopping() && errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + return + } + args = append(args, "error", err) + s.Logger.Error(message, args...) +} + // Tick executes the Validator main logic of producing claims and/or proofs // for processed epochs of all running applications. func (s *Service) Tick() []error { diff --git a/internal/prt/validation_test.go b/internal/prt/validation_test.go index d04c60e44..41bead25b 100644 --- a/internal/prt/validation_test.go +++ b/internal/prt/validation_test.go @@ -4,10 +4,13 @@ package prt import ( + "bytes" "context" + "errors" "io" "log/slog" "math/big" + "strings" "testing" "time" @@ -22,6 +25,42 @@ import ( "github.com/stretchr/testify/require" ) +func TestLogErrorUnlessShutdown(t *testing.T) { + tests := []struct { + name string + stopping bool + err error + wantError bool + }{ + {name: "ShutdownCancellation", stopping: true, err: context.Canceled, wantError: false}, + {name: "ShutdownDeadline", stopping: true, err: context.DeadlineExceeded, wantError: true}, + { + name: "ShutdownCancellationWithDeadline", + stopping: true, + err: errors.Join(context.Canceled, context.DeadlineExceeded), + wantError: true, + }, + {name: "RuntimeCancellation", stopping: false, err: context.Canceled, wantError: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var output bytes.Buffer + s := &Service{Service: service.Service{ + Logger: slog.New(slog.NewTextHandler(&output, nil)), + }} + if test.stopping { + s.SetStopping() + } + + s.logErrorUnlessShutdown("operation failed", test.err, "operation", "test") + + hasError := strings.Contains(output.String(), "level=ERROR") + require.Equal(t, test.wantError, hasError, output.String()) + }) + } +} + func TestTrySettleOperationDeadlineDoesNotCancelServiceContext(t *testing.T) { s, app := newValidationService(t) ctx, cancel := context.WithTimeout(s.Context, 50*time.Millisecond) diff --git a/internal/repository/postgres/tournament.go b/internal/repository/postgres/tournament.go index 0ca494a35..7281dad18 100644 --- a/internal/repository/postgres/tournament.go +++ b/internal/repository/postgres/tournament.go @@ -77,7 +77,19 @@ func (r *PostgresRepository) CreateTournament( whereClause, ) - sqlStr, args := insertStmt.QUERY(selectQuery).Sql() + // Tournament addresses come from the chain and may be observed again after + // an interrupted shutdown. Ignore only an exact replay of the tournament + // identity; other conflicts, such as a different root for the same epoch, + // must still surface as errors. + sqlStr, args := insertStmt. + QUERY(selectQuery). + ON_CONFLICT( + table.Tournaments.ApplicationID, + table.Tournaments.EpochIndex, + table.Tournaments.Address, + ). + DO_NOTHING(). + Sql() _, err := r.db.Exec(ctx, sqlStr, args...) return err diff --git a/internal/repository/repository.go b/internal/repository/repository.go index d79c1618d..af649162f 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -303,6 +303,9 @@ type StateHashRepository interface { } type TournamentRepository interface { + // CreateTournament is idempotent only for an exact + // (application_id, epoch_index, address) replay. Other constraint conflicts + // remain errors. CreateTournament(ctx context.Context, nameOrAddress string, t *Tournament) error UpdateTournament(ctx context.Context, nameOrAddress string, t *Tournament) error GetTournament(ctx context.Context, nameOrAddress string, address string) (*Tournament, error) diff --git a/internal/repository/repotest/tournament_test_cases.go b/internal/repository/repotest/tournament_test_cases.go index dd74f490a..8dcddcfbb 100644 --- a/internal/repository/repotest/tournament_test_cases.go +++ b/internal/repository/repotest/tournament_test_cases.go @@ -30,6 +30,40 @@ func (s *TournamentSuite) TestCreateTournament() { s.Ctx, seed.App.IApplicationAddress.String(), tournament) s.Require().NoError(err) }) + + s.Run("ExactReplayIsIdempotent", func() { + seed := s.seedWithEpoch() + first := NewTournamentBuilder(seed.App.ID). + WithEpochIndex(0).Build() + s.Require().NoError(s.Repo.CreateTournament( + s.Ctx, seed.App.IApplicationAddress.String(), first)) + + replay := *first + replay.MaxLevel++ + err := s.Repo.CreateTournament( + s.Ctx, seed.App.IApplicationAddress.String(), &replay) + s.Require().NoError(err) + + got, err := s.Repo.GetTournament( + s.Ctx, seed.App.IApplicationAddress.String(), first.Address.String()) + s.Require().NoError(err) + s.Require().NotNil(got) + s.Equal(first.MaxLevel, got.MaxLevel, "an exact replay must not overwrite the first observation") + }) + + s.Run("DifferentRootForSameEpochIsAnError", func() { + seed := s.seedWithEpoch() + first := NewTournamentBuilder(seed.App.ID). + WithEpochIndex(0).Build() + s.Require().NoError(s.Repo.CreateTournament( + s.Ctx, seed.App.IApplicationAddress.String(), first)) + + conflicting := NewTournamentBuilder(seed.App.ID). + WithEpochIndex(0).Build() + err := s.Repo.CreateTournament( + s.Ctx, seed.App.IApplicationAddress.String(), conflicting) + s.Require().Error(err, "only the exact tournament identity may be replayed") + }) } func (s *TournamentSuite) TestGetTournament() {