diff --git a/cmd/analyze_rawtx_binary_size/main.go b/cmd/analyze_rawtx_binary_size/main.go index d854881..5d55018 100644 --- a/cmd/analyze_rawtx_binary_size/main.go +++ b/cmd/analyze_rawtx_binary_size/main.go @@ -138,15 +138,20 @@ func addInnerInstructions(stats *sizeStats, values []pump_parser.InnerInstructio func addInstructions(stats *sizeStats, prefix string, values []pump_parser.Instruction) { stats.add(prefix+".count", 4) for _, value := range values { - stats.add(prefix+".program_id_index", 2) + stats.add(prefix+".program_id_index", 1) stats.add(prefix+".accounts.count", 4) - stats.add(prefix+".accounts.refs", uint64(len(value.Accounts))*2) + stats.add(prefix+".accounts.refs", uint64(len(value.Accounts))) stats.add(prefix+".data.length", 4) stats.add(prefix+".data.bytes", uint64(len(value.Data))) stats.add(prefix+".stack_height.present", 1) if value.StackHeight != nil { stats.add(prefix+".stack_height.value", 4) } + stats.add(prefix+".log_events.count", 4) + for _, event := range value.LogEvents { + stats.add(prefix+".log_events.length", 4) + stats.add(prefix+".log_events.bytes", uint64(len(event))) + } } } @@ -194,13 +199,13 @@ func addLamportBalances(stats *sizeStats, preBalances []uint64, postBalances []u func addTokenBalances(stats *sizeStats, prefix string, values []pump_parser.RawTxTokenBalanceBinary) { stats.add(prefix+".count", 4) for _, value := range values { - stats.add(prefix+".account_index", 2) - stats.add(prefix+".mint_ref", 2) + stats.add(prefix+".account_index", 1) + stats.add(prefix+".mint_ref", 1) stats.add(prefix+".owner.present", 1) if value.HasOwnerAccount { - stats.add(prefix+".owner_ref", 2) + stats.add(prefix+".owner_ref", 1) } - stats.add(prefix+".program_id_ref", 2) + stats.add(prefix+".program_id_ref", 1) stats.add(prefix+".decimals", 1) stats.add(prefix+".pre_amount.present", 1) if value.HasPreAmount { diff --git a/meta.go b/meta.go index 852b164..0fa749f 100644 --- a/meta.go +++ b/meta.go @@ -132,6 +132,7 @@ var ( metaoraPoolRemoveBalanceLiquidityDiscriminator = calculateDiscriminator("global:remove_balance_liquidity") metaoraPoolClaimFeeDiscriminator = calculateDiscriminator("global:claim_fee") metaoraPoolBootstrapLiquidityDiscriminator = calculateDiscriminator("global:bootstrap_liquidity") + metaoraPoolSwapEventDiscriminator = calculateDiscriminator("event:Swap") ) var ( diff --git a/metaorapool.go b/metaorapool.go index 4d5dc75..f68972d 100644 --- a/metaorapool.go +++ b/metaorapool.go @@ -2,8 +2,10 @@ package pump_parser import ( "bytes" + "encoding/base64" "encoding/binary" "fmt" + "strings" agbinary "github.com/gagliardetto/binary" "github.com/gagliardetto/solana-go" @@ -20,6 +22,14 @@ type metaoraPoolSwapArgs struct { MinimumOutAmount uint64 } +type metaoraPoolSwapEvent struct { + InAmount uint64 + OutAmount uint64 + TradeFee uint64 + ProtocolFee uint64 + HostFee uint64 +} + var ( meteoraVaultProgram = solana.MustPublicKeyFromBase58("24Uqj9JCLxUeoC3hGfh5W3s9FM9uCHDS2SG3LYwBpyTi") meteoraVaultDepositDiscriminator = []byte{0xf2, 0x23, 0xc6, 0x89, 0x52, 0xe1, 0xf2, 0xb6} @@ -731,6 +741,7 @@ func metaoraPoolRemoveLiquidity(tx *Tx, instruction Instruction, innerInstructio } func metaoraPoolSwap(tx *Tx, instruction Instruction, innerInstructions InnerInstructions, offset [2]uint) ([]Swap, [2]uint, error) { + swapOffset := offset var args metaoraPoolSwapArgs if err := agbinary.NewBorshDecoder(instruction.Data[8:]).Decode(&args); err != nil { return nil, increaseOffset(offset), fmt.Errorf("failed to decode meteora pools swap args: %w", err) @@ -886,10 +897,195 @@ func metaoraPoolSwap(tx *Tx, instruction Instruction, innerInstructions InnerIns EntryContract: entryContract, }, } - swaps[0].SetSwapAmountInfo( + fixedSide := fixedSwapAmountSide(event, SwapModeExactIn) + limitSide := oppositeSwapAmountSide(fixedSide) + if fixedSide == SwapAmountSideUnknown || limitSide == SwapAmountSideUnknown { + swaps[0].SetSwapAmountInfo( + SwapModeExactIn, + decimal.NewFromUint64(args.InAmount), + decimal.NewFromUint64(args.MinimumOutAmount), + ) + return swaps, offset, nil + } + + actualLimitAmount := swapAmountForSide(baseAmount, quoteAmount, limitSide) + if swapEvent, ok := metaoraPoolSwapEventFromInstruction(instruction); ok { + actualLimitAmount = decimal.NewFromUint64(swapEvent.OutAmount) + } else if swapEvent, ok := metaoraPoolSwapEventForOffset(tx, swapOffset); ok { + actualLimitAmount = decimal.NewFromUint64(swapEvent.OutAmount) + } + swaps[0].SetSwapAmountInfoDetailed( SwapModeExactIn, decimal.NewFromUint64(args.InAmount), + fixedSide, + swapMintForSide(baseMint, quoteMint, fixedSide), + SwapLimitTypeMinOut, decimal.NewFromUint64(args.MinimumOutAmount), + limitSide, + swapMintForSide(baseMint, quoteMint, limitSide), + actualLimitAmount, ) return swaps, offset, nil } + +func metaoraPoolSwapEventFromInstruction(instruction Instruction) (metaoraPoolSwapEvent, bool) { + for _, event := range instruction.LogEvents { + if swapEvent, ok := metaoraPoolDecodeSwapEventData(event); ok { + return swapEvent, true + } + } + return metaoraPoolSwapEvent{}, false +} + +func metaoraPoolSwapEventForOffset(tx *Tx, offset [2]uint) (metaoraPoolSwapEvent, bool) { + if tx == nil || tx.rawTx == nil { + return metaoraPoolSwapEvent{}, false + } + occurrence := metaoraPoolSwapInstructionOccurrence(tx.rawTx, offset) + if occurrence == 0 { + return metaoraPoolSwapEvent{}, false + } + return metaoraPoolSwapEventFromLogs(tx.rawTx.Meta.LogMessages, occurrence) +} + +func metaoraPoolSwapInstructionOccurrence(rawTx *RawTx, offset [2]uint) int { + if rawTx == nil { + return 0 + } + accountList := rawTx.getAccountList() + innerByOuter := make(map[int]InnerInstructions, len(rawTx.Meta.InnerInstructions)) + for _, inner := range rawTx.Meta.InnerInstructions { + innerByOuter[inner.Index] = inner + } + + occurrence := 0 + for i, instruction := range rawTx.Transaction.Message.Instructions { + if uint(i) == offset[0] && offset[1] == 0 { + if metaoraPoolIsSwapInstruction(accountList, instruction) { + return occurrence + 1 + } + return 0 + } + if metaoraPoolIsSwapInstruction(accountList, instruction) { + occurrence++ + } + + inner := innerByOuter[i] + for j, instruction := range inner.Instructions { + innerOffset := uint(j + 1) + if uint(i) == offset[0] && offset[1] == innerOffset { + if metaoraPoolIsSwapInstruction(accountList, instruction) { + return occurrence + 1 + } + return 0 + } + if metaoraPoolIsSwapInstruction(accountList, instruction) { + occurrence++ + } + } + } + return 0 +} + +func metaoraPoolIsSwapInstruction(accountList []solana.PublicKey, instruction Instruction) bool { + if instruction.ProgramIDIndex < 0 || instruction.ProgramIDIndex >= len(accountList) { + return false + } + if !accountList[instruction.ProgramIDIndex].Equals(metaoraPoolProgramID) { + return false + } + return len(instruction.Data) >= 8 && bytes.Equal(instruction.Data[:8], metaoraPoolSwapDiscriminator[:]) +} + +func metaoraPoolSwapEventFromLogs(logMessages []string, occurrence int) (metaoraPoolSwapEvent, bool) { + if occurrence <= 0 { + return metaoraPoolSwapEvent{}, false + } + + type frame struct { + program string + sawSwap bool + } + + targetProgram := metaoraPoolProgramID.String() + var stack []frame + seen := 0 + for _, logMessage := range logMessages { + if program, ok := metaoraPoolLogInvokeProgram(logMessage); ok { + stack = append(stack, frame{program: program}) + continue + } + + if len(stack) > 0 && stack[len(stack)-1].program == targetProgram { + if logMessage == "Program log: Instruction: Swap" { + stack[len(stack)-1].sawSwap = true + continue + } + if stack[len(stack)-1].sawSwap && strings.HasPrefix(logMessage, "Program data: ") { + event, ok := metaoraPoolDecodeSwapEventLog(strings.TrimSpace(strings.TrimPrefix(logMessage, "Program data: "))) + if ok { + seen++ + if seen == occurrence { + return event, true + } + } + } + } + + if program, ok := metaoraPoolLogFinishedProgram(logMessage); ok { + for i := len(stack) - 1; i >= 0; i-- { + if stack[i].program == program { + stack = stack[:i] + break + } + } + } + } + return metaoraPoolSwapEvent{}, false +} + +func metaoraPoolLogInvokeProgram(logMessage string) (string, bool) { + if !strings.HasPrefix(logMessage, "Program ") || !strings.Contains(logMessage, " invoke [") { + return "", false + } + rest := strings.TrimPrefix(logMessage, "Program ") + program, _, ok := strings.Cut(rest, " ") + return program, ok +} + +func metaoraPoolLogFinishedProgram(logMessage string) (string, bool) { + if !strings.HasPrefix(logMessage, "Program ") { + return "", false + } + if !strings.HasSuffix(logMessage, " success") && !strings.Contains(logMessage, " failed:") { + return "", false + } + rest := strings.TrimPrefix(logMessage, "Program ") + program, _, ok := strings.Cut(rest, " ") + return program, ok +} + +func metaoraPoolDecodeSwapEventLog(encoded string) (metaoraPoolSwapEvent, bool) { + data, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return metaoraPoolSwapEvent{}, false + } + return metaoraPoolDecodeSwapEventData(data) +} + +func metaoraPoolDecodeSwapEventData(data []byte) (metaoraPoolSwapEvent, bool) { + if len(data) < 48 { + return metaoraPoolSwapEvent{}, false + } + if !bytes.Equal(data[:8], metaoraPoolSwapEventDiscriminator[:]) { + return metaoraPoolSwapEvent{}, false + } + body := data[8:] + return metaoraPoolSwapEvent{ + InAmount: binary.LittleEndian.Uint64(body[0:8]), + OutAmount: binary.LittleEndian.Uint64(body[8:16]), + TradeFee: binary.LittleEndian.Uint64(body[16:24]), + ProtocolFee: binary.LittleEndian.Uint64(body[24:32]), + HostFee: binary.LittleEndian.Uint64(body[32:40]), + }, true +} diff --git a/metaorapool_test.go b/metaorapool_test.go new file mode 100644 index 0000000..038ab51 --- /dev/null +++ b/metaorapool_test.go @@ -0,0 +1,152 @@ +package pump_parser + +import ( + "encoding/base64" + "encoding/binary" + "testing" + + "github.com/gagliardetto/solana-go" +) + +func TestMetaoraPoolSwapEventFromLogsUsesMatchingInvocation(t *testing.T) { + firstEvent := metaoraPoolSwapEventLogForTest(10, 9, 1, 0, 0) + secondEvent := metaoraPoolSwapEventLogForTest(4013522650, 135, 8043041, 2010760, 0) + + logs := []string{ + "Program " + metaoraPoolProgramID.String() + " invoke [1]", + "Program log: Instruction: Swap", + "Program data: " + firstEvent, + "Program " + metaoraPoolProgramID.String() + " success", + "Program " + solana.TokenProgramID.String() + " invoke [1]", + "Program data: " + secondEvent, + "Program " + solana.TokenProgramID.String() + " success", + "Program " + metaoraPoolProgramID.String() + " invoke [1]", + "Program log: Instruction: Swap", + "Program data: " + secondEvent, + "Program " + metaoraPoolProgramID.String() + " success", + } + + event, ok := metaoraPoolSwapEventFromLogs(logs, 2) + if !ok { + t.Fatal("expected second swap event") + } + if event.InAmount != 4013522650 { + t.Fatalf("in amount = %d, want 4013522650", event.InAmount) + } + if event.OutAmount != 135 { + t.Fatalf("out amount = %d, want 135", event.OutAmount) + } + if event.TradeFee != 8043041 { + t.Fatalf("trade fee = %d, want 8043041", event.TradeFee) + } + if event.ProtocolFee != 2010760 { + t.Fatalf("protocol fee = %d, want 2010760", event.ProtocolFee) + } +} + +func TestMetaoraPoolSwapInstructionOccurrenceIncludesInnerInstructions(t *testing.T) { + rawTx := &RawTx{ + Transaction: Transaction{ + Message: Message{ + AccountKeys: solana.PublicKeySlice{ + metaoraPoolProgramID, + solana.MustPublicKeyFromBase58("BASDaPs2cdVTsvgPRfESDLZgek8tKRTfqbR2ksdgptsn"), + }, + Instructions: []Instruction{ + {ProgramIDIndex: 0, Data: metaoraPoolSwapInstructionDataForTest()}, + {ProgramIDIndex: 1, Data: []byte{1}}, + }, + }, + }, + Meta: Meta{ + InnerInstructions: []InnerInstructions{ + { + Index: 1, + Instructions: []Instruction{ + {ProgramIDIndex: 0, Data: metaoraPoolSwapInstructionDataForTest()}, + }, + }, + }, + }, + } + + if got := metaoraPoolSwapInstructionOccurrence(rawTx, [2]uint{1, 1}); got != 2 { + t.Fatalf("occurrence = %d, want 2", got) + } +} + +func TestAttachLogEventsToInstructions(t *testing.T) { + swapEvent := metaoraPoolSwapEventLogForTest(4013522650, 135, 8043041, 2010760, 0) + rawTx := &RawTx{ + Transaction: Transaction{ + Message: Message{ + AccountKeys: solana.PublicKeySlice{ + metaoraPoolProgramID, + solana.TokenProgramID, + }, + Instructions: []Instruction{ + {ProgramIDIndex: 0, Data: metaoraPoolSwapInstructionDataForTest()}, + }, + }, + }, + Meta: Meta{ + InnerInstructions: []InnerInstructions{ + { + Index: 0, + Instructions: []Instruction{ + {ProgramIDIndex: 1, Data: []byte{3}, StackHeight: intPtrForTest(2)}, + }, + }, + }, + LogMessages: []string{ + "Program " + metaoraPoolProgramID.String() + " invoke [1]", + "Program log: Instruction: Swap", + "Program " + solana.TokenProgramID.String() + " invoke [2]", + "Program " + solana.TokenProgramID.String() + " success", + "Program data: " + swapEvent, + "Program " + metaoraPoolProgramID.String() + " success", + }, + }, + } + + applyRawTxConvertLogOptions(rawTx, RawTxConvertOptions{ParseLogEvents: true, IgnoreLogMessages: true}) + + if len(rawTx.Meta.LogMessages) != 0 { + t.Fatalf("log messages length = %d, want 0", len(rawTx.Meta.LogMessages)) + } + if len(rawTx.Transaction.Message.Instructions[0].LogEvents) != 1 { + t.Fatalf("outer log events length = %d, want 1", len(rawTx.Transaction.Message.Instructions[0].LogEvents)) + } + if len(rawTx.Meta.InnerInstructions[0].Instructions[0].LogEvents) != 0 { + t.Fatalf("inner log events length = %d, want 0", len(rawTx.Meta.InnerInstructions[0].Instructions[0].LogEvents)) + } + + event, ok := metaoraPoolSwapEventFromInstruction(rawTx.Transaction.Message.Instructions[0]) + if !ok { + t.Fatal("expected swap event from outer instruction") + } + if event.OutAmount != 135 { + t.Fatalf("out amount = %d, want 135", event.OutAmount) + } +} + +func metaoraPoolSwapInstructionDataForTest() []byte { + data := make([]byte, 8+16) + copy(data, metaoraPoolSwapDiscriminator[:]) + return data +} + +func metaoraPoolSwapEventLogForTest(inAmount, outAmount, tradeFee, protocolFee, hostFee uint64) string { + data := make([]byte, 8+40) + copy(data, metaoraPoolSwapEventDiscriminator[:]) + binary.LittleEndian.PutUint64(data[8:16], inAmount) + binary.LittleEndian.PutUint64(data[16:24], outAmount) + binary.LittleEndian.PutUint64(data[24:32], tradeFee) + binary.LittleEndian.PutUint64(data[32:40], protocolFee) + binary.LittleEndian.PutUint64(data[40:48], hostFee) + return base64.StdEncoding.EncodeToString(data) +} + +func intPtrForTest(value int) *int { + return &value +} diff --git a/rawtx.go b/rawtx.go index 12b3f15..a47223c 100644 --- a/rawtx.go +++ b/rawtx.go @@ -1,8 +1,11 @@ package pump_parser import ( + "encoding/base64" "encoding/json" "fmt" + "strconv" + "strings" "time" bin "github.com/gagliardetto/binary" @@ -105,10 +108,11 @@ func (tx *RawTx) GetBlockTime() *pgtype.Timestamptz { } type Instruction struct { - Accounts []int `json:"accounts"` - Data solana.Base58 `json:"data"` - ProgramIDIndex int `json:"programIdIndex"` - StackHeight *int `json:"stackHeight"` + Accounts []int `json:"accounts"` + Data solana.Base58 `json:"data"` + ProgramIDIndex int `json:"programIdIndex"` + StackHeight *int `json:"stackHeight"` + LogEvents []solana.Base64 `json:"logEvents,omitempty"` } type InnerInstructions struct { Index int `json:"index"` @@ -180,6 +184,11 @@ type Transaction struct { Signatures []solana.Signature `json:"signatures"` } +type RawTxConvertOptions struct { + IgnoreLogMessages bool + ParseLogEvents bool +} + func (tx *Transaction) UnmarshalJSON(data []byte) error { if len(data) == 0 || (len(data) == 4 && string(data) == "null") { // TODO: is this an error? @@ -308,7 +317,8 @@ func marshalRpcTransactionErr(err any) string { return string(e) } -func FromRpcTransactionWithMeta(tx rpc.TransactionWithMeta, blockTime *uint64, slot uint64, index int64) (*RawTx, error) { +func FromRpcTransactionWithMeta(tx rpc.TransactionWithMeta, blockTime *uint64, slot uint64, index int64, options ...RawTxConvertOptions) (*RawTx, error) { + option := rawTxConvertOption(options) created := int64(0) if blockTime != nil { created = int64(*blockTime) @@ -523,6 +533,8 @@ func FromRpcTransactionWithMeta(tx rpc.TransactionWithMeta, blockTime *uint64, s }) } + applyRawTxConvertLogOptions(sTx, option) + return sTx, nil } @@ -833,7 +845,8 @@ func isAccountOwner(account, owner, mint solana.PublicKey) (bool, error) { return account == ata, nil } -func ConvertYellowstoneGrpcTransactionToSolanaTransaction(y *pb.SubscribeUpdateTransaction, created int64) (*RawTx, error) { +func ConvertYellowstoneGrpcTransactionToSolanaTransaction(y *pb.SubscribeUpdateTransaction, created int64, options ...RawTxConvertOptions) (*RawTx, error) { + option := rawTxConvertOption(options) sTx := &RawTx{ BlockTime: created, Slot: y.Slot, @@ -1002,6 +1015,8 @@ func ConvertYellowstoneGrpcTransactionToSolanaTransaction(y *pb.SubscribeUpdateT }) } + applyRawTxConvertLogOptions(sTx, option) + // resolve the lookups //{ // if sTx.Transaction.Message.IsVersioned() { @@ -1021,6 +1036,168 @@ func newInt16(x uint16) *int { return &y } +func rawTxConvertOption(options []RawTxConvertOptions) RawTxConvertOptions { + out := RawTxConvertOptions{ParseLogEvents: true} + if len(options) > 0 { + out = options[0] + } + return out +} + +func applyRawTxConvertLogOptions(tx *RawTx, option RawTxConvertOptions) { + if tx == nil { + return + } + if option.ParseLogEvents { + attachLogEventsToInstructions(tx, tx.Meta.LogMessages) + } + if option.IgnoreLogMessages { + tx.Meta.LogMessages = nil + } +} + +type instructionLogFrame struct { + program string + instr *Instruction +} + +type instructionLogTarget struct { + program string + stackHeight int + instr *Instruction +} + +func attachLogEventsToInstructions(tx *RawTx, logMessages []string) { + if tx == nil || len(logMessages) == 0 { + return + } + targets := rawTxInstructionLogTargets(tx) + if len(targets) == 0 { + return + } + + nextTarget := 0 + var stack []instructionLogFrame + for _, logMessage := range logMessages { + if program, stackHeight, ok := parseProgramInvokeLog(logMessage); ok { + var instr *Instruction + for nextTarget < len(targets) { + target := targets[nextTarget] + nextTarget++ + if target.program != program { + continue + } + if target.stackHeight != 0 && target.stackHeight != stackHeight { + continue + } + instr = target.instr + break + } + stack = append(stack, instructionLogFrame{program: program, instr: instr}) + continue + } + + if data, ok := parseProgramDataLog(logMessage); ok { + if len(stack) == 0 { + continue + } + top := stack[len(stack)-1] + if top.instr != nil { + top.instr.LogEvents = append(top.instr.LogEvents, data) + } + continue + } + + if program, ok := parseProgramFinishedLog(logMessage); ok { + for i := len(stack) - 1; i >= 0; i-- { + if stack[i].program == program { + stack = stack[:i] + break + } + } + } + } +} + +func rawTxInstructionLogTargets(tx *RawTx) []instructionLogTarget { + accountList := tx.getAccountList() + innerByOuter := make(map[int]*InnerInstructions, len(tx.Meta.InnerInstructions)) + for i := range tx.Meta.InnerInstructions { + inner := &tx.Meta.InnerInstructions[i] + innerByOuter[inner.Index] = inner + } + + out := make([]instructionLogTarget, 0, len(tx.Transaction.Message.Instructions)) + for i := range tx.Transaction.Message.Instructions { + instr := &tx.Transaction.Message.Instructions[i] + if instr.ProgramIDIndex >= 0 && instr.ProgramIDIndex < len(accountList) { + out = append(out, instructionLogTarget{ + program: accountList[instr.ProgramIDIndex].String(), + stackHeight: 1, + instr: instr, + }) + } + if inner := innerByOuter[i]; inner != nil { + for j := range inner.Instructions { + innerInstr := &inner.Instructions[j] + if innerInstr.ProgramIDIndex < 0 || innerInstr.ProgramIDIndex >= len(accountList) { + continue + } + stackHeight := 0 + if innerInstr.StackHeight != nil { + stackHeight = *innerInstr.StackHeight + } + out = append(out, instructionLogTarget{ + program: accountList[innerInstr.ProgramIDIndex].String(), + stackHeight: stackHeight, + instr: innerInstr, + }) + } + } + } + return out +} + +func parseProgramInvokeLog(logMessage string) (string, int, bool) { + if !strings.HasPrefix(logMessage, "Program ") { + return "", 0, false + } + rest := strings.TrimPrefix(logMessage, "Program ") + program, suffix, ok := strings.Cut(rest, " invoke [") + if !ok { + return "", 0, false + } + suffix = strings.TrimSuffix(suffix, "]") + stackHeight, err := strconv.Atoi(suffix) + if err != nil { + return "", 0, false + } + return program, stackHeight, true +} + +func parseProgramDataLog(logMessage string) (solana.Base64, bool) { + if !strings.HasPrefix(logMessage, "Program data: ") { + return nil, false + } + data, err := base64.StdEncoding.DecodeString(strings.TrimSpace(strings.TrimPrefix(logMessage, "Program data: "))) + if err != nil { + return nil, false + } + return solana.Base64(data), true +} + +func parseProgramFinishedLog(logMessage string) (string, bool) { + if !strings.HasPrefix(logMessage, "Program ") { + return "", false + } + if !strings.HasSuffix(logMessage, " success") && !strings.Contains(logMessage, " failed:") { + return "", false + } + rest := strings.TrimPrefix(logMessage, "Program ") + program, _, ok := strings.Cut(rest, " ") + return program, ok +} + func newInt(x *uint32) *int { if x == nil { return nil diff --git a/rawtx_binary.go b/rawtx_binary.go index 6123e28..6df837b 100644 --- a/rawtx_binary.go +++ b/rawtx_binary.go @@ -13,7 +13,7 @@ import ( "github.com/shopspring/decimal" ) -const rawTxBinarySchemaVersionCurrent uint16 = 7 +const rawTxBinarySchemaVersionCurrent uint16 = 10 var rawTxBinaryMagic = [4]byte{'P', 'R', 'T', 'X'} var rawTxsBinaryMagic = [4]byte{'P', 'R', 'T', 'S'} @@ -61,11 +61,11 @@ type RawTxMetaBinary struct { } type RawTxTokenBalanceBinary struct { - AccountIndex uint16 - MintAccount uint16 - OwnerAccount uint16 + AccountIndex uint8 + MintAccount uint8 + OwnerAccount uint8 HasOwnerAccount bool - ProgramIDAccount uint16 + ProgramIDAccount uint8 Decimals uint8 HasPreAmount bool PreAmount string @@ -270,10 +270,17 @@ func DecodeRawTxsBinaryReader(r io.Reader) iter.Seq2[*RawTx, error] { } func newRawTxBinaryWithAddressTable(tx *RawTx, addressTable []solana.PublicKey, addressIndex *txBinaryAddressIndex) (*RawTxBinary, error) { - accountList := tx.getAccountList() + accountList, err := rawTxBinaryEffectiveAccountList(tx) + if err != nil { + return nil, err + } if uint64(len(accountList)) > uint64(math.MaxUint32) { return nil, fmt.Errorf("account list exceeds uint32 capacity") } + accountListIndex, err := newRawTxBinaryAccountListIndex(accountList) + if err != nil { + return nil, err + } if uint64(len(tx.Transaction.Message.AccountKeys)) > uint64(math.MaxUint32) { return nil, fmt.Errorf("message account key count exceeds uint32 capacity") } @@ -299,7 +306,7 @@ func newRawTxBinaryWithAddressTable(tx *RawTx, addressTable []solana.PublicKey, out.AccountList = append(out.AccountList, ref) } - meta, err := rawTxMetaToBinary(&tx.Meta, addressIndex) + meta, err := rawTxMetaToBinary(&tx.Meta, accountListIndex) if err != nil { return nil, err } @@ -520,7 +527,7 @@ func (tx *RawTxBinary) ToRawTx() (*RawTx, error) { IndexWithinBlock: int64(tx.IndexWithinBlock), Slot: tx.Slot, Version: rawTxBinaryVersionValue(tx.Version), - Meta: rawTxMetaFromBinary(tx.Meta, tx.AddressTable), + Meta: rawTxMetaFromBinary(tx.Meta, accountList), Transaction: rawTxTransactionFromBinary(tx.Transaction, tx.AddressTable), } if tx.AccountKeyCount > 0 && tx.AccountKeyCount <= uint32(len(accountList)) { @@ -678,7 +685,7 @@ func rawTxBinaryReadTxBody(dec txBinaryBodyReader, tx *RawTxBinary, addressTable return nil } -func rawTxMetaToBinary(meta *Meta, addressIndex *txBinaryAddressIndex) (RawTxMetaBinary, error) { +func rawTxMetaToBinary(meta *Meta, accountListIndex map[solana.PublicKey]uint8) (RawTxMetaBinary, error) { out := RawTxMetaBinary{ Err: cloneTransactionParsedError(meta.Err), Fee: meta.Fee, @@ -688,7 +695,7 @@ func rawTxMetaToBinary(meta *Meta, addressIndex *txBinaryAddressIndex) (RawTxMet ComputeUnitsConsumed: meta.ComputeUnitsConsumed, } var err error - out.TokenBalances, err = rawTxTokenBalancesToBinary(meta.PreTokenBalances, meta.PostTokenBalances, addressIndex) + out.TokenBalances, err = rawTxTokenBalancesToBinary(meta.PreTokenBalances, meta.PostTokenBalances, accountListIndex) if err != nil { return out, fmt.Errorf("token_balances: %w", err) } @@ -715,86 +722,75 @@ func rawTxMessageToBinary(message *Message, addressIndex *txBinaryAddressIndex) return out, nil } -func rawTxTokenBalancesToBinary(preBalances []TokenBalance, postBalances []TokenBalance, addressIndex *txBinaryAddressIndex) ([]RawTxTokenBalanceBinary, error) { +func rawTxTokenBalancesToBinary(preBalances []TokenBalance, postBalances []TokenBalance, accountListIndex map[solana.PublicKey]uint8) ([]RawTxTokenBalanceBinary, error) { out := make([]RawTxTokenBalanceBinary, 0, len(preBalances)+len(postBalances)) - byAccountIndex := make(map[uint16]int, len(preBalances)+len(postBalances)) + preByAccountIndex := make(map[uint8]int, len(preBalances)) + postSeenByAccountIndex := make(map[uint8]struct{}, len(postBalances)) for i, balance := range preBalances { - encoded, err := rawTxTokenBalanceToBinary(balance, addressIndex) + encoded, err := rawTxTokenBalanceToBinary(balance, accountListIndex) if err != nil { return nil, fmt.Errorf("pre[%d]: %w", i, err) } - if _, exists := byAccountIndex[encoded.AccountIndex]; exists { + if _, exists := preByAccountIndex[encoded.AccountIndex]; exists { return nil, fmt.Errorf("pre[%d].account_index duplicate: %d", i, encoded.AccountIndex) } encoded.HasPreAmount = true encoded.PreAmount = balance.UITokenAmount.Amount - byAccountIndex[encoded.AccountIndex] = len(out) + preByAccountIndex[encoded.AccountIndex] = len(out) out = append(out, encoded) } for i, balance := range postBalances { - encoded, err := rawTxTokenBalanceToBinary(balance, addressIndex) + encoded, err := rawTxTokenBalanceToBinary(balance, accountListIndex) if err != nil { return nil, fmt.Errorf("post[%d]: %w", i, err) } - if existingIndex, exists := byAccountIndex[encoded.AccountIndex]; exists { - if out[existingIndex].HasPostAmount { - return nil, fmt.Errorf("post[%d].account_index duplicate: %d", i, encoded.AccountIndex) - } - if !rawTxTokenBalanceBinarySameIdentity(out[existingIndex], encoded) { - return nil, fmt.Errorf("post[%d].account_index %d identity mismatch", i, encoded.AccountIndex) - } + if _, exists := postSeenByAccountIndex[encoded.AccountIndex]; exists { + return nil, fmt.Errorf("post[%d].account_index duplicate: %d", i, encoded.AccountIndex) + } + postSeenByAccountIndex[encoded.AccountIndex] = struct{}{} + if existingIndex, exists := preByAccountIndex[encoded.AccountIndex]; exists && rawTxTokenBalanceBinarySameIdentity(out[existingIndex], encoded) { out[existingIndex].HasPostAmount = true out[existingIndex].PostAmount = balance.UITokenAmount.Amount continue } encoded.HasPostAmount = true encoded.PostAmount = balance.UITokenAmount.Amount - byAccountIndex[encoded.AccountIndex] = len(out) out = append(out, encoded) } return out, nil } -func rawTxTokenBalanceToBinary(balance TokenBalance, addressIndex *txBinaryAddressIndex) (RawTxTokenBalanceBinary, error) { +func rawTxTokenBalanceToBinary(balance TokenBalance, accountListIndex map[solana.PublicKey]uint8) (RawTxTokenBalanceBinary, error) { mintAccount, ownerAccount, programIDAccount, err := rawTxBinaryTokenBalanceAccounts(balance) if err != nil { return RawTxTokenBalanceBinary{}, err } - mint, err := addressIndex.id(mintAccount) - if err != nil { - return RawTxTokenBalanceBinary{}, fmt.Errorf("mint: %w", err) + mint, ok := accountListIndex[mintAccount] + if !ok { + return RawTxTokenBalanceBinary{}, fmt.Errorf("mint account not found in account_list: %s", mintAccount) } - programID, err := addressIndex.id(programIDAccount) - if err != nil { - return RawTxTokenBalanceBinary{}, fmt.Errorf("program_id: %w", err) - } - if mint > math.MaxUint16 { - return RawTxTokenBalanceBinary{}, fmt.Errorf("mint ref overflows uint16: %d", mint) - } - if programID > math.MaxUint16 { - return RawTxTokenBalanceBinary{}, fmt.Errorf("program_id ref overflows uint16: %d", programID) + programID, ok := accountListIndex[programIDAccount] + if !ok { + return RawTxTokenBalanceBinary{}, fmt.Errorf("program_id account not found in account_list: %s", programIDAccount) } if balance.UITokenAmount.Decimals > math.MaxUint8 { return RawTxTokenBalanceBinary{}, fmt.Errorf("decimals overflows uint8: %d", balance.UITokenAmount.Decimals) } - if balance.AccountIndex < 0 || balance.AccountIndex > math.MaxUint16 { - return RawTxTokenBalanceBinary{}, fmt.Errorf("account_index overflows uint16: %d", balance.AccountIndex) + if balance.AccountIndex < 0 || balance.AccountIndex > math.MaxUint8 { + return RawTxTokenBalanceBinary{}, fmt.Errorf("account_index overflows uint8: %d", balance.AccountIndex) } encoded := RawTxTokenBalanceBinary{ - AccountIndex: uint16(balance.AccountIndex), - MintAccount: uint16(mint), - ProgramIDAccount: uint16(programID), + AccountIndex: uint8(balance.AccountIndex), + MintAccount: mint, + ProgramIDAccount: programID, Decimals: uint8(balance.UITokenAmount.Decimals), } if ownerAccount != nil { - owner, err := addressIndex.id(*ownerAccount) - if err != nil { - return RawTxTokenBalanceBinary{}, fmt.Errorf("owner: %w", err) + owner, ok := accountListIndex[*ownerAccount] + if !ok { + return RawTxTokenBalanceBinary{}, fmt.Errorf("owner account not found in account_list: %s", *ownerAccount) } - if owner > math.MaxUint16 { - return RawTxTokenBalanceBinary{}, fmt.Errorf("owner ref overflows uint16: %d", owner) - } - encoded.OwnerAccount = uint16(owner) + encoded.OwnerAccount = owner encoded.HasOwnerAccount = true } return encoded, nil @@ -809,8 +805,8 @@ func rawTxTokenBalanceBinarySameIdentity(a, b RawTxTokenBalanceBinary) bool { a.Decimals == b.Decimals } -func rawTxMetaFromBinary(meta RawTxMetaBinary, addressTable []solana.PublicKey) Meta { - preTokenBalances, postTokenBalances := rawTxTokenBalancesFromBinary(meta.TokenBalances, addressTable) +func rawTxMetaFromBinary(meta RawTxMetaBinary, accountList []solana.PublicKey) Meta { + preTokenBalances, postTokenBalances := rawTxTokenBalancesFromBinary(meta.TokenBalances, accountList) return Meta{ Err: cloneTransactionParsedError(meta.Err), Fee: meta.Fee, @@ -838,15 +834,15 @@ func rawTxTransactionFromBinary(tx RawTxTransactionBinary, addressTable []solana return out } -func rawTxTokenBalancesFromBinary(balances []RawTxTokenBalanceBinary, addressTable []solana.PublicKey) ([]TokenBalance, []TokenBalance) { +func rawTxTokenBalancesFromBinary(balances []RawTxTokenBalanceBinary, accountList []solana.PublicKey) ([]TokenBalance, []TokenBalance) { pre := make([]TokenBalance, 0, len(balances)) post := make([]TokenBalance, 0, len(balances)) for _, balance := range balances { - mint, _ := txBinaryAddressAt(addressTable, uint32(balance.MintAccount), "token_balance.mint") - programID, _ := txBinaryAddressAt(addressTable, uint32(balance.ProgramIDAccount), "token_balance.program_id") + mint, _ := txBinaryAddressAt(accountList, uint32(balance.MintAccount), "token_balance.mint") + programID, _ := txBinaryAddressAt(accountList, uint32(balance.ProgramIDAccount), "token_balance.program_id") var owner *solana.PublicKey if balance.HasOwnerAccount { - ownerKey, _ := txBinaryAddressAt(addressTable, uint32(balance.OwnerAccount), "token_balance.owner") + ownerKey, _ := txBinaryAddressAt(accountList, uint32(balance.OwnerAccount), "token_balance.owner") owner = &ownerKey } if balance.HasPreAmount { @@ -1065,10 +1061,10 @@ func readInnerInstructions(dec txBinaryBodyReader) ([]InnerInstructions, error) func writeInstructions(enc *txBinaryEncoder, values []Instruction) error { enc.writeUint32(uint32(len(values))) for i, value := range values { - if value.ProgramIDIndex < 0 || value.ProgramIDIndex > math.MaxUint16 { - return fmt.Errorf("[%d].program_id_index overflows uint16: %d", i, value.ProgramIDIndex) + if value.ProgramIDIndex < 0 || value.ProgramIDIndex > math.MaxUint8 { + return fmt.Errorf("[%d].program_id_index overflows uint8: %d", i, value.ProgramIDIndex) } - enc.writeUint16(uint16(value.ProgramIDIndex)) + enc.writeUint8(uint8(value.ProgramIDIndex)) if err := writeAccountIndexSlice(enc, value.Accounts); err != nil { return fmt.Errorf("[%d].accounts: %w", i, err) } @@ -1077,6 +1073,10 @@ func writeInstructions(enc *txBinaryEncoder, values []Instruction) error { if value.StackHeight != nil { enc.writeUint32(uint32(*value.StackHeight)) } + enc.writeUint32(uint32(len(value.LogEvents))) + for _, event := range value.LogEvents { + writeByteSlice(enc, event) + } } return nil } @@ -1088,7 +1088,7 @@ func readInstructions(dec txBinaryBodyReader) ([]Instruction, error) { } out := make([]Instruction, 0, count) for i := uint32(0); i < count; i++ { - programIDIndex, err := dec.readUint16() + programIDIndex, err := dec.readUint8() if err != nil { return nil, err } @@ -1113,11 +1113,24 @@ func readInstructions(dec txBinaryBodyReader) ([]Instruction, error) { sh := int(rawStackHeight) stackHeight = &sh } + logEventCount, err := dec.readUint32() + if err != nil { + return nil, err + } + logEvents := make([]solana.Base64, 0, logEventCount) + for j := uint32(0); j < logEventCount; j++ { + eventData, err := readByteSlice(dec) + if err != nil { + return nil, err + } + logEvents = append(logEvents, solana.Base64(eventData)) + } out = append(out, Instruction{ ProgramIDIndex: int(programIDIndex), Accounts: accounts, Data: solana.Base58(data), StackHeight: stackHeight, + LogEvents: logEvents, }) } return out, nil @@ -1126,13 +1139,13 @@ func readInstructions(dec txBinaryBodyReader) ([]Instruction, error) { func writeTokenBalances(enc *txBinaryEncoder, values []RawTxTokenBalanceBinary) error { enc.writeUint32(uint32(len(values))) for i, value := range values { - enc.writeUint16(value.AccountIndex) - enc.writeUint16(value.MintAccount) + enc.writeUint8(value.AccountIndex) + enc.writeUint8(value.MintAccount) enc.writeBool(value.HasOwnerAccount) if value.HasOwnerAccount { - enc.writeUint16(value.OwnerAccount) + enc.writeUint8(value.OwnerAccount) } - enc.writeUint16(value.ProgramIDAccount) + enc.writeUint8(value.ProgramIDAccount) enc.writeUint8(value.Decimals) enc.writeBool(value.HasPreAmount) if value.HasPreAmount { @@ -1158,21 +1171,21 @@ func readTokenBalances(dec txBinaryBodyReader) ([]RawTxTokenBalanceBinary, error out := make([]RawTxTokenBalanceBinary, 0, count) for i := uint32(0); i < count; i++ { value := RawTxTokenBalanceBinary{} - if value.AccountIndex, err = dec.readUint16(); err != nil { + if value.AccountIndex, err = dec.readUint8(); err != nil { return nil, err } - if value.MintAccount, err = dec.readUint16(); err != nil { + if value.MintAccount, err = dec.readUint8(); err != nil { return nil, err } if value.HasOwnerAccount, err = dec.readBool(); err != nil { return nil, err } if value.HasOwnerAccount { - if value.OwnerAccount, err = dec.readUint16(); err != nil { + if value.OwnerAccount, err = dec.readUint8(); err != nil { return nil, err } } - if value.ProgramIDAccount, err = dec.readUint16(); err != nil { + if value.ProgramIDAccount, err = dec.readUint8(); err != nil { return nil, err } if value.Decimals, err = dec.readUint8(); err != nil { @@ -1336,10 +1349,10 @@ func readUint32Slice(dec txBinaryBodyReader) ([]uint32, error) { func writeAccountIndexSlice(enc *txBinaryEncoder, values []int) error { enc.writeUint32(uint32(len(values))) for i, value := range values { - if value < 0 || value > math.MaxUint16 { - return fmt.Errorf("[%d] overflows uint16: %d", i, value) + if value < 0 || value > math.MaxUint8 { + return fmt.Errorf("[%d] overflows uint8: %d", i, value) } - enc.writeUint16(uint16(value)) + enc.writeUint8(uint8(value)) } return nil } @@ -1351,7 +1364,7 @@ func readAccountIndexSlice(dec txBinaryBodyReader) ([]int, error) { } out := make([]int, 0, count) for i := uint32(0); i < count; i++ { - value, err := dec.readUint16() + value, err := dec.readUint8() if err != nil { return nil, err } @@ -1547,7 +1560,11 @@ func rawTxBinaryBuildAddressTable(txs []*RawTx) ([]solana.PublicKey, error) { if tx == nil { return nil, fmt.Errorf("tx[%d] is nil", txIndex) } - for accountIndex, account := range tx.getAccountList() { + accountList, err := rawTxBinaryEffectiveAccountList(tx) + if err != nil { + return nil, fmt.Errorf("tx[%d].account_list: %w", txIndex, err) + } + for accountIndex, account := range accountList { if err := builder.add(account); err != nil { return nil, fmt.Errorf("tx[%d].account_list[%d]: %w", txIndex, accountIndex, err) } @@ -1557,20 +1574,64 @@ func rawTxBinaryBuildAddressTable(txs []*RawTx) ([]solana.PublicKey, error) { return nil, fmt.Errorf("tx[%d].address_table_lookups[%d].account_key: %w", txIndex, lookupIndex, err) } } - for balanceIndex, balance := range tx.Meta.PreTokenBalances { - if err := rawTxBinaryAddTokenBalanceAddresses(&builder, balance); err != nil { - return nil, fmt.Errorf("tx[%d].pre_token_balances[%d]: %w", txIndex, balanceIndex, err) - } - } - for balanceIndex, balance := range tx.Meta.PostTokenBalances { - if err := rawTxBinaryAddTokenBalanceAddresses(&builder, balance); err != nil { - return nil, fmt.Errorf("tx[%d].post_token_balances[%d]: %w", txIndex, balanceIndex, err) - } - } } return builder.addresses, nil } +func rawTxBinaryEffectiveAccountList(tx *RawTx) ([]solana.PublicKey, error) { + accountList := append([]solana.PublicKey(nil), tx.getAccountList()...) + seen := make(map[solana.PublicKey]struct{}, len(accountList)) + for _, account := range accountList { + seen[account] = struct{}{} + } + for balanceIndex, balance := range tx.Meta.PreTokenBalances { + if err := rawTxBinaryAppendTokenBalanceAccounts(&accountList, seen, balance); err != nil { + return nil, fmt.Errorf("pre_token_balances[%d]: %w", balanceIndex, err) + } + } + for balanceIndex, balance := range tx.Meta.PostTokenBalances { + if err := rawTxBinaryAppendTokenBalanceAccounts(&accountList, seen, balance); err != nil { + return nil, fmt.Errorf("post_token_balances[%d]: %w", balanceIndex, err) + } + } + return accountList, nil +} + +func rawTxBinaryAppendTokenBalanceAccounts(accountList *[]solana.PublicKey, seen map[solana.PublicKey]struct{}, balance TokenBalance) error { + mintAccount, ownerAccount, programIDAccount, err := rawTxBinaryTokenBalanceAccounts(balance) + if err != nil { + return err + } + rawTxBinaryAppendAccountIfMissing(accountList, seen, mintAccount) + if ownerAccount != nil { + rawTxBinaryAppendAccountIfMissing(accountList, seen, *ownerAccount) + } + rawTxBinaryAppendAccountIfMissing(accountList, seen, programIDAccount) + return nil +} + +func rawTxBinaryAppendAccountIfMissing(accountList *[]solana.PublicKey, seen map[solana.PublicKey]struct{}, account solana.PublicKey) { + if _, exists := seen[account]; exists { + return + } + seen[account] = struct{}{} + *accountList = append(*accountList, account) +} + +func newRawTxBinaryAccountListIndex(accountList []solana.PublicKey) (map[solana.PublicKey]uint8, error) { + out := make(map[solana.PublicKey]uint8, len(accountList)) + for i, account := range accountList { + if i > math.MaxUint8 { + return nil, fmt.Errorf("account_list index overflows uint8: %d", i) + } + if _, exists := out[account]; exists { + continue + } + out[account] = uint8(i) + } + return out, nil +} + func rawTxBinarySharedBlockTime(txs []*RawTx, field string) (int64, error) { if len(txs) == 0 { return 0, nil @@ -1672,6 +1733,7 @@ func cloneInstructions(values []Instruction) []Instruction { Accounts: append([]int(nil), value.Accounts...), Data: append(solana.Base58(nil), value.Data...), ProgramIDIndex: value.ProgramIDIndex, + LogEvents: cloneBase64Slice(value.LogEvents), } if value.StackHeight != nil { stackHeight := *value.StackHeight @@ -1682,6 +1744,14 @@ func cloneInstructions(values []Instruction) []Instruction { return out } +func cloneBase64Slice(values []solana.Base64) []solana.Base64 { + out := make([]solana.Base64, 0, len(values)) + for _, value := range values { + out = append(out, append(solana.Base64(nil), value...)) + } + return out +} + func rawTxBinaryVersionID(version interface{}) uint8 { switch value := version.(type) { case solana.MessageVersion: