diff --git a/cmd/analyze_rawtx_binary_size/main.go b/cmd/analyze_rawtx_binary_size/main.go new file mode 100644 index 0000000..d854881 --- /dev/null +++ b/cmd/analyze_rawtx_binary_size/main.go @@ -0,0 +1,589 @@ +package main + +import ( + "flag" + "fmt" + "math/big" + "os" + "sort" + + pump_parser "github.com/thloyi/pump-parser" +) + +type sizeStats struct { + total uint64 + items map[string]uint64 +} + +type txInnerDataStat struct { + TxOrdinal int + BlockIndex int + IndexWithinBlock uint32 + Slot uint64 + Bytes uint64 + InstructionCount int +} + +func newSizeStats() *sizeStats { + return &sizeStats{items: make(map[string]uint64)} +} + +func (s *sizeStats) add(name string, n uint64) { + s.items[name] += n + s.total += n +} + +func main() { + filePath := flag.String("file", "testdata/rawtx-binary/rawtx-blocks-414696178-414696182.prbs", "path to RawTxBlocksBinary .prbs file") + flag.Parse() + + raw, err := os.ReadFile(*filePath) + if err != nil { + fmt.Fprintf(os.Stderr, "read file: %v\n", err) + os.Exit(1) + } + + var blocks pump_parser.RawTxBlocksBinary + if err := blocks.UnmarshalBinary(raw); err != nil { + fmt.Fprintf(os.Stderr, "decode rawtx blocks binary: %v\n", err) + os.Exit(1) + } + + stats := analyzeRawTxBlocksBinary(&blocks) + if stats.total != uint64(len(raw)) { + fmt.Fprintf(os.Stderr, "size accounting mismatch: accounted=%d file=%d\n", stats.total, len(raw)) + os.Exit(1) + } + + printReport(*filePath, len(raw), &blocks, stats) + fmt.Println() + printInnerInstructionDataDistribution(&blocks) + fmt.Println() + printBalanceAnalysis(&blocks) +} + +func analyzeRawTxBlocksBinary(blocks *pump_parser.RawTxBlocksBinary) *sizeStats { + stats := newSizeStats() + stats.add("file.magic", 4) + stats.add("file.schema_version", 2) + stats.add("address_table.count", 4) + stats.add("address_table.pubkeys", uint64(len(blocks.AddressTable))*32) + stats.add("blocks.count", 4) + stats.add("blocks.block_time", uint64(len(blocks.BlockTimes))*8) + stats.add("blocks.tx_count", uint64(len(blocks.BlockTxCounts))*4) + stats.add("txs.total_count", 4) + + for i := range blocks.Txs { + addTx(stats, &blocks.Txs[i]) + } + return stats +} + +func addTx(stats *sizeStats, tx *pump_parser.RawTxBinary) { + stats.add("tx.index_within_block", 4) + stats.add("tx.slot", 8) + stats.add("tx.version", 1) + stats.add("tx.account_key_count", 4) + stats.add("tx.account_list.count", 4) + stats.add("tx.account_list.refs", uint64(len(tx.AccountList))*4) + + addMeta(stats, &tx.Meta) + addTransaction(stats, &tx.Transaction) +} + +func addMeta(stats *sizeStats, meta *pump_parser.RawTxMetaBinary) { + addErr(stats, meta.Err) + stats.add("meta.fee", 8) + addInnerInstructions(stats, meta.InnerInstructions) + addLamportBalances(stats, meta.PreBalances, meta.PostBalances) + addTokenBalances(stats, "meta.token_balances", meta.TokenBalances) + stats.add("meta.compute_units_consumed", 8) +} + +func addErr(stats *sizeStats, errValue *pump_parser.TransactionParsedError) { + stats.add("meta.err.present", 1) + if errValue == nil { + return + } + stats.add("meta.err.index", 1) + stats.add("meta.err.variant", 4) + stats.add("meta.err.enum", 4) + stats.add("meta.err.custom_code", 4) +} + +func addTransaction(stats *sizeStats, tx *pump_parser.RawTxTransactionBinary) { + stats.add("transaction.signature.present", 1) + if tx.HasSignature { + stats.add("transaction.signature.first", 64) + } + addHeader(stats) + addInstructions(stats, "transaction.instructions", tx.Message.Instructions) + addAddressTableLookups(stats, tx.Message.AddressTableLookups) +} + +func addHeader(stats *sizeStats) { + stats.add("transaction.header.num_readonly_signed", 4) + stats.add("transaction.header.num_readonly_unsigned", 4) + stats.add("transaction.header.num_required_signatures", 4) +} + +func addInnerInstructions(stats *sizeStats, values []pump_parser.InnerInstructions) { + stats.add("meta.inner_instructions.count", 4) + for _, value := range values { + stats.add("meta.inner_instructions.index", 4) + addInstructions(stats, "meta.inner_instructions.instructions", value.Instructions) + } +} + +func addInstructions(stats *sizeStats, prefix string, values []pump_parser.Instruction) { + stats.add(prefix+".count", 4) + for _, value := range values { + stats.add(prefix+".program_id_index", 2) + stats.add(prefix+".accounts.count", 4) + stats.add(prefix+".accounts.refs", uint64(len(value.Accounts))*2) + stats.add(prefix+".data.length", 4) + stats.add(prefix+".data.bytes", uint64(len(value.Data))) + stats.add(prefix+".stack_height.present", 1) + if value.StackHeight != nil { + stats.add(prefix+".stack_height.value", 4) + } + } +} + +func addAddressTableLookups(stats *sizeStats, values []pump_parser.RawTxAddressTableLookupBinary) { + stats.add("transaction.address_table_lookups.count", 4) + for _, value := range values { + stats.add("transaction.address_table_lookups.account_key", 4) + stats.add("transaction.address_table_lookups.writable.count", 4) + stats.add("transaction.address_table_lookups.writable.indexes", uint64(len(value.WritableIndexes))) + stats.add("transaction.address_table_lookups.readonly.count", 4) + stats.add("transaction.address_table_lookups.readonly.indexes", uint64(len(value.ReadonlyIndexes))) + } +} + +func addUint64Slice(stats *sizeStats, prefix string, count int) { + stats.add(prefix+".count", 4) + stats.add(prefix+".values", uint64(count)*8) +} + +func addLamportBalances(stats *sizeStats, preBalances []uint64, postBalances []uint64) { + stats.add("meta.pre_balances.count_uvarint", uint64(uvarintLen(uint64(len(preBalances))))) + for _, value := range preBalances { + stats.add("meta.pre_balances.value_uvarint", uint64(uvarintLen(value))) + } + n := len(preBalances) + if len(postBalances) < n { + n = len(postBalances) + } + changed := 0 + for i := 0; i < n; i++ { + if preBalances[i] != postBalances[i] { + changed++ + } + } + stats.add("meta.post_balance_changes.count_uvarint", uint64(uvarintLen(uint64(changed)))) + for i := 0; i < n; i++ { + if preBalances[i] == postBalances[i] { + continue + } + stats.add("meta.post_balance_changes.index_uvarint", uint64(uvarintLen(uint64(i)))) + stats.add("meta.post_balance_changes.delta_uvarint", uint64(zigzagDeltaUvarintLen(preBalances[i], postBalances[i]))) + } +} + +func addTokenBalances(stats *sizeStats, prefix string, values []pump_parser.RawTxTokenBalanceBinary) { + stats.add(prefix+".count", 4) + for _, value := range values { + stats.add(prefix+".account_index", 2) + stats.add(prefix+".mint_ref", 2) + stats.add(prefix+".owner.present", 1) + if value.HasOwnerAccount { + stats.add(prefix+".owner_ref", 2) + } + stats.add(prefix+".program_id_ref", 2) + stats.add(prefix+".decimals", 1) + stats.add(prefix+".pre_amount.present", 1) + if value.HasPreAmount { + stats.add(prefix+".pre_amount.length", 1) + stats.add(prefix+".pre_amount.bytes", uint64(uint256ByteLen(value.PreAmount))) + } + stats.add(prefix+".post_amount.present", 1) + if value.HasPostAmount { + stats.add(prefix+".post_amount.length", 1) + stats.add(prefix+".post_amount.bytes", uint64(uint256ByteLen(value.PostAmount))) + } + } +} + +func uint256ByteLen(value string) int { + if value == "" || value == "0" { + return 0 + } + amount, ok := new(big.Int).SetString(value, 10) + if !ok || amount.Sign() <= 0 { + return 0 + } + return len(amount.Bytes()) +} + +func printReport(filePath string, fileSize int, blocks *pump_parser.RawTxBlocksBinary, stats *sizeStats) { + type row struct { + name string + bytes uint64 + } + rows := make([]row, 0, len(stats.items)) + for name, bytes := range stats.items { + rows = append(rows, row{name: name, bytes: bytes}) + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].bytes == rows[j].bytes { + return rows[i].name < rows[j].name + } + return rows[i].bytes > rows[j].bytes + }) + + fmt.Printf("file=%s\n", filePath) + fmt.Printf("bytes=%d\n", fileSize) + fmt.Printf("schema_version=%d\n", blocks.SchemaVersion) + fmt.Printf("blocks=%d\n", len(blocks.BlockTxCounts)) + fmt.Printf("txs=%d\n", len(blocks.Txs)) + fmt.Printf("address_table_entries=%d\n", len(blocks.AddressTable)) + fmt.Println() + fmt.Printf("%-56s %12s %8s\n", "field", "bytes", "pct") + fmt.Printf("%-56s %12s %8s\n", "-----", "-----", "---") + for _, row := range rows { + fmt.Printf("%-56s %12d %7.2f%%\n", row.name, row.bytes, float64(row.bytes)*100/float64(fileSize)) + } +} + +func printInnerInstructionDataDistribution(blocks *pump_parser.RawTxBlocksBinary) { + stats := collectInnerInstructionDataStats(blocks) + values := make([]uint64, 0, len(stats)) + var total uint64 + var nonZero int + for _, stat := range stats { + values = append(values, stat.Bytes) + total += stat.Bytes + if stat.Bytes > 0 { + nonZero++ + } + } + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + + fmt.Println("inner_instruction_data_bytes_per_tx") + fmt.Printf("txs=%d nonzero_txs=%d total_bytes=%d avg=%.2f\n", len(stats), nonZero, total, avg(total, len(stats))) + if len(values) > 0 { + fmt.Printf("min=%d p50=%d p75=%d p90=%d p95=%d p99=%d max=%d\n", + values[0], + percentile(values, 0.50), + percentile(values, 0.75), + percentile(values, 0.90), + percentile(values, 0.95), + percentile(values, 0.99), + values[len(values)-1], + ) + } + + fmt.Println() + fmt.Printf("%-16s %8s %8s\n", "bucket", "txs", "bytes") + fmt.Printf("%-16s %8s %8s\n", "------", "---", "-----") + for _, bucket := range innerDataBuckets(stats) { + fmt.Printf("%-16s %8d %8d\n", bucket.label, bucket.count, bucket.bytes) + } + + sort.Slice(stats, func(i, j int) bool { + if stats[i].Bytes == stats[j].Bytes { + return stats[i].TxOrdinal < stats[j].TxOrdinal + } + return stats[i].Bytes > stats[j].Bytes + }) + fmt.Println() + fmt.Printf("%-6s %-5s %-8s %-12s %-8s %-10s\n", "rank", "block", "tx_index", "slot", "bytes", "inner_ix") + fmt.Printf("%-6s %-5s %-8s %-12s %-8s %-10s\n", "----", "-----", "--------", "----", "-----", "--------") + limit := 20 + if len(stats) < limit { + limit = len(stats) + } + for i := 0; i < limit; i++ { + stat := stats[i] + fmt.Printf("%-6d %-5d %-8d %-12d %-8d %-10d\n", i+1, stat.BlockIndex, stat.IndexWithinBlock, stat.Slot, stat.Bytes, stat.InstructionCount) + } +} + +func collectInnerInstructionDataStats(blocks *pump_parser.RawTxBlocksBinary) []txInnerDataStat { + out := make([]txInnerDataStat, 0, len(blocks.Txs)) + txOffset := 0 + for blockIndex, count := range blocks.BlockTxCounts { + for i := uint32(0); i < count; i++ { + tx := &blocks.Txs[txOffset] + var bytes uint64 + var instructionCount int + for _, inner := range tx.Meta.InnerInstructions { + for _, instruction := range inner.Instructions { + bytes += uint64(len(instruction.Data)) + instructionCount++ + } + } + out = append(out, txInnerDataStat{ + TxOrdinal: txOffset, + BlockIndex: blockIndex, + IndexWithinBlock: tx.IndexWithinBlock, + Slot: tx.Slot, + Bytes: bytes, + InstructionCount: instructionCount, + }) + txOffset++ + } + } + return out +} + +func percentile(values []uint64, p float64) uint64 { + if len(values) == 0 { + return 0 + } + idx := int(float64(len(values)-1) * p) + return values[idx] +} + +func avg(total uint64, count int) float64 { + if count == 0 { + return 0 + } + return float64(total) / float64(count) +} + +type innerDataBucket struct { + label string + count int + bytes uint64 +} + +func innerDataBuckets(stats []txInnerDataStat) []innerDataBucket { + buckets := []innerDataBucket{ + {label: "0"}, + {label: "1-63"}, + {label: "64-127"}, + {label: "128-255"}, + {label: "256-511"}, + {label: "512-1023"}, + {label: "1024-2047"}, + {label: "2048-4095"}, + {label: "4096+"}, + } + for _, stat := range stats { + index := 0 + switch { + case stat.Bytes == 0: + index = 0 + case stat.Bytes < 64: + index = 1 + case stat.Bytes < 128: + index = 2 + case stat.Bytes < 256: + index = 3 + case stat.Bytes < 512: + index = 4 + case stat.Bytes < 1024: + index = 5 + case stat.Bytes < 2048: + index = 6 + case stat.Bytes < 4096: + index = 7 + default: + index = 8 + } + buckets[index].count++ + buckets[index].bytes += stat.Bytes + } + return buckets +} + +type balanceValueStats struct { + name string + count int + unique int + zeroCount int + topValues []balanceTopValue + fixedBytes uint64 + uvarintBytes uint64 + duplicateCount int +} + +type balanceTopValue struct { + value uint64 + count int +} + +type balancePairStats struct { + txCount int + pairCount int + lengthMismatchTxs int + unchangedCount int + changedCount int + currentFixedValueBytes uint64 + bothUvarintBytes uint64 + preUvarintPostDelta uint64 + preUvarintChangedDeltas uint64 +} + +func printBalanceAnalysis(blocks *pump_parser.RawTxBlocksBinary) { + preValues := make([]uint64, 0) + postValues := make([]uint64, 0) + pairs := balancePairStats{} + pairs.txCount = len(blocks.Txs) + + for _, tx := range blocks.Txs { + preValues = append(preValues, tx.Meta.PreBalances...) + postValues = append(postValues, tx.Meta.PostBalances...) + preLen := len(tx.Meta.PreBalances) + postLen := len(tx.Meta.PostBalances) + if preLen != postLen { + pairs.lengthMismatchTxs++ + } + n := preLen + if postLen < n { + n = postLen + } + pairs.currentFixedValueBytes += uint64(preLen+postLen) * 8 + pairs.preUvarintChangedDeltas += uint64(uvarintLen(uint64(n))) + for i := 0; i < preLen; i++ { + pairs.bothUvarintBytes += uint64(uvarintLen(tx.Meta.PreBalances[i])) + pairs.preUvarintPostDelta += uint64(uvarintLen(tx.Meta.PreBalances[i])) + pairs.preUvarintChangedDeltas += uint64(uvarintLen(tx.Meta.PreBalances[i])) + } + for i := 0; i < postLen; i++ { + pairs.bothUvarintBytes += uint64(uvarintLen(tx.Meta.PostBalances[i])) + } + for i := 0; i < n; i++ { + pre := tx.Meta.PreBalances[i] + post := tx.Meta.PostBalances[i] + pairs.pairCount++ + pairs.preUvarintPostDelta += uint64(zigzagDeltaUvarintLen(pre, post)) + if pre == post { + pairs.unchangedCount++ + continue + } + pairs.changedCount++ + pairs.preUvarintChangedDeltas += uint64(uvarintLen(uint64(i))) + pairs.preUvarintChangedDeltas += uint64(zigzagDeltaUvarintLen(pre, post)) + } + } + + preStats := collectBalanceValueStats("pre_balances", preValues) + postStats := collectBalanceValueStats("post_balances", postValues) + combined := append(append([]uint64(nil), preValues...), postValues...) + combinedStats := collectBalanceValueStats("pre+post_balances", combined) + + fmt.Println("balance_values_analysis") + printBalanceValueStats(preStats) + printBalanceValueStats(postStats) + printBalanceValueStats(combinedStats) + + fmt.Println() + fmt.Println("balance_encoding_estimates") + fmt.Printf("txs=%d pairs=%d length_mismatch_txs=%d unchanged_pairs=%d changed_pairs=%d unchanged_pct=%.2f%%\n", + pairs.txCount, + pairs.pairCount, + pairs.lengthMismatchTxs, + pairs.unchangedCount, + pairs.changedCount, + float64(pairs.unchangedCount)*100/float64(maxInt(pairs.pairCount, 1)), + ) + printEstimate("current_fixed_uint64_values", pairs.currentFixedValueBytes, pairs.currentFixedValueBytes) + printEstimate("both_values_uvarint", pairs.bothUvarintBytes, pairs.currentFixedValueBytes) + printEstimate("pre_uvarint_post_delta_each_index", pairs.preUvarintPostDelta, pairs.currentFixedValueBytes) + printEstimate("pre_uvarint_post_changed_delta_pairs", pairs.preUvarintChangedDeltas, pairs.currentFixedValueBytes) +} + +func collectBalanceValueStats(name string, values []uint64) balanceValueStats { + freq := make(map[uint64]int) + var zeroCount int + var uvarintBytes uint64 + for _, value := range values { + freq[value]++ + if value == 0 { + zeroCount++ + } + uvarintBytes += uint64(uvarintLen(value)) + } + top := make([]balanceTopValue, 0, len(freq)) + var duplicateCount int + for value, count := range freq { + top = append(top, balanceTopValue{value: value, count: count}) + if count > 1 { + duplicateCount += count - 1 + } + } + sort.Slice(top, func(i, j int) bool { + if top[i].count == top[j].count { + return top[i].value < top[j].value + } + return top[i].count > top[j].count + }) + if len(top) > 10 { + top = top[:10] + } + return balanceValueStats{ + name: name, + count: len(values), + unique: len(freq), + zeroCount: zeroCount, + topValues: top, + fixedBytes: uint64(len(values)) * 8, + uvarintBytes: uvarintBytes, + duplicateCount: duplicateCount, + } +} + +func printBalanceValueStats(stats balanceValueStats) { + fmt.Printf("%s: count=%d unique=%d duplicate_values=%d zero=%d zero_pct=%.2f%% fixed_bytes=%d uvarint_bytes=%d uvarint_saved=%.2f%%\n", + stats.name, + stats.count, + stats.unique, + stats.duplicateCount, + stats.zeroCount, + float64(stats.zeroCount)*100/float64(maxInt(stats.count, 1)), + stats.fixedBytes, + stats.uvarintBytes, + savedPct(stats.fixedBytes, stats.uvarintBytes), + ) + fmt.Printf("%-22s %-8s %-8s\n", "value", "count", "pct") + for _, item := range stats.topValues { + fmt.Printf("%-22d %-8d %7.2f%%\n", item.value, item.count, float64(item.count)*100/float64(maxInt(stats.count, 1))) + } +} + +func printEstimate(name string, bytes uint64, baseline uint64) { + fmt.Printf("%-38s %10d saved=%7.2f%%\n", name, bytes, savedPct(baseline, bytes)) +} + +func savedPct(baseline uint64, value uint64) float64 { + if baseline == 0 { + return 0 + } + return (float64(baseline) - float64(value)) * 100 / float64(baseline) +} + +func uvarintLen(value uint64) int { + n := 1 + for value >= 0x80 { + value >>= 7 + n++ + } + return n +} + +func zigzagDeltaUvarintLen(pre uint64, post uint64) int { + if post >= pre { + return uvarintLen((post - pre) << 1) + } + return uvarintLen(((pre - post) << 1) - 1) +} + +func maxInt(a int, b int) int { + if a > b { + return a + } + return b +} diff --git a/rawtx_binary.go b/rawtx_binary.go new file mode 100644 index 0000000..6123e28 --- /dev/null +++ b/rawtx_binary.go @@ -0,0 +1,1742 @@ +package pump_parser + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "iter" + "math" + "math/big" + + "github.com/gagliardetto/solana-go" + "github.com/shopspring/decimal" +) + +const rawTxBinarySchemaVersionCurrent uint16 = 7 + +var rawTxBinaryMagic = [4]byte{'P', 'R', 'T', 'X'} +var rawTxsBinaryMagic = [4]byte{'P', 'R', 'T', 'S'} +var rawTxBlocksBinaryMagic = [4]byte{'P', 'R', 'B', 'S'} + +type RawTxBinary struct { + SchemaVersion uint16 + AddressTable []solana.PublicKey + + BlockTime int64 + IndexWithinBlock uint32 + Slot uint64 + Version uint8 + + AccountList []uint32 + AccountKeyCount uint32 + + Meta RawTxMetaBinary + Transaction RawTxTransactionBinary +} + +type RawTxsBinary struct { + SchemaVersion uint16 + AddressTable []solana.PublicKey + BlockTime int64 + Txs []RawTxBinary +} + +type RawTxBlocksBinary struct { + SchemaVersion uint16 + AddressTable []solana.PublicKey + BlockTimes []int64 + BlockTxCounts []uint32 + Txs []RawTxBinary +} + +type RawTxMetaBinary struct { + Err *TransactionParsedError + Fee uint64 + InnerInstructions []InnerInstructions + PostBalances []uint64 + PreBalances []uint64 + TokenBalances []RawTxTokenBalanceBinary + ComputeUnitsConsumed uint64 +} + +type RawTxTokenBalanceBinary struct { + AccountIndex uint16 + MintAccount uint16 + OwnerAccount uint16 + HasOwnerAccount bool + ProgramIDAccount uint16 + Decimals uint8 + HasPreAmount bool + PreAmount string + HasPostAmount bool + PostAmount string +} + +type RawTxTransactionBinary struct { + Message RawTxMessageBinary + Signature solana.Signature + HasSignature bool +} + +type RawTxMessageBinary struct { + Header Header + Instructions []Instruction + AddressTableLookups []RawTxAddressTableLookupBinary +} + +type RawTxAddressTableLookupBinary struct { + AccountKey uint32 + WritableIndexes []uint8 + ReadonlyIndexes []uint8 +} + +func NewRawTxBinary(tx *RawTx) (*RawTxBinary, error) { + if tx == nil { + return nil, fmt.Errorf("raw tx is nil") + } + addressTable, err := rawTxBinaryBuildAddressTable([]*RawTx{tx}) + if err != nil { + return nil, err + } + addressIndex, err := newTxBinaryAddressIndex(addressTable) + if err != nil { + return nil, err + } + return newRawTxBinaryWithAddressTable(tx, addressTable, addressIndex) +} + +func NewRawTxsBinary(txs []RawTx) (*RawTxsBinary, error) { + txPtrs := make([]*RawTx, 0, len(txs)) + for i := range txs { + txPtrs = append(txPtrs, &txs[i]) + } + addressTable, err := rawTxBinaryBuildAddressTable(txPtrs) + if err != nil { + return nil, err + } + addressIndex, err := newTxBinaryAddressIndex(addressTable) + if err != nil { + return nil, err + } + blockTime, err := rawTxBinarySharedBlockTime(txPtrs, "txs") + if err != nil { + return nil, err + } + out := &RawTxsBinary{ + SchemaVersion: rawTxBinarySchemaVersionCurrent, + AddressTable: addressTable, + BlockTime: blockTime, + Txs: make([]RawTxBinary, 0, len(txPtrs)), + } + for i, tx := range txPtrs { + binaryTx, err := newRawTxBinaryWithAddressTable(tx, addressTable, addressIndex) + if err != nil { + return nil, fmt.Errorf("tx[%d], %s: %w", i, tx.TxHash(), err) + } + out.Txs = append(out.Txs, *binaryTx) + } + return out, nil +} + +func NewRawTxBlocksBinary(blocks [][]RawTx) (*RawTxBlocksBinary, error) { + var txPtrs []*RawTx + for blockIndex := range blocks { + for txIndex := range blocks[blockIndex] { + txPtrs = append(txPtrs, &blocks[blockIndex][txIndex]) + } + } + addressTable, err := rawTxBinaryBuildAddressTable(txPtrs) + if err != nil { + return nil, err + } + addressIndex, err := newTxBinaryAddressIndex(addressTable) + if err != nil { + return nil, err + } + out := &RawTxBlocksBinary{ + SchemaVersion: rawTxBinarySchemaVersionCurrent, + AddressTable: addressTable, + BlockTimes: make([]int64, 0, len(blocks)), + BlockTxCounts: make([]uint32, 0, len(blocks)), + Txs: make([]RawTxBinary, 0, len(txPtrs)), + } + for blockIndex := range blocks { + if uint64(len(blocks[blockIndex])) > uint64(math.MaxUint32) { + return nil, fmt.Errorf("block[%d] tx count exceeds uint32 capacity", blockIndex) + } + blockTime := int64(0) + if len(blocks[blockIndex]) > 0 { + blockTime = blocks[blockIndex][0].BlockTime + for txIndex := range blocks[blockIndex] { + if blocks[blockIndex][txIndex].BlockTime != blockTime { + return nil, fmt.Errorf("block[%d].tx[%d] block time mismatch: got %d want %d", blockIndex, txIndex, blocks[blockIndex][txIndex].BlockTime, blockTime) + } + } + } + out.BlockTimes = append(out.BlockTimes, blockTime) + out.BlockTxCounts = append(out.BlockTxCounts, uint32(len(blocks[blockIndex]))) + for txIndex := range blocks[blockIndex] { + binaryTx, err := newRawTxBinaryWithAddressTable(&blocks[blockIndex][txIndex], addressTable, addressIndex) + if err != nil { + return nil, fmt.Errorf("block[%d].tx[%d], %s: %w", blockIndex, txIndex, blocks[blockIndex][txIndex].TxHash(), err) + } + out.Txs = append(out.Txs, *binaryTx) + } + } + return out, nil +} + +func EncodeRawTxBinary(tx *RawTx) ([]byte, error) { + binaryTx, err := NewRawTxBinary(tx) + if err != nil { + return nil, err + } + return binaryTx.MarshalBinary() +} + +func EncodeRawTxsBinary(txs []RawTx) ([]byte, error) { + binaryTxs, err := NewRawTxsBinary(txs) + if err != nil { + return nil, err + } + return binaryTxs.MarshalBinary() +} + +func EncodeRawTxBlocksBinary(blocks [][]RawTx) ([]byte, error) { + binaryBlocks, err := NewRawTxBlocksBinary(blocks) + if err != nil { + return nil, err + } + return binaryBlocks.MarshalBinary() +} + +func DecodeRawTxBinary(data []byte) (*RawTx, error) { + var binaryTx RawTxBinary + if err := binaryTx.UnmarshalBinary(data); err != nil { + return nil, err + } + return binaryTx.ToRawTx() +} + +func DecodeRawTxsBinary(data []byte) ([]*RawTx, error) { + var binaryTxs RawTxsBinary + if err := binaryTxs.UnmarshalBinary(data); err != nil { + return nil, err + } + return binaryTxs.ToRawTxs() +} + +func DecodeRawTxBlocksBinary(data []byte) ([][]*RawTx, error) { + var binaryBlocks RawTxBlocksBinary + if err := binaryBlocks.UnmarshalBinary(data); err != nil { + return nil, err + } + return binaryBlocks.ToRawTxBlocks() +} + +func DecodeRawTxsBinaryReader(r io.Reader) iter.Seq2[*RawTx, error] { + return func(yield func(*RawTx, error) bool) { + if r == nil { + yield(nil, fmt.Errorf("raw txs binary reader is nil")) + return + } + dec := txBinaryStreamDecoder{reader: r} + header, err := rawTxBinaryReadTxsHeader(&dec) + if err != nil { + yield(nil, err) + return + } + for i := uint32(0); i < header.count; i++ { + tx := RawTxBinary{ + SchemaVersion: header.schemaVersion, + AddressTable: header.addressTable, + BlockTime: header.blockTime, + } + if err := rawTxBinaryReadTxBody(&dec, &tx, header.addressTable, &header.blockTime); err != nil { + yield(nil, fmt.Errorf("tx[%d]: %w", i, err)) + return + } + decodedTx, err := tx.ToRawTx() + if err != nil { + yield(nil, fmt.Errorf("tx[%d]: %w", i, err)) + return + } + if !yield(decodedTx, nil) { + return + } + } + } +} + +func newRawTxBinaryWithAddressTable(tx *RawTx, addressTable []solana.PublicKey, addressIndex *txBinaryAddressIndex) (*RawTxBinary, error) { + accountList := tx.getAccountList() + if uint64(len(accountList)) > uint64(math.MaxUint32) { + return nil, fmt.Errorf("account list exceeds uint32 capacity") + } + if uint64(len(tx.Transaction.Message.AccountKeys)) > uint64(math.MaxUint32) { + return nil, fmt.Errorf("message account key count exceeds uint32 capacity") + } + if tx.IndexWithinBlock < 0 || uint64(tx.IndexWithinBlock) > uint64(math.MaxUint32) { + return nil, fmt.Errorf("index within block overflows uint32: %d", tx.IndexWithinBlock) + } + + out := &RawTxBinary{ + SchemaVersion: rawTxBinarySchemaVersionCurrent, + AddressTable: addressTable, + BlockTime: tx.BlockTime, + IndexWithinBlock: uint32(tx.IndexWithinBlock), + Slot: tx.Slot, + Version: rawTxBinaryVersionID(tx.Version), + AccountList: make([]uint32, 0, len(accountList)), + AccountKeyCount: uint32(len(tx.Transaction.Message.AccountKeys)), + } + for i, account := range accountList { + ref, err := addressIndex.id(account) + if err != nil { + return nil, fmt.Errorf("account_list[%d]: %w", i, err) + } + out.AccountList = append(out.AccountList, ref) + } + + meta, err := rawTxMetaToBinary(&tx.Meta, addressIndex) + if err != nil { + return nil, err + } + out.Meta = meta + message, err := rawTxMessageToBinary(&tx.Transaction.Message, addressIndex) + if err != nil { + return nil, err + } + out.Transaction = RawTxTransactionBinary{Message: message} + if len(tx.Transaction.Signatures) > 0 { + out.Transaction.Signature = tx.Transaction.Signatures[0] + out.Transaction.HasSignature = true + } + return out, nil +} + +func (tx *RawTxBinary) MarshalBinary() ([]byte, error) { + if tx == nil { + return nil, fmt.Errorf("raw tx binary is nil") + } + enc := txBinaryEncoder{} + enc.writeBytes(rawTxBinaryMagic[:]) + if err := rawTxBinaryWriteHeader(&enc, tx.SchemaVersion, tx.AddressTable); err != nil { + return nil, err + } + if err := rawTxBinaryWriteTxBody(&enc, tx, true); err != nil { + return nil, err + } + return enc.bytes(), nil +} + +func (txs *RawTxsBinary) MarshalBinary() ([]byte, error) { + if txs == nil { + return nil, fmt.Errorf("raw txs binary is nil") + } + enc := txBinaryEncoder{} + enc.writeBytes(rawTxsBinaryMagic[:]) + if err := rawTxBinaryWriteHeader(&enc, txs.SchemaVersion, txs.AddressTable); err != nil { + return nil, err + } + enc.writeUint64(uint64(txs.BlockTime)) + enc.writeUint32(uint32(len(txs.Txs))) + for i := range txs.Txs { + if txs.Txs[i].BlockTime != txs.BlockTime { + return nil, fmt.Errorf("tx[%d] block time mismatch: got %d want %d", i, txs.Txs[i].BlockTime, txs.BlockTime) + } + if err := rawTxBinaryWriteTxBody(&enc, &txs.Txs[i], false); err != nil { + return nil, fmt.Errorf("tx[%d]: %w", i, err) + } + } + return enc.bytes(), nil +} + +func (blocks *RawTxBlocksBinary) MarshalBinary() ([]byte, error) { + if blocks == nil { + return nil, fmt.Errorf("raw tx blocks binary is nil") + } + enc := txBinaryEncoder{} + enc.writeBytes(rawTxBlocksBinaryMagic[:]) + if err := rawTxBinaryWriteHeader(&enc, blocks.SchemaVersion, blocks.AddressTable); err != nil { + return nil, err + } + if len(blocks.BlockTimes) != len(blocks.BlockTxCounts) { + return nil, fmt.Errorf("raw tx blocks block time count mismatch: block_times=%d counts=%d", len(blocks.BlockTimes), len(blocks.BlockTxCounts)) + } + enc.writeUint32(uint32(len(blocks.BlockTxCounts))) + for i, count := range blocks.BlockTxCounts { + enc.writeUint64(uint64(blocks.BlockTimes[i])) + enc.writeUint32(count) + } + enc.writeUint32(uint32(len(blocks.Txs))) + txOffset := 0 + for blockIndex, count := range blocks.BlockTxCounts { + for txIndex := uint32(0); txIndex < count; txIndex++ { + if txOffset >= len(blocks.Txs) { + return nil, fmt.Errorf("block[%d].tx[%d]: tx offset out of range", blockIndex, txIndex) + } + if blocks.Txs[txOffset].BlockTime != blocks.BlockTimes[blockIndex] { + return nil, fmt.Errorf("block[%d].tx[%d] block time mismatch: got %d want %d", blockIndex, txIndex, blocks.Txs[txOffset].BlockTime, blocks.BlockTimes[blockIndex]) + } + if err := rawTxBinaryWriteTxBody(&enc, &blocks.Txs[txOffset], false); err != nil { + return nil, fmt.Errorf("block[%d].tx[%d]: %w", blockIndex, txIndex, err) + } + txOffset++ + } + } + if txOffset != len(blocks.Txs) { + return nil, fmt.Errorf("raw tx blocks unused tx payloads: %d", len(blocks.Txs)-txOffset) + } + return enc.bytes(), nil +} + +func (tx *RawTxBinary) UnmarshalBinary(data []byte) error { + dec := txBinaryDecoder{reader: bytes.NewReader(data)} + magic, err := dec.readN(len(rawTxBinaryMagic)) + if err != nil { + return err + } + if !bytes.Equal(magic, rawTxBinaryMagic[:]) { + return fmt.Errorf("invalid raw tx binary magic") + } + schemaVersion, addressTable, err := rawTxBinaryReadHeader(&dec) + if err != nil { + return err + } + tx.SchemaVersion = schemaVersion + tx.AddressTable = addressTable + if err := rawTxBinaryReadTxBody(&dec, tx, addressTable, nil); err != nil { + return err + } + if dec.reader.Len() != 0 { + return fmt.Errorf("unexpected trailing raw tx binary data: %d bytes", dec.reader.Len()) + } + return nil +} + +func (txs *RawTxsBinary) UnmarshalBinary(data []byte) error { + dec := txBinaryDecoder{reader: bytes.NewReader(data)} + header, err := rawTxBinaryReadTxsHeader(&dec) + if err != nil { + return err + } + txs.SchemaVersion = header.schemaVersion + txs.AddressTable = header.addressTable + txs.BlockTime = header.blockTime + txs.Txs = make([]RawTxBinary, 0, header.count) + for i := uint32(0); i < header.count; i++ { + tx := RawTxBinary{SchemaVersion: header.schemaVersion, AddressTable: header.addressTable, BlockTime: header.blockTime} + if err := rawTxBinaryReadTxBody(&dec, &tx, header.addressTable, &header.blockTime); err != nil { + return fmt.Errorf("tx[%d]: %w", i, err) + } + txs.Txs = append(txs.Txs, tx) + } + if dec.reader.Len() != 0 { + return fmt.Errorf("unexpected trailing raw txs binary data: %d bytes", dec.reader.Len()) + } + return nil +} + +func (blocks *RawTxBlocksBinary) UnmarshalBinary(data []byte) error { + dec := txBinaryDecoder{reader: bytes.NewReader(data)} + magic, err := dec.readN(len(rawTxBlocksBinaryMagic)) + if err != nil { + return err + } + if !bytes.Equal(magic, rawTxBlocksBinaryMagic[:]) { + return fmt.Errorf("invalid raw tx blocks binary magic") + } + schemaVersion, addressTable, err := rawTxBinaryReadHeader(&dec) + if err != nil { + return err + } + blocks.SchemaVersion = schemaVersion + blocks.AddressTable = addressTable + blockCount, err := dec.readUint32() + if err != nil { + return err + } + blocks.BlockTimes = make([]int64, 0, blockCount) + blocks.BlockTxCounts = make([]uint32, 0, blockCount) + var totalTxCount uint64 + for i := uint32(0); i < blockCount; i++ { + blockTime, err := readInt64(&dec) + if err != nil { + return err + } + count, err := dec.readUint32() + if err != nil { + return err + } + blocks.BlockTimes = append(blocks.BlockTimes, blockTime) + blocks.BlockTxCounts = append(blocks.BlockTxCounts, count) + totalTxCount += uint64(count) + if totalTxCount > uint64(math.MaxUint32) { + return fmt.Errorf("raw tx blocks total tx count exceeds uint32 capacity") + } + } + txCount, err := dec.readUint32() + if err != nil { + return err + } + if uint64(txCount) != totalTxCount { + return fmt.Errorf("raw tx blocks tx count mismatch: header=%d blocks=%d", txCount, totalTxCount) + } + blocks.Txs = make([]RawTxBinary, 0, txCount) + var txOffset uint32 + for blockIndex, count := range blocks.BlockTxCounts { + blockTime := blocks.BlockTimes[blockIndex] + for txIndex := uint32(0); txIndex < count; txIndex++ { + tx := RawTxBinary{SchemaVersion: schemaVersion, AddressTable: addressTable, BlockTime: blockTime} + if err := rawTxBinaryReadTxBody(&dec, &tx, addressTable, &blockTime); err != nil { + return fmt.Errorf("block[%d].tx[%d]: %w", blockIndex, txIndex, err) + } + blocks.Txs = append(blocks.Txs, tx) + txOffset++ + } + } + if txOffset != txCount { + return fmt.Errorf("raw tx blocks decoded tx count mismatch: got %d want %d", txOffset, txCount) + } + if dec.reader.Len() != 0 { + return fmt.Errorf("unexpected trailing raw tx blocks binary data: %d bytes", dec.reader.Len()) + } + return nil +} + +func (tx *RawTxBinary) ToRawTx() (*RawTx, error) { + if tx == nil { + return nil, nil + } + accountList, err := rawTxBinaryResolveAccountList(tx.AddressTable, tx.AccountList) + if err != nil { + return nil, err + } + out := &RawTx{ + accountList: append([]solana.PublicKey(nil), accountList...), + BlockTime: tx.BlockTime, + IndexWithinBlock: int64(tx.IndexWithinBlock), + Slot: tx.Slot, + Version: rawTxBinaryVersionValue(tx.Version), + Meta: rawTxMetaFromBinary(tx.Meta, tx.AddressTable), + Transaction: rawTxTransactionFromBinary(tx.Transaction, tx.AddressTable), + } + if tx.AccountKeyCount > 0 && tx.AccountKeyCount <= uint32(len(accountList)) { + out.Transaction.Message.AccountKeys = append(solana.PublicKeySlice(nil), accountList[:tx.AccountKeyCount]...) + } else { + out.Transaction.Message.AccountKeys = append(solana.PublicKeySlice(nil), accountList...) + } + return out, nil +} + +func (txs *RawTxsBinary) ToRawTxs() ([]*RawTx, error) { + if txs == nil { + return nil, nil + } + out := make([]*RawTx, 0, len(txs.Txs)) + for i := range txs.Txs { + txs.Txs[i].AddressTable = txs.AddressTable + tx, err := txs.Txs[i].ToRawTx() + if err != nil { + return nil, fmt.Errorf("tx[%d]: %w", i, err) + } + out = append(out, tx) + } + return out, nil +} + +func (blocks *RawTxBlocksBinary) ToRawTxBlocks() ([][]*RawTx, error) { + if blocks == nil { + return nil, nil + } + out := make([][]*RawTx, 0, len(blocks.BlockTxCounts)) + txOffset := 0 + for blockIndex, count := range blocks.BlockTxCounts { + block := make([]*RawTx, 0, count) + for txIndex := uint32(0); txIndex < count; txIndex++ { + if txOffset >= len(blocks.Txs) { + return nil, fmt.Errorf("block[%d].tx[%d]: tx offset out of range", blockIndex, txIndex) + } + blocks.Txs[txOffset].AddressTable = blocks.AddressTable + tx, err := blocks.Txs[txOffset].ToRawTx() + if err != nil { + return nil, fmt.Errorf("block[%d].tx[%d]: %w", blockIndex, txIndex, err) + } + block = append(block, tx) + txOffset++ + } + out = append(out, block) + } + if txOffset != len(blocks.Txs) { + return nil, fmt.Errorf("raw tx blocks unused tx payloads: %d", len(blocks.Txs)-txOffset) + } + return out, nil +} + +type rawTxsBinaryHeader struct { + schemaVersion uint16 + addressTable []solana.PublicKey + blockTime int64 + count uint32 +} + +func rawTxBinaryWriteHeader(enc *txBinaryEncoder, schemaVersion uint16, addressTable []solana.PublicKey) error { + if schemaVersion != rawTxBinarySchemaVersionCurrent { + return fmt.Errorf("unsupported raw tx binary schema version: %d", schemaVersion) + } + enc.writeUint16(schemaVersion) + return enc.writeAddressTable(addressTable) +} + +func rawTxBinaryReadHeader(dec txBinaryBodyReader) (uint16, []solana.PublicKey, error) { + schemaVersion, err := dec.readUint16() + if err != nil { + return 0, nil, err + } + if schemaVersion != rawTxBinarySchemaVersionCurrent { + return 0, nil, fmt.Errorf("unsupported raw tx binary schema version: %d", schemaVersion) + } + addressTable, err := txBinaryReadAddressTable(dec) + if err != nil { + return 0, nil, err + } + return schemaVersion, addressTable, nil +} + +func rawTxBinaryReadTxsHeader(dec txBinaryBodyReader) (*rawTxsBinaryHeader, error) { + magic, err := dec.readN(len(rawTxsBinaryMagic)) + if err != nil { + return nil, err + } + if !bytes.Equal(magic, rawTxsBinaryMagic[:]) { + return nil, fmt.Errorf("invalid raw txs binary magic") + } + schemaVersion, addressTable, err := rawTxBinaryReadHeader(dec) + if err != nil { + return nil, err + } + blockTime, err := readInt64(dec) + if err != nil { + return nil, err + } + count, err := dec.readUint32() + if err != nil { + return nil, err + } + return &rawTxsBinaryHeader{schemaVersion: schemaVersion, addressTable: addressTable, blockTime: blockTime, count: count}, nil +} + +func rawTxBinaryWriteTxBody(enc *txBinaryEncoder, tx *RawTxBinary, includeBlockTime bool) error { + if includeBlockTime { + enc.writeUint64(uint64(tx.BlockTime)) + } + enc.writeUint32(tx.IndexWithinBlock) + enc.writeUint64(tx.Slot) + enc.writeUint8(tx.Version) + enc.writeUint32(tx.AccountKeyCount) + writeUint32Slice(enc, tx.AccountList) + if err := writeRawTxMetaBinary(enc, tx.Meta); err != nil { + return err + } + return writeRawTxTransactionBinary(enc, tx.Transaction) +} + +func rawTxBinaryReadTxBody(dec txBinaryBodyReader, tx *RawTxBinary, addressTable []solana.PublicKey, blockTime *int64) error { + var err error + if blockTime == nil { + if tx.BlockTime, err = readInt64(dec); err != nil { + return err + } + } else { + tx.BlockTime = *blockTime + } + if tx.IndexWithinBlock, err = dec.readUint32(); err != nil { + return err + } + if tx.Slot, err = dec.readUint64(); err != nil { + return err + } + if tx.Version, err = dec.readUint8(); err != nil { + return err + } + if tx.AccountKeyCount, err = dec.readUint32(); err != nil { + return err + } + if tx.AccountList, err = readUint32Slice(dec); err != nil { + return err + } + if tx.Meta, err = readRawTxMetaBinary(dec); err != nil { + return err + } + if tx.Transaction, err = readRawTxTransactionBinary(dec); err != nil { + return err + } + tx.SchemaVersion = rawTxBinarySchemaVersionCurrent + tx.AddressTable = addressTable + return nil +} + +func rawTxMetaToBinary(meta *Meta, addressIndex *txBinaryAddressIndex) (RawTxMetaBinary, error) { + out := RawTxMetaBinary{ + Err: cloneTransactionParsedError(meta.Err), + Fee: meta.Fee, + InnerInstructions: cloneInnerInstructions(meta.InnerInstructions), + PostBalances: append([]uint64(nil), meta.PostBalances...), + PreBalances: append([]uint64(nil), meta.PreBalances...), + ComputeUnitsConsumed: meta.ComputeUnitsConsumed, + } + var err error + out.TokenBalances, err = rawTxTokenBalancesToBinary(meta.PreTokenBalances, meta.PostTokenBalances, addressIndex) + if err != nil { + return out, fmt.Errorf("token_balances: %w", err) + } + return out, nil +} + +func rawTxMessageToBinary(message *Message, addressIndex *txBinaryAddressIndex) (RawTxMessageBinary, error) { + out := RawTxMessageBinary{ + Header: message.Header, + Instructions: cloneInstructions(message.Instructions), + } + out.AddressTableLookups = make([]RawTxAddressTableLookupBinary, 0, len(message.AddressTableLookups)) + for i, lookup := range message.AddressTableLookups { + accountKey, err := addressIndex.id(lookup.AccountKey) + if err != nil { + return out, fmt.Errorf("address_table_lookups[%d].account_key: %w", i, err) + } + out.AddressTableLookups = append(out.AddressTableLookups, RawTxAddressTableLookupBinary{ + AccountKey: accountKey, + WritableIndexes: append([]uint8(nil), lookup.WritableIndexes...), + ReadonlyIndexes: append([]uint8(nil), lookup.ReadonlyIndexes...), + }) + } + return out, nil +} + +func rawTxTokenBalancesToBinary(preBalances []TokenBalance, postBalances []TokenBalance, addressIndex *txBinaryAddressIndex) ([]RawTxTokenBalanceBinary, error) { + out := make([]RawTxTokenBalanceBinary, 0, len(preBalances)+len(postBalances)) + byAccountIndex := make(map[uint16]int, len(preBalances)+len(postBalances)) + for i, balance := range preBalances { + encoded, err := rawTxTokenBalanceToBinary(balance, addressIndex) + if err != nil { + return nil, fmt.Errorf("pre[%d]: %w", i, err) + } + if _, exists := byAccountIndex[encoded.AccountIndex]; exists { + return nil, fmt.Errorf("pre[%d].account_index duplicate: %d", i, encoded.AccountIndex) + } + encoded.HasPreAmount = true + encoded.PreAmount = balance.UITokenAmount.Amount + byAccountIndex[encoded.AccountIndex] = len(out) + out = append(out, encoded) + } + for i, balance := range postBalances { + encoded, err := rawTxTokenBalanceToBinary(balance, addressIndex) + if err != nil { + return nil, fmt.Errorf("post[%d]: %w", i, err) + } + if existingIndex, exists := byAccountIndex[encoded.AccountIndex]; exists { + if out[existingIndex].HasPostAmount { + return nil, fmt.Errorf("post[%d].account_index duplicate: %d", i, encoded.AccountIndex) + } + if !rawTxTokenBalanceBinarySameIdentity(out[existingIndex], encoded) { + return nil, fmt.Errorf("post[%d].account_index %d identity mismatch", i, encoded.AccountIndex) + } + out[existingIndex].HasPostAmount = true + out[existingIndex].PostAmount = balance.UITokenAmount.Amount + continue + } + encoded.HasPostAmount = true + encoded.PostAmount = balance.UITokenAmount.Amount + byAccountIndex[encoded.AccountIndex] = len(out) + out = append(out, encoded) + } + return out, nil +} + +func rawTxTokenBalanceToBinary(balance TokenBalance, addressIndex *txBinaryAddressIndex) (RawTxTokenBalanceBinary, error) { + mintAccount, ownerAccount, programIDAccount, err := rawTxBinaryTokenBalanceAccounts(balance) + if err != nil { + return RawTxTokenBalanceBinary{}, err + } + mint, err := addressIndex.id(mintAccount) + if err != nil { + return RawTxTokenBalanceBinary{}, fmt.Errorf("mint: %w", err) + } + programID, err := addressIndex.id(programIDAccount) + if err != nil { + return RawTxTokenBalanceBinary{}, fmt.Errorf("program_id: %w", err) + } + if mint > math.MaxUint16 { + return RawTxTokenBalanceBinary{}, fmt.Errorf("mint ref overflows uint16: %d", mint) + } + if programID > math.MaxUint16 { + return RawTxTokenBalanceBinary{}, fmt.Errorf("program_id ref overflows uint16: %d", programID) + } + if balance.UITokenAmount.Decimals > math.MaxUint8 { + return RawTxTokenBalanceBinary{}, fmt.Errorf("decimals overflows uint8: %d", balance.UITokenAmount.Decimals) + } + if balance.AccountIndex < 0 || balance.AccountIndex > math.MaxUint16 { + return RawTxTokenBalanceBinary{}, fmt.Errorf("account_index overflows uint16: %d", balance.AccountIndex) + } + encoded := RawTxTokenBalanceBinary{ + AccountIndex: uint16(balance.AccountIndex), + MintAccount: uint16(mint), + ProgramIDAccount: uint16(programID), + Decimals: uint8(balance.UITokenAmount.Decimals), + } + if ownerAccount != nil { + owner, err := addressIndex.id(*ownerAccount) + if err != nil { + return RawTxTokenBalanceBinary{}, fmt.Errorf("owner: %w", err) + } + if owner > math.MaxUint16 { + return RawTxTokenBalanceBinary{}, fmt.Errorf("owner ref overflows uint16: %d", owner) + } + encoded.OwnerAccount = uint16(owner) + encoded.HasOwnerAccount = true + } + return encoded, nil +} + +func rawTxTokenBalanceBinarySameIdentity(a, b RawTxTokenBalanceBinary) bool { + return a.AccountIndex == b.AccountIndex && + a.MintAccount == b.MintAccount && + a.OwnerAccount == b.OwnerAccount && + a.HasOwnerAccount == b.HasOwnerAccount && + a.ProgramIDAccount == b.ProgramIDAccount && + a.Decimals == b.Decimals +} + +func rawTxMetaFromBinary(meta RawTxMetaBinary, addressTable []solana.PublicKey) Meta { + preTokenBalances, postTokenBalances := rawTxTokenBalancesFromBinary(meta.TokenBalances, addressTable) + return Meta{ + Err: cloneTransactionParsedError(meta.Err), + Fee: meta.Fee, + InnerInstructions: cloneInnerInstructions(meta.InnerInstructions), + PostBalances: append([]uint64(nil), meta.PostBalances...), + PostTokenBalances: postTokenBalances, + PreBalances: append([]uint64(nil), meta.PreBalances...), + PreTokenBalances: preTokenBalances, + Rewards: nil, + ComputeUnitsConsumed: meta.ComputeUnitsConsumed, + } +} + +func rawTxTransactionFromBinary(tx RawTxTransactionBinary, addressTable []solana.PublicKey) Transaction { + out := Transaction{ + Message: Message{ + Header: tx.Message.Header, + AddressTableLookups: rawTxAddressTableLookupsFromBinary(tx.Message.AddressTableLookups, addressTable), + Instructions: cloneInstructions(tx.Message.Instructions), + }, + } + if tx.HasSignature { + out.Signatures = []solana.Signature{tx.Signature} + } + return out +} + +func rawTxTokenBalancesFromBinary(balances []RawTxTokenBalanceBinary, addressTable []solana.PublicKey) ([]TokenBalance, []TokenBalance) { + pre := make([]TokenBalance, 0, len(balances)) + post := make([]TokenBalance, 0, len(balances)) + for _, balance := range balances { + mint, _ := txBinaryAddressAt(addressTable, uint32(balance.MintAccount), "token_balance.mint") + programID, _ := txBinaryAddressAt(addressTable, uint32(balance.ProgramIDAccount), "token_balance.program_id") + var owner *solana.PublicKey + if balance.HasOwnerAccount { + ownerKey, _ := txBinaryAddressAt(addressTable, uint32(balance.OwnerAccount), "token_balance.owner") + owner = &ownerKey + } + if balance.HasPreAmount { + tb := rawTxTokenBalanceFromBinary(balance, balance.PreAmount, mint, owner, programID) + pre = append(pre, tb) + } + if balance.HasPostAmount { + tb := rawTxTokenBalanceFromBinary(balance, balance.PostAmount, mint, owner, programID) + post = append(post, tb) + } + } + return pre, post +} + +func rawTxTokenBalanceFromBinary(balance RawTxTokenBalanceBinary, amount string, mint solana.PublicKey, owner *solana.PublicKey, programID solana.PublicKey) TokenBalance { + uiAmountDecimal := rawTxBinaryUIAmountDecimal(amount, balance.Decimals) + uiAmount, _ := uiAmountDecimal.Float64() + tb := TokenBalance{ + AccountIndex: int(balance.AccountIndex), + MintAccount: mint, + OwnerAccount: owner, + ProgramIDAccount: programID, + UITokenAmount: UITokenAmount{ + Amount: amount, + Decimals: uint64(balance.Decimals), + UIAmount: uiAmount, + UIAmountString: uiAmountDecimal.String(), + }, + } + tb.ParseAccount() + return tb +} + +func rawTxAddressTableLookupsFromBinary(lookups []RawTxAddressTableLookupBinary, addressTable []solana.PublicKey) solana.MessageAddressTableLookupSlice { + out := make(solana.MessageAddressTableLookupSlice, 0, len(lookups)) + for _, lookup := range lookups { + accountKey, _ := txBinaryAddressAt(addressTable, lookup.AccountKey, "address_table_lookup.account_key") + out = append(out, solana.MessageAddressTableLookup{ + AccountKey: accountKey, + WritableIndexes: append([]uint8(nil), lookup.WritableIndexes...), + ReadonlyIndexes: append([]uint8(nil), lookup.ReadonlyIndexes...), + }) + } + return out +} + +func writeRawTxMetaBinary(enc *txBinaryEncoder, meta RawTxMetaBinary) error { + writeTransactionParsedError(enc, meta.Err) + enc.writeUint64(meta.Fee) + if err := writeInnerInstructions(enc, meta.InnerInstructions); err != nil { + return fmt.Errorf("inner_instructions: %w", err) + } + if err := writeLamportBalances(enc, meta.PreBalances, meta.PostBalances); err != nil { + return fmt.Errorf("balances: %w", err) + } + if err := writeTokenBalances(enc, meta.TokenBalances); err != nil { + return fmt.Errorf("token_balances: %w", err) + } + enc.writeUint64(meta.ComputeUnitsConsumed) + return nil +} + +func readRawTxMetaBinary(dec txBinaryBodyReader) (RawTxMetaBinary, error) { + var out RawTxMetaBinary + var err error + if out.Err, err = readTransactionParsedError(dec); err != nil { + return out, err + } + if out.Fee, err = dec.readUint64(); err != nil { + return out, err + } + if out.InnerInstructions, err = readInnerInstructions(dec); err != nil { + return out, err + } + if out.PreBalances, out.PostBalances, err = readLamportBalances(dec); err != nil { + return out, err + } + if out.TokenBalances, err = readTokenBalances(dec); err != nil { + return out, err + } + if out.ComputeUnitsConsumed, err = dec.readUint64(); err != nil { + return out, err + } + return out, nil +} + +func writeRawTxTransactionBinary(enc *txBinaryEncoder, tx RawTxTransactionBinary) error { + enc.writeBool(tx.HasSignature) + if tx.HasSignature { + enc.writeBytes(tx.Signature[:]) + } + writeHeader(enc, tx.Message.Header) + if err := writeInstructions(enc, tx.Message.Instructions); err != nil { + return fmt.Errorf("instructions: %w", err) + } + enc.writeUint32(uint32(len(tx.Message.AddressTableLookups))) + for _, lookup := range tx.Message.AddressTableLookups { + enc.writeUint32(lookup.AccountKey) + writeUint8Slice(enc, lookup.WritableIndexes) + writeUint8Slice(enc, lookup.ReadonlyIndexes) + } + return nil +} + +func readRawTxTransactionBinary(dec txBinaryBodyReader) (RawTxTransactionBinary, error) { + var out RawTxTransactionBinary + hasSignature, err := dec.readBool() + if err != nil { + return out, err + } + out.HasSignature = hasSignature + if hasSignature { + raw, err := dec.readN(64) + if err != nil { + return out, err + } + out.Signature = solana.SignatureFromBytes(raw) + } + if out.Message.Header, err = readHeader(dec); err != nil { + return out, err + } + if out.Message.Instructions, err = readInstructions(dec); err != nil { + return out, err + } + lookupCount, err := dec.readUint32() + if err != nil { + return out, err + } + out.Message.AddressTableLookups = make([]RawTxAddressTableLookupBinary, 0, lookupCount) + for i := uint32(0); i < lookupCount; i++ { + lookup := RawTxAddressTableLookupBinary{} + if lookup.AccountKey, err = dec.readUint32(); err != nil { + return out, err + } + if lookup.WritableIndexes, err = readUint8Slice(dec); err != nil { + return out, err + } + if lookup.ReadonlyIndexes, err = readUint8Slice(dec); err != nil { + return out, err + } + out.Message.AddressTableLookups = append(out.Message.AddressTableLookups, lookup) + } + return out, nil +} + +func writeTransactionParsedError(enc *txBinaryEncoder, errValue *TransactionParsedError) { + enc.writeBool(errValue != nil) + if errValue == nil { + return + } + enc.writeUint8(errValue.Index) + enc.writeUint32(uint32(errValue.Variant)) + enc.writeUint32(uint32(errValue.Enum)) + enc.writeUint32(errValue.CustomCode) +} + +func readTransactionParsedError(dec txBinaryBodyReader) (*TransactionParsedError, error) { + hasErr, err := dec.readBool() + if err != nil { + return nil, err + } + if !hasErr { + return nil, nil + } + out := &TransactionParsedError{} + if out.Index, err = dec.readUint8(); err != nil { + return nil, err + } + var variant uint32 + if variant, err = dec.readUint32(); err != nil { + return nil, err + } + out.Variant = TransactionErrorVariant(variant) + var enum uint32 + if enum, err = dec.readUint32(); err != nil { + return nil, err + } + out.Enum = InstructionErrorVariant(enum) + if out.CustomCode, err = dec.readUint32(); err != nil { + return nil, err + } + return out, nil +} + +func writeInnerInstructions(enc *txBinaryEncoder, values []InnerInstructions) error { + enc.writeUint32(uint32(len(values))) + for i, value := range values { + enc.writeUint32(uint32(value.Index)) + if err := writeInstructions(enc, value.Instructions); err != nil { + return fmt.Errorf("[%d].instructions: %w", i, err) + } + } + return nil +} + +func readInnerInstructions(dec txBinaryBodyReader) ([]InnerInstructions, error) { + count, err := dec.readUint32() + if err != nil { + return nil, err + } + out := make([]InnerInstructions, 0, count) + for i := uint32(0); i < count; i++ { + index, err := dec.readUint32() + if err != nil { + return nil, err + } + instructions, err := readInstructions(dec) + if err != nil { + return nil, err + } + out = append(out, InnerInstructions{Index: int(index), Instructions: instructions}) + } + return out, nil +} + +func writeInstructions(enc *txBinaryEncoder, values []Instruction) error { + enc.writeUint32(uint32(len(values))) + for i, value := range values { + if value.ProgramIDIndex < 0 || value.ProgramIDIndex > math.MaxUint16 { + return fmt.Errorf("[%d].program_id_index overflows uint16: %d", i, value.ProgramIDIndex) + } + enc.writeUint16(uint16(value.ProgramIDIndex)) + if err := writeAccountIndexSlice(enc, value.Accounts); err != nil { + return fmt.Errorf("[%d].accounts: %w", i, err) + } + writeByteSlice(enc, value.Data) + enc.writeBool(value.StackHeight != nil) + if value.StackHeight != nil { + enc.writeUint32(uint32(*value.StackHeight)) + } + } + return nil +} + +func readInstructions(dec txBinaryBodyReader) ([]Instruction, error) { + count, err := dec.readUint32() + if err != nil { + return nil, err + } + out := make([]Instruction, 0, count) + for i := uint32(0); i < count; i++ { + programIDIndex, err := dec.readUint16() + if err != nil { + return nil, err + } + accounts, err := readAccountIndexSlice(dec) + if err != nil { + return nil, err + } + data, err := readByteSlice(dec) + if err != nil { + return nil, err + } + hasStackHeight, err := dec.readBool() + if err != nil { + return nil, err + } + var stackHeight *int + if hasStackHeight { + rawStackHeight, err := dec.readUint32() + if err != nil { + return nil, err + } + sh := int(rawStackHeight) + stackHeight = &sh + } + out = append(out, Instruction{ + ProgramIDIndex: int(programIDIndex), + Accounts: accounts, + Data: solana.Base58(data), + StackHeight: stackHeight, + }) + } + return out, nil +} + +func writeTokenBalances(enc *txBinaryEncoder, values []RawTxTokenBalanceBinary) error { + enc.writeUint32(uint32(len(values))) + for i, value := range values { + enc.writeUint16(value.AccountIndex) + enc.writeUint16(value.MintAccount) + enc.writeBool(value.HasOwnerAccount) + if value.HasOwnerAccount { + enc.writeUint16(value.OwnerAccount) + } + enc.writeUint16(value.ProgramIDAccount) + enc.writeUint8(value.Decimals) + enc.writeBool(value.HasPreAmount) + if value.HasPreAmount { + if err := writeUint256String(enc, value.PreAmount); err != nil { + return fmt.Errorf("[%d].pre_amount: %w", i, err) + } + } + enc.writeBool(value.HasPostAmount) + if value.HasPostAmount { + if err := writeUint256String(enc, value.PostAmount); err != nil { + return fmt.Errorf("[%d].post_amount: %w", i, err) + } + } + } + return nil +} + +func readTokenBalances(dec txBinaryBodyReader) ([]RawTxTokenBalanceBinary, error) { + count, err := dec.readUint32() + if err != nil { + return nil, err + } + out := make([]RawTxTokenBalanceBinary, 0, count) + for i := uint32(0); i < count; i++ { + value := RawTxTokenBalanceBinary{} + if value.AccountIndex, err = dec.readUint16(); err != nil { + return nil, err + } + if value.MintAccount, err = dec.readUint16(); err != nil { + return nil, err + } + if value.HasOwnerAccount, err = dec.readBool(); err != nil { + return nil, err + } + if value.HasOwnerAccount { + if value.OwnerAccount, err = dec.readUint16(); err != nil { + return nil, err + } + } + if value.ProgramIDAccount, err = dec.readUint16(); err != nil { + return nil, err + } + if value.Decimals, err = dec.readUint8(); err != nil { + return nil, err + } + if value.HasPreAmount, err = dec.readBool(); err != nil { + return nil, err + } + if value.HasPreAmount { + if value.PreAmount, err = readUint256String(dec); err != nil { + return nil, err + } + } + if value.HasPostAmount, err = dec.readBool(); err != nil { + return nil, err + } + if value.HasPostAmount { + if value.PostAmount, err = readUint256String(dec); err != nil { + return nil, err + } + } + out = append(out, value) + } + return out, nil +} + +func writeHeader(enc *txBinaryEncoder, header Header) { + enc.writeUint32(uint32(header.NumReadonlySignedAccounts)) + enc.writeUint32(uint32(header.NumReadonlyUnsignedAccounts)) + enc.writeUint32(uint32(header.NumRequiredSignatures)) +} + +func readHeader(dec txBinaryBodyReader) (Header, error) { + var out Header + value, err := dec.readUint32() + if err != nil { + return out, err + } + out.NumReadonlySignedAccounts = int(value) + value, err = dec.readUint32() + if err != nil { + return out, err + } + out.NumReadonlyUnsignedAccounts = int(value) + value, err = dec.readUint32() + if err != nil { + return out, err + } + out.NumRequiredSignatures = int(value) + return out, nil +} + +func writeString(enc *txBinaryEncoder, value string) { + writeByteSlice(enc, []byte(value)) +} + +func readString(dec txBinaryBodyReader) (string, error) { + raw, err := readByteSlice(dec) + if err != nil { + return "", err + } + return string(raw), nil +} + +func writeStringSlice(enc *txBinaryEncoder, values []string) { + enc.writeUint32(uint32(len(values))) + for _, value := range values { + writeString(enc, value) + } +} + +func readStringSlice(dec txBinaryBodyReader) ([]string, error) { + count, err := dec.readUint32() + if err != nil { + return nil, err + } + out := make([]string, 0, count) + for i := uint32(0); i < count; i++ { + value, err := readString(dec) + if err != nil { + return nil, err + } + out = append(out, value) + } + return out, nil +} + +func writeByteSlice(enc *txBinaryEncoder, values []byte) { + enc.writeUint32(uint32(len(values))) + enc.writeBytes(values) +} + +func readByteSlice(dec txBinaryBodyReader) ([]byte, error) { + count, err := dec.readUint32() + if err != nil { + return nil, err + } + return dec.readN(int(count)) +} + +func writeUint256String(enc *txBinaryEncoder, value string) error { + amount, err := rawTxBinaryParseUint256(value) + if err != nil { + return err + } + raw := amount.Bytes() + enc.writeUint8(uint8(len(raw))) + enc.writeBytes(raw) + return nil +} + +func readUint256String(dec txBinaryBodyReader) (string, error) { + length, err := dec.readUint8() + if err != nil { + return "", err + } + if length > 32 { + return "", fmt.Errorf("uint256 length exceeds 32 bytes: %d", length) + } + if length == 0 { + return "0", nil + } + raw, err := dec.readN(int(length)) + if err != nil { + return "", err + } + return new(big.Int).SetBytes(raw).String(), nil +} + +func writeUint8Slice(enc *txBinaryEncoder, values []uint8) { + writeByteSlice(enc, values) +} + +func readUint8Slice(dec txBinaryBodyReader) ([]uint8, error) { + return readByteSlice(dec) +} + +func writeUint32Slice(enc *txBinaryEncoder, values []uint32) { + enc.writeUint32(uint32(len(values))) + for _, value := range values { + enc.writeUint32(value) + } +} + +func readUint32Slice(dec txBinaryBodyReader) ([]uint32, error) { + count, err := dec.readUint32() + if err != nil { + return nil, err + } + out := make([]uint32, 0, count) + for i := uint32(0); i < count; i++ { + value, err := dec.readUint32() + if err != nil { + return nil, err + } + out = append(out, value) + } + return out, nil +} + +func writeAccountIndexSlice(enc *txBinaryEncoder, values []int) error { + enc.writeUint32(uint32(len(values))) + for i, value := range values { + if value < 0 || value > math.MaxUint16 { + return fmt.Errorf("[%d] overflows uint16: %d", i, value) + } + enc.writeUint16(uint16(value)) + } + return nil +} + +func readAccountIndexSlice(dec txBinaryBodyReader) ([]int, error) { + count, err := dec.readUint32() + if err != nil { + return nil, err + } + out := make([]int, 0, count) + for i := uint32(0); i < count; i++ { + value, err := dec.readUint16() + if err != nil { + return nil, err + } + out = append(out, int(value)) + } + return out, nil +} + +func writeLamportBalances(enc *txBinaryEncoder, preBalances []uint64, postBalances []uint64) error { + if len(preBalances) != len(postBalances) { + return fmt.Errorf("pre/post balance length mismatch: pre=%d post=%d", len(preBalances), len(postBalances)) + } + writeUvarint(enc, uint64(len(preBalances))) + for _, value := range preBalances { + writeUvarint(enc, value) + } + changed := 0 + for i := range preBalances { + if preBalances[i] != postBalances[i] { + changed++ + } + } + writeUvarint(enc, uint64(changed)) + for i := range preBalances { + if preBalances[i] == postBalances[i] { + continue + } + writeUvarint(enc, uint64(i)) + if err := writeLamportDelta(enc, preBalances[i], postBalances[i]); err != nil { + return fmt.Errorf("balance[%d]: %w", i, err) + } + } + return nil +} + +func readLamportBalances(dec txBinaryBodyReader) ([]uint64, []uint64, error) { + count, err := readUvarint(dec) + if err != nil { + return nil, nil, err + } + if count > uint64(math.MaxUint32) { + return nil, nil, fmt.Errorf("balance count too large: %d", count) + } + preBalances := make([]uint64, 0, count) + for i := uint64(0); i < count; i++ { + value, err := readUvarint(dec) + if err != nil { + return nil, nil, err + } + preBalances = append(preBalances, value) + } + postBalances := append([]uint64(nil), preBalances...) + changed, err := readUvarint(dec) + if err != nil { + return nil, nil, err + } + for i := uint64(0); i < changed; i++ { + index, err := readUvarint(dec) + if err != nil { + return nil, nil, err + } + if index >= uint64(len(postBalances)) { + return nil, nil, fmt.Errorf("post balance changed index out of range: %d", index) + } + value, err := readLamportDelta(dec, preBalances[index]) + if err != nil { + return nil, nil, fmt.Errorf("balance[%d]: %w", index, err) + } + postBalances[index] = value + } + return preBalances, postBalances, nil +} + +func writeLamportDelta(enc *txBinaryEncoder, pre uint64, post uint64) error { + const maxInt64Uint = uint64(1<<63 - 1) + var encoded uint64 + if post >= pre { + diff := post - pre + if diff > maxInt64Uint { + return fmt.Errorf("positive delta overflows int64: %d", diff) + } + encoded = diff << 1 + } else { + diff := pre - post + if diff > maxInt64Uint { + return fmt.Errorf("negative delta overflows int64: %d", diff) + } + encoded = (diff << 1) | 1 + } + writeUvarint(enc, encoded) + return nil +} + +func readLamportDelta(dec txBinaryBodyReader, pre uint64) (uint64, error) { + encoded, err := readUvarint(dec) + if err != nil { + return 0, err + } + diff := encoded >> 1 + if encoded&1 == 0 { + if ^uint64(0)-pre < diff { + return 0, fmt.Errorf("positive delta overflows uint64: pre=%d delta=%d", pre, diff) + } + return pre + diff, nil + } + if pre < diff { + return 0, fmt.Errorf("negative delta underflows uint64: pre=%d delta=%d", pre, diff) + } + return pre - diff, nil +} + +func writeUvarint(enc *txBinaryEncoder, value uint64) { + var raw [binary.MaxVarintLen64]byte + n := binary.PutUvarint(raw[:], value) + enc.writeBytes(raw[:n]) +} + +func readUvarint(dec txBinaryBodyReader) (uint64, error) { + var value uint64 + for shift := uint(0); shift < 64; shift += 7 { + b, err := dec.readUint8() + if err != nil { + return 0, err + } + if b < 0x80 { + if shift == 63 && b > 1 { + return 0, fmt.Errorf("uvarint overflows uint64") + } + return value | uint64(b)<= 0: %s", value) + } + if amount.BitLen() > 256 { + return nil, fmt.Errorf("uint256 overflow: %s", value) + } + return amount, nil +} + +func rawTxBinaryUIAmountDecimal(amount string, decimals uint8) decimal.Decimal { + rawAmount, err := rawTxBinaryParseUint256(amount) + if err != nil { + return decimal.Zero + } + return decimal.NewFromBigInt(rawAmount, -int32(decimals)) +} diff --git a/rawtx_binary_test.go b/rawtx_binary_test.go new file mode 100644 index 0000000..ddb91fb --- /dev/null +++ b/rawtx_binary_test.go @@ -0,0 +1,434 @@ +package pump_parser + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/shopspring/decimal" +) + +func TestRawTxBinaryRoundTripRealFixture(t *testing.T) { + original := mustLoadRawTxFixture(t, "testdata/rpc/4sj82GCLtgTDExq7B8YrBsrrqPcE4FqT5Y1gKWmE4cHMDxs7wkCV1hik73dSZ99gZm3K4wyBZQ6U8Nmf48rM9Jri.json") + + encoded, err := EncodeRawTxBinary(original) + if err != nil { + t.Fatalf("EncodeRawTxBinary() error = %v", err) + } + + decoded, err := DecodeRawTxBinary(encoded) + if err != nil { + t.Fatalf("DecodeRawTxBinary() error = %v", err) + } + + assertRawTxAccountAccess(t, decoded) + if decoded.TxHash() != original.TxHash() { + t.Fatalf("TxHash = %s, want %s", decoded.TxHash(), original.TxHash()) + } + if decoded.Slot != original.Slot { + t.Fatalf("Slot = %d, want %d", decoded.Slot, original.Slot) + } + if decoded.IndexWithinBlock != original.IndexWithinBlock { + t.Fatalf("IndexWithinBlock = %d, want %d", decoded.IndexWithinBlock, original.IndexWithinBlock) + } + if len(decoded.Meta.PostTokenBalances) != len(original.Meta.PostTokenBalances) { + t.Fatalf("PostTokenBalances len = %d, want %d", len(decoded.Meta.PostTokenBalances), len(original.Meta.PostTokenBalances)) + } + if len(decoded.Meta.PostTokenBalances) > 0 { + got := decoded.Meta.PostTokenBalances[0] + want := original.Meta.PostTokenBalances[0] + if got.AccountIndex != want.AccountIndex { + t.Fatalf("token balance account index = %d, want %d", got.AccountIndex, want.AccountIndex) + } + wantMint := want.MintAccount + if wantMint.IsZero() && want.Mint != "" { + var err error + wantMint, err = solana.PublicKeyFromBase58(want.Mint) + if err != nil { + t.Fatalf("parse want mint: %v", err) + } + } + if got.MintAccount != wantMint { + t.Fatalf("token balance mint = %s, want %s", got.MintAccount, wantMint) + } + if got.UITokenAmount.Decimals != want.UITokenAmount.Decimals { + t.Fatalf("token balance decimals = %d, want %d", got.UITokenAmount.Decimals, want.UITokenAmount.Decimals) + } + if got.UITokenAmount.Amount != want.UITokenAmount.Amount { + t.Fatalf("token balance amount = %s, want %s", got.UITokenAmount.Amount, want.UITokenAmount.Amount) + } + } +} + +func TestRawTxsBinaryBatchAndStreamRoundTrip(t *testing.T) { + tx1 := mustLoadRawTxFixture(t, "testdata/rpc/4sj82GCLtgTDExq7B8YrBsrrqPcE4FqT5Y1gKWmE4cHMDxs7wkCV1hik73dSZ99gZm3K4wyBZQ6U8Nmf48rM9Jri.json") + tx2 := mustLoadRawTxFixture(t, "testdata/rpc/43EouSYkeVmLBZSKW1KptiQcAVvB6KX49wjztjgzkQ9iU38A6fF68k77bNy9Wn6fwjykqYsPorUKj8m6SFY7naf1.json") + tx2.BlockTime = tx1.BlockTime + original := []RawTx{*tx1, *tx2} + + encoded, err := EncodeRawTxsBinary(original) + if err != nil { + t.Fatalf("EncodeRawTxsBinary() error = %v", err) + } + + decoded, err := DecodeRawTxsBinary(encoded) + if err != nil { + t.Fatalf("DecodeRawTxsBinary() error = %v", err) + } + if len(decoded) != len(original) { + t.Fatalf("DecodeRawTxsBinary len = %d, want %d", len(decoded), len(original)) + } + for i := range decoded { + assertRawTxAccountAccess(t, decoded[i]) + if decoded[i].TxHash() != original[i].TxHash() { + t.Fatalf("decoded[%d].TxHash = %s, want %s", i, decoded[i].TxHash(), original[i].TxHash()) + } + if decoded[i].BlockTime != original[i].BlockTime { + t.Fatalf("decoded[%d].BlockTime = %d, want %d", i, decoded[i].BlockTime, original[i].BlockTime) + } + } + + var streamed int + for decodedTx, err := range DecodeRawTxsBinaryReader(bytes.NewReader(encoded)) { + if err != nil { + t.Fatalf("DecodeRawTxsBinaryReader() error = %v", err) + } + assertRawTxAccountAccess(t, decodedTx) + streamed++ + } + if streamed != len(original) { + t.Fatalf("streamed tx count = %d, want %d", streamed, len(original)) + } +} + +func TestRawTxBlocksBinaryRoundTrip(t *testing.T) { + tx1 := mustLoadRawTxFixture(t, "testdata/rpc/4sj82GCLtgTDExq7B8YrBsrrqPcE4FqT5Y1gKWmE4cHMDxs7wkCV1hik73dSZ99gZm3K4wyBZQ6U8Nmf48rM9Jri.json") + tx2 := mustLoadRawTxFixture(t, "testdata/rpc/43EouSYkeVmLBZSKW1KptiQcAVvB6KX49wjztjgzkQ9iU38A6fF68k77bNy9Wn6fwjykqYsPorUKj8m6SFY7naf1.json") + blocks := [][]RawTx{{*tx1}, {*tx2}} + + encoded, err := EncodeRawTxBlocksBinary(blocks) + if err != nil { + t.Fatalf("EncodeRawTxBlocksBinary() error = %v", err) + } + + decoded, err := DecodeRawTxBlocksBinary(encoded) + if err != nil { + t.Fatalf("DecodeRawTxBlocksBinary() error = %v", err) + } + if len(decoded) != len(blocks) { + t.Fatalf("block count = %d, want %d", len(decoded), len(blocks)) + } + for blockIndex := range decoded { + if len(decoded[blockIndex]) != len(blocks[blockIndex]) { + t.Fatalf("block[%d] tx count = %d, want %d", blockIndex, len(decoded[blockIndex]), len(blocks[blockIndex])) + } + for txIndex := range decoded[blockIndex] { + assertRawTxAccountAccess(t, decoded[blockIndex][txIndex]) + if decoded[blockIndex][txIndex].TxHash() != blocks[blockIndex][txIndex].TxHash() { + t.Fatalf("block[%d].tx[%d] hash mismatch", blockIndex, txIndex) + } + if decoded[blockIndex][txIndex].BlockTime != blocks[blockIndex][txIndex].BlockTime { + t.Fatalf("block[%d].tx[%d] block time = %d, want %d", blockIndex, txIndex, decoded[blockIndex][txIndex].BlockTime, blocks[blockIndex][txIndex].BlockTime) + } + } + } +} + +func TestRawTxBinaryPreservesAccountListHelperBehavior(t *testing.T) { + owner := mustPubKey("SysvarRent111111111111111111111111111111111") + mint := solana.WrappedSol + tokenProgram := solana.TokenProgramID + ata, _, err := solana.FindProgramAddress([][]byte{ + owner[:], + tokenProgram[:], + mint[:], + }, solana.SPLAssociatedTokenAccountProgramID) + if err != nil { + t.Fatalf("find ata: %v", err) + } + + original := &RawTx{ + accountList: []solana.PublicKey{owner, ata, mint, tokenProgram}, + BlockTime: 1710000000, + Slot: 123, + IndexWithinBlock: 7, + Transaction: Transaction{Signatures: []solana.Signature{{1, 2, 3}}}, + Meta: Meta{ + PreBalances: []uint64{2_000_000_000, 0, 0, 0}, + PostBalances: []uint64{1_500_000_000, 0, 0, 0}, + PreTokenBalances: []TokenBalance{{ + AccountIndex: 1, + MintAccount: mint, + OwnerAccount: &owner, + ProgramIDAccount: tokenProgram, + UITokenAmount: UITokenAmount{ + Amount: "100", + Decimals: 9, + UIAmount: 0.0000001, + UIAmountString: "0.0000001", + }, + }}, + PostTokenBalances: []TokenBalance{{ + AccountIndex: 1, + MintAccount: mint, + OwnerAccount: &owner, + ProgramIDAccount: tokenProgram, + UITokenAmount: UITokenAmount{ + Amount: "250", + Decimals: 9, + UIAmount: 0.00000025, + UIAmountString: "0.00000025", + }, + }}, + }, + } + + binaryForm, err := NewRawTxBinary(original) + if err != nil { + t.Fatalf("NewRawTxBinary() error = %v", err) + } + if len(binaryForm.Meta.TokenBalances) != 1 { + t.Fatalf("binary token balance count = %d, want 1", len(binaryForm.Meta.TokenBalances)) + } + if got := binaryForm.Meta.TokenBalances[0]; !got.HasPreAmount || !got.HasPostAmount || got.PreAmount != "100" || got.PostAmount != "250" { + t.Fatalf("merged binary token balance = %+v", got) + } + + encoded, err := EncodeRawTxBinary(original) + if err != nil { + t.Fatalf("EncodeRawTxBinary() error = %v", err) + } + decoded, err := DecodeRawTxBinary(encoded) + if err != nil { + t.Fatalf("DecodeRawTxBinary() error = %v", err) + } + + for _, tx := range []*RawTx{original, decoded} { + if got, err := GetSolAfterTx(tx, 0); err != nil || got != 1_500_000_000 { + t.Fatalf("GetSolAfterTx() = %d, %v; want 1500000000, nil", got, err) + } + balance, err := getTokenBalanceAfterTx(tx, 1) + if err != nil { + t.Fatalf("getTokenBalanceAfterTx() error = %v", err) + } + if balance.UITokenAmount.Amount != "250" { + t.Fatalf("getTokenBalanceAfterTx amount = %s, want 250", balance.UITokenAmount.Amount) + } + if got := getAccountBalanceAfterTx(tx, 1); !got.Equal(decimal.NewFromInt(250)) { + t.Fatalf("getAccountBalanceAfterTx() = %s, want 250", got) + } + if got := GetTokenBalanceAfterTx(tx, 0, tokenProgram, mint); !got.Equal(decimal.NewFromInt(250)) { + t.Fatalf("GetTokenBalanceAfterTx() = %s, want 250", got) + } + ataBalance, err := getAtaByOwner(tx, owner, mint) + if err != nil { + t.Fatalf("getAtaByOwner() error = %v", err) + } + if ataBalance.AccountIndex != 1 { + t.Fatalf("getAtaByOwner account index = %d, want 1", ataBalance.AccountIndex) + } + ataIndex, err := getAtaIdxByOwner(tx, owner, mint) + if err != nil { + t.Fatalf("getAtaIdxByOwner() error = %v", err) + } + if ataIndex != 1 { + t.Fatalf("getAtaIdxByOwner() = %d, want 1", ataIndex) + } + change, changeAtaIndex := tokenBalanceChange(tx, 0, tokenProgram, mint) + if !change.Equal(decimal.NewFromInt(150)) || changeAtaIndex != 1 { + t.Fatalf("tokenBalanceChange() = (%s, %d), want (150, 1)", change, changeAtaIndex) + } + } +} + +func TestRawTxBinaryTokenBalanceAmountSupportsUint256(t *testing.T) { + owner := mustPubKey("SysvarRent111111111111111111111111111111111") + mint := solana.WrappedSol + tokenProgram := solana.TokenProgramID + amount := "340282366920938463463374607431768211455" + original := &RawTx{ + accountList: []solana.PublicKey{owner, mint, tokenProgram}, + Transaction: Transaction{ + Signatures: []solana.Signature{{1, 2, 3}}, + }, + Meta: Meta{ + PreBalances: []uint64{1}, + PostBalances: []uint64{1}, + PostTokenBalances: []TokenBalance{{ + AccountIndex: 0, + MintAccount: mint, + OwnerAccount: &owner, + ProgramIDAccount: tokenProgram, + UITokenAmount: UITokenAmount{ + Amount: amount, + Decimals: 9, + }, + }}, + }, + } + + encoded, err := EncodeRawTxBinary(original) + if err != nil { + t.Fatalf("EncodeRawTxBinary() error = %v", err) + } + decoded, err := DecodeRawTxBinary(encoded) + if err != nil { + t.Fatalf("DecodeRawTxBinary() error = %v", err) + } + got := decoded.Meta.PostTokenBalances[0].UITokenAmount + if got.Amount != amount { + t.Fatalf("Amount = %s, want %s", got.Amount, amount) + } + if got.UIAmountString != "340282366920938463463374607431.768211455" { + t.Fatalf("UIAmountString = %s", got.UIAmountString) + } +} + +func TestRawTxBlocksBinarySaveSlots414696178To414696182(t *testing.T) { + rpcURL := os.Getenv("RAWTX_BINARY_RPC_URL") + if rpcURL == "" { + t.Skip("set RAWTX_BINARY_RPC_URL to run RPC-backed rawtx-binary block save test") + } + + const startSlot uint64 = 414696178 + const endSlot uint64 = 414696182 + + client := rpc.New(rpcURL) + rewards := false + version := uint64(0) + blocks := make([][]RawTx, 0, endSlot-startSlot+1) + totalTx := 0 + filteredVote := 0 + for slot := startSlot; slot <= endSlot; slot++ { + block, err := client.GetBlockWithOpts(context.Background(), slot, &rpc.GetBlockOpts{ + TransactionDetails: rpc.TransactionDetailsFull, + Rewards: &rewards, + Commitment: rpc.CommitmentFinalized, + Encoding: solana.EncodingBase64, + MaxSupportedTransactionVersion: &version, + }) + if err != nil { + t.Fatalf("get block %d: %v", slot, err) + } + var blockTime uint64 + if block.BlockTime != nil { + blockTime = uint64(*block.BlockTime) + } + + rawTxs := make([]RawTx, 0, len(block.Transactions)) + for i, tx := range block.Transactions { + totalTx++ + rawTx, err := FromRpcTransactionWithMeta(tx, &blockTime, slot, int64(i)) + if err != nil { + t.Fatalf("slot %d tx[%d] convert: %v", slot, i, err) + } + if rawTxBinaryIsVoteTx(rawTx) { + filteredVote++ + continue + } + rawTxs = append(rawTxs, *rawTx) + } + blocks = append(blocks, rawTxs) + } + + encoded, err := EncodeRawTxBlocksBinary(blocks) + if err != nil { + t.Fatalf("EncodeRawTxBlocksBinary() error = %v", err) + } + outputPath := os.Getenv("RAWTX_BINARY_OUT") + if outputPath == "" { + outputPath = filepath.Join("testdata", "rawtx-binary", "rawtx-blocks-414696178-414696182.prbs") + } + if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil { + t.Fatalf("create rawtx binary output dir: %v", err) + } + if err := os.WriteFile(outputPath, encoded, 0o644); err != nil { + t.Fatalf("write rawtx binary: %v", err) + } + + decoded, err := DecodeRawTxBlocksBinary(encoded) + if err != nil { + t.Fatalf("DecodeRawTxBlocksBinary() error = %v", err) + } + if len(decoded) != len(blocks) { + t.Fatalf("decoded block count = %d, want %d", len(decoded), len(blocks)) + } + savedTx := 0 + for blockIndex := range decoded { + if len(decoded[blockIndex]) != len(blocks[blockIndex]) { + t.Fatalf("decoded block[%d] tx count = %d, want %d", blockIndex, len(decoded[blockIndex]), len(blocks[blockIndex])) + } + for txIndex := range decoded[blockIndex] { + assertRawTxAccountAccess(t, decoded[blockIndex][txIndex]) + if rawTxBinaryIsVoteTx(decoded[blockIndex][txIndex]) { + t.Fatalf("decoded block[%d].tx[%d] is vote tx", blockIndex, txIndex) + } + savedTx++ + } + } + if savedTx == 0 { + t.Fatal("saved tx count is zero") + } + t.Logf("saved rawtx binary: path=%s bytes=%d total_tx=%d saved_tx=%d filtered_vote=%d", outputPath, len(encoded), totalTx, savedTx, filteredVote) +} + +func mustLoadRawTxFixture(t *testing.T, path string) *RawTx { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + var response RPCResponse + if err := json.Unmarshal(raw, &response); err != nil { + t.Fatalf("unmarshal fixture: %v", err) + } + return &response.Result +} + +func assertRawTxAccountAccess(t *testing.T, tx *RawTx) { + t.Helper() + accounts := tx.GetAccountList() + if len(accounts) == 0 { + t.Fatal("decoded account list is empty") + } + for _, instr := range tx.Transaction.Message.Instructions { + if instr.ProgramIDIndex < 0 || instr.ProgramIDIndex >= len(accounts) { + t.Fatalf("instruction program id index %d out of range %d", instr.ProgramIDIndex, len(accounts)) + } + for _, accountIndex := range instr.Accounts { + if accountIndex < 0 || accountIndex >= len(accounts) { + t.Fatalf("instruction account index %d out of range %d", accountIndex, len(accounts)) + } + } + } + for _, inner := range tx.Meta.InnerInstructions { + for _, instr := range inner.Instructions { + if instr.ProgramIDIndex < 0 || instr.ProgramIDIndex >= len(accounts) { + t.Fatalf("inner instruction program id index %d out of range %d", instr.ProgramIDIndex, len(accounts)) + } + for _, accountIndex := range instr.Accounts { + if accountIndex < 0 || accountIndex >= len(accounts) { + t.Fatalf("inner instruction account index %d out of range %d", accountIndex, len(accounts)) + } + } + } + } +} + +func rawTxBinaryIsVoteTx(tx *RawTx) bool { + if tx == nil { + return false + } + accountList := tx.GetAccountList() + for _, instr := range tx.Transaction.Message.Instructions { + if instr.ProgramIDIndex >= 0 && instr.ProgramIDIndex < len(accountList) && accountList[instr.ProgramIDIndex] == solana.VoteProgramID { + return true + } + } + return false +}