onsig/libsam
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

parallel parsing

Browse Source
This commit is contained in:
thloyi
2026-01-07 21:15:54 +08:00
parent b82b7d9b0e
commit d9aea3e8d7
8 changed files with 290 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
pkg/shreder/addresstables.go
View File

@@ -11,11 +11,17 @@ import (
"github.com/panjf2000/ants/v2"
)
type TableInfo struct {
overErrCount int
addresses []solana.PublicKey
}
type AddressTables struct {
rpcClient *rpc.Client
mux sync.RWMutex
loadMux sync.Mutex
tables *lru.Cache[solana.PublicKey, []solana.PublicKey]
tables *lru.Cache[solana.PublicKey, *TableInfo]
loading map[solana.PublicKey]struct{}
pool *ants.Pool
@@ -23,7 +29,7 @@ type AddressTables struct {
func NewAddressTables(rpcClient *rpc.Client) *AddressTables {
pool, _ := ants.NewPool(5, ants.WithPreAlloc(true), ants.WithNonblocking(true))
cache, _ := lru.New[solana.PublicKey, []solana.PublicKey](10000)
cache, _ := lru.New[solana.PublicKey, *TableInfo](10000)
return &AddressTables{
rpcClient: rpcClient,
tables: cache,
@@ -54,53 +60,60 @@ func (at *AddressTables) loadAddressTable(tablePubkey solana.PublicKey) ([]solan
return addresses, nil
}
func (at *AddressTables) load(tablePubkey solana.PublicKey) {
_ = at.pool.Submit(func() {
at.loadMux.Lock()
_, loading := at.loading[tablePubkey]
if loading {
at.loadMux.Unlock()
return
}
at.loading[tablePubkey] = struct{}{}
at.loadMux.Unlock()
table, err := at.loadAddressTable(tablePubkey)
if err != nil {
logger.Error("loadAddressTable failed", "err", err, "table", tablePubkey)
at.loadMux.Lock()
delete(at.loading, tablePubkey)
at.loadMux.Unlock()
return
}
at.loadMux.Lock()
delete(at.loading, tablePubkey)
at.loadMux.Unlock()
at.mux.Lock()
at.tables.Add(tablePubkey, &TableInfo{
addresses: table,
})
total := at.tables.Len()
at.mux.Unlock()
logger.Info("loadAddressTable", "table", tablePubkey.String(), "table count:", total)
})
}
func (at *AddressTables) GetAddressTable(tablePubkey solana.PublicKey, idx []uint8) []solana.PublicKey {
at.mux.RLock()
addresses, ok := at.tables.Get(tablePubkey)
if !ok {
at.mux.RUnlock()
_ = at.pool.Submit(func() {
at.loadMux.Lock()
_, loading := at.loading[tablePubkey]
if loading {
at.loadMux.Unlock()
return
}
at.loading[tablePubkey] = struct{}{}
at.loadMux.Unlock()
table, err := at.loadAddressTable(tablePubkey)
if err != nil {
logger.Error("loadAddressTable failed", "err", err, "table", tablePubkey)
at.loadMux.Lock()
delete(at.loading, tablePubkey)
at.loadMux.Unlock()
return
}
at.loadMux.Lock()
delete(at.loading, tablePubkey)
at.loadMux.Unlock()
at.mux.Lock()
at.tables.Add(tablePubkey, table)
total := at.tables.Len()
at.mux.Unlock()
logger.Info("loadAddressTable", "table", tablePubkey.String(), "table count:", total)
})
at.load(tablePubkey)
return nil
}
at.mux.RUnlock()
var result solana.PublicKeySlice = make([]solana.PublicKey, 0, len(idx))
for _, i := range idx {
if int(i) >= len(addresses) {
if int(i) >= len(addresses.addresses) {
logger.Error("over loadAddressTable failed", "idx", i, "table", tablePubkey)
//todo... update table?
continue
addresses.overErrCount++
if addresses.overErrCount > 10 {
at.load(tablePubkey)
}
break
}
result = append(result, addresses[i])
result = append(result, addresses.addresses[i])
}
return result
}