以太坊区块数据的持久化和查询
2018-08-21 本文已影响32人
swapmem
区块链数据的存储和查询需求十分简单。比如，给定一个区块号查询对应区块数据，给定一个区块哈希查询对应区块的数据，给定一个交易哈希查询这笔交易的详情，等等。这样的需求最适合KV存储引擎。以太坊区块数据的存储底层引擎采用谷歌开源的levelDB存储引擎。本文从源码入手分析以太坊区块数据的存储和查询过程，使用的go-ethereum源码，git commit hash是(6d1e292eefa70b5cb76cd03ff61fc6c4550d7c36)。和数据持久化和查询相关的代码主要分布在$GOPATH/src/github.com/ethereum/go-ethereum/core/rawdb/中。
关键数据结构
以太坊中一个区块在逻辑上可以看作是由区块头(header)和区块体(body)组成。header的结构如下
type Header struct {
ParentHash common.Hash `json:"parentHash" gencodec:"required"`
UncleHash common.Hash `json:"sha3Uncles" gencodec:"required"`
Coinbase common.Address `json:"miner" gencodec:"required"`
Root common.Hash `json:"stateRoot" gencodec:"required"`
TxHash common.Hash `json:"transactionsRoot" gencodec:"required"`
ReceiptHash common.Hash `json:"receiptsRoot" gencodec:"required"`
Bloom Bloom `json:"logsBloom" gencodec:"required"`
Difficulty *big.Int `json:"difficulty" gencodec:"required"`
Number *big.Int `json:"number" gencodec:"required"`
GasLimit uint64 `json:"gasLimit" gencodec:"required"`
GasUsed uint64 `json:"gasUsed" gencodec:"required"`
Time *big.Int `json:"timestamp" gencodec:"required"`
Extra []byte `json:"extraData" gencodec:"required"`
MixDigest common.Hash `json:"mixHash" gencodec:"required"`
Nonce BlockNonce `json:"nonce" gencodec:"required"`
}
body的结构如下
type Body struct {
Transactions []*Transaction
Uncles []*Header
}
但是以太坊一个区块的实际定义是这样的
type Block struct {
header *Header
uncles []*Header
transactions Transactions
// caches
hash atomic.Value
size atomic.Value
// Td is used by package core to store the total difficulty
// of the chain up to and including the block.
td *big.Int
// These fields are used by package eth to track
// inter-peer block relay.
ReceivedAt time.Time
ReceivedFrom interface{}
}
主要包含了header, uncles, transactions和td四个字段, 其他的几个字段主要用于性能优化和统计。
区块数据存储和查询
在进行数据同步时，节点会首先下载区块头，节点应该怎么存储从临节点下载的区块头数据呢？看下面这个函数
// WriteHeader stores a block header into the database and also stores the hash-
// to-number mapping.
func WriteHeader(db DatabaseWriter, header *types.Header) {
// Write the hash -> number mapping
var (
hash = header.Hash()
number = header.Number.Uint64()
encoded = encodeBlockNumber(number)
)
key := headerNumberKey(hash)
if err := db.Put(key, encoded); err != nil {
log.Crit("Failed to store hash to number mapping", "err", err)
}
// Write the encoded header
data, err := rlp.EncodeToBytes(header)
if err != nil {
log.Crit("Failed to RLP encode header", "err", err)
}
key = headerKey(number, hash)
if err := db.Put(key, data); err != nil {
log.Crit("Failed to store header", "err", err)
}
}
第164~166行，通过header得到区块哈希和大端编码之后的区块号。第168行得到存储区块头对应区块号的key。169~171存储一对key-value。这样做的目的是通过区块头的哈希就可以快速查询到对应的区块号。接着173行对header数据进行rlp编码,177行通过number和hash构造可以直接查询hader数据的key，178行将数据存入数据库。
我们希望通过区块号，区块哈希查询到整个区块的数据，通过交易号查询交易的详细信息和交易的回执(receipt)信息。接下来分析这些数据是如何被查询到的。
通过区块号查询整个区块的数据
首先需要根据区块号构造查询key，看下面的这几个函数
// ReadCanonicalHash retrieves the hash assigned to a canonical block number.
func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash {
data, _ := db.Get(headerHashKey(number))
if len(data) == 0 {
return common.Hash{}
}
return common.BytesToHash(data)
}
// ReadHeaderRLP retrieves a block header in its raw RLP database encoding.
func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue {
data, _ := db.Get(headerKey(number, hash))
return data
}
// ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding.
func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue {
data, _ := db.Get(blockBodyKey(number, hash))
return data
}
向ReadCanonicalHash传入区块号可以得到区块头哈希，通过ReadHeaderRLP，传入区块头哈希和区块号就可以得到Header经过RLP编码之后的值，最后ReadBodyRLP传入同样的参数就可以得到header对应的body。
至此，通过区块号查询整个区块信息的过程就理解清楚了。
通过区块哈希查询整个区块的数据
这个功能和通过区块号查询区块信息的过程基本类似，但是需要首先调用ReadheaderNumber函数得到这个header哈希对应的区块编号。
func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 {
data, _ := db.Get(headerNumberKey(hash))
if len(data) != 8 {
return nil
}
number := binary.BigEndian.Uint64(data)
return &number
}
之后就是通过header hash和number，调用ReadHeaderRLP, ReadBodyRLP得到整个区块的信息。
通过交易号查询交易详细信息
首先看交易信息是如何写入底层数据库的。
// TxLookupEntry is a positional metadata to help looking up the data content of
// a transaction or receipt given only its hash.
type TxLookupEntry struct {
BlockHash common.Hash
BlockIndex uint64
Index uint64
}
// WriteTxLookupEntries stores a positional metadata for every transaction from
// a block, enabling hash based transaction and receipt lookups.
func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) {
for i, tx := range block.Transactions() {
entry := TxLookupEntry{
BlockHash: block.Hash(),
BlockIndex: block.NumberU64(),
Index: uint64(i),
}
data, err := rlp.EncodeToBytes(entry)
if err != nil {
log.Crit("Failed to encode transaction lookup entry", "err", err)
}
if err := db.Put(txLookupKey(tx.Hash()), data); err != nil {
log.Crit("Failed to store transaction lookup entry", "err", err)
}
}
}
写入过程简单易懂，由交易号构造查询key，vaule对应一个TxLookupEntry。再看查询过程
func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash)
if blockHash == (common.Hash{}) {
return nil, common.Hash{}, 0, 0
}
body := ReadBody(db, blockHash, blockNumber)
if body == nil || len(body.Transactions) <= int(txIndex) {
log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex)
return nil, common.Hash{}, 0, 0
}
return body.Transactions[txIndex], blockHash, blockNumber, txIndex
}
// ReadTxLookupEntry retrieves the positional metadata associated with a transaction
// hash to allow retrieving the transaction or receipt by hash.
func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) {
data, _ := db.Get(txLookupKey(hash))
if len(data) == 0 {
return common.Hash{}, 0, 0
}
var entry TxLookupEntry
if err := rlp.DecodeBytes(data, &entry); err != nil {
log.Error("Invalid transaction lookup entry RLP", "hash", hash, "err", err)
return common.Hash{}, 0, 0
}
return entry.BlockHash, entry.BlockIndex, entry.Index
}
查询交易信息可以通过交易哈希调用ReadTransaction直接查询,返回的数据是交易信息和这笔交易的位置信息。
查询交易回执信息
// ReadReceipts retrieves all the transaction receipts belonging to a block.
func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
// Retrieve the flattened receipt slice
data, _ := db.Get(blockReceiptsKey(number, hash))
if len(data) == 0 {
return nil
}
// Convert the revceipts from their storage form to their internal representation
storageReceipts := []*types.ReceiptForStorage{}
if err := rlp.DecodeBytes(data, &storageReceipts); err != nil {
log.Error("Invalid receipt array RLP", "hash", hash, "err", err)
return nil
}
receipts := make(types.Receipts, len(storageReceipts))
for i, receipt := range storageReceipts {
receipts[i] = (*types.Receipt)(receipt)
}
return receipts
}
// WriteReceipts stores all the transaction receipts belonging to a block.
func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) {
// Convert the receipts into their storage form and serialize them
storageReceipts := make([]*types.ReceiptForStorage, len(receipts))
for i, receipt := range receipts {
storageReceipts[i] = (*types.ReceiptForStorage)(receipt)
}
bytes, err := rlp.EncodeToBytes(storageReceipts)
if err != nil {
log.Crit("Failed to encode block receipts", "err", err)
}
// Store the flattened receipt slice
if err := db.Put(blockReceiptsKey(number, hash), bytes); err != nil {
log.Crit("Failed to store block receipts", "err", err)
}
}
// ReadReceipt retrieves a specific transaction receipt from the database, along with
// its added positional metadata.
func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) {
blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash)
if blockHash == (common.Hash{}) {
return nil, common.Hash{}, 0, 0
}
receipts := ReadReceipts(db, blockHash, blockNumber)
if len(receipts) <= int(receiptIndex) {
log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex)
return nil, common.Hash{}, 0, 0
}
return receipts[receiptIndex], blockHash, blockNumber, receiptIndex
}
原理和其他几个类似，在这里贴出了关键的源码。
总结
header, body, transaction, transactionReceipt 是分开存放的，不是根据区块号直接找到这个区块的所有数据，也不是根据header hash直接找到该区块的所有数据。以太坊在写入数据的时候会首先写入一些元信息，主要是在区块中的位置信息。例如在写入一笔交易信息的时候会首先写入它的位置信息(区块头哈希，区块号，交易所在区块体中的位置)。
