Skip to content

Commit

Permalink
Update Committee Cache for v0.9 (#3948)
Browse files Browse the repository at this point in the history 
* Updated committee cache

* Removed shuffled indices cache

* Started testing run time

* Lint

* Fixed test
  • Loading branch information
@terencechain @prestonvanloon
terencechain authored and prestonvanloon committed Nov 7, 2019
1 parent d27bd4e commit bb64762
Show file tree
Hide file tree
Showing 16 changed files with 42 additions and 396 deletions.
1 change: 0 additions & 1 deletion beacon-chain/blockchain/forkchoice/process_attestation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,6 @@ func (s *Store) aggregateAttestation(ctx context.Context, att *ethpb.Attestation
if err != nil {
return err
}

if a, ok := s.attsQueue[root]; ok {
a, err := helpers.AggregateAttestation(a, att)
if err != nil {
Expand Down
7 changes: 7 additions & 0 deletions beacon-chain/blockchain/forkchoice/process_block.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,13 @@ func (s *Store) OnBlockNoVerifyStateTransition(ctx context.Context, b *ethpb.Bea
// Epoch boundary bookkeeping such as logging epoch summaries.
if helpers.IsEpochStart(postState.Slot) {
reportEpochMetrics(postState)

// Update committee shuffled indices at the end of every epoch
if featureconfig.Get().EnableNewCache {
if err := helpers.UpdateCommitteeCache(postState); err != nil {
return err
}
}
}

return nil
Expand Down
2 changes: 0 additions & 2 deletions beacon-chain/cache/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ go_library(
"committee.go",
"common.go",
"eth1_data.go",
"shuffled_indices.go",
],
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/cache",
visibility = ["//beacon-chain:__subpackages__"],
Expand Down Expand Up @@ -41,7 +40,6 @@ go_test(
"committee_test.go",
"eth1_data_test.go",
"feature_flag_test.go",
"shuffled_indices_test.go",
],
embed = [":go_default_library"],
race = "on",
Expand Down
58 changes: 20 additions & 38 deletions beacon-chain/cache/committee.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prysmaticlabs/prysm/shared/featureconfig"
"github.com/prysmaticlabs/prysm/shared/params"
"github.com/prysmaticlabs/prysm/shared/sliceutil"
"k8s.io/client-go/tools/cache"
)
Expand All @@ -33,15 +34,14 @@ var (
})
)

// Committee defines the committee per epoch and shard.
// Committee defines the committee per epoch and index.
type Committee struct {
CommitteeIndex uint64
CommitteeCount uint64
Epoch uint64
Committee []uint64
}

// CommitteeCache is a struct with 1 queue for looking up shuffled indices list by epoch and committee.
// CommitteeCache is a struct with 1 queue for looking up shuffled indices list by epoch and committee index.
type CommitteeCache struct {
CommitteeCache *cache.FIFO
lock sync.RWMutex
Expand All @@ -64,15 +64,17 @@ func NewCommitteeCache() *CommitteeCache {
}
}

// ShuffledIndices fetches the shuffled indices by epoch and committee. Every list of indices
// represent one committee. Returns true if the list exists with epoch and committee. Otherwise returns false, nil.
func (c *CommitteeCache) ShuffledIndices(epoch uint64, index uint64) ([]uint64, error) {
// ShuffledIndices fetches the shuffled indices by slot and committee index. Every list of indices
// represent one committee. Returns true if the list exists with slot and committee index. Otherwise returns false, nil.
func (c *CommitteeCache) ShuffledIndices(slot uint64, index uint64) ([]uint64, error) {
if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
return nil, nil
}
c.lock.RLock()
defer c.lock.RUnlock()
obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(int(epoch)))

epoch := int(slot / params.BeaconConfig().SlotsPerEpoch)
obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(epoch))
if err != nil {
return nil, err
}
Expand All @@ -89,7 +91,13 @@ func (c *CommitteeCache) ShuffledIndices(epoch uint64, index uint64) ([]uint64,
return nil, ErrNotCommittee
}

start, end := startEndIndices(item, index)
committeeCountPerSlot := uint64(1)
if item.CommitteeCount/params.BeaconConfig().SlotsPerEpoch > 1 {
committeeCountPerSlot = item.CommitteeCount / params.BeaconConfig().SlotsPerEpoch
}

indexOffSet := index + (slot%params.BeaconConfig().SlotsPerEpoch)*committeeCountPerSlot
start, end := startEndIndices(item, indexOffSet)
return item.Committee[start:end], nil
}

Expand Down Expand Up @@ -147,40 +155,14 @@ func (c *CommitteeCache) EpochInCache(wantedEpoch uint64) (bool, error) {
return false, nil
}

// CommitteeCount returns the total number of committees in a given epoch as stored in cache.
func (c *CommitteeCache) CommitteeCount(epoch uint64) (uint64, bool, error) {
if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
return 0, false, nil
}
c.lock.RLock()
defer c.lock.RUnlock()
obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(int(epoch)))
if err != nil {
return 0, false, err
}

if exists {
CommitteeCacheHit.Inc()
} else {
CommitteeCacheMiss.Inc()
return 0, false, nil
}

item, ok := obj.(*Committee)
if !ok {
return 0, false, ErrNotCommittee
}

return item.CommitteeCount, true, nil
}

// CommitteeIndex returns the committee index in a given epoch as stored in cache.
func (c *CommitteeCache) CommitteeIndex(epoch uint64) (uint64, bool, error) {
// CommitteeCountPerSlot returns the number of committees in a given slot as stored in cache.
func (c *CommitteeCache) CommitteeCountPerSlot(slot uint64) (uint64, bool, error) {
if !featureconfig.Get().EnableShuffledIndexCache && !featureconfig.Get().EnableNewCache {
return 0, false, nil
}
c.lock.RLock()
defer c.lock.RUnlock()
epoch := int(slot / params.BeaconConfig().SlotsPerEpoch)
obj, exists, err := c.CommitteeCache.GetByKey(strconv.Itoa(int(epoch)))
if err != nil {
return 0, false, err
Expand All @@ -198,7 +180,7 @@ func (c *CommitteeCache) CommitteeIndex(epoch uint64) (uint64, bool, error) {
return 0, false, ErrNotCommittee
}

return item.CommitteeIndex, true, nil
return item.CommitteeCount / params.BeaconConfig().SlotsPerEpoch, true, nil
}

// ActiveIndices returns the active indices of a given epoch stored in cache.
Expand Down
74 changes: 7 additions & 67 deletions beacon-chain/cache/committee_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"reflect"
"strconv"
"testing"

"github.com/prysmaticlabs/prysm/shared/params"
)

func TestCommitteeKeyFn_OK(t *testing.T) {
Expand Down Expand Up @@ -36,12 +38,11 @@ func TestCommitteeCache_CommitteesByEpoch(t *testing.T) {
Epoch: 1,
Committee: []uint64{1, 2, 3, 4, 5, 6},
CommitteeCount: 3,
CommitteeIndex: 1,
}

epoch := uint64(1)
slot := uint64(item.Epoch * params.BeaconConfig().SlotsPerEpoch)
committeeIndex := uint64(1)
indices, err := cache.ShuffledIndices(epoch, committeeIndex)
indices, err := cache.ShuffledIndices(slot, committeeIndex)
if err != nil {
t.Fatal(err)
}
Expand All @@ -52,11 +53,12 @@ func TestCommitteeCache_CommitteesByEpoch(t *testing.T) {
if err := cache.AddCommitteeShuffledList(item); err != nil {
t.Fatal(err)
}
wantedIndex := uint64(2)
indices, err = cache.ShuffledIndices(epoch, wantedIndex)
wantedIndex := uint64(0)
indices, err = cache.ShuffledIndices(slot, wantedIndex)
if err != nil {
t.Fatal(err)
}

start, end := startEndIndices(item, wantedIndex)
if !reflect.DeepEqual(indices, item.Committee[start:end]) {
t.Errorf(
Expand Down Expand Up @@ -143,68 +145,6 @@ func TestCommitteeCache_EpochInCache(t *testing.T) {
}
}

func TestCommitteeCache_CommitteesCount(t *testing.T) {
cache := NewCommitteeCache()

committeeCount := uint64(3)
epoch := uint64(10)
item := &Committee{Epoch: epoch, CommitteeCount: committeeCount}

_, exists, err := cache.CommitteeCount(1)
if err != nil {
t.Fatal(err)
}
if exists {
t.Error("Expected committee count not to exist in empty cache")
}

if err := cache.AddCommitteeShuffledList(item); err != nil {
t.Fatal(err)
}

count, exists, err := cache.CommitteeCount(epoch)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Error("Expected committee count to be in cache")
}
if count != committeeCount {
t.Errorf("wanted: %d, got: %d", committeeCount, count)
}
}

func TestCommitteeCache_IndexCount(t *testing.T) {
cache := NewCommitteeCache()

committeeIndex := uint64(7)
epoch := uint64(3)
item := &Committee{Epoch: epoch, CommitteeIndex: committeeIndex}

_, exists, err := cache.CommitteeIndex(1)
if err != nil {
t.Fatal(err)
}
if exists {
t.Error("Expected start index not to exist in empty cache")
}

if err := cache.AddCommitteeShuffledList(item); err != nil {
t.Fatal(err)
}

index, exists, err := cache.CommitteeIndex(epoch)
if err != nil {
t.Fatal(err)
}
if !exists {
t.Error("Expected start index to be in cache")
}
if index != committeeIndex {
t.Errorf("wanted: %d, got: %d", committeeIndex, index)
}
}

func TestCommitteeCache_ActiveIndices(t *testing.T) {
cache := NewCommitteeCache()

Expand Down
106 changes: 0 additions & 106 deletions beacon-chain/cache/shuffled_indices.go
View file

This file was deleted.

Loading

0 comments on commit bb64762

Please sign in to comment.