Refactor state module to leverage DocumentBuffer
This commit refactors the state module to remove the eventCache and introduce an eventStream instead. The eventStream is not a static array but a DocumentBuffer, and thus many functions had to be altered. This commit is best reviewed side by side with a before/after view instead of just looking at the diff. An Init() function is added to initialize the DocumentBuffer with config values. Facilities are added to both Event and EventType to allow for parsing them to and from string documents. a MakeEvent() function is added to create events of proper subtype based on a general data map input. ApplyToEvents() is added, which wraps around DocumentBuffer's Apply() method to alter the filter input type with one that Marshals and Unmarshals events. GetMatchingEvents() is now a proof of concept for DocumentBuffer's Apply() PruneCache() is now no longer needed. Signed-off-by: Ava Affine <ava@sunnypup.io>
This commit is contained in:
parent
609c50ff7d
commit
0a29b35f4b
2 changed files with 321 additions and 138 deletions
|
|
@ -9,10 +9,13 @@ package state
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
"os"
|
||||
"errors"
|
||||
"encoding/json"
|
||||
|
||||
"gitlab.com/whom/bingobot/internal/docbuf"
|
||||
"gitlab.com/whom/bingobot/internal/logging"
|
||||
)
|
||||
|
||||
|
|
@ -24,32 +27,46 @@ import (
|
|||
const (
|
||||
BadEventTypeError = "bad event type"
|
||||
EventValidationFailedError = "event failed validation: "
|
||||
BadEventNoUID = "event data has no UID field"
|
||||
BadEventNoCreated = "event data has no created field"
|
||||
BadEventStoreFilename = "failed to open event store file"
|
||||
BadEventStreamInit = "failed to initialize event stream"
|
||||
BadEventCreate = "failed to create event"
|
||||
BadUserEventCreatedParse = "failed to parse created time"
|
||||
BadChallengeEvent = "failed to make Challenge Event"
|
||||
BadRestorationEvent = "failed to make Restoration Event"
|
||||
BadUserActiveEvent = "failed to make UserActive Event"
|
||||
BadEventObjMarshal = "error marshalling event"
|
||||
BadEventObjUnmarshal = "failed to unmarshal event map"
|
||||
BadEventUnmarshal = "failed to unmarshal event"
|
||||
BadEventMissingTypeKey = "event map missing type key"
|
||||
)
|
||||
|
||||
const (
|
||||
EventTypeMapKey = "type"
|
||||
)
|
||||
|
||||
var eventMutex sync.RWMutex
|
||||
var eventSubscriptionCache = [NumEventTypes][]chan Event{}
|
||||
var eventCache = []Event{}
|
||||
var eventStream docbuf.DocumentBuffer
|
||||
var maxEventsInMemory int
|
||||
|
||||
// TODO: Configurable
|
||||
var eventChannelBufferSize = 256
|
||||
|
||||
// TODO: pruned events go to DISK
|
||||
// ASSUMES eventCache is ordered by age (oldest first)
|
||||
func pruneEventCache() {
|
||||
eventMutex.Lock()
|
||||
defer eventMutex.Unlock()
|
||||
|
||||
oldCacheInvalid := false
|
||||
newCache := []Event{}
|
||||
for _, obj := range eventCache {
|
||||
if time.Since(obj.Time()).Hours() > 24*10 {
|
||||
oldCacheInvalid = true
|
||||
} else if oldCacheInvalid {
|
||||
newCache = append(newCache, obj)
|
||||
}
|
||||
// expect filename validations in config package
|
||||
func Init(eventMemCacheSize int, eventStoreFileName string) error {
|
||||
file, err := os.OpenFile(eventStoreFileName, os.O_CREATE|os.O_RDWR, 0644)
|
||||
if err != nil {
|
||||
return errors.Join(errors.New(BadEventStoreFilename), err)
|
||||
}
|
||||
|
||||
eventCache = newCache
|
||||
if eventStream, err = docbuf.NewDocumentBuffer(
|
||||
eventMemCacheSize,
|
||||
file,
|
||||
); err != nil {
|
||||
return errors.Join(errors.New(BadEventStreamInit), err)
|
||||
}
|
||||
|
||||
maxEventsInMemory = eventMemCacheSize
|
||||
return nil
|
||||
}
|
||||
|
||||
type EventType int8
|
||||
|
|
@ -65,14 +82,123 @@ const (
|
|||
NumEventTypes
|
||||
)
|
||||
|
||||
// either returns a valid event type or NumEventTypes
|
||||
func EventTypeFromString(doc string) EventType {
|
||||
switch doc {
|
||||
case "Vote":
|
||||
return Vote
|
||||
case "Challenge":
|
||||
return Challenge
|
||||
case "Restoration":
|
||||
return Restoration
|
||||
case "UserActive":
|
||||
return UserActive
|
||||
default:
|
||||
// error case
|
||||
return NumEventTypes
|
||||
}
|
||||
}
|
||||
|
||||
func (et EventType) String() string {
|
||||
events := []string{
|
||||
"Vote",
|
||||
"Challenge",
|
||||
"Restoration",
|
||||
"UserActive",
|
||||
}
|
||||
|
||||
if et < 0 || et >= NumEventTypes {
|
||||
return ""
|
||||
}
|
||||
|
||||
return events[et]
|
||||
}
|
||||
|
||||
func (et EventType) Validate() error {
|
||||
if et < 0 || et >= NumEventTypes {
|
||||
return stringErrorType(BadEventTypeError)
|
||||
return errors.New(BadEventTypeError)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/* Allocates a specific event of type T from event type instance.
|
||||
* output event not guaranteed to be valid.
|
||||
* Call Validate() yourself.
|
||||
*/
|
||||
func (et EventType) MakeEvent(data map[string]string) (Event, error) {
|
||||
if err := et.Validate(); err != nil {
|
||||
return nil, errors.Join(errors.New(BadEventCreate), err)
|
||||
}
|
||||
|
||||
MakeUserEvent := func(data map[string]string) (*UserEvent, error) {
|
||||
user, hasUser := data[UserEventUserKey]
|
||||
if !hasUser {
|
||||
return nil, errors.New(BadEventNoUID)
|
||||
}
|
||||
|
||||
created, hasCreated := data[UserEventCreatedKey]
|
||||
if !hasCreated {
|
||||
return nil, errors.New(BadEventNoCreated)
|
||||
}
|
||||
|
||||
createdTime, err := time.Parse(time.RFC3339, created)
|
||||
if err != nil {
|
||||
return nil, errors.Join(errors.New(BadUserEventCreatedParse), err)
|
||||
}
|
||||
|
||||
return &UserEvent{
|
||||
uid: user,
|
||||
created: createdTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
switch et {
|
||||
case Vote:
|
||||
return VoteEvent(data), nil
|
||||
case Challenge:
|
||||
e, err := MakeUserEvent(data)
|
||||
if err != nil {
|
||||
return nil, errors.Join(errors.New(BadChallengeEvent), err)
|
||||
}
|
||||
return ChallengeEvent{*e}, nil
|
||||
case Restoration:
|
||||
e, err := MakeUserEvent(data)
|
||||
if err != nil {
|
||||
return nil, errors.Join(errors.New(BadRestorationEvent), err)
|
||||
}
|
||||
return RestorationEvent{*e}, nil
|
||||
case UserActive:
|
||||
e, err := MakeUserEvent(data)
|
||||
if err != nil {
|
||||
return nil, errors.Join(errors.New(BadUserActiveEvent), err)
|
||||
}
|
||||
return UserActiveEvent{*e}, nil
|
||||
default:
|
||||
return nil, errors.New(BadEventTypeError)
|
||||
}
|
||||
}
|
||||
|
||||
/* adds a new subscriber channel to the event subscription cache
|
||||
* and returns the channel that it will publish notifications on
|
||||
*/
|
||||
func (et EventType) Subscribe() (chan Event, error) {
|
||||
if err := et.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ch := make(chan Event, maxEventsInMemory)
|
||||
|
||||
eventMutex.Lock()
|
||||
defer eventMutex.Unlock()
|
||||
eventSubscriptionCache[et] = append(
|
||||
eventSubscriptionCache[et],
|
||||
ch,
|
||||
)
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
type Event interface {
|
||||
// gets EventType associated with event
|
||||
Type() EventType
|
||||
|
|
@ -88,80 +214,48 @@ type Event interface {
|
|||
Validate() error
|
||||
}
|
||||
|
||||
// TODO: Something better than this
|
||||
type stringErrorType string
|
||||
func EventToString(e Event) (string, error) {
|
||||
m := e.Data()
|
||||
m[EventTypeMapKey] = e.Type().String()
|
||||
buf, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return "", errors.Join(errors.New(BadEventObjMarshal), err)
|
||||
}
|
||||
|
||||
func (s stringErrorType) Error() string {
|
||||
return string(s)
|
||||
return string(buf), nil
|
||||
}
|
||||
|
||||
/* adds a new subscriber channel to the event subscription cache
|
||||
* and returns the channel that it will publish notifications on
|
||||
*
|
||||
* note: channel is prefilled with at most eventChannelBufferSize
|
||||
* historical events. Truncated if history exceeds event channel
|
||||
* buffer size.
|
||||
*/
|
||||
func (et EventType) SubscribeWithHistory() (chan Event, error) {
|
||||
if err := et.Validate(); err != nil {
|
||||
return nil, err
|
||||
func EventFromString(doc string) (Event, error) {
|
||||
var obj map[string]string
|
||||
if err := json.Unmarshal([]byte(doc), &obj); err != nil {
|
||||
return nil, errors.Join(errors.New(BadEventObjUnmarshal), err)
|
||||
}
|
||||
|
||||
ch := make(chan Event, eventChannelBufferSize)
|
||||
|
||||
eventMutex.Lock()
|
||||
eventSubscriptionCache[et] = append(
|
||||
eventSubscriptionCache[et],
|
||||
ch,
|
||||
)
|
||||
eventMutex.Unlock()
|
||||
|
||||
eventMutex.RLock()
|
||||
defer eventMutex.RUnlock()
|
||||
numEventsAdded := 0
|
||||
for _, ev := range slices.Backward(eventCache) {
|
||||
if numEventsAdded >= eventChannelBufferSize {
|
||||
break
|
||||
}
|
||||
|
||||
if ev.Type() == et {
|
||||
ch <- ev
|
||||
numEventsAdded += 1
|
||||
}
|
||||
et, ok := obj[EventTypeMapKey]
|
||||
if !ok {
|
||||
return nil, errors.New(BadEventMissingTypeKey)
|
||||
}
|
||||
t := EventTypeFromString(et)
|
||||
ev, err := t.MakeEvent(obj)
|
||||
if err != nil {
|
||||
return nil, errors.Join(errors.New(BadEventCreate), err)
|
||||
}
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
/* adds a new subscriber channel to the event subscription cache
|
||||
* and returns the channel that it will publish notifications on
|
||||
*/
|
||||
func (et EventType) Subscribe() (chan Event, error) {
|
||||
if err := et.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ch := make(chan Event, eventChannelBufferSize)
|
||||
|
||||
eventMutex.Lock()
|
||||
defer eventMutex.Unlock()
|
||||
eventSubscriptionCache[et] = append(
|
||||
eventSubscriptionCache[et],
|
||||
ch,
|
||||
)
|
||||
|
||||
return ch, nil
|
||||
return ev, ev.Validate()
|
||||
}
|
||||
|
||||
func PublishEvent(e Event) error {
|
||||
if err := ValidateEvent(e); err != nil {
|
||||
return stringErrorType(
|
||||
EventValidationFailedError + err.Error(),
|
||||
)
|
||||
return errors.Join(errors.New(EventValidationFailedError), err)
|
||||
}
|
||||
|
||||
doc, err := EventToString(e)
|
||||
if err != nil {
|
||||
return errors.New(BadEventUnmarshal)
|
||||
}
|
||||
|
||||
eventMutex.Lock()
|
||||
eventCache = append(eventCache, e)
|
||||
eventStream.Push(doc)
|
||||
eventMutex.Unlock()
|
||||
eventMutex.RLock()
|
||||
defer eventMutex.RUnlock()
|
||||
|
|
@ -169,8 +263,8 @@ func PublishEvent(e Event) error {
|
|||
blocking := false
|
||||
|
||||
for _, c := range eventSubscriptionCache[e.Type()] {
|
||||
if float32(len(c)) > (float32(eventChannelBufferSize) * 0.25) {
|
||||
if len(c) == eventChannelBufferSize {
|
||||
if float32(len(c)) > (float32(maxEventsInMemory) * 0.25) {
|
||||
if len(c) == maxEventsInMemory {
|
||||
logging.Warn(
|
||||
"PublishEvent() blocking -- event channel full",
|
||||
// log the event time to provide blockage timing information
|
||||
|
|
@ -205,6 +299,45 @@ func ValidateEvent(e Event) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
/* Takes a filter and applies it to each individual event
|
||||
* The filter is a function/closure that accepts one event
|
||||
* and returns true if ApplyToEvents should continue iterating
|
||||
*
|
||||
* If the filter returns false then ApplyToEvents halts and returns.
|
||||
*
|
||||
* (this calls docbuf.Apply() under the hood)
|
||||
*/
|
||||
func ApplyToEvents(
|
||||
f func(Event)bool,
|
||||
) error {
|
||||
// local variables enclosed by filter function filterWrap
|
||||
var err error
|
||||
var ev Event
|
||||
// wrap f() to be compatible with docbuf.Apply()
|
||||
filterWrap := func(doc string) bool {
|
||||
ev, err = EventFromString(doc)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return f(ev)
|
||||
}
|
||||
|
||||
eventMutex.RLock()
|
||||
defer eventMutex.RUnlock()
|
||||
|
||||
// cant reuse err or return val directly as err is set
|
||||
// by filter function if an error happens unmarshalling
|
||||
// an event. In this case apply might return nil
|
||||
err2 := eventStream.Apply(filterWrap)
|
||||
if err2 != nil {
|
||||
return err2
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/* gets all events of type T in cache
|
||||
* that also share all field values in
|
||||
* map 'filters'
|
||||
|
|
@ -219,25 +352,26 @@ func GetMatchingEvents(
|
|||
return matches, err
|
||||
}
|
||||
|
||||
eventMutex.RLock()
|
||||
defer eventMutex.RUnlock()
|
||||
|
||||
Events:
|
||||
for _, e := range eventCache {
|
||||
filter := func(e Event) bool {
|
||||
ev, er := EventToString(e)
|
||||
fmt.Printf("Checking: %s (%e)\n", ev, er)
|
||||
if e.Type() != t {
|
||||
continue
|
||||
return true
|
||||
}
|
||||
for k, v := range filters {
|
||||
val, found := e.Data()[k]
|
||||
if !found || val != v {
|
||||
continue Events
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("Found Match")
|
||||
matches = append(matches, e)
|
||||
return true
|
||||
}
|
||||
|
||||
return matches, nil
|
||||
err := ApplyToEvents(filter)
|
||||
return matches, err
|
||||
}
|
||||
|
||||
type VoteEvent map[string]string
|
||||
|
|
@ -303,8 +437,7 @@ func (ve VoteEvent) Validate() error {
|
|||
VoteStatusKey,
|
||||
} {
|
||||
if _, found := ve[key]; !found {
|
||||
return stringErrorType(
|
||||
VoteMissingKeyError + key)
|
||||
return errors.New(VoteMissingKeyError + key)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -312,15 +445,15 @@ func (ve VoteEvent) Validate() error {
|
|||
if status != VoteStatusTimeout &&
|
||||
status != VoteStatusInProgress &&
|
||||
status != VoteStatusFinalized {
|
||||
return stringErrorType(VoteBadStatusError + status)
|
||||
return errors.New(VoteBadStatusError + status)
|
||||
}
|
||||
|
||||
result, hasResult := ve[VoteResultKey]
|
||||
if hasResult && status == VoteStatusInProgress {
|
||||
return stringErrorType(VoteNotFinishedError)
|
||||
return errors.New(VoteNotFinishedError)
|
||||
}
|
||||
if status != VoteStatusInProgress && !hasResult {
|
||||
return stringErrorType(VoteMissingResultError)
|
||||
return errors.New(VoteMissingResultError)
|
||||
}
|
||||
|
||||
if hasResult &&
|
||||
|
|
@ -329,7 +462,7 @@ func (ve VoteEvent) Validate() error {
|
|||
result != VoteResultTie &&
|
||||
result != VoteResultTimeout) {
|
||||
|
||||
return stringErrorType(VoteBadResultError + result)
|
||||
return errors.New(VoteBadResultError + result)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -353,7 +486,7 @@ func (ue UserEvent) Time() time.Time {
|
|||
func (ue UserEvent) Data() map[string]string {
|
||||
return map[string]string{
|
||||
UserEventUserKey: ue.uid,
|
||||
UserEventCreatedKey: ue.created.Local().String(),
|
||||
UserEventCreatedKey: ue.created.Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue