diff --git a/internal/state/state.go b/internal/state/state.go index caba329..ab8469b 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -9,10 +9,13 @@ package state import ( "fmt" - "slices" "sync" "time" + "os" + "errors" + "encoding/json" + "gitlab.com/whom/bingobot/internal/docbuf" "gitlab.com/whom/bingobot/internal/logging" ) @@ -24,32 +27,46 @@ import ( const ( BadEventTypeError = "bad event type" EventValidationFailedError = "event failed validation: " + BadEventNoUID = "event data has no UID field" + BadEventNoCreated = "event data has no created field" + BadEventStoreFilename = "failed to open event store file" + BadEventStreamInit = "failed to initialize event stream" + BadEventCreate = "failed to create event" + BadUserEventCreatedParse = "failed to parse created time" + BadChallengeEvent = "failed to make Challenge Event" + BadRestorationEvent = "failed to make Restoration Event" + BadUserActiveEvent = "failed to make UserActive Event" + BadEventObjMarshal = "error marshalling event" + BadEventObjUnmarshal = "failed to unmarshal event map" + BadEventUnmarshal = "failed to unmarshal event" + BadEventMissingTypeKey = "event map missing type key" +) + +const ( + EventTypeMapKey = "type" ) var eventMutex sync.RWMutex var eventSubscriptionCache = [NumEventTypes][]chan Event{} -var eventCache = []Event{} +var eventStream docbuf.DocumentBuffer +var maxEventsInMemory int -// TODO: Configurable -var eventChannelBufferSize = 256 - -// TODO: pruned events go to DISK -// ASSUMES eventCache is ordered by age (oldest first) -func pruneEventCache() { - eventMutex.Lock() - defer eventMutex.Unlock() - - oldCacheInvalid := false - newCache := []Event{} - for _, obj := range eventCache { - if time.Since(obj.Time()).Hours() > 24*10 { - oldCacheInvalid = true - } else if oldCacheInvalid { - newCache = append(newCache, obj) - } +// expect filename validations in config package +func Init(eventMemCacheSize int, eventStoreFileName string) error { + file, err := os.OpenFile(eventStoreFileName, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return errors.Join(errors.New(BadEventStoreFilename), err) } - eventCache = newCache + if eventStream, err = docbuf.NewDocumentBuffer( + eventMemCacheSize, + file, + ); err != nil { + return errors.Join(errors.New(BadEventStreamInit), err) + } + + maxEventsInMemory = eventMemCacheSize + return nil } type EventType int8 @@ -65,14 +82,123 @@ const ( NumEventTypes ) +// either returns a valid event type or NumEventTypes +func EventTypeFromString(doc string) EventType { + switch doc { + case "Vote": + return Vote + case "Challenge": + return Challenge + case "Restoration": + return Restoration + case "UserActive": + return UserActive + default: + // error case + return NumEventTypes + } +} + +func (et EventType) String() string { + events := []string{ + "Vote", + "Challenge", + "Restoration", + "UserActive", + } + + if et < 0 || et >= NumEventTypes { + return "" + } + + return events[et] +} + func (et EventType) Validate() error { if et < 0 || et >= NumEventTypes { - return stringErrorType(BadEventTypeError) + return errors.New(BadEventTypeError) } return nil } +/* Allocates a specific event of type T from event type instance. + * output event not guaranteed to be valid. + * Call Validate() yourself. + */ +func (et EventType) MakeEvent(data map[string]string) (Event, error) { + if err := et.Validate(); err != nil { + return nil, errors.Join(errors.New(BadEventCreate), err) + } + + MakeUserEvent := func(data map[string]string) (*UserEvent, error) { + user, hasUser := data[UserEventUserKey] + if !hasUser { + return nil, errors.New(BadEventNoUID) + } + + created, hasCreated := data[UserEventCreatedKey] + if !hasCreated { + return nil, errors.New(BadEventNoCreated) + } + + createdTime, err := time.Parse(time.RFC3339, created) + if err != nil { + return nil, errors.Join(errors.New(BadUserEventCreatedParse), err) + } + + return &UserEvent{ + uid: user, + created: createdTime, + }, nil + } + + switch et { + case Vote: + return VoteEvent(data), nil + case Challenge: + e, err := MakeUserEvent(data) + if err != nil { + return nil, errors.Join(errors.New(BadChallengeEvent), err) + } + return ChallengeEvent{*e}, nil + case Restoration: + e, err := MakeUserEvent(data) + if err != nil { + return nil, errors.Join(errors.New(BadRestorationEvent), err) + } + return RestorationEvent{*e}, nil + case UserActive: + e, err := MakeUserEvent(data) + if err != nil { + return nil, errors.Join(errors.New(BadUserActiveEvent), err) + } + return UserActiveEvent{*e}, nil + default: + return nil, errors.New(BadEventTypeError) + } +} + +/* adds a new subscriber channel to the event subscription cache + * and returns the channel that it will publish notifications on + */ +func (et EventType) Subscribe() (chan Event, error) { + if err := et.Validate(); err != nil { + return nil, err + } + + ch := make(chan Event, maxEventsInMemory) + + eventMutex.Lock() + defer eventMutex.Unlock() + eventSubscriptionCache[et] = append( + eventSubscriptionCache[et], + ch, + ) + + return ch, nil +} + type Event interface { // gets EventType associated with event Type() EventType @@ -88,80 +214,48 @@ type Event interface { Validate() error } -// TODO: Something better than this -type stringErrorType string +func EventToString(e Event) (string, error) { + m := e.Data() + m[EventTypeMapKey] = e.Type().String() + buf, err := json.Marshal(m) + if err != nil { + return "", errors.Join(errors.New(BadEventObjMarshal), err) + } -func (s stringErrorType) Error() string { - return string(s) + return string(buf), nil } -/* adds a new subscriber channel to the event subscription cache - * and returns the channel that it will publish notifications on - * - * note: channel is prefilled with at most eventChannelBufferSize - * historical events. Truncated if history exceeds event channel - * buffer size. - */ -func (et EventType) SubscribeWithHistory() (chan Event, error) { - if err := et.Validate(); err != nil { - return nil, err +func EventFromString(doc string) (Event, error) { + var obj map[string]string + if err := json.Unmarshal([]byte(doc), &obj); err != nil { + return nil, errors.Join(errors.New(BadEventObjUnmarshal), err) } - ch := make(chan Event, eventChannelBufferSize) - - eventMutex.Lock() - eventSubscriptionCache[et] = append( - eventSubscriptionCache[et], - ch, - ) - eventMutex.Unlock() - - eventMutex.RLock() - defer eventMutex.RUnlock() - numEventsAdded := 0 - for _, ev := range slices.Backward(eventCache) { - if numEventsAdded >= eventChannelBufferSize { - break - } - - if ev.Type() == et { - ch <- ev - numEventsAdded += 1 - } + et, ok := obj[EventTypeMapKey] + if !ok { + return nil, errors.New(BadEventMissingTypeKey) + } + t := EventTypeFromString(et) + ev, err := t.MakeEvent(obj) + if err != nil { + return nil, errors.Join(errors.New(BadEventCreate), err) } - return ch, nil -} - -/* adds a new subscriber channel to the event subscription cache - * and returns the channel that it will publish notifications on - */ -func (et EventType) Subscribe() (chan Event, error) { - if err := et.Validate(); err != nil { - return nil, err - } - - ch := make(chan Event, eventChannelBufferSize) - - eventMutex.Lock() - defer eventMutex.Unlock() - eventSubscriptionCache[et] = append( - eventSubscriptionCache[et], - ch, - ) - - return ch, nil + return ev, ev.Validate() } func PublishEvent(e Event) error { if err := ValidateEvent(e); err != nil { - return stringErrorType( - EventValidationFailedError + err.Error(), - ) + return errors.Join(errors.New(EventValidationFailedError), err) + } + + doc, err := EventToString(e) + if err != nil { + return errors.New(BadEventUnmarshal) } eventMutex.Lock() - eventCache = append(eventCache, e) + eventStream.Push(doc) eventMutex.Unlock() eventMutex.RLock() defer eventMutex.RUnlock() @@ -169,8 +263,8 @@ func PublishEvent(e Event) error { blocking := false for _, c := range eventSubscriptionCache[e.Type()] { - if float32(len(c)) > (float32(eventChannelBufferSize) * 0.25) { - if len(c) == eventChannelBufferSize { + if float32(len(c)) > (float32(maxEventsInMemory) * 0.25) { + if len(c) == maxEventsInMemory { logging.Warn( "PublishEvent() blocking -- event channel full", // log the event time to provide blockage timing information @@ -205,6 +299,45 @@ func ValidateEvent(e Event) error { return nil } +/* Takes a filter and applies it to each individual event + * The filter is a function/closure that accepts one event + * and returns true if ApplyToEvents should continue iterating + * + * If the filter returns false then ApplyToEvents halts and returns. + * + * (this calls docbuf.Apply() under the hood) + */ +func ApplyToEvents( + f func(Event)bool, +) error { + // local variables enclosed by filter function filterWrap + var err error + var ev Event + // wrap f() to be compatible with docbuf.Apply() + filterWrap := func(doc string) bool { + ev, err = EventFromString(doc) + if err != nil { + return false + } + return f(ev) + } + + eventMutex.RLock() + defer eventMutex.RUnlock() + + // cant reuse err or return val directly as err is set + // by filter function if an error happens unmarshalling + // an event. In this case apply might return nil + err2 := eventStream.Apply(filterWrap) + if err2 != nil { + return err2 + } + if err != nil { + return err + } + return nil +} + /* gets all events of type T in cache * that also share all field values in * map 'filters' @@ -219,25 +352,26 @@ func GetMatchingEvents( return matches, err } - eventMutex.RLock() - defer eventMutex.RUnlock() - -Events: - for _, e := range eventCache { + filter := func(e Event) bool { + ev, er := EventToString(e) + fmt.Printf("Checking: %s (%e)\n", ev, er) if e.Type() != t { - continue + return true } for k, v := range filters { val, found := e.Data()[k] if !found || val != v { - continue Events + return true } } + fmt.Println("Found Match") matches = append(matches, e) + return true } - return matches, nil + err := ApplyToEvents(filter) + return matches, err } type VoteEvent map[string]string @@ -303,8 +437,7 @@ func (ve VoteEvent) Validate() error { VoteStatusKey, } { if _, found := ve[key]; !found { - return stringErrorType( - VoteMissingKeyError + key) + return errors.New(VoteMissingKeyError + key) } } @@ -312,15 +445,15 @@ func (ve VoteEvent) Validate() error { if status != VoteStatusTimeout && status != VoteStatusInProgress && status != VoteStatusFinalized { - return stringErrorType(VoteBadStatusError + status) + return errors.New(VoteBadStatusError + status) } result, hasResult := ve[VoteResultKey] if hasResult && status == VoteStatusInProgress { - return stringErrorType(VoteNotFinishedError) + return errors.New(VoteNotFinishedError) } if status != VoteStatusInProgress && !hasResult { - return stringErrorType(VoteMissingResultError) + return errors.New(VoteMissingResultError) } if hasResult && @@ -329,7 +462,7 @@ func (ve VoteEvent) Validate() error { result != VoteResultTie && result != VoteResultTimeout) { - return stringErrorType(VoteBadResultError + result) + return errors.New(VoteBadResultError + result) } return nil @@ -353,7 +486,7 @@ func (ue UserEvent) Time() time.Time { func (ue UserEvent) Data() map[string]string { return map[string]string{ UserEventUserKey: ue.uid, - UserEventCreatedKey: ue.created.Local().String(), + UserEventCreatedKey: ue.created.Format(time.RFC3339), } } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index e775e74..e6d4073 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "gitlab.com/whom/bingobot/internal/docbuf" "gitlab.com/whom/bingobot/internal/logging" ) @@ -17,12 +18,71 @@ const TestTok = "TEST_NAME" var loggingInitialized = false func SetupTest(t *testing.T) { + maxEventsInMemory = 271 + + var err error // have to set up logger if !loggingInitialized { logging.Init() loggingInitialized = true } + b := docbuf.NewReadWriteSeekString() + eventStream, err = docbuf.NewDocumentBuffer(300, &b) + if err != nil { + t.Fatalf("error allocating buffer: %e", err) + } +} + +func TestEventMarshalUnmarshal(t *testing.T) { + tm := time.Now() + cev := ChallengeEvent{UserEvent{ + uid: TestTok, + created: tm, + }} + + vev := VoteEvent(map[string]string{ + VoteActionKey: "a", + VoteRequesterKey: "r", + VoteCreatedKey: "c", + VoteStatusKey: VoteStatusFinalized, + VoteResultKey: VoteResultFail, + }) + + cstr, err := EventToString(cev); + if err != nil { + t.Fatalf("error marshalling challenge: %e, %s", err, cstr) + } + + t.Logf("cstr: %s\n", cstr) + + vstr, err := EventToString(vev); + if err != nil { + t.Fatalf("error marshalling vote: %e, %s", err, vstr) + } + + t.Logf("vstr: %s\n", vstr) + + if ev, err := EventFromString(cstr); err != nil || + ev.Data()[UserEventUserKey] != cev.Data()[UserEventUserKey] || + ev.Data()[UserEventCreatedKey] != cev.Data()[UserEventCreatedKey] { + t.Fatalf("error unmarshalling challenge: %e, %v!=%v", err, ev, cev) + } + + if ev, err := EventFromString(vstr); err != nil || + fmt.Sprint(ev) != fmt.Sprint(vev) { + t.Fatalf("error unmarshalling vote: %e, %v!=%v", err, ev, vev) + } +} + +func TestPubSub(t *testing.T) { + SetupTest(t) + + c, e := UserActive.Subscribe() + if e != nil { + t.Errorf("Error subscribing to UserActive events: %e", e) + } + old, _ := time.Parse( time.RFC3339, VeryOldVote, @@ -47,40 +107,21 @@ func SetupTest(t *testing.T) { created: time.Now(), }}) - if len(eventCache) != 272 { - t.Errorf("Unexpected number of events in cache: %d", - len(eventCache)) - } -} - -func CleanupTest() { - eventSubscriptionCache = [NumEventTypes][]chan Event{} - eventCache = []Event{} -} - -func TestPubSub(t *testing.T) { - SetupTest(t) - - c, e := UserActive.SubscribeWithHistory() - if e != nil { - t.Errorf("Error subscribing to UserActive events: %e", e) - } - Loop: for i := 0; true; i++ { select { case e, ok := <-c: if !ok { - t.Errorf("Subscription Channel Closed") + t.Fatalf("Subscription Channel Closed") } if e.Type() != UserActive { - t.Errorf("Non UserActive Event in UserActive subscription: %v", e.Type()) + t.Fatalf("Non UserActive Event in UserActive subscription: %v", e.Type()) } default: - if i == eventChannelBufferSize { + if i == maxEventsInMemory { break Loop } else { - t.Errorf("Unexpected number of events in channel: %d", i) + t.Fatalf("Unexpected number of events in channel: %d", i) } } } @@ -93,18 +134,40 @@ Loop: select { case e, ok := <-c: if !ok || e.Data()[UserEventUserKey] != "uniqueToken" { - t.Errorf("didnt read correct event from channel: %v", e) + t.Fatalf("didnt read correct event from channel: %v", e) } default: - t.Errorf("New event not published to subscription!") + t.Fatalf("New event not published to subscription!") } - - CleanupTest() } func TestFilterCache(t *testing.T) { SetupTest(t) + old, _ := time.Parse( + time.RFC3339, + VeryOldVote, + ) + + for i := range 270 { + if err := PublishEvent(UserActiveEvent{UserEvent{ + uid: fmt.Sprintf("%d", i), + created: old, + }}); err != nil { + t.Errorf("Failed to add event: %e", err) + } + } + + PublishEvent(UserActiveEvent{UserEvent{ + uid: fmt.Sprintf(TestTok), + created: time.Now(), + }}) + + PublishEvent(ChallengeEvent{UserEvent{ + uid: fmt.Sprintf(TestTok), + created: time.Now(), + }}) + events, err := GetMatchingEvents( UserActive, map[string]string{ @@ -123,19 +186,6 @@ func TestFilterCache(t *testing.T) { if events[0].Type() != UserActive { t.Errorf("Got wrong event!: %+v", events[0]) } - - CleanupTest() -} - -func TestPruneCache(t *testing.T) { - SetupTest(t) - pruneEventCache() - - if len(eventCache) != 2 { - t.Errorf("Incorrect number of remaining events: %d", len(eventCache)) - } - - CleanupTest() } func TestVoteEventValidations(t *testing.T) {