From 1000bbf46bebdc432a4ae14147f1e3f37ffa3af4 Mon Sep 17 00:00:00 2001 From: gempir Date: Fri, 16 Apr 2021 18:03:41 +0200 Subject: [PATCH 1/3] remove all message type configs, better handle joining/leaving --- api/admin.go | 81 +------------------------------------------------ api/docs.go | 14 +-------- api/server.go | 19 ++---------- bot/commands.go | 44 ++++++++------------------- bot/main.go | 69 +++++------------------------------------ config/main.go | 36 ---------------------- helix/user.go | 8 +++-- main.go | 2 +- 8 files changed, 29 insertions(+), 244 deletions(-) diff --git a/api/admin.go b/api/admin.go index a41d8a2..bf8c28c 100644 --- a/api/admin.go +++ b/api/admin.go @@ -4,9 +4,6 @@ import ( "encoding/json" "fmt" "net/http" - "strings" - - "github.com/gempir/justlog/config" ) func (s *Server) authenticateAdmin(w http.ResponseWriter, r *http.Request) bool { @@ -20,15 +17,6 @@ func (s *Server) authenticateAdmin(w http.ResponseWriter, r *http.Request) bool return true } -type channelConfigsDeleteRequest struct { - MessageTypes bool `json:"messageTypes,omitempty"` -} - -// swagger:model -type channelConfigsRequest struct { - config.ChannelConfig -} - // swagger:route POST /admin/channelConfigs/{channelID} admin channelConfigs // // Will set the messageTypes logged for a channel @@ -51,73 +39,6 @@ type channelConfigsRequest struct { // 400: // 405: -// swagger:route DELETE /admin/channelConfigs/{channelID} admin deleteChannelConfigs -// -// Will reset the messageTypes logged for a channel -// -// Consumes: -// - application/json -// -// Produces: -// - application/json -// - text/plain -// -// Security: -// - api_key: -// -// Schemes: https -// -// Responses: -// 200: -// 400: -// 405: -func (s *Server) writeChannelConfigs(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost && r.Method != http.MethodDelete { - http.Error(w, "We'll see, we'll see. The winner gets tea.", http.StatusMethodNotAllowed) - return - } - - channelID := strings.TrimPrefix(r.URL.String(), "/admin/channelConfigs/") - - if _, ok := s.cfg.ChannelConfigs[channelID]; !ok { - http.Error(w, "Uhhhhhh... unkown channel", http.StatusBadRequest) - return - } - - if r.Method == http.MethodDelete { - var request channelConfigsDeleteRequest - - err := json.NewDecoder(r.Body).Decode(&request) - if err != nil { - http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest) - return - } - - if request.MessageTypes { - s.cfg.ResetMessageTypes(channelID) - s.bot.UpdateMessageTypesToLog() - writeJSON("Doubters? Reset "+channelID+" messageTypes", http.StatusOK, w, r) - return - } - - http.Error(w, "Uhhhhhh...", http.StatusBadRequest) - return - } - - var request channelConfigsRequest - - err := json.NewDecoder(r.Body).Decode(&request) - if err != nil { - http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest) - return - } - - s.cfg.SetMessageTypes(channelID, request.MessageTypes) - s.bot.UpdateMessageTypesToLog() - - writeJSON("Doubters? Updated "+channelID+" messageTypes to "+fmt.Sprintf("%v", request.MessageTypes), http.StatusOK, w, r) -} - type channelsDeleteRequest struct { Channels []string `json:"channels"` } @@ -191,7 +112,7 @@ func (s *Server) writeChannels(w http.ResponseWriter, r *http.Request) { return } for _, userData := range data { - s.bot.Depart(userData.Login) + s.bot.Part(userData.Login) } writeJSON(fmt.Sprintf("Doubters? Removed channels %v", request.Channels), http.StatusOK, w, r) diff --git a/api/docs.go b/api/docs.go index 5f61e2d..73a9232 100644 --- a/api/docs.go +++ b/api/docs.go @@ -197,18 +197,6 @@ type ChannelUserIdLogsYearMonthParams struct { LogParams } -// swagger:parameters channelConfigs -type ChannelConfigsParameters struct { - // in:body - Body channelConfigsRequest -} - -// swagger:parameters deleteChannelConfigs -type DeleteChannelConfigsParameters struct { - // in:body - Body channelConfigsDeleteRequest -} - // swagger:parameters addChannels type AddChannelsParameters struct { // in:body @@ -231,4 +219,4 @@ type ListLogsParams struct { ChannelId string `json:"channelid"` // in: query Userid string `json:"userid"` -} \ No newline at end of file +} diff --git a/api/server.go b/api/server.go index aae957f..6eee653 100644 --- a/api/server.go +++ b/api/server.go @@ -30,12 +30,11 @@ type Server struct { cfg *config.Config fileLogger *filelog.Logger helixClient helix.TwitchApiClient - channels []string assetsHandler http.Handler } // NewServer create api Server -func NewServer(cfg *config.Config, bot *bot.Bot, fileLogger *filelog.Logger, helixClient helix.TwitchApiClient, channels []string, assets fs.FS) Server { +func NewServer(cfg *config.Config, bot *bot.Bot, fileLogger *filelog.Logger, helixClient helix.TwitchApiClient, assets fs.FS) Server { build, err := fs.Sub(assets, "web/build") if err != nil { log.Fatal("failed to read public assets") @@ -48,16 +47,10 @@ func NewServer(cfg *config.Config, bot *bot.Bot, fileLogger *filelog.Logger, hel cfg: cfg, fileLogger: fileLogger, helixClient: helixClient, - channels: channels, assetsHandler: http.FileServer(http.FS(build)), } } -// AddChannel adds a channel to the collection to output on the channels endpoint -func (s *Server) AddChannel(channel string) { - s.channels = append(s.channels, channel) -} - const ( responseTypeJSON = "json" responseTypeText = "text" @@ -133,14 +126,6 @@ func (s *Server) route(w http.ResponseWriter, r *http.Request) { return } - if strings.HasPrefix(url, "/admin/channelConfigs/") { - success := s.authenticateAdmin(w, r) - if success { - s.writeChannelConfigs(w, r) - } - return - } - if strings.HasPrefix(url, "/admin/channels") { success := s.authenticateAdmin(w, r) if success { @@ -283,7 +268,7 @@ func reverseSlice(input []string) []string { func (s *Server) writeAllChannels(w http.ResponseWriter, r *http.Request) { response := new(AllChannelsJSON) response.Channels = []channel{} - users, err := s.helixClient.GetUsersByUserIds(s.channels) + users, err := s.helixClient.GetUsersByUserIds(s.cfg.Channels) if err != nil { log.Error(err) diff --git a/bot/commands.go b/bot/commands.go index 887c30b..2a9436c 100644 --- a/bot/commands.go +++ b/bot/commands.go @@ -2,7 +2,6 @@ package bot import ( "fmt" - "strconv" "strings" twitch "github.com/gempir/go-twitch-irc/v2" @@ -19,8 +18,8 @@ func (b *Bot) handlePrivateMessageCommands(message twitch.PrivateMessage) { if strings.HasPrefix(message.Message, "!justlog join ") { b.handleJoin(message) } - if strings.HasPrefix(message.Message, "!justlog messageType ") { - b.handleMessageType(message) + if strings.HasPrefix(message.Message, "!justlog part ") { + b.handlePart(message) } } } @@ -37,47 +36,28 @@ func (b *Bot) handleJoin(message twitch.PrivateMessage) { ids := []string{} for _, user := range users { ids = append(ids, user.ID) - log.Infof("[bot] joining %s", user.Login) b.Join(user.Login) } b.cfg.AddChannels(ids...) b.Say(message.Channel, fmt.Sprintf("%s, added channels: %v", message.User.DisplayName, ids)) } -func (b *Bot) handleMessageType(message twitch.PrivateMessage) { - input := strings.TrimPrefix(message.Message, "!justlog messageType ") +func (b *Bot) handlePart(message twitch.PrivateMessage) { + input := strings.TrimPrefix(message.Message, "!justlog part ") - parts := strings.Split(input, " ") - if len(parts) < 2 { - return - } - - users, err := b.helixClient.GetUsersByUsernames([]string{parts[0]}) + users, err := b.helixClient.GetUsersByUsernames(strings.Split(input, ",")) if err != nil { log.Error(err) - return + b.Say(message.Channel, message.User.DisplayName+", something went wrong requesting the userids") } - if parts[1] == "reset" { - b.cfg.ResetMessageTypes(users[parts[0]].ID) - b.UpdateMessageTypesToLog() - log.Infof("[bot] setting %s config messageTypes to default", parts[0]) - } else { - var messageTypes []twitch.MessageType - for _, msgType := range strings.Split(parts[1], ",") { - messageType, err := strconv.Atoi(msgType) - if err != nil { - log.Error(err) - return - } - - messageTypes = append(messageTypes, twitch.MessageType(messageType)) - } - - b.cfg.SetMessageTypes(users[parts[0]].ID, messageTypes) - b.UpdateMessageTypesToLog() - log.Infof("[bot] setting %s config messageTypes to %v", parts[0], messageTypes) + ids := []string{} + for _, user := range users { + ids = append(ids, user.ID) + b.Part(user.Login) } + b.cfg.RemoveChannels(ids...) + b.Say(message.Channel, fmt.Sprintf("%s, removed channels: %v", message.User.DisplayName, ids)) } func contains(arr []string, str string) bool { diff --git a/bot/main.go b/bot/main.go index 082c722..74ae865 100644 --- a/bot/main.go +++ b/bot/main.go @@ -15,13 +15,12 @@ import ( // Bot basic logging bot type Bot struct { - startTime time.Time - cfg *config.Config - helixClient helix.TwitchApiClient - fileLogger *filelog.Logger - worker []*worker - channels map[string]helix.UserData - messageTypesToLog map[string][]twitch.MessageType + startTime time.Time + cfg *config.Config + helixClient helix.TwitchApiClient + fileLogger *filelog.Logger + worker []*worker + channels map[string]helix.UserData } type worker struct { @@ -54,7 +53,6 @@ func (b *Bot) Say(channel, text string) { func (b *Bot) Connect() { b.startTime = time.Now() client := b.newClient() - b.UpdateMessageTypesToLog() b.initialJoins() if strings.HasPrefix(b.cfg.Username, "justinfan") { @@ -66,32 +64,7 @@ func (b *Bot) Connect() { log.Fatal(client.Connect()) } -func (b *Bot) shouldLog(channelName string, receivedMsgType twitch.MessageType) bool { - for _, msgType := range b.messageTypesToLog[channelName] { - if msgType == receivedMsgType { - return true - } - } - - return false -} - -// UpdateMessageTypesToLog reload the config -func (b *Bot) UpdateMessageTypesToLog() { - messageTypesToLog := make(map[string][]twitch.MessageType) - - for _, channel := range b.channels { - if _, ok := b.cfg.ChannelConfigs[channel.ID]; ok && b.cfg.ChannelConfigs[channel.ID].MessageTypes != nil { - messageTypesToLog[channel.Login] = b.cfg.ChannelConfigs[channel.ID].MessageTypes - } else { - messageTypesToLog[channel.Login] = []twitch.MessageType{twitch.PRIVMSG, twitch.CLEARCHAT, twitch.USERNOTICE} - } - } - - b.messageTypesToLog = messageTypesToLog -} - -func (b *Bot) Depart(channelNames ...string) { +func (b *Bot) Part(channelNames ...string) { for _, channelName := range channelNames { log.Info("[bot] leaving " + channelName) @@ -142,10 +115,6 @@ func (b *Bot) initialJoins() { func (b *Bot) handlePrivateMessage(message twitch.PrivateMessage) { go func() { - if !b.shouldLog(message.Channel, message.GetType()) { - return - } - err := b.fileLogger.LogPrivateMessageForUser(message.User, message) if err != nil { log.Error(err.Error()) @@ -153,10 +122,6 @@ func (b *Bot) handlePrivateMessage(message twitch.PrivateMessage) { }() go func() { - if !b.shouldLog(message.Channel, message.GetType()) { - return - } - err := b.fileLogger.LogPrivateMessageForChannel(message) if err != nil { log.Error(err.Error()) @@ -168,10 +133,6 @@ func (b *Bot) handlePrivateMessage(message twitch.PrivateMessage) { func (b *Bot) handleUserNotice(message twitch.UserNoticeMessage) { go func() { - if !b.shouldLog(message.Channel, message.GetType()) { - return - } - err := b.fileLogger.LogUserNoticeMessageForUser(message.User.ID, message) if err != nil { log.Error(err.Error()) @@ -180,10 +141,6 @@ func (b *Bot) handleUserNotice(message twitch.UserNoticeMessage) { if _, ok := message.Tags["msg-param-recipient-id"]; ok { go func() { - if !b.shouldLog(message.Channel, message.GetType()) { - return - } - err := b.fileLogger.LogUserNoticeMessageForUser(message.Tags["msg-param-recipient-id"], message) if err != nil { log.Error(err.Error()) @@ -192,10 +149,6 @@ func (b *Bot) handleUserNotice(message twitch.UserNoticeMessage) { } go func() { - if !b.shouldLog(message.Channel, message.GetType()) { - return - } - err := b.fileLogger.LogUserNoticeMessageForChannel(message) if err != nil { log.Error(err.Error()) @@ -205,10 +158,6 @@ func (b *Bot) handleUserNotice(message twitch.UserNoticeMessage) { func (b *Bot) handleClearChat(message twitch.ClearChatMessage) { go func() { - if !b.shouldLog(message.Channel, message.GetType()) { - return - } - err := b.fileLogger.LogClearchatMessageForUser(message.TargetUserID, message) if err != nil { log.Error(err.Error()) @@ -216,10 +165,6 @@ func (b *Bot) handleClearChat(message twitch.ClearChatMessage) { }() go func() { - if !b.shouldLog(message.Channel, message.GetType()) { - return - } - err := b.fileLogger.LogClearchatMessageForChannel(message) if err != nil { log.Error(err.Error()) diff --git a/config/main.go b/config/main.go index e11f083..398f96d 100644 --- a/config/main.go +++ b/config/main.go @@ -6,7 +6,6 @@ import ( "os" "strings" - twitch "github.com/gempir/go-twitch-irc/v2" log "github.com/sirupsen/logrus" ) @@ -25,12 +24,6 @@ type Config struct { ClientID string `json:"clientID"` ClientSecret string `json:"clientSecret"` LogLevel string `json:"logLevel"` - ChannelConfigs map[string]ChannelConfig `json:"channelConfigs"` -} - -// ChannelConfig config for individual channels -type ChannelConfig struct { - MessageTypes []twitch.MessageType `json:"messageTypes,omitempty"` } // NewConfig create configuration from file @@ -70,34 +63,6 @@ func (cfg *Config) RemoveChannels(channelIDs ...string) { cfg.persistConfig() } -// SetMessageTypes sets recorded message types for a channel -func (cfg *Config) SetMessageTypes(channelID string, messageTypes []twitch.MessageType) { - if _, ok := cfg.ChannelConfigs[channelID]; ok { - channelCfg := cfg.ChannelConfigs[channelID] - channelCfg.MessageTypes = messageTypes - - cfg.ChannelConfigs[channelID] = channelCfg - } else { - cfg.ChannelConfigs[channelID] = ChannelConfig{ - MessageTypes: messageTypes, - } - } - - cfg.persistConfig() -} - -// ResetMessageTypes removed message type option and therefore resets it -func (cfg *Config) ResetMessageTypes(channelID string) { - if _, ok := cfg.ChannelConfigs[channelID]; ok { - channelCfg := cfg.ChannelConfigs[channelID] - channelCfg.MessageTypes = nil - - cfg.ChannelConfigs[channelID] = channelCfg - } - - cfg.persistConfig() -} - func appendIfMissing(slice []string, i string) []string { for _, ele := range slice { if ele == i { @@ -129,7 +94,6 @@ func loadConfiguration(filePath string) *Config { Username: "justinfan777777", OAuth: "oauth:777777777", Channels: []string{}, - ChannelConfigs: make(map[string]ChannelConfig), Admins: []string{"gempir"}, LogLevel: "info", Archive: true, diff --git a/helix/user.go b/helix/user.go index 567de85..41ad247 100644 --- a/helix/user.go +++ b/helix/user.go @@ -157,8 +157,9 @@ func (c *Client) GetUsersByUsernames(usernames []string) (map[string]UserData, e var filteredUsernames []string for _, username := range usernames { - if _, ok := userCacheByUsername[strings.ToLower(username)]; !ok { - filteredUsernames = append(filteredUsernames, strings.ToLower(username)) + username = strings.ToLower(username) + if _, ok := userCacheByUsername[username]; !ok { + filteredUsernames = append(filteredUsernames, username) } } @@ -197,11 +198,12 @@ func (c *Client) GetUsersByUsernames(usernames []string) (map[string]UserData, e result := make(map[string]UserData) for _, username := range usernames { + username = strings.ToLower(username) if _, ok := userCacheByUsername[username]; !ok { log.Warningf("Could not find username, channel might be banned: %s", username) continue } - result[strings.ToLower(username)] = *userCacheByUsername[strings.ToLower(username)] + result[username] = *userCacheByUsername[username] } return result, nil diff --git a/main.go b/main.go index 869a2c7..d3982f9 100644 --- a/main.go +++ b/main.go @@ -34,7 +34,7 @@ func main() { bot := bot.NewBot(cfg, &helixClient, &fileLogger) - apiServer := api.NewServer(cfg, bot, &fileLogger, &helixClient, cfg.Channels, assets) + apiServer := api.NewServer(cfg, bot, &fileLogger, &helixClient, assets) go apiServer.Init() bot.Connect() From 3c699bae5b79a00a9b273a3ed24f67197732df7a Mon Sep 17 00:00:00 2001 From: gempir Date: Fri, 16 Apr 2021 18:27:19 +0200 Subject: [PATCH 2/3] don't log clearchats for massbans --- bot/main.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bot/main.go b/bot/main.go index 74ae865..8b9c59f 100644 --- a/bot/main.go +++ b/bot/main.go @@ -3,6 +3,7 @@ package bot import ( "math/rand" "strings" + "sync" "time" "github.com/gempir/justlog/config" @@ -21,6 +22,7 @@ type Bot struct { fileLogger *filelog.Logger worker []*worker channels map[string]helix.UserData + clearchats sync.Map } type worker struct { @@ -157,6 +159,25 @@ func (b *Bot) handleUserNotice(message twitch.UserNoticeMessage) { } func (b *Bot) handleClearChat(message twitch.ClearChatMessage) { + count, ok := b.clearchats.Load(message.RoomID) + if !ok { + count = 0 + } + newCount := count.(int) + 1 + b.clearchats.Store(message.RoomID, newCount) + + go func() { + time.Sleep(time.Second * 1) + count, ok := b.clearchats.Load(message.RoomID) + if ok { + b.clearchats.Store(message.RoomID, count.(int)-1) + } + }() + + if newCount > 50 { + return + } + go func() { err := b.fileLogger.LogClearchatMessageForUser(message.TargetUserID, message) if err != nil { From 1a891ca8ba1ab03f173b995a6ef424646e678b68 Mon Sep 17 00:00:00 2001 From: gempir Date: Fri, 16 Apr 2021 18:34:11 +0200 Subject: [PATCH 3/3] update swagger --- web/build/swagger.json | 82 ----------------------------------------- web/public/swagger.json | 82 ----------------------------------------- 2 files changed, 164 deletions(-) diff --git a/web/build/swagger.json b/web/build/swagger.json index dd1dd34..66c5f2b 100644 --- a/web/build/swagger.json +++ b/web/build/swagger.json @@ -39,51 +39,6 @@ "admin" ], "operationId": "channelConfigs", - "parameters": [ - { - "name": "Body", - "in": "body", - "schema": { - "$ref": "#/definitions/channelConfigsRequest" - } - } - ], - "responses": { - "200": {}, - "400": {}, - "405": {} - } - }, - "delete": { - "security": [ - { - "api_key": [] - } - ], - "description": "Will reset the messageTypes logged for a channel", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json", - "text/plain" - ], - "schemes": [ - "https" - ], - "tags": [ - "admin" - ], - "operationId": "deleteChannelConfigs", - "parameters": [ - { - "name": "Body", - "in": "body", - "schema": { - "$ref": "#/definitions/channelConfigsDeleteRequest" - } - } - ], "responses": { "200": {}, "400": {}, @@ -1201,20 +1156,6 @@ }, "x-go-package": "github.com/gempir/justlog/api" }, - "ChannelConfig": { - "description": "ChannelConfig config for individual channels", - "type": "object", - "properties": { - "messageTypes": { - "type": "array", - "items": { - "$ref": "#/definitions/MessageType" - }, - "x-go-name": "MessageTypes" - } - }, - "x-go-package": "github.com/gempir/justlog/config" - }, "MessageType": { "description": "MessageType different message types possible to receive via IRC", "type": "integer", @@ -1255,16 +1196,6 @@ }, "x-go-package": "github.com/gempir/justlog/api" }, - "channelConfigsDeleteRequest": { - "type": "object", - "properties": { - "messageTypes": { - "type": "boolean", - "x-go-name": "MessageTypes" - } - }, - "x-go-package": "github.com/gempir/justlog/api" - }, "channelConfigsJoinRequest": { "type": "object", "properties": { @@ -1278,19 +1209,6 @@ }, "x-go-package": "github.com/gempir/justlog/api" }, - "channelConfigsRequest": { - "type": "object", - "properties": { - "messageTypes": { - "type": "array", - "items": { - "$ref": "#/definitions/MessageType" - }, - "x-go-name": "MessageTypes" - } - }, - "x-go-package": "github.com/gempir/justlog/api" - }, "channelsDeleteRequest": { "type": "object", "properties": { diff --git a/web/public/swagger.json b/web/public/swagger.json index dd1dd34..66c5f2b 100644 --- a/web/public/swagger.json +++ b/web/public/swagger.json @@ -39,51 +39,6 @@ "admin" ], "operationId": "channelConfigs", - "parameters": [ - { - "name": "Body", - "in": "body", - "schema": { - "$ref": "#/definitions/channelConfigsRequest" - } - } - ], - "responses": { - "200": {}, - "400": {}, - "405": {} - } - }, - "delete": { - "security": [ - { - "api_key": [] - } - ], - "description": "Will reset the messageTypes logged for a channel", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json", - "text/plain" - ], - "schemes": [ - "https" - ], - "tags": [ - "admin" - ], - "operationId": "deleteChannelConfigs", - "parameters": [ - { - "name": "Body", - "in": "body", - "schema": { - "$ref": "#/definitions/channelConfigsDeleteRequest" - } - } - ], "responses": { "200": {}, "400": {}, @@ -1201,20 +1156,6 @@ }, "x-go-package": "github.com/gempir/justlog/api" }, - "ChannelConfig": { - "description": "ChannelConfig config for individual channels", - "type": "object", - "properties": { - "messageTypes": { - "type": "array", - "items": { - "$ref": "#/definitions/MessageType" - }, - "x-go-name": "MessageTypes" - } - }, - "x-go-package": "github.com/gempir/justlog/config" - }, "MessageType": { "description": "MessageType different message types possible to receive via IRC", "type": "integer", @@ -1255,16 +1196,6 @@ }, "x-go-package": "github.com/gempir/justlog/api" }, - "channelConfigsDeleteRequest": { - "type": "object", - "properties": { - "messageTypes": { - "type": "boolean", - "x-go-name": "MessageTypes" - } - }, - "x-go-package": "github.com/gempir/justlog/api" - }, "channelConfigsJoinRequest": { "type": "object", "properties": { @@ -1278,19 +1209,6 @@ }, "x-go-package": "github.com/gempir/justlog/api" }, - "channelConfigsRequest": { - "type": "object", - "properties": { - "messageTypes": { - "type": "array", - "items": { - "$ref": "#/definitions/MessageType" - }, - "x-go-name": "MessageTypes" - } - }, - "x-go-package": "github.com/gempir/justlog/api" - }, "channelsDeleteRequest": { "type": "object", "properties": {