Fijxu/justlog
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

query helix for userids

Browse source
This commit is contained in:
gempir 2018-12-02 19:23:54 +01:00
parent bd73490f45
commit 146511c52b
10 changed files with 385 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
ansible/group_vars/all.yml
View file

@ -14,4 +14,11 @@ oauth: !vault |
6137316537306234353665333065636636663363383038640a653735356262613362666561383033
33343936646165393262353239633335653563373438353765636463373439653235643535333633
6163656464313337350a323039656238346538666132646530386261643336613238356264363132
36623964383333333965343132626662623832626663656631366563613139373832
36623964383333333965343132626662623832626663656631366563613139373832
clientID: !vault |
$ANSIBLE_VAULT;1.1;AES256
39663965636532653361383032343166653338323638613035333265376266316565363436303339
3235363539356363346537373234613631333534326234610a353239306566373737386630633336
36303362366362613564643065666531663464633061663230623330366232646439613361363966
6661313364356636610a356337323334376533613030393438383363343930613239653663306437
36663430303738343333323439363034663335643839326335613039343435396566

1
ansible/roles/bot/templates/config.j2
View file

@ -1,6 +1,7 @@
{
"admin": "gempir",
"logsDirectory": "/var/justlog",
"clientID": "{{ clientID }}",
"username": "{{ username }}",
"oauth": "{{ oauth }}",
"channels": [

26
api/logs.go
View file

@ -14,7 +14,7 @@ import (
"github.com/gempir/go-twitch-irc"
"github.com/labstack/echo"
"github.com/labstack/gommon/log"
log "github.com/sirupsen/logrus"
)
type RandomQuoteJSON struct {
@ -119,6 +119,30 @@ func (s *Server) getRandomQuote(c echo.Context) error {
return c.String(http.StatusOK, lineSplit[1])
}
func (s *Server) getUserLogsByName(c echo.Context) error {
channel := strings.ToLower(c.Param("channel"))
username := strings.ToLower(c.Param("username"))
userMap, err := s.helixClient.GetUsersByUsernames([]string{channel, username})
if err != nil {
log.Error(err)
return c.JSON(http.StatusInternalServerError, "Failure fetching userIDs")
}
names := c.ParamNames()
names = append(names, "channelid")
names = append(names, "userid")
values := c.ParamValues()
values = append(values, userMap[channel].ID)
values = append(values, userMap[username].ID)
c.SetParamNames(names...)
c.SetParamValues(values...)
return s.getUserLogs(c)
}
func (s *Server) getUserLogs(c echo.Context) error {
channelID := c.Param("channelid")
userID := c.Param("userid")

146
api/main.go
View file

@ -1,146 +0,0 @@
package api
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
twitch "github.com/gempir/go-twitch-irc"
jsoniter "github.com/json-iterator/go"
"github.com/labstack/echo"
)
type order string
var (
orderDesc order = "DESC"
orderAsc order = "ASC"
)
type chatLog struct {
Messages []chatMessage `json:"messages"`
}
type chatMessage struct {
Text string `json:"text"`
Username string `json:"username"`
Channel string `json:"channel"`
Timestamp timestamp `json:"timestamp"`
Type twitch.MessageType `json:"type"`
}
type timestamp struct {
time.Time
}
func (t timestamp) MarshalJSON() ([]byte, error) {
return []byte("\"" + t.UTC().Format(time.RFC3339) + "\""), nil
}
func (t *timestamp) UnmarshalJSON(data []byte) error {
goTime, err := time.Parse(time.RFC3339, strings.TrimSuffix(strings.TrimPrefix(string(data[:]), "\""), "\""))
if err != nil {
return err
}
*t = timestamp{
goTime,
}
return nil
}
func parseFromTo(from, to string, limit float64) (time.Time, time.Time, error) {
var fromTime time.Time
var toTime time.Time
if from == "" && to == "" {
fromTime = time.Now().AddDate(0, -1, 0)
toTime = time.Now()
} else if from == "" && to != "" {
var err error
toTime, err = parseTimestamp(to)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse to timestamp: %s", err)
}
fromTime = toTime.AddDate(0, -1, 0)
} else if from != "" && to == "" {
var err error
fromTime, err = parseTimestamp(from)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse from timestamp: %s", err)
}
toTime = fromTime.AddDate(0, 1, 0)
} else {
var err error
fromTime, err = parseTimestamp(from)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse from timestamp: %s", err)
}
toTime, err = parseTimestamp(to)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse to timestamp: %s", err)
}
if toTime.Sub(fromTime).Hours() > limit {
return fromTime, toTime, errors.New("Timespan too big")
}
}
return fromTime, toTime, nil
}
func writeTextResponse(c echo.Context, cLog *chatLog) error {
c.Response().WriteHeader(http.StatusOK)
for _, cMessage := range cLog.Messages {
switch cMessage.Type {
case twitch.PRIVMSG:
c.Response().Write([]byte(fmt.Sprintf("[%s] #%s %s: %s\r\n", cMessage.Timestamp.Format("2006-01-2 15:04:05"), cMessage.Channel, cMessage.Username, cMessage.Text)))
case twitch.CLEARCHAT:
c.Response().Write([]byte(fmt.Sprintf("[%s] #%s %s\r\n", cMessage.Timestamp.Format("2006-01-2 15:04:05"), cMessage.Channel, cMessage.Text)))
}
}
return nil
}
func writeJSONResponse(c echo.Context, logResult *chatLog) error {
_, stream := c.QueryParams()["stream"]
if stream {
c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8)
c.Response().WriteHeader(http.StatusOK)
return json.NewEncoder(c.Response()).Encode(logResult)
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
data, err := json.Marshal(logResult)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
return c.Blob(http.StatusOK, echo.MIMEApplicationJSONCharsetUTF8, data)
}
func parseTimestamp(timestamp string) (time.Time, error) {
i, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return time.Now(), err
}
return time.Unix(i, 0), nil
}
func buildOrder(c echo.Context) order {
dataOrder := orderAsc
_, reverse := c.QueryParams()["reverse"]
if reverse {
dataOrder = orderDesc
}
return dataOrder
}

150
api/server.go
View file

@ -1,9 +1,19 @@
package api
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/gempir/justlog/helix"
twitch "github.com/gempir/go-twitch-irc"
"github.com/gempir/justlog/filelog"
jsoniter "github.com/json-iterator/go"
"github.com/labstack/echo"
"github.com/labstack/echo/middleware"
)
@ -12,14 +22,16 @@ type Server struct {
listenAddress string
logPath string
fileLogger *filelog.Logger
helixClient *helix.Client
channels []string
}
func NewServer(logPath string, listenAddress string, fileLogger *filelog.Logger) Server {
func NewServer(logPath string, listenAddress string, fileLogger *filelog.Logger, helixClient *helix.Client) Server {
return Server{
listenAddress: listenAddress,
logPath: logPath,
fileLogger: fileLogger,
helixClient: helixClient,
channels: []string{},
}
}
@ -41,14 +53,146 @@ func (s *Server) Init() {
e.Use(middleware.CORSWithConfig(DefaultCORSConfig))
e.GET("/", func(c echo.Context) error {
return c.String(http.StatusOK, "Hello, World!")
return c.String(http.StatusOK, "Welcome to justlog")
})
e.GET("/channelid/:channelid/user/:userid", s.getCurrentUserLogs)
e.GET("/channelid", s.getAllChannels)
e.GET("/channelid/:channelid/user/:userid", s.getCurrentUserLogs)
e.GET("/channelid/:channelid", s.getCurrentChannelLogs)
e.GET("/channelid/:channelid/:year/:month/:day", s.getChannelLogs)
e.GET("/channelid/:channelid/userid/:userid/:year/:month", s.getUserLogs)
e.GET("/channel/:channel/user/:username/:year/:month", s.getUserLogsByName)
e.GET("/channelid/:channelid/userid/:userid/random", s.getRandomQuote)
e.Logger.Fatal(e.Start(s.listenAddress))
}
type order string
var (
orderDesc order = "DESC"
orderAsc order = "ASC"
)
type chatLog struct {
Messages []chatMessage `json:"messages"`
}
type chatMessage struct {
Text string `json:"text"`
Username string `json:"username"`
Channel string `json:"channel"`
Timestamp timestamp `json:"timestamp"`
Type twitch.MessageType `json:"type"`
}
type timestamp struct {
time.Time
}
func (t timestamp) MarshalJSON() ([]byte, error) {
return []byte("\"" + t.UTC().Format(time.RFC3339) + "\""), nil
}
func (t *timestamp) UnmarshalJSON(data []byte) error {
goTime, err := time.Parse(time.RFC3339, strings.TrimSuffix(strings.TrimPrefix(string(data[:]), "\""), "\""))
if err != nil {
return err
}
*t = timestamp{
goTime,
}
return nil
}
func parseFromTo(from, to string, limit float64) (time.Time, time.Time, error) {
var fromTime time.Time
var toTime time.Time
if from == "" && to == "" {
fromTime = time.Now().AddDate(0, -1, 0)
toTime = time.Now()
} else if from == "" && to != "" {
var err error
toTime, err = parseTimestamp(to)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse to timestamp: %s", err)
}
fromTime = toTime.AddDate(0, -1, 0)
} else if from != "" && to == "" {
var err error
fromTime, err = parseTimestamp(from)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse from timestamp: %s", err)
}
toTime = fromTime.AddDate(0, 1, 0)
} else {
var err error
fromTime, err = parseTimestamp(from)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse from timestamp: %s", err)
}
toTime, err = parseTimestamp(to)
if err != nil {
return fromTime, toTime, fmt.Errorf("Can't parse to timestamp: %s", err)
}
if toTime.Sub(fromTime).Hours() > limit {
return fromTime, toTime, errors.New("Timespan too big")
}
}
return fromTime, toTime, nil
}
func writeTextResponse(c echo.Context, cLog *chatLog) error {
c.Response().WriteHeader(http.StatusOK)
for _, cMessage := range cLog.Messages {
switch cMessage.Type {
case twitch.PRIVMSG:
c.Response().Write([]byte(fmt.Sprintf("[%s] #%s %s: %s\r\n", cMessage.Timestamp.Format("2006-01-2 15:04:05"), cMessage.Channel, cMessage.Username, cMessage.Text)))
case twitch.CLEARCHAT:
c.Response().Write([]byte(fmt.Sprintf("[%s] #%s %s\r\n", cMessage.Timestamp.Format("2006-01-2 15:04:05"), cMessage.Channel, cMessage.Text)))
}
}
return nil
}
func writeJSONResponse(c echo.Context, logResult *chatLog) error {
_, stream := c.QueryParams()["stream"]
if stream {
c.Response().Header().Set(echo.HeaderContentType, echo.MIMEApplicationJSONCharsetUTF8)
c.Response().WriteHeader(http.StatusOK)
return json.NewEncoder(c.Response()).Encode(logResult)
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
data, err := json.Marshal(logResult)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
return c.Blob(http.StatusOK, echo.MIMEApplicationJSONCharsetUTF8, data)
}
func parseTimestamp(timestamp string) (time.Time, error) {
i, err := strconv.ParseInt(timestamp, 10, 64)
if err != nil {
return time.Now(), err
}
return time.Unix(i, 0), nil
}
func buildOrder(c echo.Context) order {
dataOrder := orderAsc
_, reverse := c.QueryParams()["reverse"]
if reverse {
dataOrder = orderDesc
}
return dataOrder
}

2
config.json
View file

@ -3,5 +3,7 @@
"logsDirectory": "./logs",
"username": "gempbot",
"oauth": "cg74a9xb4le868tctmelk7eidqtrra",
"clientID": "862bsdm5ev5x4swg1vzqtwlokxglqv",
"logLevel": "debug",
"channels": ["gempir", "pajlada"]
}

2
filelog/channellog.go
View file

@ -44,7 +44,7 @@ func (l *Logger) ReadLogForChannel(channelID string, year int, month int, day in
f, err := os.Open(filename)
if err != nil {
return []string{}, errors.New("file not found")
return []string{}, errors.New("file not found: " + filename)
}
defer f.Close()

5
filelog/userlog.go
View file

@ -10,6 +10,7 @@ import (
"strings"
"github.com/gempir/go-twitch-irc"
log "github.com/sirupsen/logrus"
)
type Logger struct {
@ -50,9 +51,10 @@ func (l *Logger) ReadLogForUser(channelID string, userID string, year int, month
filename = filename + ".gz"
}
log.Debug("Opening " + filename)
f, err := os.Open(filename)
if err != nil {
return []string{}, errors.New("file not found")
return []string{}, errors.New("file not found: " + filename)
}
defer f.Close()
@ -77,6 +79,7 @@ func (l *Logger) ReadLogForUser(channelID string, userID string, year int, month
for scanner.Scan() {
line := scanner.Text()
log.Debug(line)
content = append(content, line)
}

166
helix/user.go
View file

@ -1 +1,167 @@
package helix
import (
"encoding/json"
"io/ioutil"
"net/http"
log "github.com/sirupsen/logrus"
)
type Client struct {
clientID string
httpClient *http.Client
}
var (
userCacheByID map[string]UserData
userCacheByUsername map[string]UserData
)
func init() {
userCacheByID = map[string]UserData{}
userCacheByUsername = map[string]UserData{}
}
func NewClient(clientID string) Client {
return Client{
clientID: clientID,
httpClient: &http.Client{},
}
}
type userResponse struct {
Data []UserData `json:"data"`
}
type UserData struct {
ID string `json:"id"`
Login string `json:"login"`
DisplayName string `json:"display_name"`
Type string `json:"type"`
BroadcasterType string `json:"broadcaster_type"`
Description string `json:"description"`
ProfileImageURL string `json:"profile_image_url"`
OfflineImageURL string `json:"offline_image_url"`
ViewCount int `json:"view_count"`
Email string `json:"email"`
}
func (c *Client) GetUsersByUserIds(userIDs []string) (map[string]UserData, error) {
var filteredUserIDs []string
for _, id := range userIDs {
if _, ok := userCacheByID[id]; !ok {
filteredUserIDs = append(filteredUserIDs, id)
}
}
if len(filteredUserIDs) == 1 {
params := "?id=" + filteredUserIDs[0]
err := c.makeRequest(params)
if err != nil {
return nil, err
}
} else if len(filteredUserIDs) > 1 {
var params string
for index, id := range filteredUserIDs {
if index == 0 {
params += "?id=" + id
} else {
params += "&id=" + id
}
}
err := c.makeRequest(params)
if err != nil {
return nil, err
}
}
result := make(map[string]UserData)
for _, id := range userIDs {
result[id] = userCacheByID[id]
}
return result, nil
}
func (c *Client) GetUsersByUsernames(usernames []string) (map[string]UserData, error) {
var filteredUsernames []string
for _, username := range usernames {
if _, ok := userCacheByUsername[username]; !ok {
filteredUsernames = append(filteredUsernames, username)
}
}
if len(filteredUsernames) == 1 {
params := "?login=" + filteredUsernames[0]
err := c.makeRequest(params)
if err != nil {
return nil, err
}
} else if len(filteredUsernames) > 1 {
var params string
for index, id := range filteredUsernames {
if index == 0 {
params += "?login=" + id
} else {
params += "&login=" + id
}
}
err := c.makeRequest(params)
if err != nil {
return nil, err
}
}
result := make(map[string]UserData)
for _, username := range usernames {
result[username] = userCacheByUsername[username]
}
return result, nil
}
func (c *Client) makeRequest(parameters string) error {
request, err := http.NewRequest("GET", "https://api.twitch.tv/helix/users"+parameters, nil)
if err != nil {
return err
}
request.Header.Set("Client-ID", c.clientID)
response, err := c.httpClient.Do(request)
if err != nil {
return err
}
log.Infof("%d GET https://api.twitch.tv/helix/users%s", response.StatusCode, parameters)
defer response.Body.Close()
contents, err := ioutil.ReadAll(response.Body)
if err != nil {
return err
}
var userResp userResponse
err = json.Unmarshal(contents, &userResp)
if err != nil {
return err
}
for _, user := range userResp.Data {
userCacheByID[user.ID] = user
userCacheByUsername[user.Login] = user
}
return nil
}

32
main.go
View file

@ -11,6 +11,7 @@ import (
"github.com/gempir/go-twitch-irc"
"github.com/gempir/justlog/api"
"github.com/gempir/justlog/filelog"
"github.com/gempir/justlog/helix"
"github.com/gempir/justlog/humanize"
log "github.com/sirupsen/logrus"
@ -23,6 +24,8 @@ type config struct {
ListenAddress string `json:"listenAddress"`
Admin string `json:"admin"`
Channels []string `json:"channels"`
ClientID string `json:"clientID"`
LogLevel string `json:"logLevel"`
}
var (
@ -36,8 +39,10 @@ func main() {
flag.Parse()
cfg = loadConfiguration(*configFile)
setupLogger(cfg)
twitchClient := twitch.NewClient(cfg.Username, "oauth:"+cfg.OAuth)
fileLogger := filelog.NewFileLogger(cfg.LogsDirectory)
helixClient := helix.NewClient(cfg.ClientID)
if strings.HasPrefix(cfg.Username, "justinfan") {
log.Info("Bot joining anonymous")
@ -45,7 +50,7 @@ func main() {
log.Info("Bot joining as user " + cfg.Username)
}
apiServer := api.NewServer(cfg.LogsDirectory, cfg.ListenAddress, &fileLogger)
apiServer := api.NewServer(cfg.LogsDirectory, cfg.ListenAddress, &fileLogger, &helixClient)
go apiServer.Init()
for _, channel := range cfg.Channels {
@ -96,6 +101,23 @@ func main() {
log.Fatal(twitchClient.Connect())
}
func setupLogger(cfg config) {
switch cfg.LogLevel {
case "fatal":
log.SetLevel(log.FatalLevel)
case "panic":
log.SetLevel(log.PanicLevel)
case "error":
log.SetLevel(log.ErrorLevel)
case "warn":
log.SetLevel(log.WarnLevel)
case "info":
log.SetLevel(log.InfoLevel)
case "debug":
log.SetLevel(log.DebugLevel)
}
}
func loadConfiguration(file string) config {
log.Info("Loading config from " + file)
@ -107,6 +129,7 @@ func loadConfiguration(file string) config {
OAuth: "oauth:777777777",
Channels: []string{},
Admin: "gempir",
LogLevel: "info",
}
configFile, err := os.Open(file)
@ -121,8 +144,15 @@ func loadConfiguration(file string) config {
log.Fatal(err)
}
// normalize
cfg.LogsDirectory = strings.TrimSuffix(cfg.LogsDirectory, "/")
cfg.OAuth = strings.TrimPrefix(cfg.OAuth, "oauth:")
cfg.LogLevel = strings.ToLower(cfg.LogLevel)
// ensure required
if cfg.ClientID == "" {
log.Fatal("No clientID specified")
}
return cfg
}