How to get buffered logging with Golang logrus

I am working with github.com/sirupsen/logrus module and have a requirement that the logs I generate should be buffered(time stamp preserved). I'll also start a timer when the operation begins and various steps of the operation will keep logging to this buffer.

At the end, if the total time spent in the operation is more than some threshold, I'll log everything else just discard the buffer.

I am just trying to find out how to elegantly implement this with logrus.

I have tried something like this:

type BufferedLog struct {
  Message string
  Level   logrus.Level
  Time    time.Time
}

type BufferingFormatter struct {
  bufferMutex sync.Mutex
  buffer      []*BufferedLog
}

func (f *BufferingFormatter) Format(entry *logrus.Entry) ([]byte, error) {

  bl := &BufferedLog{
    Message: entry.Message,
	Level:   entry.Level,
	Time:    entry.Time,
  }

  f.buffer = append(f.buffer, bl)
  return nil, nil
}

func PrintBufferedLogs(buffer []*BufferedLog) {
  now := time.Now()
  for _, log := range buffer {
	if now.Sub(log.Time) >= 5 * time.Second {
		data, err := json.Marshal(log)
		if err != nil {
			logrus.Errorf("error marshaling log entry to JSON: %v", err)
			continue
		}
		logrus.Print(string(data))
	}
  }
}

func NewBufferingLogger() *logrus.Logger {
  logger := logrus.New()
  bufferingFormatter := &BufferingFormatter{buffer: make([]*BufferedLog, 0)}
  logger.Formatter = bufferingFormatter
  return logger
}

And this is my main():

func main() {
  logrus.SetLevel(logrus.DebugLevel)
  logrus.SetFormatter(&logrus.TextFormatter{
	FullTimestamp: true,
  })
  bufferingLogger := NewBufferingLogger()
  bufferingLogger.SetLevel(logrus.DebugLevel)

  // Log some messages using the custom bufferingLogger
  bufferingLogger.Info("This is an informational message.")
  bufferingLogger.Warn("This is a warning message.")
  bufferingLogger.Error("This is an error message.")
  bufferingLogger.Debug("This is a debug message.")
  time.Sleep(5 * time.Second)
  PrintBufferedLogs(bufferingLogger.Formatter.(*BufferingFormatter).buffer) 
}

It generates this output:

INFO {"message":"This is an informational message.","level":"info","time":"2023-07-25T11:41:17.545041+05:30"} 
INFO {"message":"This is a warning message.","level":"warning","time":"2023-07-25T11:41:17.545056+05:30"} 
INFO {"message":"This is an error message.","level":"error","time":"2023-07-25T11:41:17.545059+05:30"} 
INFO {"message":"This is a debug message.","level":"debug","time":"2023-07-25T11:41:17.545061+05:30"} 

There are still lingering INFO in the output and this is changing the global logrus behavior as well.

What I am trying to do is to have this buffered logging as a separate logger and keep global logrus the way it is.

This looks like an overkill. Any better ideas ?

Your approach looks correct but a few points are missing.

Independent Loggers: Instead of changing global logrus logger, you can create a new logger instance using logrus.New(). Each instance of logrus Logger is independent and doesn't interfere with each other. Therefore, you can create a separate logger with a buffering formatter that won't impact the global logger.

Printing Buffer Logs: You can utilise a separate method to flush out logs after some operations are completed. This method should scan the buffered logs, and log them with your original (global) logger.

Using Correct Level: You should log the buffered logs with the correct level they were logged with. In your code, you used logrus.Print(), which doesn't respect the level of logs.

Clear Buffer: After printing buffered logs, you should clear the buffer to prevent duplicated logs in the next flush.

The updated code will be similar to this

package main

import (
	"bytes"
	"fmt"
	"time"

	"github.com/sirupsen/logrus"
)

type MyFormatter struct {
}

func (f *MyFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	b := &bytes.Buffer{}
	fmt.Fprintf(b, "{\"Level\":\"%s\",\"Time\":\"%s\",\"Message\":\"%s\"}\n", entry.Level, entry.Time.Format(time.RFC3339), entry.Message)
	return b.Bytes(), nil
}

type BufferedLog struct {
	Message string
	Level   logrus.Level
	Time    time.Time
}

func NewBufferingLogger() *logrus.Logger {
	logger := logrus.New()
	logger.Formatter = new(MyFormatter)
	logger.Level = logrus.DebugLevel
	return logger
}

func main() {
	bufferingLogger := NewBufferingLogger()

	// Log some messages using the custom bufferingLogger
	bufferingLogger.Info("This is an informational message.")
	bufferingLogger.Warn("This is a warning message.")
	bufferingLogger.Error("This is an error message.")
	bufferingLogger.Debug("This is a debug message.")
}

Based on @ekundayo-blessing-funminiyi 's answer I modified his code a bit to get this

var buflogger *BufferedLogger
type BufferedLog struct {
Message		string			`json:"message"`
Level		logrus.Level	`json:"level"`
Time		time.Time		`json:"time"`
TimeSpent	time.Time		`json:"time_spent,omitempty"`
}
type BufferingFormatter struct {
bufferMutex sync.Mutex
buffer []*BufferedLog
}
func init() {
logrus.SetLevel(logrus.DebugLevel)
logrus.SetFormatter(&logrus.JSONFormatter{})
once := sync.Once{}
once.Do(func() {
buflogger = NewBufferingLogger()
})
}
func (f *BufferingFormatter) Format(entry *logrus.Entry) ([]byte, error) 
{
f.bufferMutex.Lock()
defer f.bufferMutex.Unlock()
bl := &BufferedLog{
Message: entry.Message,
Level:   entry.Level,
Time:    entry.Time.UTC(),
}
f.buffer = append(f.buffer, bl)
return nil, nil
}
func (f *BufferingFormatter)PrintBufferedLogs() {
f.bufferMutex.Lock()
defer f.bufferMutex.Unlock()
dumpAll := false
now := time.Now()
for _, log := range f.buffer {
if dumpAll || (now.Sub(log.Time) >= 5*time.Minute) {
dumpAll = true
logrus.WithTime(log.Time).Log(log.Level, log.Message)
}
}
// Flush the buffer to avoid memory leaks
f.buffer = nil
}
func NewBufferingLogger() *BufferedLogger {
if buflogger != nil {
return buflogger
}
bufferingFormatter := &BufferingFormatter{buffer: make([]*BufferedLog, 0)}
logger := logrus.New()
logger.Level = logrus.DebugLevel
logger.Formatter = bufferingFormatter
buflogger =  &BufferedLogger{
Logger:             logger,
bufferingFormatter: bufferingFormatter,
}
return buflogger
}
type BufferedLogger struct {
*logrus.Logger
bufferingFormatter *BufferingFormatter
}
func (bl *BufferedLogger) PrintBufferedLogs() {
bl.bufferingFormatter.PrintBufferedLogs()
}