From bef0d38f1ef790a275c874fc0797952c3ee8c88a Mon Sep 17 00:00:00 2001 From: Matthieu 'JP' DERASSE Date: Sat, 10 Dec 2022 14:09:00 +0000 Subject: [PATCH] feat(log): Handle multiple log formatter for file --- log/hooks/file/config.go | 8 ++++ log/hooks/file/enum.go | 34 +++++++++++++++ log/hooks/file/file_hook.go | 82 +++++++++++++++---------------------- log/hooks/file/hook.go | 7 +++- log/log.go | 36 ++++++++-------- 5 files changed, 98 insertions(+), 69 deletions(-) create mode 100644 log/hooks/file/enum.go diff --git a/log/hooks/file/config.go b/log/hooks/file/config.go index f7c61a2..f68511a 100644 --- a/log/hooks/file/config.go +++ b/log/hooks/file/config.go @@ -4,8 +4,10 @@ import ( "io/fs" "os" "path/filepath" + "strings" "time" + "git.dev.m-and-m.ovh/mderasse/gocommon/convert" "github.com/asaskevich/govalidator" "github.com/juju/errors" ) @@ -17,6 +19,7 @@ type ConfigStruct struct { RotateTime *string `yaml:"rotate_time"` RotateMaxAge *string `yaml:"rotate_max_age"` RotateMaxFile *int `yaml:"rotate_max_file"` + OutputFormat *string `yaml:"output_format"` } // IsValid will check that the File configuration is valid. @@ -64,5 +67,10 @@ func (c *ConfigStruct) IsValid() error { return errors.NotValidf("rotate_max_age and rotate_max_file cannot be defined at the same time in File configuration") } } + + if c.OutputFormat != nil && !OutputFormat(*c.OutputFormat).IsValid() { + return errors.NotValidf("output_format is invalid in File configuration. (Possible Values:Allowed values: %s", strings.Join(convert.StringerSliceToStringSlice(GetListOutputFormat()), ", ")) + } + return nil } diff --git a/log/hooks/file/enum.go b/log/hooks/file/enum.go new file mode 100644 index 0000000..a51d943 --- /dev/null +++ b/log/hooks/file/enum.go @@ -0,0 +1,34 @@ +package file + +// OutputFormat Enum + +// OutputFormat is the format that will be use to store log in the file. +type OutputFormat string + +//nolint:exported // keeping the enum simple and readable. +const ( + OutputFormat_TEXT OutputFormat = "TEXT" + OutputFormat_JSON OutputFormat = "JSON" +) + +// IsValid check if the gaven OutputFormat is part of the list of handled provider name. +func (e OutputFormat) IsValid() bool { + for _, v := range GetListOutputFormat() { + if e == v { + return true + } + } + return false +} + +func (e OutputFormat) String() string { + return string(e) +} + +// GetListOutputFormat return a the list of possible OutputFormat. +func GetListOutputFormat() []OutputFormat { + return []OutputFormat{ + OutputFormat_TEXT, + OutputFormat_JSON, + } +} diff --git a/log/hooks/file/file_hook.go b/log/hooks/file/file_hook.go index 0162b90..4201260 100644 --- a/log/hooks/file/file_hook.go +++ b/log/hooks/file/file_hook.go @@ -1,11 +1,9 @@ package file import ( - "bytes" "fmt" "io" "os" - "strings" "sync" "time" @@ -21,15 +19,18 @@ var BufSize uint = 8192 // Hook will write logs to a file. type Hook struct { Level logrus.Level - w io.Writer buf chan logrus.Entry - wg sync.WaitGroup + f logrus.Formatter mu sync.RWMutex synchronous bool + w io.Writer + wg sync.WaitGroup } +// XXX: Maybe just take a formatter in input + // NewFileHook creates a hook to be added to an instance of logger. -func NewFileHook(w io.Writer) *Hook { +func NewFileHook(w io.Writer, of OutputFormat) *Hook { if w == nil { logrus.Error("Can't create File Hook with an empty writer") return nil @@ -38,6 +39,7 @@ func NewFileHook(w io.Writer) *Hook { hook := &Hook{ Level: logrus.DebugLevel, synchronous: true, + f: handleFormat(of), w: w, } @@ -47,7 +49,7 @@ func NewFileHook(w io.Writer) *Hook { // NewAsyncFileHook creates a hook to be added to an instance of logger. // The hook created will be asynchronous, and it's the responsibility of the user to call the Flush method // before exiting to empty the log queue. -func NewAsyncFileHook(w io.Writer) *Hook { +func NewAsyncFileHook(w io.Writer, of OutputFormat) *Hook { if w == nil { logrus.Error("Can't create File Hook with an empty writer") return nil @@ -56,8 +58,10 @@ func NewAsyncFileHook(w io.Writer) *Hook { hook := &Hook{ Level: logrus.DebugLevel, buf: make(chan logrus.Entry, BufSize), + f: handleFormat(of), w: w, } + go hook.fire() // Log in background return hook @@ -76,6 +80,23 @@ func (hook *Hook) Flush() { hook.wg.Wait() } +// handleFormat will take a OutputFormat and will transform it in a formatter. +func handleFormat(of OutputFormat) logrus.Formatter { + + if of == OutputFormat_JSON { + return &logrus.JSONFormatter{ + PrettyPrint: false, + TimestampFormat: time.RFC3339Nano, + } + } + + return &logrus.TextFormatter{ + DisableColors: true, + TimestampFormat: time.RFC3339Nano, + QuoteEmptyFields: true, + } +} + // Fire is called when a log event is fired. // We assume the entry will be altered by another hook, // otherwise we might be logging something wrong to Graylog. @@ -109,8 +130,13 @@ func (hook *Hook) writeEntry(entry *logrus.Entry) { return } - message := constructMessage(entry) - _, err := hook.w.Write([]byte(fmt.Sprintf("%s\n", message))) + message, err := hook.f.Format(entry) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to format log line. Error: %s\n", err) + return + } + + _, err = hook.w.Write(message) if err != nil { fmt.Fprintf(os.Stderr, "Failed to write in log file. Error: %s\nLog Line: %s", err, message) } @@ -126,43 +152,3 @@ func (hook *Hook) Levels() []logrus.Level { } return levels } - -func constructMessage(entry *logrus.Entry) string { - var messageParts []string - - // handle time - messageParts = append(messageParts, fmt.Sprintf("time=\"%s\"", entry.Time.UTC().Format(time.RFC3339))) - - // handle log level - messageParts = append(messageParts, fmt.Sprintf("level=%s", entry.Level.String())) - - // handle actual message - messageParts = append(messageParts, fmt.Sprintf("msg=\"%s\"", string(bytes.TrimSpace([]byte(entry.Message))))) - - // add extra data except err_* - for k, v := range entry.Data { - if !strings.HasPrefix(k, "err_") { - messageParts = append(messageParts, fmt.Sprintf("%s=\"%s\"", k, v)) - } - } - - message := strings.Join(messageParts, " ") - - // handle error stack - // XXX: Is that really how it work ?? I doubt but let see - if _, exist := entry.Data["err_code"]; exist { - - // get error infos - //ns := entry.Data["err_ns"] - ctx := entry.Data["err_ctx"] - //id := entry.Data["err_id"] - - // get stack and clean it a bit - tSt := entry.Data["err_stack"] - st := strings.Replace(tSt.(string), "\n", "\n\t\t", -1) - - message = fmt.Sprintf("%s\nerror:\n\tcontext:%s\n\tstacktrace:\n\t\t%s", message, ctx, st) - } - - return message -} diff --git a/log/hooks/file/hook.go b/log/hooks/file/hook.go index e84fb3b..8a69e48 100644 --- a/log/hooks/file/hook.go +++ b/log/hooks/file/hook.go @@ -21,6 +21,11 @@ func NewHook(c *ConfigStruct) (logrus.Hook, error) { return nil, errors.Trace(err) } + outputFormat := OutputFormat_TEXT + if c.OutputFormat != nil { + outputFormat = OutputFormat(*c.OutputFormat) + } + var w io.Writer if c.Rotate { rotateTime, _ := time.ParseDuration(*c.RotateTime) @@ -65,7 +70,7 @@ func NewHook(c *ConfigStruct) (logrus.Hook, error) { w = fh } - h := NewAsyncFileHook(w) + h := NewAsyncFileHook(w, outputFormat) return h, nil } diff --git a/log/log.go b/log/log.go index af80a25..1f6efd5 100644 --- a/log/log.go +++ b/log/log.go @@ -3,7 +3,6 @@ package log import ( "io" "os" - "time" "github.com/juju/errors" "github.com/sirupsen/logrus" @@ -13,43 +12,43 @@ import ( ) // InitLog will try to initialize logger by trying to retrieve config from multiple source. -func InitLog() error { +func InitLog() (*logrus.Logger, error) { // loading configuration c, err := loadConfig() if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } return initFromSource(c) } // InitLogFromCustomVaultSecret will initialize logger with a vault secret. -func InitLogFromCustomVaultSecret(secret string) error { +func InitLogFromCustomVaultSecret(secret string) (*logrus.Logger, error) { c, err := loadConfigFromVault(secret) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } return initFromSource(c) } // InitLogFromCustomFile will initialize logger with a config file. -func InitLogFromCustomFile(path string) error { +func InitLogFromCustomFile(path string) (*logrus.Logger, error) { c, err := loadConfigFromFile(path) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } return initFromSource(c) } -func initFromSource(c *ConfigStruct) error { +func initFromSource(c *ConfigStruct) (*logrus.Logger, error) { err := c.applyEnv() if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } c.applyDefault() @@ -58,16 +57,16 @@ func initFromSource(c *ConfigStruct) error { } // InitLogFromCustomConfig will initialize logger from a gaven config. -func InitLogFromCustomConfig(c *ConfigStruct) error { +func InitLogFromCustomConfig(c *ConfigStruct) (*logrus.Logger, error) { err := c.IsValid() if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } level, err := logrus.ParseLevel(*c.Level) if err != nil { - return err + return nil, err } // init logger @@ -84,23 +83,20 @@ func InitLogFromCustomConfig(c *ConfigStruct) error { case ProviderName_FILE: hook, err := file.NewHook(c.FileConfig) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } log.AddHook(hook) case ProviderName_GELF: hook, err := gelf.NewHook(c.GelfConfig) if err != nil { - return errors.Trace(err) + return nil, errors.Trace(err) } log.AddHook(hook) case ProviderName_NONE: - // Do Nothing + fallthrough default: + return nil, errors.BadRequestf("Provider is not handled.") } - for i := 0; i < 500; i++ { - log.Infof("Test %d", i) - time.Sleep(1 * time.Second) - } - return nil + return log, nil }