From 5591da9c47b97d91047a8997637906a48c7a45a6 Mon Sep 17 00:00:00 2001 From: Matthieu 'JP' DERASSE Date: Sat, 26 Nov 2022 20:45:32 +0000 Subject: [PATCH] feat(log): Working on log library --- convert/convert.go | 20 +++ go.mod | 17 +- go.sum | 32 ++++ log/config.go | 177 ++++++++++++++++++++ log/enum.go | 36 ++++ log/hooks/file/config.go | 42 +++++ log/hooks/file/file.go | 354 +++++++++++++++++++++++++++++++++++++++ log/hooks/file/hook.go | 110 ++++++++++++ log/hooks/gelf/config.go | 32 ++++ log/hooks/gelf/hook.go | 28 ++++ log/log.go | 106 ++++++++++++ 11 files changed, 952 insertions(+), 2 deletions(-) create mode 100644 convert/convert.go create mode 100644 go.sum create mode 100644 log/config.go create mode 100644 log/enum.go create mode 100644 log/hooks/file/config.go create mode 100644 log/hooks/file/file.go create mode 100644 log/hooks/file/hook.go create mode 100644 log/hooks/gelf/config.go create mode 100644 log/hooks/gelf/hook.go create mode 100644 log/log.go diff --git a/convert/convert.go b/convert/convert.go new file mode 100644 index 0000000..1877f80 --- /dev/null +++ b/convert/convert.go @@ -0,0 +1,20 @@ +package convert + +// ToPointer will convert any type of data to his pointer. +func ToPointer[T any](v T) *T { + return &v +} + +// Stringer is a interface that any "enum" style struct need to follow. +type Stringer interface { + String() string +} + +// StringerSliceToStringSlice will take a slice of stringer and convert it to a slice of string. +func StringerSliceToStringSlice[E Stringer](v []E) []string { + output := make([]string, len(v)) + for i, data := range v { + output[i] = data.String() + } + return output +} diff --git a/go.mod b/go.mod index 8654108..f86b56b 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,16 @@ -module git.home.m-and-m.ovh/mderasse/common +module git.dev.m-and-m.ovh/mderasse/gocommon -go 1.18 +go 1.19 + +require ( + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d + github.com/juju/errors v1.0.0 + github.com/sirupsen/logrus v1.9.0 + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c +) + +require ( + github.com/gemnasium/logrus-graylog-hook/v3 v3.1.0 // indirect + github.com/pkg/errors v0.8.1 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..765f4df --- /dev/null +++ b/go.sum @@ -0,0 +1,32 @@ +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gemnasium/logrus-graylog-hook/v3 v3.1.0 h1:SLtCnpI5ZZaz4l7RSatEhppB1BBhUEu+DqGANJzJdEA= +github.com/gemnasium/logrus-graylog-hook/v3 v3.1.0/go.mod h1:wi1zWv9tIvyLSMLCAzgRP+YR24oLVQVBHfPPKjtht44= +github.com/juju/errors v1.0.0 h1:yiq7kjCLll1BiaRuNY53MGI0+EQ3rF6GB+wvboZDefM= +github.com/juju/errors v1.0.0/go.mod h1:B5x9thDqx0wIMH3+aLIMP9HjItInYWObRovoCFM5Qe8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/log/config.go b/log/config.go new file mode 100644 index 0000000..ebb16d3 --- /dev/null +++ b/log/config.go @@ -0,0 +1,177 @@ +package log + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/juju/errors" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + + "git.dev.m-and-m.ovh/mderasse/gocommon/convert" + "git.dev.m-and-m.ovh/mderasse/gocommon/log/hooks/file" + "git.dev.m-and-m.ovh/mderasse/gocommon/log/hooks/gelf" +) + +// envPrefix is the prefix that will be used for any environnement variable used to configure the logging system. +const envPrefix = "LOG_" + +// CONFIG_FILE is the default file that will be searched to apply configuration from the filesystem. +const defaultConfigFile = "log.yaml" + +// SECRET_NAME is the default name of the secret that will be searched in vault. +const defaultSecretName = "log" + +// ConfigStruct represent the configuration of our logger system. +type ConfigStruct struct { + Level *string `yaml:"level"` + EnableStdOut *bool `yaml:"ensable_std_out"` + Provider *ProviderName `yaml:"provider"` + FileConfig *file.ConfigStruct `yaml:"file_config"` + GelfConfig *gelf.ConfigStruct `yaml:"gelf_config"` +} + +func newDefaultConfig() *ConfigStruct { + return &ConfigStruct{ + Level: convert.ToPointer("debug"), + EnableStdOut: convert.ToPointer(true), + Provider: convert.ToPointer(ProviderName_NONE), + } +} + +func loadConfigFromVault(secret string) (*ConfigStruct, error) { + return nil, os.ErrNotExist +} + +// loadConfigFromFile will read the given file and return a config struct. +func loadConfigFromFile(path string) (*ConfigStruct, error) { + + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return nil, errors.Trace(err) + } + + //nolint:gosec // we did compute the file path + f, err := os.ReadFile(path) + if err != nil { + return nil, errors.Trace(err) + } + + var config ConfigStruct + + if err := yaml.Unmarshal(f, &config); err != nil { + return nil, errors.Trace(err) + } + + return &config, nil +} + +func loadConfig() (*ConfigStruct, error) { + + c, err := loadConfigFromVault(defaultSecretName) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else if err == nil { + return c, nil + } + + c, err = loadConfigFromFile(defaultConfigFile) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } else if err == nil { + return c, nil + } + + return newDefaultConfig(), nil +} + +// applyEnv will retrieve info from environment variable and overide those got by config / vault. +// validity of the value is not checked here and will be check in a IsValid method. +func (c *ConfigStruct) applyEnv() error { + + if v := os.Getenv(fmt.Sprintf("%s%s", envPrefix, "LEVEL")); v != "" { + c.Level = convert.ToPointer(v) + } + + if v := os.Getenv(fmt.Sprintf("%s%s", envPrefix, "ENABLE_STDOUT")); v != "" { + b, err := strconv.ParseBool(v) + if err != nil { + return errors.NewNotValid(err, fmt.Sprintf("Invalid %s%s environment variable. Should be a boolean", envPrefix, "ENABLE_STDOUT")) + } + c.EnableStdOut = convert.ToPointer(b) + } + + if v := os.Getenv(fmt.Sprintf("%s%s", envPrefix, "PROVIDER")); v != "" { + p := ProviderName(v) + c.Provider = convert.ToPointer(p) + } + + return nil +} + +// applyDefault will check the gaven config and will apply default on empty field. +func (c *ConfigStruct) applyDefault() { + + defaultConfig := newDefaultConfig() + + if c.Level == nil || *c.Level == "" { + c.Level = defaultConfig.Level + } + + if c.EnableStdOut == nil { + c.EnableStdOut = defaultConfig.EnableStdOut + } + + if c.Provider == nil || *c.Provider == "" { + c.Provider = defaultConfig.Provider + } +} + +// IsValid will check a config struct. +func (c *ConfigStruct) IsValid() error { + + if c.EnableStdOut == nil { + return errors.NotValidf("EnableStdOut is empty") + } + + if c.Level == nil { + return errors.NotValidf("Level is empty") + } else if _, err := logrus.ParseLevel(*c.Level); err != nil { + return errors.NotValidf("Invalid Level. Allowed values: %s", strings.Join(convert.StringerSliceToStringSlice(logrus.AllLevels), ", ")) + } + + if c.Provider == nil { + return errors.NotValidf("Provider is empty") + } else if !c.Provider.IsValid() { + return errors.NotValidf("Invalid Provider. Allowed values: %s", strings.Join(convert.StringerSliceToStringSlice(GetListProviderName()), ", ")) + } + + // Validating configuration for the chosen provider. + switch *c.Provider { + case ProviderName_FILE: + if c.FileConfig == nil { + return errors.NotValidf("file configuration is empty") + } + err := c.FileConfig.IsValid() + if err != nil { + return errors.Trace(err) + } + case ProviderName_GELF: + if c.GelfConfig == nil { + return errors.NotValidf("GELF configuration is empty") + } + err := c.GelfConfig.IsValid() + if err != nil { + return errors.Trace(err) + } + case ProviderName_NONE: + if !*c.EnableStdOut { + return errors.NotValidf("Provider set to none with StdOut disabled make no sense") + } + default: + return errors.NotValidf("Provider not handled") + } + + return nil +} diff --git a/log/enum.go b/log/enum.go new file mode 100644 index 0000000..0fd9160 --- /dev/null +++ b/log/enum.go @@ -0,0 +1,36 @@ +package log + +// Provider Name Enum + +// ProviderName is a provider that will receive the logs. +type ProviderName string + +//nolint:exported // keeping the enum simple and readable. +const ( + ProviderName_NONE ProviderName = "NONE" + ProviderName_FILE ProviderName = "FILE" + ProviderName_GELF ProviderName = "GELF" +) + +// IsValid check if the gaven ProviderName is part of the list of handled provider name. +func (e ProviderName) IsValid() bool { + for _, v := range GetListProviderName() { + if e == v { + return true + } + } + return false +} + +func (e ProviderName) String() string { + return string(e) +} + +// GetListProviderName return a the list of possible ProviderName. +func GetListProviderName() []ProviderName { + return []ProviderName{ + ProviderName_NONE, + ProviderName_FILE, + ProviderName_GELF, + } +} diff --git a/log/hooks/file/config.go b/log/hooks/file/config.go new file mode 100644 index 0000000..4799fdd --- /dev/null +++ b/log/hooks/file/config.go @@ -0,0 +1,42 @@ +package file + +import ( + "github.com/asaskevich/govalidator" + "github.com/juju/errors" +) + +// ConfigStruct is the configuration for File Provider. +type ConfigStruct struct { + Filename string `yaml:"filename"` + MaxLines int `yaml:"max_lines"` + MaxSize int `yaml:"max_size"` + Daily bool `yaml:"daily"` + MaxDays int `yaml:"max_days"` + Rotate bool `yaml:"rotate"` + Perm string `yaml:"perm"` + RotatePerm string `yaml:"rotate_perm"` +} + +// IsValid will check that the File configuration is valid. +func (c *ConfigStruct) IsValid() error { + + if c.Filename == "" { + return errors.NotValidf("Path is empty in File configuration") + } else if isValid, _ := govalidator.IsFilePath(c.Filename); !isValid { + return errors.NotValidf("Path is invalid in File configuration") + } + + return nil +} + +// ApplyDefault will check config and apply default on mandatory values. +func (c *ConfigStruct) ApplyDefault() { + + if c.Perm == "" { + c.Perm = "0660" + } + + if c.RotatePerm == "" { + c.RotatePerm = "0440" + } +} diff --git a/log/hooks/file/file.go b/log/hooks/file/file.go new file mode 100644 index 0000000..327ab39 --- /dev/null +++ b/log/hooks/file/file.go @@ -0,0 +1,354 @@ +// Based on https://github.com/gogap/logrus_mate under License Apache License 2.0 +package file + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// RFC5424 log message levels. +const ( + LevelError = iota + LevelWarn + LevelInfo + LevelDebug +) + +const ( + y1 = `0123456789` + y2 = `0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789` + y3 = `0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999` + y4 = `0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789` + mo1 = `000000000111` + mo2 = `123456789012` + d1 = `0000000001111111111222222222233` + d2 = `1234567890123456789012345678901` + h1 = `000000000011111111112222` + h2 = `012345678901234567890123` + mi1 = `000000000011111111112222222222333333333344444444445555555555` + mi2 = `012345678901234567890123456789012345678901234567890123456789` + s1 = `000000000011111111112222222222333333333344444444445555555555` + s2 = `012345678901234567890123456789012345678901234567890123456789` + ns1 = `0123456789` +) + +// fileLogWriter implements LoggerInterface. +// It writes messages by lines limit, file size limit, or time frequency. +type fileLogWriter struct { + sync.RWMutex // write log order by order and atomic incr maxLinesCurLines and maxSizeCurSize + // The opened file + Filename string + fileWriter *os.File + + // Rotate at line + MaxLines int + maxLinesCurLines int + + // Rotate at size + MaxSize int + maxSizeCurSize int + + // Rotate daily + Daily bool + MaxDays int + dailyOpenDate int + dailyOpenTime time.Time + + Rotate bool + Perm string + RotatePerm string + + fileNameOnly, suffix string // like "project.log", project is fileNameOnly and .log is suffix +} + +func InitFileLogWriter(c *ConfigStruct) (*fileLogWriter, error) { + + w := fileLogWriter{ + Filename: c.Filename, + MaxLines: c.MaxLines, + MaxSize: c.MaxSize, + Daily: c.Daily, + MaxDays: c.MaxDays, + Rotate: c.Rotate, + Perm: c.Perm, + RotatePerm: c.RotatePerm, + } + + w.suffix = filepath.Ext(w.Filename) + w.fileNameOnly = strings.TrimSuffix(w.Filename, w.suffix) + if w.suffix == "" { + w.suffix = ".log" + } + err := w.startLogger() + return &w, err +} + +// start file logger. create log file and set to locker-inside file writer. +func (w *fileLogWriter) startLogger() error { + file, err := w.createLogFile() + if err != nil { + return err + } + if w.fileWriter != nil { + w.fileWriter.Close() + } + w.fileWriter = file + return w.initFd() +} + +func (w *fileLogWriter) needRotate(size int, day int) bool { + return (w.MaxLines > 0 && w.maxLinesCurLines >= w.MaxLines) || + (w.MaxSize > 0 && w.maxSizeCurSize >= w.MaxSize) || + (w.Daily && day != w.dailyOpenDate) + +} + +// WriteMsg write logger message into file. +func (w *fileLogWriter) WriteMsg(when time.Time, msg string, level int) error { + h, d := formatTimeHeader(when) + msg = string(h) + msg + "\n" + if w.Rotate { + w.RLock() + if w.needRotate(len(msg), d) { + w.RUnlock() + w.Lock() + if w.needRotate(len(msg), d) { + if err := w.doRotate(when); err != nil { + fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) + } + } + w.Unlock() + } else { + w.RUnlock() + } + } + + w.Lock() + _, err := w.fileWriter.Write([]byte(msg)) + if err == nil { + w.maxLinesCurLines++ + w.maxSizeCurSize += len(msg) + } + w.Unlock() + return err +} + +func (w *fileLogWriter) createLogFile() (*os.File, error) { + // Open the log file + perm, err := strconv.ParseInt(w.Perm, 8, 64) + if err != nil { + return nil, err + } + fd, err := os.OpenFile(w.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(perm)) + if err == nil { + // Make sure file perm is user set perm cause of `os.OpenFile` will obey umask + os.Chmod(w.Filename, os.FileMode(perm)) + } + return fd, err +} + +func (w *fileLogWriter) initFd() error { + fd := w.fileWriter + fInfo, err := fd.Stat() + if err != nil { + return fmt.Errorf("get stat err: %s", err) + } + w.maxSizeCurSize = int(fInfo.Size()) + w.dailyOpenTime = time.Now() + w.dailyOpenDate = w.dailyOpenTime.Day() + w.maxLinesCurLines = 0 + if w.Rotate { + if w.Daily { + go w.dailyRotate(w.dailyOpenTime) + } + if fInfo.Size() > 0 && w.MaxLines > 0 { + count, err := w.lines() + if err != nil { + return err + } + w.maxLinesCurLines = count + } + } + return nil +} + +func (w *fileLogWriter) dailyRotate(openTime time.Time) { + y, m, d := openTime.Add(24 * time.Hour).Date() + nextDay := time.Date(y, m, d, 0, 0, 0, 0, openTime.Location()) + tm := time.NewTimer(time.Duration(nextDay.UnixNano() - openTime.UnixNano() + 100)) + <-tm.C + w.Lock() + if w.needRotate(0, time.Now().Day()) { + if err := w.doRotate(time.Now()); err != nil { + fmt.Fprintf(os.Stderr, "FileLogWriter(%q): %s\n", w.Filename, err) + } + } + w.Unlock() +} + +func (w *fileLogWriter) lines() (int, error) { + fd, err := os.Open(w.Filename) + if err != nil { + return 0, err + } + defer fd.Close() + + buf := make([]byte, 32768) // 32k + count := 0 + lineSep := []byte{'\n'} + + for { + c, err := fd.Read(buf) + if err != nil && err != io.EOF { + return count, err + } + + count += bytes.Count(buf[:c], lineSep) + + if err == io.EOF { + break + } + } + + return count, nil +} + +// DoRotate means it need to write file in new file. +// new file name like xx.2013-01-01.log (daily) or xx.001.log (by line or size) +func (w *fileLogWriter) doRotate(logTime time.Time) error { + // file exists + // Find the next available number + num := 1 + fName := "" + rotatePerm, err := strconv.ParseInt(w.RotatePerm, 8, 64) + if err != nil { + return err + } + + _, err = os.Lstat(w.Filename) + if err != nil { + //even if the file is not exist or other ,we should RESTART the logger + goto RESTART_LOGGER + } + + if w.MaxLines > 0 || w.MaxSize > 0 { + for ; err == nil && num <= 999; num++ { + fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", logTime.Format("2006-01-02"), num, w.suffix) + _, err = os.Lstat(fName) + } + } else { + fName = fmt.Sprintf("%s.%s%s", w.fileNameOnly, w.dailyOpenTime.Format("2006-01-02"), w.suffix) + _, err = os.Lstat(fName) + for ; err == nil && num <= 999; num++ { + fName = w.fileNameOnly + fmt.Sprintf(".%s.%03d%s", w.dailyOpenTime.Format("2006-01-02"), num, w.suffix) + _, err = os.Lstat(fName) + } + } + // return error if the last file checked still existed + if err == nil { + return fmt.Errorf("Rotate: Cannot find free log number to rename %s", w.Filename) + } + + // close fileWriter before rename + w.fileWriter.Close() + + // Rename the file to its new found name + // even if occurs error,we MUST guarantee to restart new logger + err = os.Rename(w.Filename, fName) + if err != nil { + goto RESTART_LOGGER + } + + err = os.Chmod(fName, os.FileMode(rotatePerm)) + +RESTART_LOGGER: + + startLoggerErr := w.startLogger() + go w.deleteOldLog() + + if startLoggerErr != nil { + return fmt.Errorf("Rotate StartLogger: %s", startLoggerErr) + } + if err != nil { + return fmt.Errorf("Rotate: %s", err) + } + return nil +} + +func (w *fileLogWriter) deleteOldLog() { + dir := filepath.Dir(w.Filename) + filepath.Walk(dir, func(path string, info os.FileInfo, err error) (returnErr error) { + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stderr, "Unable to delete old log '%s', error: %v\n", path, r) + } + }() + + if info == nil { + return + } + + if !info.IsDir() && info.ModTime().Add(24*time.Hour*time.Duration(w.MaxDays)).Before(time.Now()) { + if strings.HasPrefix(filepath.Base(path), filepath.Base(w.fileNameOnly)) && + strings.HasSuffix(filepath.Base(path), w.suffix) { + os.Remove(path) + } + } + return + }) +} + +// Destroy close the file description, close file writer. +func (w *fileLogWriter) Destroy() { + w.fileWriter.Close() +} + +// Flush flush file logger. +// there are no buffering messages in file logger in memory. +// flush file means sync file from disk. +func (w *fileLogWriter) Flush() { + w.fileWriter.Sync() +} + +func formatTimeHeader(when time.Time) ([]byte, int) { + y, mo, d := when.Date() + h, mi, s := when.Clock() + ns := when.Nanosecond() / 1000000 + //len("2006/01/02 15:04:05.123 ")==24 + var buf [24]byte + + buf[0] = y1[y/1000%10] + buf[1] = y2[y/100] + buf[2] = y3[y-y/100*100] + buf[3] = y4[y-y/100*100] + buf[4] = '/' + buf[5] = mo1[mo-1] + buf[6] = mo2[mo-1] + buf[7] = '/' + buf[8] = d1[d-1] + buf[9] = d2[d-1] + buf[10] = ' ' + buf[11] = h1[h] + buf[12] = h2[h] + buf[13] = ':' + buf[14] = mi1[mi] + buf[15] = mi2[mi] + buf[16] = ':' + buf[17] = s1[s] + buf[18] = s2[s] + buf[19] = '.' + buf[20] = ns1[ns/100] + buf[21] = ns1[ns%100/10] + buf[22] = ns1[ns%10] + + buf[23] = ' ' + + return buf[0:], d +} diff --git a/log/hooks/file/hook.go b/log/hooks/file/hook.go new file mode 100644 index 0000000..7a29ae6 --- /dev/null +++ b/log/hooks/file/hook.go @@ -0,0 +1,110 @@ +// Based on https://github.com/gogap/logrus_mate under License Apache License 2.0 +package file + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/juju/errors" + "github.com/sirupsen/logrus" +) + +func NewHook(c *ConfigStruct) (logrus.Hook, error) { + + c.ApplyDefault() + + err := c.IsValid() + if err != nil { + return nil, errors.Trace(err) + } + + dir := filepath.Dir(c.Filename) + + err = os.MkdirAll(dir, 0755) + if err != nil { + return nil, nil + } + + w, err := InitFileLogWriter(c) + if err != nil { + return nil, errors.Trace(err) + } + + return &FileHook{W: w}, nil +} + +type FileHook struct { + W *fileLogWriter +} + +func (p *FileHook) Fire(entry *logrus.Entry) (err error) { + message, err := getMessage(entry) + + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) + return err + } + + now := time.Now() + + switch entry.Level { + case logrus.PanicLevel: + fallthrough + case logrus.FatalLevel: + fallthrough + case logrus.ErrorLevel: + return p.W.WriteMsg(now, fmt.Sprintf("[ERROR] %s", message), LevelError) + case logrus.WarnLevel: + return p.W.WriteMsg(now, fmt.Sprintf("[WARN] %s", message), LevelWarn) + case logrus.InfoLevel: + return p.W.WriteMsg(now, fmt.Sprintf("[INFO] %s", message), LevelInfo) + case logrus.DebugLevel: + return p.W.WriteMsg(now, fmt.Sprintf("[DEBUG] %s", message), LevelDebug) + default: + return nil + } + + return +} + +func (p *FileHook) Levels() []logrus.Level { + return []logrus.Level{ + logrus.PanicLevel, + logrus.FatalLevel, + logrus.ErrorLevel, + logrus.WarnLevel, + logrus.InfoLevel, + logrus.DebugLevel, + } +} + +func getMessage(entry *logrus.Entry) (message string, err error) { + message = message + fmt.Sprintf("%s", entry.Message) + for k, v := range entry.Data { + if !strings.HasPrefix(k, "err_") { + message = message + fmt.Sprintf("\n%v:%v", k, v) + } + } + if errCode, exist := entry.Data["err_code"]; exist { + + ns, _ := entry.Data["err_ns"] + ctx, _ := entry.Data["err_ctx"] + id, _ := entry.Data["err_id"] + tSt, _ := entry.Data["err_stack"] + st, _ := tSt.(string) + st = strings.Replace(st, "\n", "\n\t\t", -1) + + buf := bytes.NewBuffer(nil) + buf.WriteString(fmt.Sprintf("\tid:\n\t\t%s#%d:%s\n", ns, errCode, id)) + buf.WriteString(fmt.Sprintf("\tcontext:\n\t\t%s\n", ctx)) + buf.WriteString(fmt.Sprintf("\tstacktrace:\n\t\t%s", st)) + + message = message + fmt.Sprintf("%v", buf.String()) + } + + return +} diff --git a/log/hooks/gelf/config.go b/log/hooks/gelf/config.go new file mode 100644 index 0000000..04ce500 --- /dev/null +++ b/log/hooks/gelf/config.go @@ -0,0 +1,32 @@ +package gelf + +import ( + "strconv" + + "github.com/asaskevich/govalidator" + "github.com/juju/errors" +) + +// ConfigStruct is the configuration for GELF Provider. +type ConfigStruct struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + ExtrasFields map[string]interface{} `yaml:"extra_fields"` +} + +// IsValid will check that the Gelf configuration is valid. +func (c *ConfigStruct) IsValid() error { + if c.Host == "" { + return errors.NotValidf("Host is empty in GELF configuration") + } else if !govalidator.IsDNSName(c.Host) && !govalidator.IsIP(c.Host) { + return errors.NotValidf("Host is invalid in GELF configuration. Should be a DNS name or an IP") + } + + if c.Port == 0 { + return errors.NotValidf("Port is empty in GELF configuration") + } else if !govalidator.IsPort(strconv.Itoa(c.Port)) { + return errors.NotValidf("Port is invalid in GELF configuration") + } + + return nil +} diff --git a/log/hooks/gelf/hook.go b/log/hooks/gelf/hook.go new file mode 100644 index 0000000..e5ecce8 --- /dev/null +++ b/log/hooks/gelf/hook.go @@ -0,0 +1,28 @@ +package gelf + +import ( + "fmt" + + "github.com/juju/errors" + "github.com/sirupsen/logrus" + + graylog "github.com/gemnasium/logrus-graylog-hook/v3" +) + +// NewHook will create a Gelf Hook for logrus. +func NewHook(c *ConfigStruct) (logrus.Hook, error) { + + err := c.IsValid() + if err != nil { + return nil, errors.Trace(err) + } + + hook := graylog.NewAsyncGraylogHook( + fmt.Sprintf("%s:%d", c.Host, c.Port), + c.ExtrasFields, + ) + + defer hook.Flush() + + return hook, nil +} diff --git a/log/log.go b/log/log.go new file mode 100644 index 0000000..c648773 --- /dev/null +++ b/log/log.go @@ -0,0 +1,106 @@ +package log + +import ( + "io" + "os" + "time" + + "github.com/juju/errors" + "github.com/sirupsen/logrus" + + "git.dev.m-and-m.ovh/mderasse/gocommon/log/hooks/file" + "git.dev.m-and-m.ovh/mderasse/gocommon/log/hooks/gelf" +) + +// InitLog will try to initialize logger by trying to retrieve config from multiple source. +func InitLog() error { + + // loading configuration + c, err := loadConfig() + if err != nil { + return errors.Trace(err) + } + + return initFromSource(c) +} + +// InitLogFromCustomVaultSecret will initialize logger with a vault secret. +func InitLogFromCustomVaultSecret(secret string) error { + + c, err := loadConfigFromVault(secret) + if err != nil { + return errors.Trace(err) + } + + return initFromSource(c) +} + +// InitLogFromCustomFile will initialize logger with a config file. +func InitLogFromCustomFile(path string) error { + + c, err := loadConfigFromFile(path) + if err != nil { + return errors.Trace(err) + } + + return initFromSource(c) +} + +func initFromSource(c *ConfigStruct) error { + err := c.applyEnv() + if err != nil { + return errors.Trace(err) + } + + c.applyDefault() + + return InitLogFromCustomConfig(c) +} + +// InitLogFromCustomConfig will initialize logger from a gaven config. +func InitLogFromCustomConfig(c *ConfigStruct) error { + + err := c.IsValid() + if err != nil { + return errors.Trace(err) + } + + level, err := logrus.ParseLevel(*c.Level) + if err != nil { + return err + } + + // init logger + log := logrus.New() + log.SetLevel(level) + + if *c.EnableStdOut { + log.SetOutput(os.Stdout) + } else { + log.SetOutput(io.Discard) + } + + switch *c.Provider { + case ProviderName_FILE: + hook, err := file.NewHook(c.FileConfig) + if err != nil { + return errors.Trace(err) + } + log.AddHook(hook) + case ProviderName_GELF: + hook, err := gelf.NewHook(c.GelfConfig) + if err != nil { + return errors.Trace(err) + } + log.AddHook(hook) + case ProviderName_NONE: + // Do Nothing + default: + } + + for i := 0; i < 50; i++ { + log.Infof("Test %d", i) + time.Sleep(1 * time.Second) + } + return nil +}